Modify subex to take JSON split into "data"

Currently no way to reassemble the data on the other side
Much of the potential data cannot be interacted with meaningfully, only the string functionality is implemented
Should rename data to something else

1 files changed, 60 insertions, 29 deletions

diff --git a/subex/main.go b/subex/main.go
index 3ae0618..9dbe5df 100644
--- a/subex/main.go
+++ b/subex/main.go
@@ -3,24 +3,36 @@ package subex
 import (
 	"os"
 	"fmt"
-	"io"
+	"bufio"
+	"main/walk"
 )
 
+// A part of an insertion, either a datum or a slot from which to load
+// TODO rename this
 type TransducerOutput interface {
-	build(Store) string
+	// Given the current store, return the []Datum produced by the TransducerOutput
+	build(Store) []walk.Datum
 }
 
-type TransducerReplacementRune rune
-func (replacement TransducerReplacementRune) build(store Store) string {
-	return string(replacement)
+// A TransducerOutput which is just a datum literal
+type TransducerReplacementRune struct {
+	datum walk.Datum
+}
+func (replacement TransducerReplacementRune) build(store Store) []walk.Datum {
+	return []walk.Datum{replacement.datum}
 }
 
-type TransducerReplacementLoad rune
-func (replacement TransducerReplacementLoad) build(store Store) string {
-	return store[rune(replacement)]
+// A TransducerOutput which is a slot that is loaded from
+type TransducerReplacementLoad struct {
+	datum walk.Datum
+}
+func (replacement TransducerReplacementLoad) build(store Store) []walk.Datum {
+	return store[replacement.datum]
 }
 
-type Store map[rune]string
+// Where slots are stored
+type Store map[walk.Datum][]walk.Datum
+// Return a new store with all the data from this one
 func (store Store) clone() Store {
 	newStore := make(Store)
 	for key, val := range store {
@@ -28,29 +40,36 @@ func (store Store) clone() Store {
 	}
 	return newStore
 }
-func (store Store) withValue(key rune, value string) Store {
+// Return a copy of this store but with an additional slot set
+func (store Store) withValue(key walk.Datum, value []walk.Datum) Store {
 	newStore := store.clone()
 	newStore[key] = value
 	return newStore
 }
 
+// Compile the SubexAST into a transducer SubexState that can be run
 func CompileTransducer(transducerAst SubexAST) SubexState {
 	return transducerAst.compileWith(SubexNoneState{})
 }
 
+// One branch of subex execution
 type SubexBranch struct {
+	// Content of slots in this branch
 	store Store
+	// State in this branch
 	state SubexState
-	output string
+	// Output so far in this branch
+	output []walk.Datum
 }
-func (pair SubexBranch) eat(char rune) []SubexBranch {
+// Read a single character and return all the branches resulting from this branch consuming it
+func (pair SubexBranch) eat(char walk.Datum) []SubexBranch {
 	states := pair.state.eat(pair.store, char)
 	for i := range states {
-		states[i].output = pair.output + states[i].output
+		states[i].output = append(pair.output, states[i].output...)
 	}
 	return states
 }
-func (pair SubexBranch) accepting() []string {
+func (pair SubexBranch) accepting() [][]walk.Datum {
 	return pair.state.accepting(pair.store)
 }
 
@@ -59,6 +78,8 @@ func equalStates(left SubexBranch, right SubexBranch) bool {
 	return left.state == right.state
 }
 
+// If two branches have the same state, only the first has a chance of being successful
+// This function removes all of the pointless execution branches to save execution time
 func pruneStates(states []SubexBranch) (newStates []SubexBranch) {
 	outer: for _, state := range states {
 		for _, newState := range newStates {
@@ -71,44 +92,54 @@ func pruneStates(states []SubexBranch) (newStates []SubexBranch) {
 	return newStates
 }
 
-func RunTransducer(transducer SubexState, input string) (output string, err bool) {
+// Run the subex transducer
+func RunTransducer(transducer SubexState, input <-chan walk.Datum) (output []walk.Datum, err bool) {
 	states := []SubexBranch{{
 		state: transducer,
-		output: "",
+		output: nil,
 		store: make(Store),
 	}}
-	for _, char := range input {
-		fmt.Printf("Running with %d states\n", len(states))
+	for piece := range input {
 		var newStates []SubexBranch
 		for _, state := range states {
-			newStates = append(newStates, state.eat(char)...)
+			newStates = append(newStates, state.eat(piece)...)
 		}
 		states = pruneStates(newStates)
 	}
 	for _, state := range states {
 		outputEnds := state.accepting()
 		for _, outputEnd := range outputEnds {
-			return state.output + outputEnd, false
+			return append(state.output, outputEnd...), false
 		}
 	}
-	return "", true
+	return nil, true
 }
 
 func Main() {
 	if len(os.Args) != 2 {
 		panic("Expected: program [subex]")
 	}
-	inputBytes, inputErr := io.ReadAll(os.Stdin)
-	input := string(inputBytes)
-	if inputErr != nil {
-		fmt.Println("Error reading")
+	stdin := bufio.NewReader(os.Stdin);
+	jsonStream := walk.Json(stdin);
+	var tokens []walk.WalkValue;
+	for token := range jsonStream {
+		tokens = append(tokens, token.Value);
 	}
 	program := os.Args[1]
 	ast := Parse(program)
 	transducer := CompileTransducer(ast)
-	output, err := RunTransducer(transducer, input)
-	if err {
-		output = input
+	pieces := make(chan walk.Datum)
+	go func(out chan<- walk.Datum, input []walk.WalkValue) {
+		for _, value := range input {
+			value.Pieces(out)
+		}
+		close(out)
+	}(pieces, tokens)
+	output, err := RunTransducer(transducer, pieces)
+	// TODO recombine data into values and then convert into items with empty paths
+	if !err {
+		fmt.Print(output)
+	} else {
+		fmt.Print("Error")
 	}
-	fmt.Print(output)
 }