1 files changed, 60 insertions, 29 deletions

diff --git a/subex/main.go b/subex/main.go
index 3ae0618..9dbe5df 100644
--- a/subex/main.go
+++ b/subex/main.go
@@ -3,24 +3,36 @@ package subex
 import (
 	"os"
 	"fmt"
-	"io"
+	"bufio"
+	"main/walk"
 )
 
+// A part of an insertion, either a datum or a slot from which to load
+// TODO rename this
 type TransducerOutput interface {
-	build(Store) string
+	// Given the current store, return the []Datum produced by the TransducerOutput
+	build(Store) []walk.Datum
 }
 
-type TransducerReplacementRune rune
-func (replacement TransducerReplacementRune) build(store Store) string {
-	return string(replacement)
+// A TransducerOutput which is just a datum literal
+type TransducerReplacementRune struct {
+	datum walk.Datum
+}
+func (replacement TransducerReplacementRune) build(store Store) []walk.Datum {
+	return []walk.Datum{replacement.datum}
 }
 
-type TransducerReplacementLoad rune
-func (replacement TransducerReplacementLoad) build(store Store) string {
-	return store[rune(replacement)]
+// A TransducerOutput which is a slot that is loaded from
+type TransducerReplacementLoad struct {
+	datum walk.Datum
+}
+func (replacement TransducerReplacementLoad) build(store Store) []walk.Datum {
+	return store[replacement.datum]
 }
 
-type Store map[rune]string
+// Where slots are stored
+type Store map[walk.Datum][]walk.Datum
+// Return a new store with all the data from this one
 func (store Store) clone() Store {
 	newStore := make(Store)
 	for key, val := range store {
@@ -28,29 +40,36 @@ func (store Store) clone() Store {
 	}
 	return newStore
 }
-func (store Store) withValue(key rune, value string) Store {
+// Return a copy of this store but with an additional slot set
+func (store Store) withValue(key walk.Datum, value []walk.Datum) Store {
 	newStore := store.clone()
 	newStore[key] = value
 	return newStore
 }
 
+// Compile the SubexAST into a transducer SubexState that can be run
 func CompileTransducer(transducerAst SubexAST) SubexState {
 	return transducerAst.compileWith(SubexNoneState{})
 }
 
+// One branch of subex execution
 type SubexBranch struct {
+	// Content of slots in this branch
 	store Store
+	// State in this branch
 	state SubexState
-	output string
+	// Output so far in this branch
+	output []walk.Datum
 }
-func (pair SubexBranch) eat(char rune) []SubexBranch {
+// Read a single character and return all the branches resulting from this branch consuming it
+func (pair SubexBranch) eat(char walk.Datum) []SubexBranch {
 	states := pair.state.eat(pair.store, char)
 	for i := range states {
-		states[i].output = pair.output + states[i].output
+		states[i].output = append(pair.output, states[i].output...)
 	}
 	return states
 }
-func (pair SubexBranch) accepting() []string {
+func (pair SubexBranch) accepting() [][]walk.Datum {
 	return pair.state.accepting(pair.store)
 }
 
@@ -59,6 +78,8 @@ func equalStates(left SubexBranch, right SubexBranch) bool {
 	return left.state == right.state
 }
 
+// If two branches have the same state, only the first has a chance of being successful
+// This function removes all of the pointless execution branches to save execution time
 func pruneStates(states []SubexBranch) (newStates []SubexBranch) {
 	outer: for _, state := range states {
 		for _, newState := range newStates {
@@ -71,44 +92,54 @@ func pruneStates(states []SubexBranch) (newStates []SubexBranch) {
 	return newStates
 }
 
-func RunTransducer(transducer SubexState, input string) (output string, err bool) {
+// Run the subex transducer
+func RunTransducer(transducer SubexState, input <-chan walk.Datum) (output []walk.Datum, err bool) {
 	states := []SubexBranch{{
 		state: transducer,
-		output: "",
+		output: nil,
 		store: make(Store),
 	}}
-	for _, char := range input {
-		fmt.Printf("Running with %d states\n", len(states))
+	for piece := range input {
 		var newStates []SubexBranch
 		for _, state := range states {
-			newStates = append(newStates, state.eat(char)...)
+			newStates = append(newStates, state.eat(piece)...)
 		}
 		states = pruneStates(newStates)
 	}
 	for _, state := range states {
 		outputEnds := state.accepting()
 		for _, outputEnd := range outputEnds {
-			return state.output + outputEnd, false
+			return append(state.output, outputEnd...), false
 		}
 	}
-	return "", true
+	return nil, true
 }
 
 func Main() {
 	if len(os.Args) != 2 {
 		panic("Expected: program [subex]")
 	}
-	inputBytes, inputErr := io.ReadAll(os.Stdin)
-	input := string(inputBytes)
-	if inputErr != nil {
-		fmt.Println("Error reading")
+	stdin := bufio.NewReader(os.Stdin);
+	jsonStream := walk.Json(stdin);
+	var tokens []walk.WalkValue;
+	for token := range jsonStream {
+		tokens = append(tokens, token.Value);
 	}
 	program := os.Args[1]
 	ast := Parse(program)
 	transducer := CompileTransducer(ast)
-	output, err := RunTransducer(transducer, input)
-	if err {
-		output = input
+	pieces := make(chan walk.Datum)
+	go func(out chan<- walk.Datum, input []walk.WalkValue) {
+		for _, value := range input {
+			value.Pieces(out)
+		}
+		close(out)
+	}(pieces, tokens)
+	output, err := RunTransducer(transducer, pieces)
+	// TODO recombine data into values and then convert into items with empty paths
+	if !err {
+		fmt.Print(output)
+	} else {
+		fmt.Print("Error")
 	}
-	fmt.Print(output)
 }