1 files changed, 49 insertions, 34 deletions
diff --git a/subex/subexstate.go b/subex/subexstate.go
index 2e613e8..9e0d61a 100644
--- a/subex/subexstate.go
+++ b/subex/subexstate.go
@@ -1,35 +1,41 @@
 package subex
 
 import (
-	"strings"
+	"main/walk"
 )
 
+// A state of execution for the transducer
 type SubexState interface {
-	eat(store Store, char rune) []SubexBranch
-	accepting(store Store) []string
+	// Eat a datum and transition to any number of new states
+	eat(store Store, char walk.Datum) []SubexBranch
+	// Find accepting states reachable through epsilon transitions and return their outputs
+	accepting(store Store) [][]walk.Datum
 }
 
+// Try first, if it fails then try second
 type SubexGroupState struct {
 	first, second SubexState
 }
-func (state SubexGroupState) eat(store Store, char rune) []SubexBranch {
+func (state SubexGroupState) eat(store Store, char walk.Datum) []SubexBranch {
 	otherStore := store.clone()
 	return append(state.first.eat(store, char), state.second.eat(otherStore, char)...)
 }
-func (state SubexGroupState) accepting(store Store) []string {
+func (state SubexGroupState) accepting(store Store) [][]walk.Datum {
 	return append(state.first.accepting(store), state.second.accepting(store)...)
 }
 
+// Run the match machine and store the output in a slot for later use
+// Output nothing
 type SubexStoreState struct {
 	match SubexState
 	slot rune
 	next SubexState
-	toStore string
+	toStore []walk.Datum
 }
-func (state SubexStoreState) eat(store Store, char rune) (nextStates []SubexBranch) {
+func (state SubexStoreState) eat(store Store, char walk.Datum) (nextStates []SubexBranch) {
 	acceptedOutputs := state.match.accepting(store)
 	for _, acceptedOutput := range acceptedOutputs {
-		nextStore := store.withValue(state.slot, state.toStore + acceptedOutput)
+		nextStore := store.withValue(state.slot, append(state.toStore, acceptedOutput...))
 		nextStates = append(nextStates, state.next.eat(nextStore.clone(), char)...)
 	}
 	nextMatchStates := state.match.eat(store.clone(), char)
@@ -39,107 +45,116 @@ func (state SubexStoreState) eat(store Store, char rune) (nextStates []SubexBran
 				match: matchState.state,
 				slot: state.slot,
 				next: state.next,
-				toStore: state.toStore + matchState.output,
+				toStore: append(state.toStore, matchState.output...),
 			},
-			output: "",
+			output: nil,
 			store: store.clone(),
 		})
 	}
 	return nextStates
 }
-func (state SubexStoreState) accepting(store Store) (outputs []string) {
+func (state SubexStoreState) accepting(store Store) (outputs [][]walk.Datum) {
 	acceptedOutputs := state.match.accepting(store)
 	for _, acceptedOutput := range acceptedOutputs {
-		nextStore := store.withValue(state.slot, state.toStore + acceptedOutput)
+		nextStore := store.withValue(state.slot, append(state.toStore, acceptedOutput...))
 		outputs = append(outputs, state.next.accepting(nextStore)...)
 	}
 	return outputs
 }
 
+// Don't read in anything, just output the series of data and slots specified
 type SubexOutputState struct {
 	content []TransducerOutput
 	next SubexState
 }
-func (state SubexOutputState) build(store Store) string {
-	var builder strings.Builder
+// Given a store, return what is outputted by an epsilon transition from this state
+func (state SubexOutputState) build(store Store) []walk.Datum {
+	var result []walk.Datum
 	for _, part := range state.content {
-		builder.WriteString(part.build(store))
+		result = append(result, part.build(store)...)
 	}
-	return builder.String()
+	return result
 }
-func (state SubexOutputState) eat(store Store, char rune) []SubexBranch {
+func (state SubexOutputState) eat(store Store, char walk.Datum) []SubexBranch {
 	content := state.build(store)
 	nextStates := state.next.eat(store, char)
 	for i := range nextStates {
-		nextStates[i].output = content + nextStates[i].output
+		nextStates[i].output = append(content, nextStates[i].output...)
 	}
 	return nextStates
 }
-func (state SubexOutputState) accepting(store Store) []string {
+func (state SubexOutputState) accepting(store Store) [][]walk.Datum {
 	content := state.build(store)
 	outputs := state.next.accepting(store)
 	for i := range outputs {
-		outputs[i] = content + outputs[i]
+		outputs[i] = append(content, outputs[i]...)
 	}
 	return outputs
 }
 
+// A final state, transitions to nothing but is accepting
 type SubexNoneState struct {}
-func (state SubexNoneState) eat(store Store, char rune) []SubexBranch {
+func (state SubexNoneState) eat(store Store, char walk.Datum) []SubexBranch {
 	return nil
 }
-func (state SubexNoneState) accepting(store Store) []string {
-	return []string{""}
+func (state SubexNoneState) accepting(store Store) [][]walk.Datum {
+	return [][]walk.Datum{nil}
 }
 
+// Read in a specific datum and output it
+// TODO rename to better reflect datum instead of rune
 type SubexCopyRuneState struct {
-	rune rune
+	rune walk.Datum
 	next SubexState
 }
-func (state SubexCopyRuneState) eat(store Store, char rune) []SubexBranch {
+func (state SubexCopyRuneState) eat(store Store, char walk.Datum) []SubexBranch {
+	// TODO can I compare Datum values with == ?
 	if char == state.rune {
 		return []SubexBranch{{
 			state: state.next,
-			output: string(char),
+			output: []walk.Datum{char},
 			store: store,
 		}}
 	}
 	return nil
 }
-func (state SubexCopyRuneState) accepting(store Store) []string {
+func (state SubexCopyRuneState) accepting(store Store) [][]walk.Datum {
 	return nil
 }
 
+// Read in any datum and output it
 type SubexCopyAnyState struct {
 	next SubexState
 }
-func (state SubexCopyAnyState) eat(store Store, char rune) []SubexBranch {
+func (state SubexCopyAnyState) eat(store Store, char walk.Datum) []SubexBranch {
 	return []SubexBranch{{
 		state: state.next,
-		output: string(char),
+		output: []walk.Datum{char},
 		store: store,
 	}}
 }
-func (state SubexCopyAnyState) accepting(store Store) []string {
+func (state SubexCopyAnyState) accepting(store Store) [][]walk.Datum {
 	return nil
 }
 
+// Read in a datum and apply a map to generate a datum to output
+// If the input isn't in the map transition to nothing
 type SubexRangeState struct {
-	parts map[rune]rune
+	parts map[walk.Datum]walk.Datum
 	next SubexState
 }
-func (state SubexRangeState) eat(store Store, char rune) []SubexBranch {
+func (state SubexRangeState) eat(store Store, char walk.Datum) []SubexBranch {
 	out, exists := state.parts[char]
 	if !exists {
 		return nil
 	} else {
 		return []SubexBranch{{
 			state: state.next,
-			output: string(out),
+			output: []walk.Datum{out},
 			store: store,
 		}}
 	}
 }
-func (state SubexRangeState) accepting(store Store) []string {
+func (state SubexRangeState) accepting(store Store) [][]walk.Datum {
 	return nil
 }