diff options
| -rw-r--r-- | main/command.go | 37 | ||||
| -rw-r--r-- | main/lex.go | 2 | ||||
| -rw-r--r-- | main/main_test.go | 69 | ||||
| -rw-r--r-- | main/parse.go | 4 | ||||
| -rw-r--r-- | subex/filter.go | 20 | ||||
| -rw-r--r-- | subex/parse.go | 72 | ||||
| -rw-r--r-- | subex/subexast.go | 12 | ||||
| -rw-r--r-- | subex/subexstate.go | 78 |
8 files changed, 180 insertions, 114 deletions
diff --git a/main/command.go b/main/command.go index 38e1c95..bbbb036 100644 --- a/main/command.go +++ b/main/command.go @@ -138,9 +138,42 @@ func (cmd MergeCommand) String() string { return "m" } -type FullMergeCommand struct {} +type FullMergeCommand struct { + subex subex.Transducer +} func (cmd FullMergeCommand) exec(state *ProgramState) { - panic("Unimplemented") + _, notOk := runSubex(cmd.subex, state.value) + if notOk { + state.pc++ + return + } + if !state.start { + state.pc += 2 + return + } + + for { + item, err := state.Read() + if err != nil { + panic("Missing next value") + } + + _, nonTerminal := runSubex(cmd.subex, []walk.Value{item.Value}) + + state.value = append( + state.value[:len(state.value) - 1], + walk.Merge(state.value[len(state.value) - 1], item.Value)... + ) + + if !nonTerminal && item.End { + state.prevStart = item.PrevStart + state.start = item.Start + state.end = item.End + state.nextEnd = item.NextEnd + state.pc += 2 + return + } + } } func (cmd FullMergeCommand) String() string { return "M" diff --git a/main/lex.go b/main/lex.go index 8e66890..da517cc 100644 --- a/main/lex.go +++ b/main/lex.go @@ -180,7 +180,7 @@ func lexCommand(l *lexer) stateFunc { case '}': l.emit(TokenRBrace) return lexCommand - case 's', 'S': + case 's', 'S', 'M': l.emit(TokenCommand) return lexSubstitution case 'x', 'X', 'y', 'Y', 'z', 'Z', 'n', 'N': diff --git a/main/main_test.go b/main/main_test.go index 1510497..076693d 100644 --- a/main/main_test.go +++ b/main/main_test.go @@ -5,9 +5,11 @@ import ( "testing" ) -var miscInput string = `{"something":{"nested":"Here is my test value"},"array":["Hello","world","these","are","values"],"people":[{"first_name":"Charlie","last_name":"Johnson","age":22},{"first_name":"Tom","last_name":"Johnson","age":18},{"first_name":"Charlie","last_name":"Chaplin","age":122},{"first_name":"John","last_name":"Johnson","age":48}]}` +const miscInput string = `{"something":{"nested":"Here is my test value"},"array":["Hello","world","these","are","values"],"people":[{"first_name":"Charlie","last_name":"Johnson","age":22},{"first_name":"Tom","last_name":"Johnson","age":18},{"first_name":"Charlie","last_name":"Chaplin","age":122},{"first_name":"John","last_name":"Johnson","age":48}]}` +const mixedArray string = `{"array":["first",null,3,{"name":"second"},"third"]}` +const mixedArray2 string = `{"array":["first",null,3,"second",{"name":"third"}]}` -func TestMain(t *testing.T) { +func TestSpecificCases(t *testing.T) { type test struct { name string program string @@ -86,6 +88,69 @@ func TestMain(t *testing.T) { input: miscInput, expected: `["Charlie Johnson","Tom Johnson","Charlie Chaplin","John Johnson"]`, }, + { + name: "Get full names with merge full command", + program: "s/#(\"people\"$_ :(): )-/p M/#( \"people\" @( . #()# )@ )#/{ s/#( \"people\"$_ @( . #[ \"first_name\" \".{-0}$a\" | \"last_name\" \".{-0}$b\" | .. $_]- `\"$a $b\"` )@ )-/p }", + quiet: true, + input: miscInput, + expected: `["Charlie Johnson","Tom Johnson","Charlie Chaplin","John Johnson"]`, + }, + { + name: "Verbose concat array values", + program: "as/#( \"array\"$_ :(): )-/{ :s N/#( .$_ . )-/{ es/.{-0}:():/be mbs } :em s/:( -( ~(.{-0}` `)-{-0} ~(.{-0})- )~ )-/p }", + quiet: true, + input: miscInput, + expected: `"Hello world these are values"`, + }, + { + name: "Short concat array values", + program: "M/#( \"array\" :(): )#/{ s/#( \"array\"$_ :( .{-0} )- )-/ s/-( ~(.{-0}` `)-{-0} ~(.{-0})- )~/p }", + quiet: true, + input: miscInput, + expected: `"Hello world these are values"`, + }, + { + name: "Drop first element of array", + program: `s/#( "people" @( 0 . )@ )#/d`, + input: miscInput, + expected: `{"something":{"nested":"Here is my test value"},"array":["Hello","world","these","are","values"],"people":[{"first_name":"Tom","last_name":"Johnson","age":18},{"first_name":"Charlie","last_name":"Chaplin","age":122},{"first_name":"John","last_name":"Johnson","age":48}]}`, + }, + { + name: "Drop last element of array", + program: `M/#( "people" @( . , )@ )#/{ Ed }`, + input: miscInput, + expected: `{"something":{"nested":"Here is my test value"},"array":["Hello","world","these","are","values"],"people":[{"first_name":"Charlie","last_name":"Johnson","age":22},{"first_name":"Tom","last_name":"Johnson","age":18},{"first_name":"Charlie","last_name":"Chaplin","age":122}]}`, + }, + { + name: "Drop last element of simple array", + program: `s/#( "array" @( . . )@ )#/{ Ed }`, + input: miscInput, + expected: `{"something":{"nested":"Here is my test value"},"array":["Hello","world","these","are"],"people":[{"first_name":"Charlie","last_name":"Johnson","age":22},{"first_name":"Tom","last_name":"Johnson","age":18},{"first_name":"Charlie","last_name":"Chaplin","age":122},{"first_name":"John","last_name":"Johnson","age":48}]}`, + }, + { + name: "Drop last element of mixed array", + program: `M/#( "array" @( . , )@ )#/{ Ed }`, + input: mixedArray, + expected: `{"array":["first",null,3,{"name":"second"}]}`, + }, + { + name: "Drop last element of mixed array 2", + program: `M/#( "array" @( . , )@ )#/{ Ed }`, + input: mixedArray2, + expected: `{"array":["first",null,3,"second"]}`, + }, + { + name: "Prepend to array", + program: "as/#( \"array\" :( `\"First\"` ): )#/", + input: miscInput, + expected: `{"something":{"nested":"Here is my test value"},"array":["First","Hello","world","these","are","values"],"people":[{"first_name":"Charlie","last_name":"Johnson","age":22},{"first_name":"Tom","last_name":"Johnson","age":18},{"first_name":"Charlie","last_name":"Chaplin","age":122},{"first_name":"John","last_name":"Johnson","age":48}]}`, + }, + { + name: "Append to array", + program: "es/#( \"array\" :( `\"Last\"` ): )#/", + input: miscInput, + expected: `{"something":{"nested":"Here is my test value"},"array":["Hello","world","these","are","values","Last"],"people":[{"first_name":"Charlie","last_name":"Johnson","age":22},{"first_name":"Tom","last_name":"Johnson","age":18},{"first_name":"Charlie","last_name":"Chaplin","age":122},{"first_name":"John","last_name":"Johnson","age":48}]}`, + }, } for _, test := range tests { diff --git a/main/parse.go b/main/parse.go index 36917ac..36bd3ee 100644 --- a/main/parse.go +++ b/main/parse.go @@ -77,6 +77,10 @@ func (p *parser) parseBasicCommand(commands []Command, commandChar rune) []Comma return append(commands, SubstituteAppendNextCommand {subex}, JumpCommand {len(commands) + 3}) case 'm': return append(commands, MergeCommand{}) + case 'M': + ast := p.parseSubex() + subex := subex.CompileTransducer(ast) + return append(commands, FullMergeCommand {subex}, JumpCommand {len(commands) + 3}) case 's': ast := p.parseSubex() subex := subex.CompileTransducer(ast) diff --git a/subex/filter.go b/subex/filter.go index ae4b8ab..309d6c7 100644 --- a/subex/filter.go +++ b/subex/filter.go @@ -27,6 +27,26 @@ func (_ anyBoolFilter) valueFilter(value walk.Value) bool { return isBool } +type simpleValueFilter struct {} +func (_ simpleValueFilter) valueFilter(value walk.Value) bool { + switch value := value.(type) { + case walk.NullValue: + return true + case walk.BoolValue: + return true + case walk.NumberValue: + return true + case walk.StringValue: + return true + case walk.ArrayValue: + return len(value) == 0 + case walk.MapValue: + return len(value) == 0 + default: + panic("Invalid value type") + } +} + type anyValueFilter struct {} func (_ anyValueFilter) valueFilter(value walk.Value) bool { return true diff --git a/subex/parse.go b/subex/parse.go index b6bf2f6..e91008a 100644 --- a/subex/parse.go +++ b/subex/parse.go @@ -55,6 +55,12 @@ func (s Structure) String() string { } } +type DestructureMethod int +const ( + Normal DestructureMethod = iota + Iterate +) + type RuneReader interface { Next() rune Rewind() @@ -361,8 +367,14 @@ func parseRuneReplacement(l RuneReader, end rune) (output SubexAST) { // } func parseDestructure(l RuneReader, destructure Structure, inType Type) (lhs SubexAST, outType Type) { - if !accept(l, "(") { - panic("Missing ( after destructure start") + var method rune + switch l.Next() { + case '(': + method = ')' + case '[': + method = ']' + default: + panic("Missing ( or [ after destructure start") } var innerInType Type @@ -390,8 +402,22 @@ func parseDestructure(l RuneReader, destructure Structure, inType Type) (lhs Sub resolveTypes(inType, expectedInType) lhs, innerOutType := parseSubex(l, 0, innerInType) - if !accept(l, ")") { - panic("Missing matching )") + if !accept(l, string(method)) { + panic("Missing matching ) or ]") + } + + switch method { + case ')': + case ']': + lhs = SubexASTRepeat { + Content: lhs, + Acceptable: []ConvexRange{{ + Start: -1, + End: 0, + }}, + } + default: + panic("Invalid method") } var structure Structure @@ -487,20 +513,6 @@ func parseSubex(l RuneReader, minPower int, inType Type) (lhs SubexAST, outType case ')', ']', '|', ';', '{', '+', '*', '/', '!', '=', '$': l.Rewind() return SubexASTEmpty{}, inType - // case '=': - // replacement := parseReplacement(l) - // lhs = SubexASTOutput{replacement} - // case '^': - // replacement := parseReplacement(l) - // replacement = append( - // []OutputContentAST{OutputValueLiteralAST {walk.NewAtomStringTerminal()}}, - // replacement... - // ) - // replacement = append( - // replacement, - // OutputValueLiteralAST {walk.NewAtomStringTerminal()}, - // ) - // lhs = SubexASTOutput {replacement} case '.': outType = inType if inType == RuneType { @@ -508,6 +520,17 @@ func parseSubex(l RuneReader, minPower int, inType Type) (lhs SubexAST, outType } else { lhs = SubexASTCopyAnyValue{} } + case ',': + switch inType { + case ValueType: + outType = inType + lhs = SubexASTCopyAnySimpleValue{} + case RuneType: + outType = inType + lhs = SubexASTCopyRune{','} + default: + panic("Invalid inType") + } case '?': outType = inType lhs = SubexASTCopyBool{} @@ -569,14 +592,10 @@ func parseSubex(l RuneReader, minPower int, inType Type) (lhs SubexAST, outType lhs = SubexASTProduct {lhs} resolveTypes(inType, ValueType) outType = resolveTypes(outType, ValueType) - // case r == '/' && minPower <= 4: - // lhs = SubexASTReciprocal {lhs} case r == '!' && minPower <= 4: lhs = SubexASTNot {lhs} resolveTypes(inType, ValueType) outType = resolveTypes(outType, ValueType) - // case r == '=' && minPower <= 4: - // lhs = SubexASTEqual {lhs} case r == '$' && minPower <= 4: slot := l.Next() if slot == eof { @@ -608,15 +627,6 @@ func parseSubex(l RuneReader, minPower int, inType Type) (lhs SubexAST, outType panic("Missing subex after |") } lhs = SubexASTOr{lhs, rhs} - /*case r == ';' && minPower <= 10: - rhs := parseSubex(l, 11, inType, outType) - if rhs == nil { - panic("Missing subex after ;") - } - lhs = SubexASTJoin { - Content: lhs, - Delimiter: rhs, - }*/ default: l.Rewind() break loop diff --git a/subex/subexast.go b/subex/subexast.go index d08ddac..655a783 100644 --- a/subex/subexast.go +++ b/subex/subexast.go @@ -238,6 +238,18 @@ func (ast SubexASTCopyNumber) String() string { return "%" } +// Read in a null, bool, number, string or empty array or map and output it unchanged +type SubexASTCopyAnySimpleValue struct {} +func (ast SubexASTCopyAnySimpleValue) compileWith(next SubexState, slotMap *SlotMap, inType Type, outType Type) SubexState { + if inType != ValueType || outType != ValueType { + panic("Invalid types for SubexASTCopyAnySimpleValue") + } + return &SubexCopyState { + next: next, + filter: simpleValueFilter{}, + } +} + // Read in any single Atom and output it unchanged type SubexASTCopyAnyValue struct {} func (ast SubexASTCopyAnyValue) compileWith(next SubexState, slotMap *SlotMap, inType Type, outType Type) SubexState { diff --git a/subex/subexstate.go b/subex/subexstate.go index 1e1e94e..8f27a10 100644 --- a/subex/subexstate.go +++ b/subex/subexstate.go @@ -146,84 +146,6 @@ func (state SubexStoreRunesEndState) epsilon(aux auxiliaryState) []SubexBranch { }} } -/* -// A part of an output literal, either an Atom or a slot from which to load -type OutputContent interface { - // Given the current store, return the ValueList produced by the TransducerOutput - buildValues(Store) walk.ValueList - // Given the current store, return the RuneList produced by the TransducerOutput - buildRunes(Store) walk.RuneList -} - -// An OutputContent which is just a Value literal -type OutputValueLiteral struct { - value walk.Value -} -func (replacement OutputValueLiteral) buildValues(store Store) walk.ValueList { - return walk.ValueList{replacement.value} -} -func (replacement OutputValueLiteral) buildRunes(store Store) walk.RuneList { - // TODO: serialise to JSON - panic("Unimplemented!") -} - -// An OutputContent which is just a rune literal -type OutputRuneLiteral struct { - rune walk.StringRuneAtom -} -func (replacement OutputRuneLiteral) buildValues(store Store) walk.ValueList { - // TODO: Try to deserialise - panic("Unimplemented!") -} -func (replacement OutputRuneLiteral) buildRunes(store Store) walk.RuneList { - return walk.RuneList {replacement.rune} -} - -// An OutputContent which is a slot that is loaded from -type OutputLoad struct { - slot int -} -func (replacement OutputLoad) buildValues(store Store) walk.ValueList { - values, isValues := store[replacement.slot].(walk.ValueList) - if !isValues { - panic("Tried to output non-values list") - } - return values -} -func (replacement OutputLoad) buildRunes(store Store) walk.RuneList { - runes, isRunes := store[replacement.slot].(walk.RuneList) - if !isRunes { - panic("Tried to output non-runes as runes") - } - return runes -} - -// Don't read in anything, just output the series of data and slots specified -type SubexOutputState struct { - content []OutputContent - next SubexState -} -// Given a store, return what is outputted by an epsilon transition from this state -// TODO: separate into buildValues and buildRunes -func (state SubexOutputState) build(store Store) walk.ValueList { - var result walk.ValueList - for _, part := range state.content { - result = append(result, part.buildValues(store)...) - } - return result -} -func (state SubexOutputState) eat(aux auxiliaryState, char walk.Value) []SubexBranch { - content := state.build(aux.store) - nextStates := state.next.eat(aux.topAppend(content), char) - return nextStates -} -func (state SubexOutputState) accepting(aux auxiliaryState) []OutputStack { - content := state.build(aux.store) - outputStacks := state.next.accepting(aux.topAppend(content)) - return outputStacks -} -*/ - type SubexOutputValueLiteralState struct { literal walk.Scalar next SubexState |