diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 241b0bfa5..29d5a832f 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -28,6 +28,8 @@ Special thanks to external contributors on this release: ### IMPROVEMENTS +- [\#7338](https://github.com/tendermint/tendermint/pull/7338) pubsub: Performance improvements for the event query API (backport of #7319) (@creachadair) + ### BUG FIXES - [\#7310](https://github.com/tendermint/tendermint/issues/7310) pubsub: Report a non-nil error when shutting down (fixes #7306). diff --git a/internal/inspect/inspect_test.go b/internal/inspect/inspect_test.go index e2808ce85..342daf5f2 100644 --- a/internal/inspect/inspect_test.go +++ b/internal/inspect/inspect_test.go @@ -114,7 +114,7 @@ func TestBlock(t *testing.T) { func TestTxSearch(t *testing.T) { testHash := []byte("test") testTx := []byte("tx") - testQuery := fmt.Sprintf("tx.hash='%s'", string(testHash)) + testQuery := fmt.Sprintf("tx.hash = '%s'", string(testHash)) testTxResult := &abcitypes.TxResult{ Height: 1, Index: 100, diff --git a/internal/state/indexer/block/kv/kv.go b/internal/state/indexer/block/kv/kv.go index d52f06c96..26fdcf1fc 100644 --- a/internal/state/indexer/block/kv/kv.go +++ b/internal/state/indexer/block/kv/kv.go @@ -14,6 +14,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/internal/state/indexer" "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" "github.com/tendermint/tendermint/types" ) @@ -91,10 +92,7 @@ func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, default: } - conditions, err := q.Conditions() - if err != nil { - return nil, fmt.Errorf("failed to parse query conditions: %w", err) - } + conditions := q.Syntax() // If there is an exact height query, return the result immediately // (if it exists). @@ -158,7 +156,7 @@ func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, continue } - startKey, err := orderedcode.Append(nil, c.CompositeKey, fmt.Sprintf("%v", c.Operand)) + startKey, err := orderedcode.Append(nil, c.Tag, c.Arg.Value()) if err != nil { return nil, err } @@ -327,7 +325,7 @@ iter: // matched. func (idx *BlockerIndexer) match( ctx context.Context, - c query.Condition, + c syntax.Condition, startKeyBz []byte, filteredHeights map[string][]byte, firstRun bool, @@ -342,7 +340,7 @@ func (idx *BlockerIndexer) match( tmpHeights := make(map[string][]byte) switch { - case c.Op == query.OpEqual: + case c.Op == syntax.TEq: it, err := dbm.IteratePrefix(idx.store, startKeyBz) if err != nil { return nil, fmt.Errorf("failed to create prefix iterator: %w", err) @@ -361,8 +359,8 @@ func (idx *BlockerIndexer) match( return nil, err } - case c.Op == query.OpExists: - prefix, err := orderedcode.Append(nil, c.CompositeKey) + case c.Op == syntax.TExists: + prefix, err := orderedcode.Append(nil, c.Tag) if err != nil { return nil, err } @@ -389,8 +387,8 @@ func (idx *BlockerIndexer) match( return nil, err } - case c.Op == query.OpContains: - prefix, err := orderedcode.Append(nil, c.CompositeKey) + case c.Op == syntax.TContains: + prefix, err := orderedcode.Append(nil, c.Tag) if err != nil { return nil, err } @@ -408,7 +406,7 @@ func (idx *BlockerIndexer) match( continue } - if strings.Contains(eventValue, c.Operand.(string)) { + if strings.Contains(eventValue, c.Arg.Value()) { tmpHeights[string(it.Value())] = it.Value() } diff --git a/internal/state/indexer/block/kv/kv_test.go b/internal/state/indexer/block/kv/kv_test.go index 024df332c..650723dbf 100644 --- a/internal/state/indexer/block/kv/kv_test.go +++ b/internal/state/indexer/block/kv/kv_test.go @@ -94,39 +94,39 @@ func TestBlockIndexer(t *testing.T) { results []int64 }{ "block.height = 100": { - q: query.MustParse("block.height = 100"), + q: query.MustCompile(`block.height = 100`), results: []int64{}, }, "block.height = 5": { - q: query.MustParse("block.height = 5"), + q: query.MustCompile(`block.height = 5`), results: []int64{5}, }, "begin_event.key1 = 'value1'": { - q: query.MustParse("begin_event.key1 = 'value1'"), + q: query.MustCompile(`begin_event.key1 = 'value1'`), results: []int64{}, }, "begin_event.proposer = 'FCAA001'": { - q: query.MustParse("begin_event.proposer = 'FCAA001'"), + q: query.MustCompile(`begin_event.proposer = 'FCAA001'`), results: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, }, "end_event.foo <= 5": { - q: query.MustParse("end_event.foo <= 5"), + q: query.MustCompile(`end_event.foo <= 5`), results: []int64{2, 4}, }, "end_event.foo >= 100": { - q: query.MustParse("end_event.foo >= 100"), + q: query.MustCompile(`end_event.foo >= 100`), results: []int64{1}, }, "block.height > 2 AND end_event.foo <= 8": { - q: query.MustParse("block.height > 2 AND end_event.foo <= 8"), + q: query.MustCompile(`block.height > 2 AND end_event.foo <= 8`), results: []int64{4, 6, 8}, }, "begin_event.proposer CONTAINS 'FFFFFFF'": { - q: query.MustParse("begin_event.proposer CONTAINS 'FFFFFFF'"), + q: query.MustCompile(`begin_event.proposer CONTAINS 'FFFFFFF'`), results: []int64{}, }, "begin_event.proposer CONTAINS 'FCAA001'": { - q: query.MustParse("begin_event.proposer CONTAINS 'FCAA001'"), + q: query.MustCompile(`begin_event.proposer CONTAINS 'FCAA001'`), results: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, }, } diff --git a/internal/state/indexer/block/kv/util.go b/internal/state/indexer/block/kv/util.go index c0b88018e..fff88046c 100644 --- a/internal/state/indexer/block/kv/util.go +++ b/internal/state/indexer/block/kv/util.go @@ -6,7 +6,7 @@ import ( "strconv" "github.com/google/orderedcode" - "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" "github.com/tendermint/tendermint/types" ) @@ -85,10 +85,10 @@ func parseValueFromEventKey(key []byte) (string, error) { return eventValue, nil } -func lookForHeight(conditions []query.Condition) (int64, bool) { +func lookForHeight(conditions []syntax.Condition) (int64, bool) { for _, c := range conditions { - if c.CompositeKey == types.BlockHeightKey && c.Op == query.OpEqual { - return c.Operand.(int64), true + if c.Tag == types.BlockHeightKey && c.Op == syntax.TEq { + return int64(c.Arg.Number()), true } } diff --git a/internal/state/indexer/query_range.go b/internal/state/indexer/query_range.go index b4edf53c5..4c026955d 100644 --- a/internal/state/indexer/query_range.go +++ b/internal/state/indexer/query_range.go @@ -3,7 +3,7 @@ package indexer import ( "time" - "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" ) // QueryRanges defines a mapping between a composite event key and a QueryRange. @@ -77,32 +77,32 @@ func (qr QueryRange) UpperBoundValue() interface{} { // LookForRanges returns a mapping of QueryRanges and the matching indexes in // the provided query conditions. -func LookForRanges(conditions []query.Condition) (ranges QueryRanges, indexes []int) { +func LookForRanges(conditions []syntax.Condition) (ranges QueryRanges, indexes []int) { ranges = make(QueryRanges) for i, c := range conditions { if IsRangeOperation(c.Op) { - r, ok := ranges[c.CompositeKey] + r, ok := ranges[c.Tag] if !ok { - r = QueryRange{Key: c.CompositeKey} + r = QueryRange{Key: c.Tag} } switch c.Op { - case query.OpGreater: - r.LowerBound = c.Operand + case syntax.TGt: + r.LowerBound = conditionArg(c) - case query.OpGreaterEqual: + case syntax.TGeq: r.IncludeLowerBound = true - r.LowerBound = c.Operand + r.LowerBound = conditionArg(c) - case query.OpLess: - r.UpperBound = c.Operand + case syntax.TLt: + r.UpperBound = conditionArg(c) - case query.OpLessEqual: + case syntax.TLeq: r.IncludeUpperBound = true - r.UpperBound = c.Operand + r.UpperBound = conditionArg(c) } - ranges[c.CompositeKey] = r + ranges[c.Tag] = r indexes = append(indexes, i) } } @@ -112,12 +112,26 @@ func LookForRanges(conditions []query.Condition) (ranges QueryRanges, indexes [] // IsRangeOperation returns a boolean signifying if a query Operator is a range // operation or not. -func IsRangeOperation(op query.Operator) bool { +func IsRangeOperation(op syntax.Token) bool { switch op { - case query.OpGreater, query.OpGreaterEqual, query.OpLess, query.OpLessEqual: + case syntax.TGt, syntax.TGeq, syntax.TLt, syntax.TLeq: return true default: return false } } + +func conditionArg(c syntax.Condition) interface{} { + if c.Arg == nil { + return nil + } + switch c.Arg.Type { + case syntax.TNumber: + return int64(c.Arg.Number()) + case syntax.TTime, syntax.TDate: + return c.Arg.Time() + default: + return c.Arg.Value() // string + } +} diff --git a/internal/state/indexer/sink/kv/kv_test.go b/internal/state/indexer/sink/kv/kv_test.go index 7d7552946..47b1f5364 100644 --- a/internal/state/indexer/sink/kv/kv_test.go +++ b/internal/state/indexer/sink/kv/kv_test.go @@ -111,39 +111,39 @@ func TestBlockFuncs(t *testing.T) { results []int64 }{ "block.height = 100": { - q: query.MustParse("block.height = 100"), + q: query.MustCompile(`block.height = 100`), results: []int64{}, }, "block.height = 5": { - q: query.MustParse("block.height = 5"), + q: query.MustCompile(`block.height = 5`), results: []int64{5}, }, "begin_event.key1 = 'value1'": { - q: query.MustParse("begin_event.key1 = 'value1'"), + q: query.MustCompile(`begin_event.key1 = 'value1'`), results: []int64{}, }, "begin_event.proposer = 'FCAA001'": { - q: query.MustParse("begin_event.proposer = 'FCAA001'"), + q: query.MustCompile(`begin_event.proposer = 'FCAA001'`), results: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, }, "end_event.foo <= 5": { - q: query.MustParse("end_event.foo <= 5"), + q: query.MustCompile(`end_event.foo <= 5`), results: []int64{2, 4}, }, "end_event.foo >= 100": { - q: query.MustParse("end_event.foo >= 100"), + q: query.MustCompile(`end_event.foo >= 100`), results: []int64{1}, }, "block.height > 2 AND end_event.foo <= 8": { - q: query.MustParse("block.height > 2 AND end_event.foo <= 8"), + q: query.MustCompile(`block.height > 2 AND end_event.foo <= 8`), results: []int64{4, 6, 8}, }, "begin_event.proposer CONTAINS 'FFFFFFF'": { - q: query.MustParse("begin_event.proposer CONTAINS 'FFFFFFF'"), + q: query.MustCompile(`begin_event.proposer CONTAINS 'FFFFFFF'`), results: []int64{}, }, "begin_event.proposer CONTAINS 'FCAA001'": { - q: query.MustParse("begin_event.proposer CONTAINS 'FCAA001'"), + q: query.MustCompile(`begin_event.proposer CONTAINS 'FCAA001'`), results: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, }, } @@ -175,7 +175,7 @@ func TestTxSearchWithCancelation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - results, err := indexer.SearchTxEvents(ctx, query.MustParse("account.number = 1")) + results, err := indexer.SearchTxEvents(ctx, query.MustCompile(`account.number = 1`)) assert.NoError(t, err) assert.Empty(t, results) } @@ -249,7 +249,7 @@ func TestTxSearchDeprecatedIndexing(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.q, func(t *testing.T) { - results, err := indexer.SearchTxEvents(ctx, query.MustParse(tc.q)) + results, err := indexer.SearchTxEvents(ctx, query.MustCompile(tc.q)) require.NoError(t, err) for _, txr := range results { for _, tr := range tc.results { @@ -273,7 +273,7 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { ctx := context.Background() - results, err := indexer.SearchTxEvents(ctx, query.MustParse("account.number >= 1")) + results, err := indexer.SearchTxEvents(ctx, query.MustCompile(`account.number >= 1`)) assert.NoError(t, err) assert.Len(t, results, 1) @@ -330,7 +330,7 @@ func TestTxSearchMultipleTxs(t *testing.T) { ctx := context.Background() - results, err := indexer.SearchTxEvents(ctx, query.MustParse("account.number >= 1")) + results, err := indexer.SearchTxEvents(ctx, query.MustCompile(`account.number >= 1`)) assert.NoError(t, err) require.Len(t, results, 3) diff --git a/internal/state/indexer/tx/kv/kv.go b/internal/state/indexer/tx/kv/kv.go index f0550f8f3..4bcff958b 100644 --- a/internal/state/indexer/tx/kv/kv.go +++ b/internal/state/indexer/tx/kv/kv.go @@ -14,6 +14,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" indexer "github.com/tendermint/tendermint/internal/state/indexer" "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" "github.com/tendermint/tendermint/types" ) @@ -148,10 +149,7 @@ func (txi *TxIndex) Search(ctx context.Context, q *query.Query) ([]*abci.TxResul filteredHashes := make(map[string][]byte) // get a list of conditions (like "tx.height > 5") - conditions, err := q.Conditions() - if err != nil { - return nil, fmt.Errorf("error during parsing conditions from query: %w", err) - } + conditions := q.Syntax() // if there is a hash condition, return the result immediately hash, ok, err := lookForHash(conditions) @@ -238,10 +236,10 @@ hashes: return results, nil } -func lookForHash(conditions []query.Condition) (hash []byte, ok bool, err error) { +func lookForHash(conditions []syntax.Condition) (hash []byte, ok bool, err error) { for _, c := range conditions { - if c.CompositeKey == types.TxHashKey { - decoded, err := hex.DecodeString(c.Operand.(string)) + if c.Tag == types.TxHashKey { + decoded, err := hex.DecodeString(c.Arg.Value()) return decoded, true, err } } @@ -249,10 +247,10 @@ func lookForHash(conditions []query.Condition) (hash []byte, ok bool, err error) } // lookForHeight returns a height if there is an "height=X" condition. -func lookForHeight(conditions []query.Condition) (height int64) { +func lookForHeight(conditions []syntax.Condition) (height int64) { for _, c := range conditions { - if c.CompositeKey == types.TxHeightKey && c.Op == query.OpEqual { - return c.Operand.(int64) + if c.Tag == types.TxHeightKey && c.Op == syntax.TEq { + return int64(c.Arg.Number()) } } return 0 @@ -265,7 +263,7 @@ func lookForHeight(conditions []query.Condition) (height int64) { // NOTE: filteredHashes may be empty if no previous condition has matched. func (txi *TxIndex) match( ctx context.Context, - c query.Condition, + c syntax.Condition, startKeyBz []byte, filteredHashes map[string][]byte, firstRun bool, @@ -279,7 +277,7 @@ func (txi *TxIndex) match( tmpHashes := make(map[string][]byte) switch { - case c.Op == query.OpEqual: + case c.Op == syntax.TEq: it, err := dbm.IteratePrefix(txi.store, startKeyBz) if err != nil { panic(err) @@ -301,10 +299,10 @@ func (txi *TxIndex) match( panic(err) } - case c.Op == query.OpExists: + case c.Op == syntax.TExists: // XXX: can't use startKeyBz here because c.Operand is nil // (e.g. "account.owner//" won't match w/ a single row) - it, err := dbm.IteratePrefix(txi.store, prefixFromCompositeKey(c.CompositeKey)) + it, err := dbm.IteratePrefix(txi.store, prefixFromCompositeKey(c.Tag)) if err != nil { panic(err) } @@ -325,11 +323,11 @@ func (txi *TxIndex) match( panic(err) } - case c.Op == query.OpContains: + case c.Op == syntax.TContains: // XXX: startKey does not apply here. // For example, if startKey = "account.owner/an/" and search query = "account.owner CONTAINS an" // we can't iterate with prefix "account.owner/an/" because we might miss keys like "account.owner/Ulan/" - it, err := dbm.IteratePrefix(txi.store, prefixFromCompositeKey(c.CompositeKey)) + it, err := dbm.IteratePrefix(txi.store, prefixFromCompositeKey(c.Tag)) if err != nil { panic(err) } @@ -341,7 +339,7 @@ func (txi *TxIndex) match( if err != nil { continue } - if strings.Contains(value, c.Operand.(string)) { + if strings.Contains(value, c.Arg.Value()) { tmpHashes[string(it.Value())] = it.Value() } @@ -577,8 +575,8 @@ func prefixFromCompositeKeyAndValue(compositeKey, value string) []byte { } // a small utility function for getting a keys prefix based on a condition and a height -func prefixForCondition(c query.Condition, height int64) []byte { - key := prefixFromCompositeKeyAndValue(c.CompositeKey, fmt.Sprintf("%v", c.Operand)) +func prefixForCondition(c syntax.Condition, height int64) []byte { + key := prefixFromCompositeKeyAndValue(c.Tag, c.Arg.Value()) if height > 0 { var err error key, err = orderedcode.Append(key, height) diff --git a/internal/state/indexer/tx/kv/kv_bench_test.go b/internal/state/indexer/tx/kv/kv_bench_test.go index 3f4e63ee1..93724beba 100644 --- a/internal/state/indexer/tx/kv/kv_bench_test.go +++ b/internal/state/indexer/tx/kv/kv_bench_test.go @@ -60,7 +60,7 @@ func BenchmarkTxSearch(b *testing.B) { } } - txQuery := query.MustParse("transfer.address = 'address_43' AND transfer.amount = 50") + txQuery := query.MustCompile(`transfer.address = 'address_43' AND transfer.amount = 50`) b.ResetTimer() diff --git a/internal/state/indexer/tx/kv/kv_test.go b/internal/state/indexer/tx/kv/kv_test.go index c8ab2b0f2..9bb8bfb7b 100644 --- a/internal/state/indexer/tx/kv/kv_test.go +++ b/internal/state/indexer/tx/kv/kv_test.go @@ -132,7 +132,7 @@ func TestTxSearch(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.q, func(t *testing.T) { - results, err := indexer.Search(ctx, query.MustParse(tc.q)) + results, err := indexer.Search(ctx, query.MustCompile(tc.q)) assert.NoError(t, err) assert.Len(t, results, tc.resultsLength) @@ -158,7 +158,7 @@ func TestTxSearchWithCancelation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - results, err := indexer.Search(ctx, query.MustParse("account.number = 1")) + results, err := indexer.Search(ctx, query.MustCompile(`account.number = 1`)) assert.NoError(t, err) assert.Empty(t, results) } @@ -231,7 +231,7 @@ func TestTxSearchDeprecatedIndexing(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.q, func(t *testing.T) { - results, err := indexer.Search(ctx, query.MustParse(tc.q)) + results, err := indexer.Search(ctx, query.MustCompile(tc.q)) require.NoError(t, err) for _, txr := range results { for _, tr := range tc.results { @@ -255,7 +255,7 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { ctx := context.Background() - results, err := indexer.Search(ctx, query.MustParse("account.number >= 1")) + results, err := indexer.Search(ctx, query.MustCompile(`account.number >= 1`)) assert.NoError(t, err) assert.Len(t, results, 1) @@ -312,7 +312,7 @@ func TestTxSearchMultipleTxs(t *testing.T) { ctx := context.Background() - results, err := indexer.Search(ctx, query.MustParse("account.number >= 1")) + results, err := indexer.Search(ctx, query.MustCompile(`account.number >= 1`)) assert.NoError(t, err) require.Len(t, results, 3) diff --git a/libs/pubsub/example_test.go b/libs/pubsub/example_test.go index fd4a94382..4be1d97c4 100644 --- a/libs/pubsub/example_test.go +++ b/libs/pubsub/example_test.go @@ -26,7 +26,7 @@ func TestExample(t *testing.T) { ctx := context.Background() - subscription, err := s.Subscribe(ctx, "example-client", query.MustParse("abci.account.name='John'")) + subscription, err := s.Subscribe(ctx, "example-client", query.MustCompile("abci.account.name='John'")) require.NoError(t, err) events := []abci.Event{ diff --git a/libs/pubsub/pubsub_test.go b/libs/pubsub/pubsub_test.go index d3855be7a..36359a143 100644 --- a/libs/pubsub/pubsub_test.go +++ b/libs/pubsub/pubsub_test.go @@ -33,7 +33,7 @@ func TestSubscribe(t *testing.T) { }) ctx := context.Background() - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}) + subscription, err := s.Subscribe(ctx, clientID, query.All) require.NoError(t, err) require.Equal(t, 1, s.NumClients()) @@ -79,14 +79,14 @@ func TestSubscribeWithCapacity(t *testing.T) { ctx := context.Background() require.Panics(t, func() { - _, err = s.Subscribe(ctx, clientID, query.Empty{}, -1) + _, err = s.Subscribe(ctx, clientID, query.All, -1) require.NoError(t, err) }) require.Panics(t, func() { - _, err = s.Subscribe(ctx, clientID, query.Empty{}, 0) + _, err = s.Subscribe(ctx, clientID, query.All, 0) require.NoError(t, err) }) - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}, 1) + subscription, err := s.Subscribe(ctx, clientID, query.All, 1) require.NoError(t, err) err = s.Publish(ctx, "Aggamon") require.NoError(t, err) @@ -105,7 +105,7 @@ func TestSubscribeUnbuffered(t *testing.T) { }) ctx := context.Background() - subscription, err := s.SubscribeUnbuffered(ctx, clientID, query.Empty{}) + subscription, err := s.SubscribeUnbuffered(ctx, clientID, query.All) require.NoError(t, err) published := make(chan struct{}) @@ -140,7 +140,7 @@ func TestSlowClientIsRemovedWithErrOutOfCapacity(t *testing.T) { }) ctx := context.Background() - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}) + subscription, err := s.Subscribe(ctx, clientID, query.All) require.NoError(t, err) err = s.Publish(ctx, "Fat Cobra") require.NoError(t, err) @@ -163,7 +163,7 @@ func TestDifferentClients(t *testing.T) { ctx := context.Background() - subscription1, err := s.Subscribe(ctx, "client-1", query.MustParse("tm.events.type='NewBlock'")) + subscription1, err := s.Subscribe(ctx, "client-1", query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) events := []abci.Event{ @@ -179,7 +179,7 @@ func TestDifferentClients(t *testing.T) { subscription2, err := s.Subscribe( ctx, "client-2", - query.MustParse("tm.events.type='NewBlock' AND abci.account.name='Igor'"), + query.MustCompile("tm.events.type='NewBlock' AND abci.account.name='Igor'"), ) require.NoError(t, err) @@ -201,7 +201,7 @@ func TestDifferentClients(t *testing.T) { subscription3, err := s.Subscribe( ctx, "client-3", - query.MustParse("tm.events.type='NewRoundStep' AND abci.account.name='Igor' AND abci.invoice.number = 10"), + query.MustCompile("tm.events.type='NewRoundStep' AND abci.account.name='Igor' AND abci.invoice.number = 10"), ) require.NoError(t, err) @@ -252,7 +252,7 @@ func TestSubscribeDuplicateKeys(t *testing.T) { } for i, tc := range testCases { - sub, err := s.Subscribe(ctx, fmt.Sprintf("client-%d", i), query.MustParse(tc.query)) + sub, err := s.Subscribe(ctx, fmt.Sprintf("client-%d", i), query.MustCompile(tc.query)) require.NoError(t, err) events := []abci.Event{ @@ -296,7 +296,7 @@ func TestClientSubscribesTwice(t *testing.T) { }) ctx := context.Background() - q := query.MustParse("tm.events.type='NewBlock'") + q := query.MustCompile("tm.events.type='NewBlock'") subscription1, err := s.Subscribe(ctx, clientID, q) require.NoError(t, err) @@ -331,11 +331,11 @@ func TestUnsubscribe(t *testing.T) { }) ctx := context.Background() - subscription, err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + subscription, err := s.Subscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) err = s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{ Subscriber: clientID, - Query: query.MustParse("tm.events.type='NewBlock'")}) + Query: query.MustCompile("tm.events.type='NewBlock'")}) require.NoError(t, err) err = s.Publish(ctx, "Nick Fury") @@ -357,16 +357,16 @@ func TestClientUnsubscribesTwice(t *testing.T) { }) ctx := context.Background() - _, err = s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + _, err = s.Subscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) err = s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{ Subscriber: clientID, - Query: query.MustParse("tm.events.type='NewBlock'")}) + Query: query.MustCompile("tm.events.type='NewBlock'")}) require.NoError(t, err) err = s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{ Subscriber: clientID, - Query: query.MustParse("tm.events.type='NewBlock'")}) + Query: query.MustCompile("tm.events.type='NewBlock'")}) require.Equal(t, pubsub.ErrSubscriptionNotFound, err) err = s.UnsubscribeAll(ctx, clientID) require.Equal(t, pubsub.ErrSubscriptionNotFound, err) @@ -384,11 +384,11 @@ func TestResubscribe(t *testing.T) { }) ctx := context.Background() - _, err = s.Subscribe(ctx, clientID, query.Empty{}) + _, err = s.Subscribe(ctx, clientID, query.All) require.NoError(t, err) - err = s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{Subscriber: clientID, Query: query.Empty{}}) + err = s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{Subscriber: clientID, Query: query.All}) require.NoError(t, err) - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}) + subscription, err := s.Subscribe(ctx, clientID, query.All) require.NoError(t, err) err = s.Publish(ctx, "Cable") @@ -408,9 +408,9 @@ func TestUnsubscribeAll(t *testing.T) { }) ctx := context.Background() - subscription1, err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + subscription1, err := s.Subscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) - subscription2, err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlockHeader'")) + subscription2, err := s.Subscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlockHeader'")) require.NoError(t, err) err = s.UnsubscribeAll(ctx, clientID) @@ -470,7 +470,7 @@ func benchmarkNClients(n int, b *testing.B) { subscription, err := s.Subscribe( ctx, clientID, - query.MustParse(fmt.Sprintf("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = %d", i)), + query.MustCompile(fmt.Sprintf("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = %d", i)), ) if err != nil { b.Fatal(err) @@ -516,7 +516,7 @@ func benchmarkNClientsOneQuery(n int, b *testing.B) { }) ctx := context.Background() - q := query.MustParse("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = 1") + q := query.MustCompile("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = 1") for i := 0; i < n; i++ { id := fmt.Sprintf("clientID-%d", i+1) subscription, err := s.Subscribe(ctx, id, q) diff --git a/libs/pubsub/query/Makefile b/libs/pubsub/query/Makefile deleted file mode 100644 index aef42b2df..000000000 --- a/libs/pubsub/query/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -gen_query_parser: - go get -u -v github.com/pointlander/peg - peg -inline -switch query.peg - -fuzzy_test: - go get -u -v github.com/dvyukov/go-fuzz/go-fuzz - go get -u -v github.com/dvyukov/go-fuzz/go-fuzz-build - go-fuzz-build github.com/tendermint/tendermint/libs/pubsub/query/fuzz_test - go-fuzz -bin=./fuzz_test-fuzz.zip -workdir=./fuzz_test/output - -.PHONY: gen_query_parser fuzzy_test diff --git a/libs/pubsub/query/bench_test.go b/libs/pubsub/query/bench_test.go new file mode 100644 index 000000000..894c16628 --- /dev/null +++ b/libs/pubsub/query/bench_test.go @@ -0,0 +1,58 @@ +package query_test + +import ( + "testing" + + "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/libs/pubsub/query" +) + +const testQuery = `tm.events.type='NewBlock' AND abci.account.name='Igor'` + +var testEvents = []types.Event{ + { + Type: "tm.events", + Attributes: []types.EventAttribute{{ + Key: "index", + Value: "25", + }, { + Key: "type", + Value: "NewBlock", + }}, + }, + { + Type: "abci.account", + Attributes: []types.EventAttribute{{ + Key: "name", + Value: "Anya", + }, { + Key: "name", + Value: "Igor", + }}, + }, +} + +func BenchmarkParseCustom(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := query.New(testQuery) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkMatchCustom(b *testing.B) { + q, err := query.New(testQuery) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + ok, err := q.Matches(testEvents) + if err != nil { + b.Fatal(err) + } else if !ok { + b.Error("no match") + } + } +} diff --git a/libs/pubsub/query/empty.go b/libs/pubsub/query/empty.go deleted file mode 100644 index dd6b3f3b2..000000000 --- a/libs/pubsub/query/empty.go +++ /dev/null @@ -1,18 +0,0 @@ -package query - -import ( - "github.com/tendermint/tendermint/abci/types" -) - -// Empty query matches any set of events. -type Empty struct { -} - -// Matches always returns true. -func (Empty) Matches(events []types.Event) (bool, error) { - return true, nil -} - -func (Empty) String() string { - return "empty" -} diff --git a/libs/pubsub/query/empty_test.go b/libs/pubsub/query/empty_test.go deleted file mode 100644 index 4bb3067d6..000000000 --- a/libs/pubsub/query/empty_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package query_test - -import ( - "testing" - - "github.com/stretchr/testify/require" - abci "github.com/tendermint/tendermint/abci/types" - "github.com/tendermint/tendermint/libs/pubsub/query" -) - -func TestEmptyQueryMatchesAnything(t *testing.T) { - q := query.Empty{} - - testCases := []struct { - events []abci.Event - }{ - { - []abci.Event{}, - }, - { - []abci.Event{ - { - Type: "Asher", - Attributes: []abci.EventAttribute{{Key: "Roth"}}, - }, - }, - }, - { - []abci.Event{ - { - Type: "Route", - Attributes: []abci.EventAttribute{{Key: "66"}}, - }, - }, - }, - { - []abci.Event{ - { - Type: "Route", - Attributes: []abci.EventAttribute{{Key: "66"}}, - }, - { - Type: "Billy", - Attributes: []abci.EventAttribute{{Key: "Blue"}}, - }, - }, - }, - } - - for _, tc := range testCases { - match, err := q.Matches(tc.events) - require.Nil(t, err) - require.True(t, match) - } -} diff --git a/libs/pubsub/query/fuzz_test/main.go b/libs/pubsub/query/fuzz_test/main.go deleted file mode 100644 index 7a46116b5..000000000 --- a/libs/pubsub/query/fuzz_test/main.go +++ /dev/null @@ -1,30 +0,0 @@ -package fuzz_test - -import ( - "fmt" - - "github.com/tendermint/tendermint/libs/pubsub/query" -) - -func Fuzz(data []byte) int { - sdata := string(data) - q0, err := query.New(sdata) - if err != nil { - return 0 - } - - sdata1 := q0.String() - q1, err := query.New(sdata1) - if err != nil { - panic(err) - } - - sdata2 := q1.String() - if sdata1 != sdata2 { - fmt.Printf("q0: %q\n", sdata1) - fmt.Printf("q1: %q\n", sdata2) - panic("query changed") - } - - return 1 -} diff --git a/libs/pubsub/query/parser_test.go b/libs/pubsub/query/parser_test.go deleted file mode 100644 index a08a0d16d..000000000 --- a/libs/pubsub/query/parser_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package query_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/tendermint/tendermint/libs/pubsub/query" -) - -// TODO: fuzzy testing? -func TestParser(t *testing.T) { - cases := []struct { - query string - valid bool - }{ - {"tm.events.type='NewBlock'", true}, - {"tm.events.type = 'NewBlock'", true}, - {"tm.events.name = ''", true}, - {"tm.events.type='TIME'", true}, - {"tm.events.type='DATE'", true}, - {"tm.events.type='='", true}, - {"tm.events.type='TIME", false}, - {"tm.events.type=TIME'", false}, - {"tm.events.type==", false}, - {"tm.events.type=NewBlock", false}, - {">==", false}, - {"tm.events.type 'NewBlock' =", false}, - {"tm.events.type>'NewBlock'", false}, - {"", false}, - {"=", false}, - {"='NewBlock'", false}, - {"tm.events.type=", false}, - - {"tm.events.typeNewBlock", false}, - {"tm.events.type'NewBlock'", false}, - {"'NewBlock'", false}, - {"NewBlock", false}, - {"", false}, - - {"tm.events.type='NewBlock' AND abci.account.name='Igor'", true}, - {"tm.events.type='NewBlock' AND", false}, - {"tm.events.type='NewBlock' AN", false}, - {"tm.events.type='NewBlock' AN tm.events.type='NewBlockHeader'", false}, - {"AND tm.events.type='NewBlock' ", false}, - - {"abci.account.name CONTAINS 'Igor'", true}, - - {"tx.date > DATE 2013-05-03", true}, - {"tx.date < DATE 2013-05-03", true}, - {"tx.date <= DATE 2013-05-03", true}, - {"tx.date >= DATE 2013-05-03", true}, - {"tx.date >= DAT 2013-05-03", false}, - {"tx.date <= DATE2013-05-03", false}, - {"tx.date <= DATE -05-03", false}, - {"tx.date >= DATE 20130503", false}, - {"tx.date >= DATE 2013+01-03", false}, - // incorrect year, month, day - {"tx.date >= DATE 0013-01-03", false}, - {"tx.date >= DATE 2013-31-03", false}, - {"tx.date >= DATE 2013-01-83", false}, - - {"tx.date > TIME 2013-05-03T14:45:00+07:00", true}, - {"tx.date < TIME 2013-05-03T14:45:00-02:00", true}, - {"tx.date <= TIME 2013-05-03T14:45:00Z", true}, - {"tx.date >= TIME 2013-05-03T14:45:00Z", true}, - {"tx.date >= TIME2013-05-03T14:45:00Z", false}, - {"tx.date = IME 2013-05-03T14:45:00Z", false}, - {"tx.date = TIME 2013-05-:45:00Z", false}, - {"tx.date >= TIME 2013-05-03T14:45:00", false}, - {"tx.date >= TIME 0013-00-00T14:45:00Z", false}, - {"tx.date >= TIME 2013+05=03T14:45:00Z", false}, - - {"account.balance=100", true}, - {"account.balance >= 200", true}, - {"account.balance >= -300", false}, - {"account.balance >>= 400", false}, - {"account.balance=33.22.1", false}, - - {"slashing.amount EXISTS", true}, - {"slashing.amount EXISTS AND account.balance=100", true}, - {"account.balance=100 AND slashing.amount EXISTS", true}, - {"slashing EXISTS", true}, - - {"hash='136E18F7E4C348B780CF873A0BF43922E5BAFA63'", true}, - {"hash=136E18F7E4C348B780CF873A0BF43922E5BAFA63", false}, - } - - for _, c := range cases { - _, err := query.New(c.query) - if c.valid { - assert.NoErrorf(t, err, "Query was '%s'", c.query) - } else { - assert.Errorf(t, err, "Query was '%s'", c.query) - } - } -} diff --git a/libs/pubsub/query/peg.go b/libs/pubsub/query/peg.go deleted file mode 100644 index 816589f02..000000000 --- a/libs/pubsub/query/peg.go +++ /dev/null @@ -1,3 +0,0 @@ -package query - -//go:generate peg -inline -switch query.peg diff --git a/libs/pubsub/query/query.go b/libs/pubsub/query/query.go index 7b1dfe0f9..e874f037c 100644 --- a/libs/pubsub/query/query.go +++ b/libs/pubsub/query/query.go @@ -1,527 +1,327 @@ -// Package query provides a parser for a custom query format: +// Package query implements the custom query format used to filter event +// subscriptions in Tendermint. // -// abci.invoice.number=22 AND abci.invoice.owner=Ivan +// Query expressions describe properties of events and their attributes, using +// strings like: // -// See query.peg for the grammar, which is a https://en.wikipedia.org/wiki/Parsing_expression_grammar. -// More: https://github.com/PhilippeSigaud/Pegged/wiki/PEG-Basics +// abci.invoice.number = 22 AND abci.invoice.owner = 'Ivan' +// +// Query expressions can handle attribute values encoding numbers, strings, +// dates, and timestamps. The complete query grammar is described in the +// query/syntax package. // -// It has a support for numbers (integer and floating point), dates and times. package query import ( "fmt" - "reflect" "regexp" "strconv" "strings" "time" "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" ) -var ( - numRegex = regexp.MustCompile(`([0-9\.]+)`) -) +// All is a query that matches all events. +var All *Query -// Query holds the query string and the query parser. +// A Query is the compiled form of a query. type Query struct { - str string - parser *QueryParser + ast syntax.Query + conds []condition } -// Condition represents a single condition within a query and consists of composite key -// (e.g. "tx.gas"), operator (e.g. "=") and operand (e.g. "7"). -type Condition struct { - CompositeKey string - Op Operator - Operand interface{} -} - -// New parses the given string and returns a query or error if the string is -// invalid. -func New(s string) (*Query, error) { - p := &QueryParser{Buffer: fmt.Sprintf(`"%s"`, s)} - p.Init() - if err := p.Parse(); err != nil { +// New parses and compiles the query expression into an executable query. +func New(query string) (*Query, error) { + ast, err := syntax.Parse(query) + if err != nil { return nil, err } - return &Query{str: s, parser: p}, nil + return Compile(ast) } -// MustParse turns the given string into a query or panics; for tests or others -// cases where you know the string is valid. -func MustParse(s string) *Query { - q, err := New(s) +// MustCompile compiles the query expression into an executable query. +// In case of error, MustCompile will panic. +// +// This is intended for use in program initialization; use query.New if you +// need to check errors. +func MustCompile(query string) *Query { + q, err := New(query) if err != nil { - panic(fmt.Sprintf("failed to parse %s: %v", s, err)) + panic(err) } return q } -// String returns the original string. -func (q *Query) String() string { - return q.str -} - -// Operator is an operator that defines some kind of relation between composite key and -// operand (equality, etc.). -type Operator uint8 - -const ( - // "<=" - OpLessEqual Operator = iota - // ">=" - OpGreaterEqual - // "<" - OpLess - // ">" - OpGreater - // "=" - OpEqual - // "CONTAINS"; used to check if a string contains a certain sub string. - OpContains - // "EXISTS"; used to check if a certain event attribute is present. - OpExists -) - -const ( - // DateLayout defines a layout for all dates (`DATE date`) - DateLayout = "2006-01-02" - // TimeLayout defines a layout for all times (`TIME time`) - TimeLayout = time.RFC3339 -) - -// Conditions returns a list of conditions. It returns an error if there is any -// error with the provided grammar in the Query. -func (q *Query) Conditions() ([]Condition, error) { - var ( - eventAttr string - op Operator - ) - - conditions := make([]Condition, 0) - buffer, begin, end := q.parser.Buffer, 0, 0 - - // tokens must be in the following order: tag ("tx.gas") -> operator ("=") -> operand ("7") - for token := range q.parser.Tokens() { - switch token.pegRule { - case rulePegText: - begin, end = int(token.begin), int(token.end) - - case ruletag: - eventAttr = buffer[begin:end] - - case rulele: - op = OpLessEqual - - case rulege: - op = OpGreaterEqual - - case rulel: - op = OpLess - - case ruleg: - op = OpGreater - - case ruleequal: - op = OpEqual - - case rulecontains: - op = OpContains - - case ruleexists: - op = OpExists - conditions = append(conditions, Condition{eventAttr, op, nil}) - - case rulevalue: - // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") - valueWithoutSingleQuotes := buffer[begin+1 : end-1] - conditions = append(conditions, Condition{eventAttr, op, valueWithoutSingleQuotes}) - - case rulenumber: - number := buffer[begin:end] - if strings.ContainsAny(number, ".") { // if it looks like a floating-point number - value, err := strconv.ParseFloat(number, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", - err, number, - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - } else { - value, err := strconv.ParseInt(number, 10, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", - err, number, - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - } - - case ruletime: - value, err := time.Parse(TimeLayout, buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - - case ruledate: - value, err := time.Parse("2006-01-02", buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) +// Compile compiles the given query AST so it can be used to match events. +func Compile(ast syntax.Query) (*Query, error) { + conds := make([]condition, len(ast)) + for i, q := range ast { + cond, err := compileCondition(q) + if err != nil { + return nil, fmt.Errorf("compile %s: %w", q, err) } + conds[i] = cond } - - return conditions, nil + return &Query{ast: ast, conds: conds}, nil } -// Matches returns true if the query matches against any event in the given set -// of events, false otherwise. For each event, a match exists if the query is -// matched against *any* value in a slice of values. An error is returned if -// any attempted event match returns an error. -// -// For example, query "name=John" matches events = {"name": ["John", "Eric"]}. -// More examples could be found in parser_test.go and query_test.go. -func (q *Query) Matches(rawEvents []types.Event) (bool, error) { - if len(rawEvents) == 0 { - return false, nil +// Matches satisfies part of the pubsub.Query interface. This implementation +// never reports an error. A nil *Query matches all events. +func (q *Query) Matches(events []types.Event) (bool, error) { + if q == nil { + return true, nil } + return q.matchesEvents(events), nil +} - events := flattenEvents(rawEvents) - - var ( - eventAttr string - op Operator - ) - - buffer, begin, end := q.parser.Buffer, 0, 0 - - // tokens must be in the following order: - - // tag ("tx.gas") -> operator ("=") -> operand ("7") - for token := range q.parser.Tokens() { - switch token.pegRule { - case rulePegText: - begin, end = int(token.begin), int(token.end) - - case ruletag: - eventAttr = buffer[begin:end] - - case rulele: - op = OpLessEqual - - case rulege: - op = OpGreaterEqual - - case rulel: - op = OpLess - - case ruleg: - op = OpGreater - - case ruleequal: - op = OpEqual - - case rulecontains: - op = OpContains - case ruleexists: - op = OpExists - if strings.Contains(eventAttr, ".") { - // Searching for a full "type.attribute" event. - _, ok := events[eventAttr] - if !ok { - return false, nil - } - } else { - foundEvent := false - - loop: - for compositeKey := range events { - if strings.Index(compositeKey, eventAttr) == 0 { - foundEvent = true - break loop - } - } - if !foundEvent { - return false, nil - } - } - - case rulevalue: - // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") - valueWithoutSingleQuotes := buffer[begin+1 : end-1] - - // see if the triplet (event attribute, operator, operand) matches any event - // "tx.gas", "=", "7", { "tx.gas": 7, "tx.ID": "4AE393495334" } - match, err := match(eventAttr, op, reflect.ValueOf(valueWithoutSingleQuotes), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - - case rulenumber: - number := buffer[begin:end] - if strings.ContainsAny(number, ".") { // if it looks like a floating-point number - value, err := strconv.ParseFloat(number, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", - err, number, - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } else { - value, err := strconv.ParseInt(number, 10, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", - err, number, - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } - - case ruletime: - value, err := time.Parse(TimeLayout, buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - - case ruledate: - value, err := time.Parse("2006-01-02", buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } +// String matches part of the pubsub.Query interface. +func (q *Query) String() string { + if q == nil { + return "" } - - return true, nil + return q.ast.String() } -// match returns true if the given triplet (attribute, operator, operand) matches -// any value in an event for that attribute. If any match fails with an error, -// that error is returned. -// -// First, it looks up the key in the events and if it finds one, tries to compare -// all the values from it to the operand using the operator. -// -// "tx.gas", "=", "7", {"tx": [{"gas": 7, "ID": "4AE393495334"}]} -func match(attr string, op Operator, operand reflect.Value, events map[string][]string) (bool, error) { - // look up the tag from the query in tags - values, ok := events[attr] - if !ok { - return false, nil +// Syntax returns the syntax tree representation of q. +func (q *Query) Syntax() syntax.Query { + if q == nil { + return nil } + return q.ast +} - for _, value := range values { - // return true if any value in the set of the event's values matches - match, err := matchValue(value, op, operand) - if err != nil { - return false, err - } - - if match { - return true, nil +// matchesEvents reports whether all the conditions match the given events. +func (q *Query) matchesEvents(events []types.Event) bool { + for _, cond := range q.conds { + if !cond.matchesAny(events) { + return false } } - - return false, nil + return len(events) != 0 } -// matchValue will attempt to match a string value against an operator an -// operand. A boolean is returned representing the match result. It will return -// an error if the value cannot be parsed and matched against the operand type. -func matchValue(value string, op Operator, operand reflect.Value) (bool, error) { - switch operand.Kind() { - case reflect.Struct: // time - operandAsTime := operand.Interface().(time.Time) - - // try our best to convert value from events to time.Time - var ( - v time.Time - err error - ) +// A condition is a compiled match condition. A condition matches an event if +// the event has the designated type, contains an attribute with the given +// name, and the match function returns true for the attribute value. +type condition struct { + tag string // e.g., "tx.hash" + match func(s string) bool +} - if strings.ContainsAny(value, "T") { - v, err = time.Parse(TimeLayout, value) - } else { - v, err = time.Parse(DateLayout, value) - } - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to time.Time: %w", value, err) +// findAttr returns a slice of attribute values from event matching the +// condition tag, and reports whether the event type strictly equals the +// condition tag. +func (c condition) findAttr(event types.Event) ([]string, bool) { + if !strings.HasPrefix(c.tag, event.Type) { + return nil, false // type does not match tag + } else if len(c.tag) == len(event.Type) { + return nil, true // type == tag + } + var vals []string + for _, attr := range event.Attributes { + fullName := event.Type + "." + attr.Key + if fullName == c.tag { + vals = append(vals, attr.Value) } + } + return vals, false +} - switch op { - case OpLessEqual: - return (v.Before(operandAsTime) || v.Equal(operandAsTime)), nil - case OpGreaterEqual: - return (v.Equal(operandAsTime) || v.After(operandAsTime)), nil - case OpLess: - return v.Before(operandAsTime), nil - case OpGreater: - return v.After(operandAsTime), nil - case OpEqual: - return v.Equal(operandAsTime), nil +// matchesAny reports whether c matches at least one of the given events. +func (c condition) matchesAny(events []types.Event) bool { + for _, event := range events { + if c.matchesEvent(event) { + return true } + } + return false +} - case reflect.Float64: - var v float64 - - operandFloat64 := operand.Interface().(float64) - filteredValue := numRegex.FindString(value) - - // try our best to convert value from tags to float64 - v, err := strconv.ParseFloat(filteredValue, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) +// matchesEvent reports whether c matches the given event. +func (c condition) matchesEvent(event types.Event) bool { + vs, tagEqualsType := c.findAttr(event) + if len(vs) == 0 { + // As a special case, a condition tag that exactly matches the event type + // is matched against an empty string. This allows existence checks to + // work for type-only queries. + if tagEqualsType { + return c.match("") } + return false + } - switch op { - case OpLessEqual: - return v <= operandFloat64, nil - case OpGreaterEqual: - return v >= operandFloat64, nil - case OpLess: - return v < operandFloat64, nil - case OpGreater: - return v > operandFloat64, nil - case OpEqual: - return v == operandFloat64, nil + // At this point, we have candidate values. + for _, v := range vs { + if c.match(v) { + return true } + } + return false +} - case reflect.Int64: - var v int64 - - operandInt := operand.Interface().(int64) - filteredValue := numRegex.FindString(value) - - // if value looks like float, we try to parse it as float - if strings.ContainsAny(filteredValue, ".") { - v1, err := strconv.ParseFloat(filteredValue, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) - } - - v = int64(v1) - } else { - var err error - // try our best to convert value from tags to int64 - v, err = strconv.ParseInt(filteredValue, 10, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to int64: %w", filteredValue, err) - } - } +func compileCondition(cond syntax.Condition) (condition, error) { + out := condition{tag: cond.Tag} - switch op { - case OpLessEqual: - return v <= operandInt, nil - case OpGreaterEqual: - return v >= operandInt, nil - case OpLess: - return v < operandInt, nil - case OpGreater: - return v > operandInt, nil - case OpEqual: - return v == operandInt, nil - } + // Handle existence checks separately to simplify the logic below for + // comparisons that take arguments. + if cond.Op == syntax.TExists { + out.match = func(string) bool { return true } + return out, nil + } - case reflect.String: - switch op { - case OpEqual: - return value == operand.String(), nil - case OpContains: - return strings.Contains(value, operand.String()), nil - } + // All the other operators require an argument. + if cond.Arg == nil { + return condition{}, fmt.Errorf("missing argument for %v", cond.Op) + } + // Precompile the argument value matcher. + argType := cond.Arg.Type + var argValue interface{} + + switch argType { + case syntax.TString: + argValue = cond.Arg.Value() + case syntax.TNumber: + argValue = cond.Arg.Number() + case syntax.TTime, syntax.TDate: + argValue = cond.Arg.Time() default: - return false, fmt.Errorf("unknown kind of operand %v", operand.Kind()) + return condition{}, fmt.Errorf("unknown argument type %v", argType) } - return false, nil + mcons := opTypeMap[cond.Op][argType] + if mcons == nil { + return condition{}, fmt.Errorf("invalid op/arg combination (%v, %v)", cond.Op, argType) + } + out.match = mcons(argValue) + return out, nil } -func flattenEvents(events []types.Event) map[string][]string { - flattened := make(map[string][]string) +// TODO(creachadair): The existing implementation allows anything number shaped +// to be treated as a number. This preserves the parts of that behavior we had +// tests for, but we should probably get rid of that. +var extractNum = regexp.MustCompile(`^\d+(\.\d+)?`) - for _, event := range events { - if len(event.Type) == 0 { - continue - } +func parseNumber(s string) (float64, error) { + return strconv.ParseFloat(extractNum.FindString(s), 64) +} - for _, attr := range event.Attributes { - if len(attr.Key) == 0 { - continue +// A map of operator ⇒ argtype ⇒ match-constructor. +// An entry does not exist if the combination is not valid. +// +// Disable the dupl lint for this map. The result isn't even correct. +//nolint:dupl +var opTypeMap = map[syntax.Token]map[syntax.Token]func(interface{}) func(string) bool{ + syntax.TContains: { + syntax.TString: func(v interface{}) func(string) bool { + return func(s string) bool { + return strings.Contains(s, v.(string)) } - - compositeEvent := fmt.Sprintf("%s.%s", event.Type, attr.Key) - flattened[compositeEvent] = append(flattened[compositeEvent], attr.Value) - } - } - - return flattened + }, + }, + syntax.TEq: { + syntax.TString: func(v interface{}) func(string) bool { + return func(s string) bool { return s == v.(string) } + }, + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w == v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && ts.Equal(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && ts.Equal(v.(time.Time)) + } + }, + }, + syntax.TLt: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w < v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && ts.Before(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && ts.Before(v.(time.Time)) + } + }, + }, + syntax.TLeq: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w <= v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && !ts.After(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && !ts.After(v.(time.Time)) + } + }, + }, + syntax.TGt: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w > v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && ts.After(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && ts.After(v.(time.Time)) + } + }, + }, + syntax.TGeq: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w >= v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && !ts.Before(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && !ts.Before(v.(time.Time)) + } + }, + }, } diff --git a/libs/pubsub/query/query.peg b/libs/pubsub/query/query.peg deleted file mode 100644 index e2cfd0826..000000000 --- a/libs/pubsub/query/query.peg +++ /dev/null @@ -1,35 +0,0 @@ -package query - -type QueryParser Peg { -} - -e <- '\"' condition ( ' '+ and ' '+ condition )* '\"' !. - -condition <- tag ' '* (le ' '* (number / time / date) - / ge ' '* (number / time / date) - / l ' '* (number / time / date) - / g ' '* (number / time / date) - / equal ' '* (number / time / date / value) - / contains ' '* value - / exists - ) - -tag <- < (![ \t\n\r\\()"'=><] .)+ > -value <- < '\'' (!["'] .)* '\''> -number <- < ('0' - / [1-9] digit* ('.' digit*)?) > -digit <- [0-9] -time <- "TIME " < year '-' month '-' day 'T' digit digit ':' digit digit ':' digit digit (('-' / '+') digit digit ':' digit digit / 'Z') > -date <- "DATE " < year '-' month '-' day > -year <- ('1' / '2') digit digit digit -month <- ('0' / '1') digit -day <- ('0' / '1' / '2' / '3') digit -and <- "AND" - -equal <- "=" -contains <- "CONTAINS" -exists <- "EXISTS" -le <- "<=" -ge <- ">=" -l <- "<" -g <- ">" diff --git a/libs/pubsub/query/query.peg.go b/libs/pubsub/query/query.peg.go deleted file mode 100644 index a8e14c869..000000000 --- a/libs/pubsub/query/query.peg.go +++ /dev/null @@ -1,1871 +0,0 @@ -// nolint -package query - -import ( - "fmt" - "math" - "sort" - "strconv" -) - -const endSymbol rune = 1114112 - -/* The rule types inferred from the grammar are below. */ -type pegRule uint8 - -const ( - ruleUnknown pegRule = iota - rulee - rulecondition - ruletag - rulevalue - rulenumber - ruledigit - ruletime - ruledate - ruleyear - rulemonth - ruleday - ruleand - ruleequal - rulecontains - ruleexists - rulele - rulege - rulel - ruleg - rulePegText - - rulePre - ruleIn - ruleSuf -) - -var rul3s = [...]string{ - "Unknown", - "e", - "condition", - "tag", - "value", - "number", - "digit", - "time", - "date", - "year", - "month", - "day", - "and", - "equal", - "contains", - "exists", - "le", - "ge", - "l", - "g", - "PegText", - - "Pre_", - "_In_", - "_Suf", -} - -type node32 struct { - token32 - up, next *node32 -} - -func (node *node32) print(depth int, buffer string) { - for node != nil { - for c := 0; c < depth; c++ { - fmt.Printf(" ") - } - fmt.Printf("\x1B[34m%v\x1B[m %v\n", rul3s[node.pegRule], strconv.Quote(string(([]rune(buffer)[node.begin:node.end])))) - if node.up != nil { - node.up.print(depth+1, buffer) - } - node = node.next - } -} - -func (node *node32) Print(buffer string) { - node.print(0, buffer) -} - -type element struct { - node *node32 - down *element -} - -/* ${@} bit structure for abstract syntax tree */ -type token32 struct { - pegRule - begin, end, next uint32 -} - -func (t *token32) isZero() bool { - return t.pegRule == ruleUnknown && t.begin == 0 && t.end == 0 && t.next == 0 -} - -func (t *token32) isParentOf(u token32) bool { - return t.begin <= u.begin && t.end >= u.end && t.next > u.next -} - -func (t *token32) getToken32() token32 { - return token32{pegRule: t.pegRule, begin: uint32(t.begin), end: uint32(t.end), next: uint32(t.next)} -} - -func (t *token32) String() string { - return fmt.Sprintf("\x1B[34m%v\x1B[m %v %v %v", rul3s[t.pegRule], t.begin, t.end, t.next) -} - -type tokens32 struct { - tree []token32 - ordered [][]token32 -} - -func (t *tokens32) trim(length int) { - t.tree = t.tree[0:length] -} - -func (t *tokens32) Print() { - for _, token := range t.tree { - fmt.Println(token.String()) - } -} - -func (t *tokens32) Order() [][]token32 { - if t.ordered != nil { - return t.ordered - } - - depths := make([]int32, 1, math.MaxInt16) - for i, token := range t.tree { - if token.pegRule == ruleUnknown { - t.tree = t.tree[:i] - break - } - depth := int(token.next) - if length := len(depths); depth >= length { - depths = depths[:depth+1] - } - depths[depth]++ - } - depths = append(depths, 0) - - ordered, pool := make([][]token32, len(depths)), make([]token32, len(t.tree)+len(depths)) - for i, depth := range depths { - depth++ - ordered[i], pool, depths[i] = pool[:depth], pool[depth:], 0 - } - - for i, token := range t.tree { - depth := token.next - token.next = uint32(i) - ordered[depth][depths[depth]] = token - depths[depth]++ - } - t.ordered = ordered - return ordered -} - -type state32 struct { - token32 - depths []int32 - leaf bool -} - -func (t *tokens32) AST() *node32 { - tokens := t.Tokens() - stack := &element{node: &node32{token32: <-tokens}} - for token := range tokens { - if token.begin == token.end { - continue - } - node := &node32{token32: token} - for stack != nil && stack.node.begin >= token.begin && stack.node.end <= token.end { - stack.node.next = node.up - node.up = stack.node - stack = stack.down - } - stack = &element{node: node, down: stack} - } - return stack.node -} - -func (t *tokens32) PreOrder() (<-chan state32, [][]token32) { - s, ordered := make(chan state32, 6), t.Order() - go func() { - var states [8]state32 - for i := range states { - states[i].depths = make([]int32, len(ordered)) - } - depths, state, depth := make([]int32, len(ordered)), 0, 1 - write := func(t token32, leaf bool) { - S := states[state] - state, S.pegRule, S.begin, S.end, S.next, S.leaf = (state+1)%8, t.pegRule, t.begin, t.end, uint32(depth), leaf - copy(S.depths, depths) - s <- S - } - - states[state].token32 = ordered[0][0] - depths[0]++ - state++ - a, b := ordered[depth-1][depths[depth-1]-1], ordered[depth][depths[depth]] - depthFirstSearch: - for { - for { - if i := depths[depth]; i > 0 { - if c, j := ordered[depth][i-1], depths[depth-1]; a.isParentOf(c) && - (j < 2 || !ordered[depth-1][j-2].isParentOf(c)) { - if c.end != b.begin { - write(token32{pegRule: ruleIn, begin: c.end, end: b.begin}, true) - } - break - } - } - - if a.begin < b.begin { - write(token32{pegRule: rulePre, begin: a.begin, end: b.begin}, true) - } - break - } - - next := depth + 1 - if c := ordered[next][depths[next]]; c.pegRule != ruleUnknown && b.isParentOf(c) { - write(b, false) - depths[depth]++ - depth, a, b = next, b, c - continue - } - - write(b, true) - depths[depth]++ - c, parent := ordered[depth][depths[depth]], true - for { - if c.pegRule != ruleUnknown && a.isParentOf(c) { - b = c - continue depthFirstSearch - } else if parent && b.end != a.end { - write(token32{pegRule: ruleSuf, begin: b.end, end: a.end}, true) - } - - depth-- - if depth > 0 { - a, b, c = ordered[depth-1][depths[depth-1]-1], a, ordered[depth][depths[depth]] - parent = a.isParentOf(b) - continue - } - - break depthFirstSearch - } - } - - close(s) - }() - return s, ordered -} - -func (t *tokens32) PrintSyntax() { - tokens, ordered := t.PreOrder() - max := -1 - for token := range tokens { - if !token.leaf { - fmt.Printf("%v", token.begin) - for i, leaf, depths := 0, int(token.next), token.depths; i < leaf; i++ { - fmt.Printf(" \x1B[36m%v\x1B[m", rul3s[ordered[i][depths[i]-1].pegRule]) - } - fmt.Printf(" \x1B[36m%v\x1B[m\n", rul3s[token.pegRule]) - } else if token.begin == token.end { - fmt.Printf("%v", token.begin) - for i, leaf, depths := 0, int(token.next), token.depths; i < leaf; i++ { - fmt.Printf(" \x1B[31m%v\x1B[m", rul3s[ordered[i][depths[i]-1].pegRule]) - } - fmt.Printf(" \x1B[31m%v\x1B[m\n", rul3s[token.pegRule]) - } else { - for c, end := token.begin, token.end; c < end; c++ { - if i := int(c); max+1 < i { - for j := max; j < i; j++ { - fmt.Printf("skip %v %v\n", j, token.String()) - } - max = i - } else if i := int(c); i <= max { - for j := i; j <= max; j++ { - fmt.Printf("dupe %v %v\n", j, token.String()) - } - } else { - max = int(c) - } - fmt.Printf("%v", c) - for i, leaf, depths := 0, int(token.next), token.depths; i < leaf; i++ { - fmt.Printf(" \x1B[34m%v\x1B[m", rul3s[ordered[i][depths[i]-1].pegRule]) - } - fmt.Printf(" \x1B[34m%v\x1B[m\n", rul3s[token.pegRule]) - } - fmt.Printf("\n") - } - } -} - -func (t *tokens32) PrintSyntaxTree(buffer string) { - tokens, _ := t.PreOrder() - for token := range tokens { - for c := 0; c < int(token.next); c++ { - fmt.Printf(" ") - } - fmt.Printf("\x1B[34m%v\x1B[m %v\n", rul3s[token.pegRule], strconv.Quote(string(([]rune(buffer)[token.begin:token.end])))) - } -} - -func (t *tokens32) Add(rule pegRule, begin, end, depth uint32, index int) { - t.tree[index] = token32{pegRule: rule, begin: uint32(begin), end: uint32(end), next: uint32(depth)} -} - -func (t *tokens32) Tokens() <-chan token32 { - s := make(chan token32, 16) - go func() { - for _, v := range t.tree { - s <- v.getToken32() - } - close(s) - }() - return s -} - -func (t *tokens32) Error() []token32 { - ordered := t.Order() - length := len(ordered) - tokens, length := make([]token32, length), length-1 - for i := range tokens { - o := ordered[length-i] - if len(o) > 1 { - tokens[i] = o[len(o)-2].getToken32() - } - } - return tokens -} - -func (t *tokens32) Expand(index int) { - tree := t.tree - if index >= len(tree) { - expanded := make([]token32, 2*len(tree)) - copy(expanded, tree) - t.tree = expanded - } -} - -type QueryParser struct { - Buffer string - buffer []rune - rules [21]func() bool - Parse func(rule ...int) error - Reset func() - Pretty bool - tokens32 -} - -type textPosition struct { - line, symbol int -} - -type textPositionMap map[int]textPosition - -func translatePositions(buffer []rune, positions []int) textPositionMap { - length, translations, j, line, symbol := len(positions), make(textPositionMap, len(positions)), 0, 1, 0 - sort.Ints(positions) - -search: - for i, c := range buffer { - if c == '\n' { - line, symbol = line+1, 0 - } else { - symbol++ - } - if i == positions[j] { - translations[positions[j]] = textPosition{line, symbol} - for j++; j < length; j++ { - if i != positions[j] { - continue search - } - } - break search - } - } - - return translations -} - -type parseError struct { - p *QueryParser - max token32 -} - -func (e *parseError) Error() string { - tokens, error := []token32{e.max}, "\n" - positions, p := make([]int, 2*len(tokens)), 0 - for _, token := range tokens { - positions[p], p = int(token.begin), p+1 - positions[p], p = int(token.end), p+1 - } - translations := translatePositions(e.p.buffer, positions) - format := "parse error near %v (line %v symbol %v - line %v symbol %v):\n%v\n" - if e.p.Pretty { - format = "parse error near \x1B[34m%v\x1B[m (line %v symbol %v - line %v symbol %v):\n%v\n" - } - for _, token := range tokens { - begin, end := int(token.begin), int(token.end) - error += fmt.Sprintf(format, - rul3s[token.pegRule], - translations[begin].line, translations[begin].symbol, - translations[end].line, translations[end].symbol, - strconv.Quote(string(e.p.buffer[begin:end]))) - } - - return error -} - -func (p *QueryParser) PrintSyntaxTree() { - p.tokens32.PrintSyntaxTree(p.Buffer) -} - -func (p *QueryParser) Highlighter() { - p.PrintSyntax() -} - -func (p *QueryParser) Init() { - p.buffer = []rune(p.Buffer) - if len(p.buffer) == 0 || p.buffer[len(p.buffer)-1] != endSymbol { - p.buffer = append(p.buffer, endSymbol) - } - - tree := tokens32{tree: make([]token32, math.MaxInt16)} - var max token32 - position, depth, tokenIndex, buffer, _rules := uint32(0), uint32(0), 0, p.buffer, p.rules - - p.Parse = func(rule ...int) error { - r := 1 - if len(rule) > 0 { - r = rule[0] - } - matches := p.rules[r]() - p.tokens32 = tree - if matches { - p.trim(tokenIndex) - return nil - } - return &parseError{p, max} - } - - p.Reset = func() { - position, tokenIndex, depth = 0, 0, 0 - } - - add := func(rule pegRule, begin uint32) { - tree.Expand(tokenIndex) - tree.Add(rule, begin, position, depth, tokenIndex) - tokenIndex++ - if begin != position && position > max.end { - max = token32{rule, begin, position, depth} - } - } - - matchDot := func() bool { - if buffer[position] != endSymbol { - position++ - return true - } - return false - } - - /*matchChar := func(c byte) bool { - if buffer[position] == c { - position++ - return true - } - return false - }*/ - - /*matchRange := func(lower byte, upper byte) bool { - if c := buffer[position]; c >= lower && c <= upper { - position++ - return true - } - return false - }*/ - - _rules = [...]func() bool{ - nil, - /* 0 e <- <('"' condition (' '+ and ' '+ condition)* '"' !.)> */ - func() bool { - position0, tokenIndex0, depth0 := position, tokenIndex, depth - { - position1 := position - depth++ - if buffer[position] != rune('"') { - goto l0 - } - position++ - if !_rules[rulecondition]() { - goto l0 - } - l2: - { - position3, tokenIndex3, depth3 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l3 - } - position++ - l4: - { - position5, tokenIndex5, depth5 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l5 - } - position++ - goto l4 - l5: - position, tokenIndex, depth = position5, tokenIndex5, depth5 - } - { - position6 := position - depth++ - { - position7, tokenIndex7, depth7 := position, tokenIndex, depth - if buffer[position] != rune('a') { - goto l8 - } - position++ - goto l7 - l8: - position, tokenIndex, depth = position7, tokenIndex7, depth7 - if buffer[position] != rune('A') { - goto l3 - } - position++ - } - l7: - { - position9, tokenIndex9, depth9 := position, tokenIndex, depth - if buffer[position] != rune('n') { - goto l10 - } - position++ - goto l9 - l10: - position, tokenIndex, depth = position9, tokenIndex9, depth9 - if buffer[position] != rune('N') { - goto l3 - } - position++ - } - l9: - { - position11, tokenIndex11, depth11 := position, tokenIndex, depth - if buffer[position] != rune('d') { - goto l12 - } - position++ - goto l11 - l12: - position, tokenIndex, depth = position11, tokenIndex11, depth11 - if buffer[position] != rune('D') { - goto l3 - } - position++ - } - l11: - depth-- - add(ruleand, position6) - } - if buffer[position] != rune(' ') { - goto l3 - } - position++ - l13: - { - position14, tokenIndex14, depth14 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l14 - } - position++ - goto l13 - l14: - position, tokenIndex, depth = position14, tokenIndex14, depth14 - } - if !_rules[rulecondition]() { - goto l3 - } - goto l2 - l3: - position, tokenIndex, depth = position3, tokenIndex3, depth3 - } - if buffer[position] != rune('"') { - goto l0 - } - position++ - { - position15, tokenIndex15, depth15 := position, tokenIndex, depth - if !matchDot() { - goto l15 - } - goto l0 - l15: - position, tokenIndex, depth = position15, tokenIndex15, depth15 - } - depth-- - add(rulee, position1) - } - return true - l0: - position, tokenIndex, depth = position0, tokenIndex0, depth0 - return false - }, - /* 1 condition <- <(tag ' '* ((le ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number))) / (ge ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number))) / ((&('E' | 'e') exists) | (&('=') (equal ' '* ((&('\'') value) | (&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('>') (g ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('<') (l ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('C' | 'c') (contains ' '* value)))))> */ - func() bool { - position16, tokenIndex16, depth16 := position, tokenIndex, depth - { - position17 := position - depth++ - { - position18 := position - depth++ - { - position19 := position - depth++ - { - position22, tokenIndex22, depth22 := position, tokenIndex, depth - { - switch buffer[position] { - case '<': - if buffer[position] != rune('<') { - goto l22 - } - position++ - break - case '>': - if buffer[position] != rune('>') { - goto l22 - } - position++ - break - case '=': - if buffer[position] != rune('=') { - goto l22 - } - position++ - break - case '\'': - if buffer[position] != rune('\'') { - goto l22 - } - position++ - break - case '"': - if buffer[position] != rune('"') { - goto l22 - } - position++ - break - case ')': - if buffer[position] != rune(')') { - goto l22 - } - position++ - break - case '(': - if buffer[position] != rune('(') { - goto l22 - } - position++ - break - case '\\': - if buffer[position] != rune('\\') { - goto l22 - } - position++ - break - case '\r': - if buffer[position] != rune('\r') { - goto l22 - } - position++ - break - case '\n': - if buffer[position] != rune('\n') { - goto l22 - } - position++ - break - case '\t': - if buffer[position] != rune('\t') { - goto l22 - } - position++ - break - default: - if buffer[position] != rune(' ') { - goto l22 - } - position++ - break - } - } - - goto l16 - l22: - position, tokenIndex, depth = position22, tokenIndex22, depth22 - } - if !matchDot() { - goto l16 - } - l20: - { - position21, tokenIndex21, depth21 := position, tokenIndex, depth - { - position24, tokenIndex24, depth24 := position, tokenIndex, depth - { - switch buffer[position] { - case '<': - if buffer[position] != rune('<') { - goto l24 - } - position++ - break - case '>': - if buffer[position] != rune('>') { - goto l24 - } - position++ - break - case '=': - if buffer[position] != rune('=') { - goto l24 - } - position++ - break - case '\'': - if buffer[position] != rune('\'') { - goto l24 - } - position++ - break - case '"': - if buffer[position] != rune('"') { - goto l24 - } - position++ - break - case ')': - if buffer[position] != rune(')') { - goto l24 - } - position++ - break - case '(': - if buffer[position] != rune('(') { - goto l24 - } - position++ - break - case '\\': - if buffer[position] != rune('\\') { - goto l24 - } - position++ - break - case '\r': - if buffer[position] != rune('\r') { - goto l24 - } - position++ - break - case '\n': - if buffer[position] != rune('\n') { - goto l24 - } - position++ - break - case '\t': - if buffer[position] != rune('\t') { - goto l24 - } - position++ - break - default: - if buffer[position] != rune(' ') { - goto l24 - } - position++ - break - } - } - - goto l21 - l24: - position, tokenIndex, depth = position24, tokenIndex24, depth24 - } - if !matchDot() { - goto l21 - } - goto l20 - l21: - position, tokenIndex, depth = position21, tokenIndex21, depth21 - } - depth-- - add(rulePegText, position19) - } - depth-- - add(ruletag, position18) - } - l26: - { - position27, tokenIndex27, depth27 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l27 - } - position++ - goto l26 - l27: - position, tokenIndex, depth = position27, tokenIndex27, depth27 - } - { - position28, tokenIndex28, depth28 := position, tokenIndex, depth - { - position30 := position - depth++ - if buffer[position] != rune('<') { - goto l29 - } - position++ - if buffer[position] != rune('=') { - goto l29 - } - position++ - depth-- - add(rulele, position30) - } - l31: - { - position32, tokenIndex32, depth32 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l32 - } - position++ - goto l31 - l32: - position, tokenIndex, depth = position32, tokenIndex32, depth32 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l29 - } - break - case 'T', 't': - if !_rules[ruletime]() { - goto l29 - } - break - default: - if !_rules[rulenumber]() { - goto l29 - } - break - } - } - - goto l28 - l29: - position, tokenIndex, depth = position28, tokenIndex28, depth28 - { - position35 := position - depth++ - if buffer[position] != rune('>') { - goto l34 - } - position++ - if buffer[position] != rune('=') { - goto l34 - } - position++ - depth-- - add(rulege, position35) - } - l36: - { - position37, tokenIndex37, depth37 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l37 - } - position++ - goto l36 - l37: - position, tokenIndex, depth = position37, tokenIndex37, depth37 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l34 - } - break - case 'T', 't': - if !_rules[ruletime]() { - goto l34 - } - break - default: - if !_rules[rulenumber]() { - goto l34 - } - break - } - } - - goto l28 - l34: - position, tokenIndex, depth = position28, tokenIndex28, depth28 - { - switch buffer[position] { - case 'E', 'e': - { - position40 := position - depth++ - { - position41, tokenIndex41, depth41 := position, tokenIndex, depth - if buffer[position] != rune('e') { - goto l42 - } - position++ - goto l41 - l42: - position, tokenIndex, depth = position41, tokenIndex41, depth41 - if buffer[position] != rune('E') { - goto l16 - } - position++ - } - l41: - { - position43, tokenIndex43, depth43 := position, tokenIndex, depth - if buffer[position] != rune('x') { - goto l44 - } - position++ - goto l43 - l44: - position, tokenIndex, depth = position43, tokenIndex43, depth43 - if buffer[position] != rune('X') { - goto l16 - } - position++ - } - l43: - { - position45, tokenIndex45, depth45 := position, tokenIndex, depth - if buffer[position] != rune('i') { - goto l46 - } - position++ - goto l45 - l46: - position, tokenIndex, depth = position45, tokenIndex45, depth45 - if buffer[position] != rune('I') { - goto l16 - } - position++ - } - l45: - { - position47, tokenIndex47, depth47 := position, tokenIndex, depth - if buffer[position] != rune('s') { - goto l48 - } - position++ - goto l47 - l48: - position, tokenIndex, depth = position47, tokenIndex47, depth47 - if buffer[position] != rune('S') { - goto l16 - } - position++ - } - l47: - { - position49, tokenIndex49, depth49 := position, tokenIndex, depth - if buffer[position] != rune('t') { - goto l50 - } - position++ - goto l49 - l50: - position, tokenIndex, depth = position49, tokenIndex49, depth49 - if buffer[position] != rune('T') { - goto l16 - } - position++ - } - l49: - { - position51, tokenIndex51, depth51 := position, tokenIndex, depth - if buffer[position] != rune('s') { - goto l52 - } - position++ - goto l51 - l52: - position, tokenIndex, depth = position51, tokenIndex51, depth51 - if buffer[position] != rune('S') { - goto l16 - } - position++ - } - l51: - depth-- - add(ruleexists, position40) - } - break - case '=': - { - position53 := position - depth++ - if buffer[position] != rune('=') { - goto l16 - } - position++ - depth-- - add(ruleequal, position53) - } - l54: - { - position55, tokenIndex55, depth55 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l55 - } - position++ - goto l54 - l55: - position, tokenIndex, depth = position55, tokenIndex55, depth55 - } - { - switch buffer[position] { - case '\'': - if !_rules[rulevalue]() { - goto l16 - } - break - case 'D', 'd': - if !_rules[ruledate]() { - goto l16 - } - break - case 'T', 't': - if !_rules[ruletime]() { - goto l16 - } - break - default: - if !_rules[rulenumber]() { - goto l16 - } - break - } - } - - break - case '>': - { - position57 := position - depth++ - if buffer[position] != rune('>') { - goto l16 - } - position++ - depth-- - add(ruleg, position57) - } - l58: - { - position59, tokenIndex59, depth59 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l59 - } - position++ - goto l58 - l59: - position, tokenIndex, depth = position59, tokenIndex59, depth59 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l16 - } - break - case 'T', 't': - if !_rules[ruletime]() { - goto l16 - } - break - default: - if !_rules[rulenumber]() { - goto l16 - } - break - } - } - - break - case '<': - { - position61 := position - depth++ - if buffer[position] != rune('<') { - goto l16 - } - position++ - depth-- - add(rulel, position61) - } - l62: - { - position63, tokenIndex63, depth63 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l63 - } - position++ - goto l62 - l63: - position, tokenIndex, depth = position63, tokenIndex63, depth63 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l16 - } - break - case 'T', 't': - if !_rules[ruletime]() { - goto l16 - } - break - default: - if !_rules[rulenumber]() { - goto l16 - } - break - } - } - - break - default: - { - position65 := position - depth++ - { - position66, tokenIndex66, depth66 := position, tokenIndex, depth - if buffer[position] != rune('c') { - goto l67 - } - position++ - goto l66 - l67: - position, tokenIndex, depth = position66, tokenIndex66, depth66 - if buffer[position] != rune('C') { - goto l16 - } - position++ - } - l66: - { - position68, tokenIndex68, depth68 := position, tokenIndex, depth - if buffer[position] != rune('o') { - goto l69 - } - position++ - goto l68 - l69: - position, tokenIndex, depth = position68, tokenIndex68, depth68 - if buffer[position] != rune('O') { - goto l16 - } - position++ - } - l68: - { - position70, tokenIndex70, depth70 := position, tokenIndex, depth - if buffer[position] != rune('n') { - goto l71 - } - position++ - goto l70 - l71: - position, tokenIndex, depth = position70, tokenIndex70, depth70 - if buffer[position] != rune('N') { - goto l16 - } - position++ - } - l70: - { - position72, tokenIndex72, depth72 := position, tokenIndex, depth - if buffer[position] != rune('t') { - goto l73 - } - position++ - goto l72 - l73: - position, tokenIndex, depth = position72, tokenIndex72, depth72 - if buffer[position] != rune('T') { - goto l16 - } - position++ - } - l72: - { - position74, tokenIndex74, depth74 := position, tokenIndex, depth - if buffer[position] != rune('a') { - goto l75 - } - position++ - goto l74 - l75: - position, tokenIndex, depth = position74, tokenIndex74, depth74 - if buffer[position] != rune('A') { - goto l16 - } - position++ - } - l74: - { - position76, tokenIndex76, depth76 := position, tokenIndex, depth - if buffer[position] != rune('i') { - goto l77 - } - position++ - goto l76 - l77: - position, tokenIndex, depth = position76, tokenIndex76, depth76 - if buffer[position] != rune('I') { - goto l16 - } - position++ - } - l76: - { - position78, tokenIndex78, depth78 := position, tokenIndex, depth - if buffer[position] != rune('n') { - goto l79 - } - position++ - goto l78 - l79: - position, tokenIndex, depth = position78, tokenIndex78, depth78 - if buffer[position] != rune('N') { - goto l16 - } - position++ - } - l78: - { - position80, tokenIndex80, depth80 := position, tokenIndex, depth - if buffer[position] != rune('s') { - goto l81 - } - position++ - goto l80 - l81: - position, tokenIndex, depth = position80, tokenIndex80, depth80 - if buffer[position] != rune('S') { - goto l16 - } - position++ - } - l80: - depth-- - add(rulecontains, position65) - } - l82: - { - position83, tokenIndex83, depth83 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l83 - } - position++ - goto l82 - l83: - position, tokenIndex, depth = position83, tokenIndex83, depth83 - } - if !_rules[rulevalue]() { - goto l16 - } - break - } - } - - } - l28: - depth-- - add(rulecondition, position17) - } - return true - l16: - position, tokenIndex, depth = position16, tokenIndex16, depth16 - return false - }, - /* 2 tag <- <<(!((&('<') '<') | (&('>') '>') | (&('=') '=') | (&('\'') '\'') | (&('"') '"') | (&(')') ')') | (&('(') '(') | (&('\\') '\\') | (&('\r') '\r') | (&('\n') '\n') | (&('\t') '\t') | (&(' ') ' ')) .)+>> */ - nil, - /* 3 value <- <<('\'' (!('"' / '\'') .)* '\'')>> */ - func() bool { - position85, tokenIndex85, depth85 := position, tokenIndex, depth - { - position86 := position - depth++ - { - position87 := position - depth++ - if buffer[position] != rune('\'') { - goto l85 - } - position++ - l88: - { - position89, tokenIndex89, depth89 := position, tokenIndex, depth - { - position90, tokenIndex90, depth90 := position, tokenIndex, depth - { - position91, tokenIndex91, depth91 := position, tokenIndex, depth - if buffer[position] != rune('"') { - goto l92 - } - position++ - goto l91 - l92: - position, tokenIndex, depth = position91, tokenIndex91, depth91 - if buffer[position] != rune('\'') { - goto l90 - } - position++ - } - l91: - goto l89 - l90: - position, tokenIndex, depth = position90, tokenIndex90, depth90 - } - if !matchDot() { - goto l89 - } - goto l88 - l89: - position, tokenIndex, depth = position89, tokenIndex89, depth89 - } - if buffer[position] != rune('\'') { - goto l85 - } - position++ - depth-- - add(rulePegText, position87) - } - depth-- - add(rulevalue, position86) - } - return true - l85: - position, tokenIndex, depth = position85, tokenIndex85, depth85 - return false - }, - /* 4 number <- <<('0' / ([1-9] digit* ('.' digit*)?))>> */ - func() bool { - position93, tokenIndex93, depth93 := position, tokenIndex, depth - { - position94 := position - depth++ - { - position95 := position - depth++ - { - position96, tokenIndex96, depth96 := position, tokenIndex, depth - if buffer[position] != rune('0') { - goto l97 - } - position++ - goto l96 - l97: - position, tokenIndex, depth = position96, tokenIndex96, depth96 - if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l93 - } - position++ - l98: - { - position99, tokenIndex99, depth99 := position, tokenIndex, depth - if !_rules[ruledigit]() { - goto l99 - } - goto l98 - l99: - position, tokenIndex, depth = position99, tokenIndex99, depth99 - } - { - position100, tokenIndex100, depth100 := position, tokenIndex, depth - if buffer[position] != rune('.') { - goto l100 - } - position++ - l102: - { - position103, tokenIndex103, depth103 := position, tokenIndex, depth - if !_rules[ruledigit]() { - goto l103 - } - goto l102 - l103: - position, tokenIndex, depth = position103, tokenIndex103, depth103 - } - goto l101 - l100: - position, tokenIndex, depth = position100, tokenIndex100, depth100 - } - l101: - } - l96: - depth-- - add(rulePegText, position95) - } - depth-- - add(rulenumber, position94) - } - return true - l93: - position, tokenIndex, depth = position93, tokenIndex93, depth93 - return false - }, - /* 5 digit <- <[0-9]> */ - func() bool { - position104, tokenIndex104, depth104 := position, tokenIndex, depth - { - position105 := position - depth++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l104 - } - position++ - depth-- - add(ruledigit, position105) - } - return true - l104: - position, tokenIndex, depth = position104, tokenIndex104, depth104 - return false - }, - /* 6 time <- <(('t' / 'T') ('i' / 'I') ('m' / 'M') ('e' / 'E') ' ' <(year '-' month '-' day 'T' digit digit ':' digit digit ':' digit digit ((('-' / '+') digit digit ':' digit digit) / 'Z'))>)> */ - func() bool { - position106, tokenIndex106, depth106 := position, tokenIndex, depth - { - position107 := position - depth++ - { - position108, tokenIndex108, depth108 := position, tokenIndex, depth - if buffer[position] != rune('t') { - goto l109 - } - position++ - goto l108 - l109: - position, tokenIndex, depth = position108, tokenIndex108, depth108 - if buffer[position] != rune('T') { - goto l106 - } - position++ - } - l108: - { - position110, tokenIndex110, depth110 := position, tokenIndex, depth - if buffer[position] != rune('i') { - goto l111 - } - position++ - goto l110 - l111: - position, tokenIndex, depth = position110, tokenIndex110, depth110 - if buffer[position] != rune('I') { - goto l106 - } - position++ - } - l110: - { - position112, tokenIndex112, depth112 := position, tokenIndex, depth - if buffer[position] != rune('m') { - goto l113 - } - position++ - goto l112 - l113: - position, tokenIndex, depth = position112, tokenIndex112, depth112 - if buffer[position] != rune('M') { - goto l106 - } - position++ - } - l112: - { - position114, tokenIndex114, depth114 := position, tokenIndex, depth - if buffer[position] != rune('e') { - goto l115 - } - position++ - goto l114 - l115: - position, tokenIndex, depth = position114, tokenIndex114, depth114 - if buffer[position] != rune('E') { - goto l106 - } - position++ - } - l114: - if buffer[position] != rune(' ') { - goto l106 - } - position++ - { - position116 := position - depth++ - if !_rules[ruleyear]() { - goto l106 - } - if buffer[position] != rune('-') { - goto l106 - } - position++ - if !_rules[rulemonth]() { - goto l106 - } - if buffer[position] != rune('-') { - goto l106 - } - position++ - if !_rules[ruleday]() { - goto l106 - } - if buffer[position] != rune('T') { - goto l106 - } - position++ - if !_rules[ruledigit]() { - goto l106 - } - if !_rules[ruledigit]() { - goto l106 - } - if buffer[position] != rune(':') { - goto l106 - } - position++ - if !_rules[ruledigit]() { - goto l106 - } - if !_rules[ruledigit]() { - goto l106 - } - if buffer[position] != rune(':') { - goto l106 - } - position++ - if !_rules[ruledigit]() { - goto l106 - } - if !_rules[ruledigit]() { - goto l106 - } - { - position117, tokenIndex117, depth117 := position, tokenIndex, depth - { - position119, tokenIndex119, depth119 := position, tokenIndex, depth - if buffer[position] != rune('-') { - goto l120 - } - position++ - goto l119 - l120: - position, tokenIndex, depth = position119, tokenIndex119, depth119 - if buffer[position] != rune('+') { - goto l118 - } - position++ - } - l119: - if !_rules[ruledigit]() { - goto l118 - } - if !_rules[ruledigit]() { - goto l118 - } - if buffer[position] != rune(':') { - goto l118 - } - position++ - if !_rules[ruledigit]() { - goto l118 - } - if !_rules[ruledigit]() { - goto l118 - } - goto l117 - l118: - position, tokenIndex, depth = position117, tokenIndex117, depth117 - if buffer[position] != rune('Z') { - goto l106 - } - position++ - } - l117: - depth-- - add(rulePegText, position116) - } - depth-- - add(ruletime, position107) - } - return true - l106: - position, tokenIndex, depth = position106, tokenIndex106, depth106 - return false - }, - /* 7 date <- <(('d' / 'D') ('a' / 'A') ('t' / 'T') ('e' / 'E') ' ' <(year '-' month '-' day)>)> */ - func() bool { - position121, tokenIndex121, depth121 := position, tokenIndex, depth - { - position122 := position - depth++ - { - position123, tokenIndex123, depth123 := position, tokenIndex, depth - if buffer[position] != rune('d') { - goto l124 - } - position++ - goto l123 - l124: - position, tokenIndex, depth = position123, tokenIndex123, depth123 - if buffer[position] != rune('D') { - goto l121 - } - position++ - } - l123: - { - position125, tokenIndex125, depth125 := position, tokenIndex, depth - if buffer[position] != rune('a') { - goto l126 - } - position++ - goto l125 - l126: - position, tokenIndex, depth = position125, tokenIndex125, depth125 - if buffer[position] != rune('A') { - goto l121 - } - position++ - } - l125: - { - position127, tokenIndex127, depth127 := position, tokenIndex, depth - if buffer[position] != rune('t') { - goto l128 - } - position++ - goto l127 - l128: - position, tokenIndex, depth = position127, tokenIndex127, depth127 - if buffer[position] != rune('T') { - goto l121 - } - position++ - } - l127: - { - position129, tokenIndex129, depth129 := position, tokenIndex, depth - if buffer[position] != rune('e') { - goto l130 - } - position++ - goto l129 - l130: - position, tokenIndex, depth = position129, tokenIndex129, depth129 - if buffer[position] != rune('E') { - goto l121 - } - position++ - } - l129: - if buffer[position] != rune(' ') { - goto l121 - } - position++ - { - position131 := position - depth++ - if !_rules[ruleyear]() { - goto l121 - } - if buffer[position] != rune('-') { - goto l121 - } - position++ - if !_rules[rulemonth]() { - goto l121 - } - if buffer[position] != rune('-') { - goto l121 - } - position++ - if !_rules[ruleday]() { - goto l121 - } - depth-- - add(rulePegText, position131) - } - depth-- - add(ruledate, position122) - } - return true - l121: - position, tokenIndex, depth = position121, tokenIndex121, depth121 - return false - }, - /* 8 year <- <(('1' / '2') digit digit digit)> */ - func() bool { - position132, tokenIndex132, depth132 := position, tokenIndex, depth - { - position133 := position - depth++ - { - position134, tokenIndex134, depth134 := position, tokenIndex, depth - if buffer[position] != rune('1') { - goto l135 - } - position++ - goto l134 - l135: - position, tokenIndex, depth = position134, tokenIndex134, depth134 - if buffer[position] != rune('2') { - goto l132 - } - position++ - } - l134: - if !_rules[ruledigit]() { - goto l132 - } - if !_rules[ruledigit]() { - goto l132 - } - if !_rules[ruledigit]() { - goto l132 - } - depth-- - add(ruleyear, position133) - } - return true - l132: - position, tokenIndex, depth = position132, tokenIndex132, depth132 - return false - }, - /* 9 month <- <(('0' / '1') digit)> */ - func() bool { - position136, tokenIndex136, depth136 := position, tokenIndex, depth - { - position137 := position - depth++ - { - position138, tokenIndex138, depth138 := position, tokenIndex, depth - if buffer[position] != rune('0') { - goto l139 - } - position++ - goto l138 - l139: - position, tokenIndex, depth = position138, tokenIndex138, depth138 - if buffer[position] != rune('1') { - goto l136 - } - position++ - } - l138: - if !_rules[ruledigit]() { - goto l136 - } - depth-- - add(rulemonth, position137) - } - return true - l136: - position, tokenIndex, depth = position136, tokenIndex136, depth136 - return false - }, - /* 10 day <- <(((&('3') '3') | (&('2') '2') | (&('1') '1') | (&('0') '0')) digit)> */ - func() bool { - position140, tokenIndex140, depth140 := position, tokenIndex, depth - { - position141 := position - depth++ - { - switch buffer[position] { - case '3': - if buffer[position] != rune('3') { - goto l140 - } - position++ - break - case '2': - if buffer[position] != rune('2') { - goto l140 - } - position++ - break - case '1': - if buffer[position] != rune('1') { - goto l140 - } - position++ - break - default: - if buffer[position] != rune('0') { - goto l140 - } - position++ - break - } - } - - if !_rules[ruledigit]() { - goto l140 - } - depth-- - add(ruleday, position141) - } - return true - l140: - position, tokenIndex, depth = position140, tokenIndex140, depth140 - return false - }, - /* 11 and <- <(('a' / 'A') ('n' / 'N') ('d' / 'D'))> */ - nil, - /* 12 equal <- <'='> */ - nil, - /* 13 contains <- <(('c' / 'C') ('o' / 'O') ('n' / 'N') ('t' / 'T') ('a' / 'A') ('i' / 'I') ('n' / 'N') ('s' / 'S'))> */ - nil, - /* 14 exists <- <(('e' / 'E') ('x' / 'X') ('i' / 'I') ('s' / 'S') ('t' / 'T') ('s' / 'S'))> */ - nil, - /* 15 le <- <('<' '=')> */ - nil, - /* 16 ge <- <('>' '=')> */ - nil, - /* 17 l <- <'<'> */ - nil, - /* 18 g <- <'>'> */ - nil, - nil, - } - p.rules = _rules -} diff --git a/libs/pubsub/query/query_test.go b/libs/pubsub/query/query_test.go index 87f61aafe..883e771c6 100644 --- a/libs/pubsub/query/query_test.go +++ b/libs/pubsub/query/query_test.go @@ -6,242 +6,271 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" - abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/libs/pubsub" "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" ) -func expandEvents(flattenedEvents map[string][]string) []abci.Event { - events := make([]abci.Event, len(flattenedEvents)) +var _ pubsub.Query = (*query.Query)(nil) - for composite, values := range flattenedEvents { - tokens := strings.Split(composite, ".") - - attrs := make([]abci.EventAttribute, len(values)) - for i, v := range values { - attrs[i] = abci.EventAttribute{ - Key: tokens[len(tokens)-1], - Value: v, - } - } - - events = append(events, abci.Event{ - Type: strings.Join(tokens[:len(tokens)-1], "."), - Attributes: attrs, - }) - } - - return events +// Example events from the OpenAPI documentation: +// https://github.com/tendermint/tendermint/blob/master/rpc/openapi/openapi.yaml +// +// Redactions: +// +// - Add an explicit "tm" event for the built-in attributes. +// - Remove Index fields (not relevant to tests). +// - Add explicit balance values (to use in tests). +// +var apiEvents = []types.Event{ + { + Type: "tm", + Attributes: []types.EventAttribute{ + {Key: "event", Value: "Tx"}, + {Key: "hash", Value: "XYZ"}, + {Key: "height", Value: "5"}, + }, + }, + { + Type: "rewards.withdraw", + Attributes: []types.EventAttribute{ + {Key: "address", Value: "AddrA"}, + {Key: "source", Value: "SrcX"}, + {Key: "amount", Value: "100"}, + {Key: "balance", Value: "1500"}, + }, + }, + { + Type: "rewards.withdraw", + Attributes: []types.EventAttribute{ + {Key: "address", Value: "AddrB"}, + {Key: "source", Value: "SrcY"}, + {Key: "amount", Value: "45"}, + {Key: "balance", Value: "999"}, + }, + }, + { + Type: "transfer", + Attributes: []types.EventAttribute{ + {Key: "sender", Value: "AddrC"}, + {Key: "recipient", Value: "AddrD"}, + {Key: "amount", Value: "160"}, + }, + }, } -func TestMatches(t *testing.T) { +func TestCompiledMatches(t *testing.T) { var ( txDate = "2017-01-01" txTime = "2018-05-03T14:45:00Z" ) + //nolint:lll testCases := []struct { - s string - events map[string][]string - err bool - matches bool - matchErr bool + s string + events []types.Event + matches bool }{ - {"tm.events.type='NewBlock'", map[string][]string{"tm.events.type": {"NewBlock"}}, false, true, false}, - {"tx.gas > 7", map[string][]string{"tx.gas": {"8"}}, false, true, false}, - {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8stake"}}, false, true, false}, - {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8.045stake"}}, false, true, false}, - {"transfer.amount > 7.043", map[string][]string{"transfer.amount": {"8.045stake"}}, false, true, false}, - {"transfer.amount > 8.045", map[string][]string{"transfer.amount": {"8.045stake"}}, false, false, false}, - {"tx.gas > 7 AND tx.gas < 9", map[string][]string{"tx.gas": {"8"}}, false, true, false}, - {"body.weight >= 3.5", map[string][]string{"body.weight": {"3.5"}}, false, true, false}, - {"account.balance < 1000.0", map[string][]string{"account.balance": {"900"}}, false, true, false}, - {"apples.kg <= 4", map[string][]string{"apples.kg": {"4.0"}}, false, true, false}, - {"body.weight >= 4.5", map[string][]string{"body.weight": {fmt.Sprintf("%v", float32(4.5))}}, false, true, false}, - { - "oranges.kg < 4 AND watermellons.kg > 10", - map[string][]string{"oranges.kg": {"3"}, "watermellons.kg": {"12"}}, - false, - true, - false, - }, - {"peaches.kg < 4", map[string][]string{"peaches.kg": {"5"}}, false, false, false}, - { - "tx.date > DATE 2017-01-01", - map[string][]string{"tx.date": {time.Now().Format(query.DateLayout)}}, - false, - true, - false, - }, - {"tx.date = DATE 2017-01-01", map[string][]string{"tx.date": {txDate}}, false, true, false}, - {"tx.date = DATE 2018-01-01", map[string][]string{"tx.date": {txDate}}, false, false, false}, - { - "tx.time >= TIME 2013-05-03T14:45:00Z", - map[string][]string{"tx.time": {time.Now().Format(query.TimeLayout)}}, - false, - true, - false, - }, - {"tx.time = TIME 2013-05-03T14:45:00Z", map[string][]string{"tx.time": {txTime}}, false, false, false}, - {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Igor,Ivan"}}, false, true, false}, - {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Pavel,Ivan"}}, false, false, false}, - {"abci.owner.name = 'Igor'", map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, false, true, false}, - { - "abci.owner.name = 'Ivan'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - false, - true, - false, - }, - { - "abci.owner.name = 'Ivan' AND abci.owner.name = 'Igor'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - false, - true, - false, - }, - { - "abci.owner.name = 'Ivan' AND abci.owner.name = 'John'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - false, - false, - false, - }, - { - "tm.events.type='NewBlock'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - true, - false, - }, - { - "app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - true, - false, - }, - { - "tm.events.type='NewBlock' AND app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - true, - false, - }, - { - "tm.events.type='NewHeader' AND app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - false, - false, - }, - {"slash EXISTS", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - false, - true, - false, - }, - {"sl EXISTS", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - false, - true, - false, - }, - {"slash EXISTS", - map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, - "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, - false, - false, - false, - }, - {"slash.reason EXISTS AND slash.power > 1000", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - false, - true, - false, - }, - {"slash.reason EXISTS AND slash.power > 1000", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"500"}}, - false, - false, - false, - }, - {"slash.reason EXISTS", - map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, - "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, - false, - false, - false, - }, + {`tm.events.type='NewBlock'`, + newTestEvents(`tm|events.type=NewBlock`), + true}, + {`tx.gas > 7`, + newTestEvents(`tx|gas=8`), + true}, + {`transfer.amount > 7`, + newTestEvents(`transfer|amount=8stake`), + true}, + {`transfer.amount > 7`, + newTestEvents(`transfer|amount=8.045`), + true}, + {`transfer.amount > 7.043`, + newTestEvents(`transfer|amount=8.045stake`), + true}, + {`transfer.amount > 8.045`, + newTestEvents(`transfer|amount=8.045stake`), + false}, + {`tx.gas > 7 AND tx.gas < 9`, + newTestEvents(`tx|gas=8`), + true}, + {`body.weight >= 3.5`, + newTestEvents(`body|weight=3.5`), + true}, + {`account.balance < 1000.0`, + newTestEvents(`account|balance=900`), + true}, + {`apples.kg <= 4`, + newTestEvents(`apples|kg=4.0`), + true}, + {`body.weight >= 4.5`, + newTestEvents(`body|weight=4.5`), + true}, + {`oranges.kg < 4 AND watermellons.kg > 10`, + newTestEvents(`oranges|kg=3`, `watermellons|kg=12`), + true}, + {`peaches.kg < 4`, + newTestEvents(`peaches|kg=5`), + false}, + {`tx.date > DATE 2017-01-01`, + newTestEvents(`tx|date=` + time.Now().Format(syntax.DateFormat)), + true}, + {`tx.date = DATE 2017-01-01`, + newTestEvents(`tx|date=` + txDate), + true}, + {`tx.date = DATE 2018-01-01`, + newTestEvents(`tx|date=` + txDate), + false}, + {`tx.time >= TIME 2013-05-03T14:45:00Z`, + newTestEvents(`tx|time=` + time.Now().Format(syntax.TimeFormat)), + true}, + {`tx.time = TIME 2013-05-03T14:45:00Z`, + newTestEvents(`tx|time=` + txTime), + false}, + {`abci.owner.name CONTAINS 'Igor'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name CONTAINS 'Igor'`, + newTestEvents(`abci|owner.name=Pavel|owner.name=Ivan`), + false}, + {`abci.owner.name = 'Igor'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name = 'Ivan'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name = 'Ivan' AND abci.owner.name = 'Igor'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name = 'Ivan' AND abci.owner.name = 'John'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + false}, + {`tm.events.type='NewBlock'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + true}, + {`app.name = 'fuzzed'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + true}, + {`tm.events.type='NewBlock' AND app.name = 'fuzzed'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + true}, + {`tm.events.type='NewHeader' AND app.name = 'fuzzed'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + false}, + {`slash EXISTS`, + newTestEvents(`slash|reason=missing_signature|power=6000`), + true}, + {`slash EXISTS`, + newTestEvents(`transfer|recipient=cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz|sender=cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5`), + false}, + {`slash.reason EXISTS AND slash.power > 1000`, + newTestEvents(`slash|reason=missing_signature|power=6000`), + true}, + {`slash.reason EXISTS AND slash.power > 1000`, + newTestEvents(`slash|reason=missing_signature|power=500`), + false}, + {`slash.reason EXISTS`, + newTestEvents(`transfer|recipient=cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz|sender=cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5`), + false}, + + // Test cases based on the OpenAPI examples. + {`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrA'`, + apiEvents, true}, + {`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrA' AND rewards.withdraw.source = 'SrcY'`, + apiEvents, true}, + {`tm.event = 'Tx' AND transfer.sender = 'AddrA'`, + apiEvents, false}, + {`tm.event = 'Tx' AND transfer.sender = 'AddrC'`, + apiEvents, true}, + {`tm.event = 'Tx' AND transfer.sender = 'AddrZ'`, + apiEvents, false}, + {`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrZ'`, + apiEvents, false}, + {`tm.event = 'Tx' AND rewards.withdraw.source = 'W'`, + apiEvents, false}, } - for _, tc := range testCases { - q, err := query.New(tc.s) - if !tc.err { - require.Nil(t, err) - } - require.NotNil(t, q, "Query '%s' should not be nil", tc.s) + // NOTE: The original implementation allowed arbitrary prefix matches on + // attribute tags, e.g., "sl" would match "slash". + // + // That is weird and probably wrong: "foo.ba" should not match "foo.bar", + // or there is no way to distinguish the case where there were two values + // for "foo.bar" or one value each for "foo.ba" and "foo.bar". + // + // Apart from a single test case, I could not find any attested usage of + // this implementation detail. It isn't documented in the OpenAPI docs and + // is not shown in any of the example inputs. + // + // On that basis, I removed that test case. This implementation still does + // correctly handle variable type/attribute splits ("x", "y.z" / "x.y", "z") + // since that was required by the original "flattened" event representation. - rawEvents := expandEvents(tc.events) + for i, tc := range testCases { + t.Run(fmt.Sprintf("%02d", i+1), func(t *testing.T) { + c, err := query.New(tc.s) + if err != nil { + t.Fatalf("NewCompiled %#q: unexpected error: %v", tc.s, err) + } - if tc.matches { - match, err := q.Matches(rawEvents) - require.Nil(t, err, "Query '%s' should not error on match %v", tc.s, tc.events) - require.True(t, match, "Query '%s' should match %v", tc.s, tc.events) - } else { - match, err := q.Matches(rawEvents) - require.Equal(t, tc.matchErr, err != nil, "Unexpected error for query '%s' match %v", tc.s, tc.events) - require.False(t, match, "Query '%s' should not match %v", tc.s, tc.events) - } + got, err := c.Matches(tc.events) + if err != nil { + t.Errorf("Query: %#q\nInput: %+v\nMatches: got error %v", + tc.s, tc.events, err) + } + if got != tc.matches { + t.Errorf("Query: %#q\nInput: %+v\nMatches: got %v, want %v", + tc.s, tc.events, got, tc.matches) + } + }) } } -func TestMustParse(t *testing.T) { - require.Panics(t, func() { query.MustParse("=") }) - require.NotPanics(t, func() { query.MustParse("tm.events.type='NewBlock'") }) +func TestAllMatchesAll(t *testing.T) { + events := newTestEvents( + ``, + `Asher|Roth=`, + `Route|66=`, + `Rilly|Blue=`, + ) + for i := 0; i < len(events); i++ { + match, err := query.All.Matches(events[:i]) + if err != nil { + t.Errorf("Matches failed: %v", err) + } else if !match { + t.Errorf("Did not match on %+v ", events[:i]) + } + } } -func TestConditions(t *testing.T) { - txTime, err := time.Parse(time.RFC3339, "2013-05-03T14:45:00Z") - require.NoError(t, err) - - testCases := []struct { - s string - conditions []query.Condition - }{ - { - s: "tm.events.type='NewBlock'", - conditions: []query.Condition{ - {CompositeKey: "tm.events.type", Op: query.OpEqual, Operand: "NewBlock"}, - }, - }, - { - s: "tx.gas > 7 AND tx.gas < 9", - conditions: []query.Condition{ - {CompositeKey: "tx.gas", Op: query.OpGreater, Operand: int64(7)}, - {CompositeKey: "tx.gas", Op: query.OpLess, Operand: int64(9)}, - }, - }, - { - s: "tx.time >= TIME 2013-05-03T14:45:00Z", - conditions: []query.Condition{ - {CompositeKey: "tx.time", Op: query.OpGreaterEqual, Operand: txTime}, - }, - }, - { - s: "slashing EXISTS", - conditions: []query.Condition{ - {CompositeKey: "slashing", Op: query.OpExists}, - }, - }, +// newTestEvent constructs an Event message from a template string. +// The format is "type|attr1=val1|attr2=val2|...". +func newTestEvent(s string) types.Event { + var event types.Event + parts := strings.Split(s, "|") + event.Type = parts[0] + if len(parts) == 1 { + return event // type only, no attributes } + for _, kv := range parts[1:] { + key, val := splitKV(kv) + event.Attributes = append(event.Attributes, types.EventAttribute{ + Key: key, + Value: val, + }) + } + return event +} - for _, tc := range testCases { - q, err := query.New(tc.s) - require.Nil(t, err) - - c, err := q.Conditions() - require.NoError(t, err) - require.Equal(t, tc.conditions, c) +// newTestEvents constructs a slice of Event messages by applying newTestEvent +// to each element of ss. +func newTestEvents(ss ...string) []types.Event { + events := make([]types.Event, len(ss)) + for i, s := range ss { + events[i] = newTestEvent(s) } + return events +} + +func splitKV(s string) (key, value string) { + kv := strings.SplitN(s, "=", 2) + return kv[0], kv[1] } diff --git a/libs/pubsub/query/syntax/doc.go b/libs/pubsub/query/syntax/doc.go new file mode 100644 index 000000000..e7a9896c4 --- /dev/null +++ b/libs/pubsub/query/syntax/doc.go @@ -0,0 +1,34 @@ +// Package syntax defines a scanner and parser for the Tendermint event filter +// query language. A query selects events by their types and attribute values. +// +// Grammar +// +// The grammar of the query language is defined by the following EBNF: +// +// query = conditions EOF +// conditions = condition {"AND" condition} +// condition = tag comparison +// comparison = equal / order / contains / "EXISTS" +// equal = "=" (date / number / time / value) +// order = cmp (date / number / time) +// contains = "CONTAINS" value +// cmp = "<" / "<=" / ">" / ">=" +// +// The lexical terms are defined here using RE2 regular expression notation: +// +// // The name of an event attribute (type.value) +// tag = #'\w+(\.\w+)*' +// +// // A datestamp (YYYY-MM-DD) +// date = #'DATE \d{4}-\d{2}-\d{2}' +// +// // A number with optional fractional parts (0, 10, 3.25) +// number = #'\d+(\.\d+)?' +// +// // An RFC3339 timestamp (2021-11-23T22:04:19-09:00) +// time = #'TIME \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}([-+]\d{2}:\d{2}|Z)' +// +// // A quoted literal string value ('a b c') +// value = #'\'[^\']*\'' +// +package syntax diff --git a/libs/pubsub/query/syntax/parser.go b/libs/pubsub/query/syntax/parser.go new file mode 100644 index 000000000..a100ec79c --- /dev/null +++ b/libs/pubsub/query/syntax/parser.go @@ -0,0 +1,213 @@ +package syntax + +import ( + "fmt" + "io" + "math" + "strconv" + "strings" + "time" +) + +// Parse parses the specified query string. It is shorthand for constructing a +// parser for s and calling its Parse method. +func Parse(s string) (Query, error) { + return NewParser(strings.NewReader(s)).Parse() +} + +// Query is the root of the parse tree for a query. A query is the conjunction +// of one or more conditions. +type Query []Condition + +func (q Query) String() string { + ss := make([]string, len(q)) + for i, cond := range q { + ss[i] = cond.String() + } + return strings.Join(ss, " AND ") +} + +// A Condition is a single conditional expression, consisting of a tag, a +// comparison operator, and an optional argument. The type of the argument +// depends on the operator. +type Condition struct { + Tag string + Op Token + Arg *Arg + + opText string +} + +func (c Condition) String() string { + s := c.Tag + " " + c.opText + if c.Arg != nil { + return s + " " + c.Arg.String() + } + return s +} + +// An Arg is the argument of a comparison operator. +type Arg struct { + Type Token + text string +} + +func (a *Arg) String() string { + if a == nil { + return "" + } + switch a.Type { + case TString: + return "'" + a.text + "'" + case TTime: + return "TIME " + a.text + case TDate: + return "DATE " + a.text + default: + return a.text + } +} + +// Number returns the value of the argument text as a number, or a NaN if the +// text does not encode a valid number value. +func (a *Arg) Number() float64 { + if a == nil { + return -1 + } + v, err := strconv.ParseFloat(a.text, 64) + if err == nil && v >= 0 { + return v + } + return math.NaN() +} + +// Time returns the value of the argument text as a time, or the zero value if +// the text does not encode a timestamp or datestamp. +func (a *Arg) Time() time.Time { + var ts time.Time + if a == nil { + return ts + } + var err error + switch a.Type { + case TDate: + ts, err = ParseDate(a.text) + case TTime: + ts, err = ParseTime(a.text) + } + if err == nil { + return ts + } + return time.Time{} +} + +// Value returns the value of the argument text as a string, or "". +func (a *Arg) Value() string { + if a == nil { + return "" + } + return a.text +} + +// Parser is a query expression parser. The grammar for query expressions is +// defined in the syntax package documentation. +type Parser struct { + scanner *Scanner +} + +// NewParser constructs a new parser that reads the input from r. +func NewParser(r io.Reader) *Parser { + return &Parser{scanner: NewScanner(r)} +} + +// Parse parses the complete input and returns the resulting query. +func (p *Parser) Parse() (Query, error) { + cond, err := p.parseCond() + if err != nil { + return nil, err + } + conds := []Condition{cond} + for p.scanner.Next() != io.EOF { + if tok := p.scanner.Token(); tok != TAnd { + return nil, fmt.Errorf("offset %d: got %v, want %v", p.scanner.Pos(), tok, TAnd) + } + cond, err := p.parseCond() + if err != nil { + return nil, err + } + conds = append(conds, cond) + } + return conds, nil +} + +// parseCond parses a conditional expression: tag OP value. +func (p *Parser) parseCond() (Condition, error) { + var cond Condition + if err := p.require(TTag); err != nil { + return cond, err + } + cond.Tag = p.scanner.Text() + if err := p.require(TLeq, TGeq, TLt, TGt, TEq, TContains, TExists); err != nil { + return cond, err + } + cond.Op = p.scanner.Token() + cond.opText = p.scanner.Text() + + var err error + switch cond.Op { + case TLeq, TGeq, TLt, TGt: + err = p.require(TNumber, TTime, TDate) + case TEq: + err = p.require(TNumber, TTime, TDate, TString) + case TContains: + err = p.require(TString) + case TExists: + // no argument + return cond, nil + default: + return cond, fmt.Errorf("offset %d: unexpected operator %v", p.scanner.Pos(), cond.Op) + } + if err != nil { + return cond, err + } + cond.Arg = &Arg{Type: p.scanner.Token(), text: p.scanner.Text()} + return cond, nil +} + +// require advances the scanner and requires that the resulting token is one of +// the specified token types. +func (p *Parser) require(tokens ...Token) error { + if err := p.scanner.Next(); err != nil { + return fmt.Errorf("offset %d: %w", p.scanner.Pos(), err) + } + got := p.scanner.Token() + for _, tok := range tokens { + if tok == got { + return nil + } + } + return fmt.Errorf("offset %d: got %v, wanted %s", p.scanner.Pos(), got, tokLabel(tokens)) +} + +// tokLabel makes a human-readable summary string for the given token types. +func tokLabel(tokens []Token) string { + if len(tokens) == 1 { + return tokens[0].String() + } + last := len(tokens) - 1 + ss := make([]string, len(tokens)-1) + for i, tok := range tokens[:last] { + ss[i] = tok.String() + } + return strings.Join(ss, ", ") + " or " + tokens[last].String() +} + +// ParseDate parses s as a date string in the format used by DATE values. +func ParseDate(s string) (time.Time, error) { + return time.Parse("2006-01-02", s) +} + +// ParseTime parses s as a timestamp in the format used by TIME values. +func ParseTime(s string) (time.Time, error) { + return time.Parse(time.RFC3339, s) +} diff --git a/libs/pubsub/query/syntax/scanner.go b/libs/pubsub/query/syntax/scanner.go new file mode 100644 index 000000000..332e3f7b1 --- /dev/null +++ b/libs/pubsub/query/syntax/scanner.go @@ -0,0 +1,312 @@ +package syntax + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strings" + "time" + "unicode" +) + +// Token is the type of a lexical token in the query grammar. +type Token byte + +const ( + TInvalid = iota // invalid or unknown token + TTag // field tag: x.y + TString // string value: 'foo bar' + TNumber // number: 0, 15.5, 100 + TTime // timestamp: TIME yyyy-mm-ddThh:mm:ss([-+]hh:mm|Z) + TDate // datestamp: DATE yyyy-mm-dd + TAnd // operator: AND + TContains // operator: CONTAINS + TExists // operator: EXISTS + TEq // operator: = + TLt // operator: < + TLeq // operator: <= + TGt // operator: > + TGeq // operator: >= + + // Do not reorder these values without updating the scanner code. +) + +var tString = [...]string{ + TInvalid: "invalid token", + TTag: "tag", + TString: "string", + TNumber: "number", + TTime: "timestamp", + TDate: "datestamp", + TAnd: "AND operator", + TContains: "CONTAINS operator", + TExists: "EXISTS operator", + TEq: "= operator", + TLt: "< operator", + TLeq: "<= operator", + TGt: "> operator", + TGeq: ">= operator", +} + +func (t Token) String() string { + v := int(t) + if v > len(tString) { + return "unknown token type" + } + return tString[v] +} + +const ( + // TimeFormat is the format string used for timestamp values. + TimeFormat = time.RFC3339 + + // DateFormat is the format string used for datestamp values. + DateFormat = "2006-01-02" +) + +// Scanner reads lexical tokens of the query language from an input stream. +// Each call to Next advances the scanner to the next token, or reports an +// error. +type Scanner struct { + r *bufio.Reader + buf bytes.Buffer + tok Token + err error + + pos, last, end int +} + +// NewScanner constructs a new scanner that reads from r. +func NewScanner(r io.Reader) *Scanner { return &Scanner{r: bufio.NewReader(r)} } + +// Next advances s to the next token in the input, or reports an error. At the +// end of input, Next returns io.EOF. +func (s *Scanner) Next() error { + s.buf.Reset() + s.pos = s.end + s.tok = TInvalid + s.err = nil + + for { + ch, err := s.rune() + if err != nil { + return s.fail(err) + } + if unicode.IsSpace(ch) { + s.pos = s.end + continue // skip whitespace + } + if '0' <= ch && ch <= '9' { + return s.scanNumber(ch) + } else if isTagRune(ch) { + return s.scanTagLike(ch) + } + switch ch { + case '\'': + return s.scanString(ch) + case '<', '>', '=': + return s.scanCompare(ch) + default: + return s.invalid(ch) + } + } +} + +// Token returns the type of the current input token. +func (s *Scanner) Token() Token { return s.tok } + +// Text returns the text of the current input token. +func (s *Scanner) Text() string { return s.buf.String() } + +// Pos returns the start offset of the current token in the input. +func (s *Scanner) Pos() int { return s.pos } + +// Err returns the last error reported by Next, if any. +func (s *Scanner) Err() error { return s.err } + +// scanNumber scans for numbers with optional fractional parts. +// Examples: 0, 1, 3.14 +func (s *Scanner) scanNumber(first rune) error { + s.buf.WriteRune(first) + if err := s.scanWhile(isDigit); err != nil { + return err + } + + ch, err := s.rune() + if err != nil && err != io.EOF { + return err + } + if ch == '.' { + s.buf.WriteRune(ch) + if err := s.scanWhile(isDigit); err != nil { + return err + } + } else { + s.unrune() + } + s.tok = TNumber + return nil +} + +func (s *Scanner) scanString(first rune) error { + // discard opening quote + for { + ch, err := s.rune() + if err != nil { + return s.fail(err) + } else if ch == first { + // discard closing quote + s.tok = TString + return nil + } + s.buf.WriteRune(ch) + } +} + +func (s *Scanner) scanCompare(first rune) error { + s.buf.WriteRune(first) + switch first { + case '=': + s.tok = TEq + return nil + case '<': + s.tok = TLt + case '>': + s.tok = TGt + default: + return s.invalid(first) + } + + ch, err := s.rune() + if err == io.EOF { + return nil // the assigned token is correct + } else if err != nil { + return s.fail(err) + } + if ch == '=' { + s.buf.WriteRune(ch) + s.tok++ // depends on token order + return nil + } + s.unrune() + return nil +} + +func (s *Scanner) scanTagLike(first rune) error { + s.buf.WriteRune(first) + var hasSpace bool + for { + ch, err := s.rune() + if err == io.EOF { + break + } else if err != nil { + return s.fail(err) + } + if !isTagRune(ch) { + hasSpace = ch == ' ' // to check for TIME, DATE + break + } + s.buf.WriteRune(ch) + } + + text := s.buf.String() + switch text { + case "TIME": + if hasSpace { + return s.scanTimestamp() + } + s.tok = TTag + case "DATE": + if hasSpace { + return s.scanDatestamp() + } + s.tok = TTag + case "AND": + s.tok = TAnd + case "EXISTS": + s.tok = TExists + case "CONTAINS": + s.tok = TContains + default: + s.tok = TTag + } + s.unrune() + return nil +} + +func (s *Scanner) scanTimestamp() error { + s.buf.Reset() // discard "TIME" label + if err := s.scanWhile(isTimeRune); err != nil { + return err + } + if ts, err := time.Parse(TimeFormat, s.buf.String()); err != nil { + return s.fail(fmt.Errorf("invalid TIME value: %w", err)) + } else if y := ts.Year(); y < 1900 || y > 2999 { + return s.fail(fmt.Errorf("timestamp year %d out of range", ts.Year())) + } + s.tok = TTime + return nil +} + +func (s *Scanner) scanDatestamp() error { + s.buf.Reset() // discard "DATE" label + if err := s.scanWhile(isDateRune); err != nil { + return err + } + if ts, err := time.Parse(DateFormat, s.buf.String()); err != nil { + return s.fail(fmt.Errorf("invalid DATE value: %w", err)) + } else if y := ts.Year(); y < 1900 || y > 2999 { + return s.fail(fmt.Errorf("datestamp year %d out of range", ts.Year())) + } + s.tok = TDate + return nil +} + +func (s *Scanner) scanWhile(ok func(rune) bool) error { + for { + ch, err := s.rune() + if err == io.EOF { + return nil + } else if err != nil { + return s.fail(err) + } else if !ok(ch) { + s.unrune() + return nil + } + s.buf.WriteRune(ch) + } +} + +func (s *Scanner) rune() (rune, error) { + ch, nb, err := s.r.ReadRune() + s.last = nb + s.end += nb + return ch, err +} + +func (s *Scanner) unrune() { + _ = s.r.UnreadRune() + s.end -= s.last +} + +func (s *Scanner) fail(err error) error { + s.err = err + return err +} + +func (s *Scanner) invalid(ch rune) error { + return s.fail(fmt.Errorf("invalid input %c at offset %d", ch, s.end)) +} + +func isDigit(r rune) bool { return '0' <= r && r <= '9' } + +func isTagRune(r rune) bool { + return r == '.' || r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) +} + +func isTimeRune(r rune) bool { + return strings.ContainsRune("-T:+Z", r) || isDigit(r) +} + +func isDateRune(r rune) bool { return isDigit(r) || r == '-' } diff --git a/libs/pubsub/query/syntax/syntax_test.go b/libs/pubsub/query/syntax/syntax_test.go new file mode 100644 index 000000000..ac95fd8b1 --- /dev/null +++ b/libs/pubsub/query/syntax/syntax_test.go @@ -0,0 +1,190 @@ +package syntax_test + +import ( + "io" + "reflect" + "strings" + "testing" + + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" +) + +func TestScanner(t *testing.T) { + tests := []struct { + input string + want []syntax.Token + }{ + // Empty inputs + {"", nil}, + {" ", nil}, + {"\t\n ", nil}, + + // Numbers + {`0 123`, []syntax.Token{syntax.TNumber, syntax.TNumber}}, + {`0.32 3.14`, []syntax.Token{syntax.TNumber, syntax.TNumber}}, + + // Tags + {`foo foo.bar`, []syntax.Token{syntax.TTag, syntax.TTag}}, + + // Strings (values) + {` '' x 'x' 'x y'`, []syntax.Token{syntax.TString, syntax.TTag, syntax.TString, syntax.TString}}, + {` 'you are not your job' `, []syntax.Token{syntax.TString}}, + + // Comparison operators + {`< <= = > >=`, []syntax.Token{ + syntax.TLt, syntax.TLeq, syntax.TEq, syntax.TGt, syntax.TGeq, + }}, + + // Mixed values of various kinds. + {`x AND y`, []syntax.Token{syntax.TTag, syntax.TAnd, syntax.TTag}}, + {`x.y CONTAINS 'z'`, []syntax.Token{syntax.TTag, syntax.TContains, syntax.TString}}, + {`foo EXISTS`, []syntax.Token{syntax.TTag, syntax.TExists}}, + {`and AND`, []syntax.Token{syntax.TTag, syntax.TAnd}}, + + // Timestamp + {`TIME 2021-11-23T15:16:17Z`, []syntax.Token{syntax.TTime}}, + + // Datestamp + {`DATE 2021-11-23`, []syntax.Token{syntax.TDate}}, + } + + for _, test := range tests { + s := syntax.NewScanner(strings.NewReader(test.input)) + var got []syntax.Token + for s.Next() == nil { + got = append(got, s.Token()) + } + if err := s.Err(); err != io.EOF { + t.Errorf("Next: unexpected error: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Logf("Scanner input: %q", test.input) + t.Errorf("Wrong tokens:\ngot: %+v\nwant: %+v", got, test.want) + } + } +} + +func TestScannerErrors(t *testing.T) { + tests := []struct { + input string + }{ + {`'incomplete string`}, + {`-23`}, + {`&`}, + {`DATE xyz-pdq`}, + {`DATE xyzp-dq-zv`}, + {`DATE 0000-00-00`}, + {`DATE 0000-00-000`}, + {`DATE 2021-01-99`}, + {`TIME 2021-01-01T34:56:78Z`}, + {`TIME 2021-01-99T14:56:08Z`}, + {`TIME 2021-01-99T34:56:08`}, + {`TIME 2021-01-99T34:56:11+3`}, + } + for _, test := range tests { + s := syntax.NewScanner(strings.NewReader(test.input)) + if err := s.Next(); err == nil { + t.Errorf("Next: got %v (%#q), want error", s.Token(), s.Text()) + } + } +} + +// These parser tests were copied from the original implementation of the query +// parser, and are preserved here as a compatibility check. +func TestParseValid(t *testing.T) { + tests := []struct { + input string + valid bool + }{ + {"tm.events.type='NewBlock'", true}, + {"tm.events.type = 'NewBlock'", true}, + {"tm.events.name = ''", true}, + {"tm.events.type='TIME'", true}, + {"tm.events.type='DATE'", true}, + {"tm.events.type='='", true}, + {"tm.events.type='TIME", false}, + {"tm.events.type=TIME'", false}, + {"tm.events.type==", false}, + {"tm.events.type=NewBlock", false}, + {">==", false}, + {"tm.events.type 'NewBlock' =", false}, + {"tm.events.type>'NewBlock'", false}, + {"", false}, + {"=", false}, + {"='NewBlock'", false}, + {"tm.events.type=", false}, + + {"tm.events.typeNewBlock", false}, + {"tm.events.type'NewBlock'", false}, + {"'NewBlock'", false}, + {"NewBlock", false}, + {"", false}, + + {"tm.events.type='NewBlock' AND abci.account.name='Igor'", true}, + {"tm.events.type='NewBlock' AND", false}, + {"tm.events.type='NewBlock' AN", false}, + {"tm.events.type='NewBlock' AN tm.events.type='NewBlockHeader'", false}, + {"AND tm.events.type='NewBlock' ", false}, + + {"abci.account.name CONTAINS 'Igor'", true}, + + {"tx.date > DATE 2013-05-03", true}, + {"tx.date < DATE 2013-05-03", true}, + {"tx.date <= DATE 2013-05-03", true}, + {"tx.date >= DATE 2013-05-03", true}, + {"tx.date >= DAT 2013-05-03", false}, + {"tx.date <= DATE2013-05-03", false}, + {"tx.date <= DATE -05-03", false}, + {"tx.date >= DATE 20130503", false}, + {"tx.date >= DATE 2013+01-03", false}, + // incorrect year, month, day + {"tx.date >= DATE 0013-01-03", false}, + {"tx.date >= DATE 2013-31-03", false}, + {"tx.date >= DATE 2013-01-83", false}, + + {"tx.date > TIME 2013-05-03T14:45:00+07:00", true}, + {"tx.date < TIME 2013-05-03T14:45:00-02:00", true}, + {"tx.date <= TIME 2013-05-03T14:45:00Z", true}, + {"tx.date >= TIME 2013-05-03T14:45:00Z", true}, + {"tx.date >= TIME2013-05-03T14:45:00Z", false}, + {"tx.date = IME 2013-05-03T14:45:00Z", false}, + {"tx.date = TIME 2013-05-:45:00Z", false}, + {"tx.date >= TIME 2013-05-03T14:45:00", false}, + {"tx.date >= TIME 0013-00-00T14:45:00Z", false}, + {"tx.date >= TIME 2013+05=03T14:45:00Z", false}, + + {"account.balance=100", true}, + {"account.balance >= 200", true}, + {"account.balance >= -300", false}, + {"account.balance >>= 400", false}, + {"account.balance=33.22.1", false}, + + {"slashing.amount EXISTS", true}, + {"slashing.amount EXISTS AND account.balance=100", true}, + {"account.balance=100 AND slashing.amount EXISTS", true}, + {"slashing EXISTS", true}, + + {"hash='136E18F7E4C348B780CF873A0BF43922E5BAFA63'", true}, + {"hash=136E18F7E4C348B780CF873A0BF43922E5BAFA63", false}, + } + + for _, test := range tests { + q, err := syntax.Parse(test.input) + if test.valid != (err == nil) { + t.Errorf("Parse %#q: valid %v got err=%v", test.input, test.valid, err) + } + + // For valid queries, check that the query round-trips. + if test.valid { + qstr := q.String() + r, err := syntax.Parse(qstr) + if err != nil { + t.Errorf("Reparse %#q failed: %v", qstr, err) + } + if rstr := r.String(); rstr != qstr { + t.Errorf("Reparse diff\nold: %#q\nnew: %#q", qstr, rstr) + } + } + } +} diff --git a/types/event_bus_test.go b/types/event_bus_test.go index 9ca075391..73dc13d0c 100644 --- a/types/event_bus_test.go +++ b/types/event_bus_test.go @@ -35,7 +35,7 @@ func TestEventBusPublishEventTx(t *testing.T) { // PublishEventTx adds 3 composite keys, so the query below should work query := fmt.Sprintf("tm.event='Tx' AND tx.height=1 AND tx.hash='%X' AND testType.baz=1", tx.Hash()) - txsSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + txsSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustCompile(query)) require.NoError(t, err) done := make(chan struct{}) @@ -89,7 +89,7 @@ func TestEventBusPublishEventNewBlock(t *testing.T) { // PublishEventNewBlock adds the tm.event compositeKey, so the query below should work query := "tm.event='NewBlock' AND testType.baz=1 AND testType.foz=2" - blocksSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + blocksSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustCompile(query)) require.NoError(t, err) done := make(chan struct{}) @@ -186,7 +186,7 @@ func TestEventBusPublishEventTxDuplicateKeys(t *testing.T) { } for i, tc := range testCases { - sub, err := eventBus.Subscribe(context.Background(), fmt.Sprintf("client-%d", i), tmquery.MustParse(tc.query)) + sub, err := eventBus.Subscribe(context.Background(), fmt.Sprintf("client-%d", i), tmquery.MustCompile(tc.query)) require.NoError(t, err) done := make(chan struct{}) @@ -250,7 +250,7 @@ func TestEventBusPublishEventNewBlockHeader(t *testing.T) { // PublishEventNewBlockHeader adds the tm.event compositeKey, so the query below should work query := "tm.event='NewBlockHeader' AND testType.baz=1 AND testType.foz=2" - headersSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + headersSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustCompile(query)) require.NoError(t, err) done := make(chan struct{}) @@ -290,7 +290,7 @@ func TestEventBusPublishEventNewEvidence(t *testing.T) { ev := NewMockDuplicateVoteEvidence(1, time.Now(), "test-chain-id") query := "tm.event='NewEvidence'" - evSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + evSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustCompile(query)) require.NoError(t, err) done := make(chan struct{}) @@ -327,7 +327,7 @@ func TestEventBusPublish(t *testing.T) { const numEventsExpected = 14 - sub, err := eventBus.Subscribe(context.Background(), "test", tmquery.Empty{}, numEventsExpected) + sub, err := eventBus.Subscribe(context.Background(), "test", tmquery.All, numEventsExpected) require.NoError(t, err) done := make(chan struct{}) diff --git a/types/events.go b/types/events.go index 46f150abd..7bb183fb7 100644 --- a/types/events.go +++ b/types/events.go @@ -232,11 +232,11 @@ var ( ) func EventQueryTxFor(tx Tx) tmpubsub.Query { - return tmquery.MustParse(fmt.Sprintf("%s='%s' AND %s='%X'", EventTypeKey, EventTxValue, TxHashKey, tx.Hash())) + return tmquery.MustCompile(fmt.Sprintf("%s='%s' AND %s='%X'", EventTypeKey, EventTxValue, TxHashKey, tx.Hash())) } func QueryForEvent(eventValue string) tmpubsub.Query { - return tmquery.MustParse(fmt.Sprintf("%s='%s'", EventTypeKey, eventValue)) + return tmquery.MustCompile(fmt.Sprintf("%s='%s'", EventTypeKey, eventValue)) } // BlockEventPublisher publishes all block related events diff --git a/types/events_test.go b/types/events_test.go index dcd998ace..bd4bde264 100644 --- a/types/events_test.go +++ b/types/events_test.go @@ -10,18 +10,18 @@ import ( func TestQueryTxFor(t *testing.T) { tx := Tx("foo") assert.Equal(t, - fmt.Sprintf("tm.event='Tx' AND tx.hash='%X'", tx.Hash()), + fmt.Sprintf("tm.event = 'Tx' AND tx.hash = '%X'", tx.Hash()), EventQueryTxFor(tx).String(), ) } func TestQueryForEvent(t *testing.T) { assert.Equal(t, - "tm.event='NewBlock'", + "tm.event = 'NewBlock'", QueryForEvent(EventNewBlockValue).String(), ) assert.Equal(t, - "tm.event='NewEvidence'", + "tm.event = 'NewEvidence'", QueryForEvent(EventNewEvidenceValue).String(), ) }