You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

275 lines
7.9 KiB

abci: Refactor tagging events using list of lists (#3643) ## PR This PR introduces a fundamental breaking change to the structure of ABCI response and tx tags and the way they're processed. Namely, the SDK can support more complex and aggregated events for distribution and slashing. In addition, block responses can include duplicate keys in events. Implement new Event type. An event has a type and a list of KV pairs (ie. list-of-lists). Typical events may look like: "rewards": [{"amount": "5000uatom", "validator": "...", "recipient": "..."}] "sender": [{"address": "...", "balance": "100uatom"}] The events are indexed by {even.type}.{even.attribute[i].key}/.... In this case a client would subscribe or query for rewards.recipient='...' ABCI response types and related types now include Events []Event instead of Tags []cmn.KVPair. PubSub logic now publishes/matches against map[string][]string instead of map[string]string to support duplicate keys in response events (from #1385). A match is successful if the value is found in the slice of strings. closes: #1859 closes: #2905 ## Commits: * Implement Event ABCI type and updates responses to use events * Update messages_test.go * Update kvstore.go * Update event_bus.go * Update subscription.go * Update pubsub.go * Update kvstore.go * Update query logic to handle slice of strings in events * Update Empty#Matches and unit tests * Update pubsub logic * Update EventBus#Publish * Update kv tx indexer * Update godocs * Update ResultEvent to use slice of strings; update RPC * Update more tests * Update abci.md * Check for key in validateAndStringifyEvents * Fix KV indexer to skip empty keys * Fix linting errors * Update CHANGELOG_PENDING.md * Update docs/spec/abci/abci.md Co-Authored-By: Federico Kunze <31522760+fedekunze@users.noreply.github.com> * Update abci/types/types.proto Co-Authored-By: Ethan Buchman <ethan@coinculture.info> * Update docs/spec/abci/abci.md Co-Authored-By: Ethan Buchman <ethan@coinculture.info> * Update libs/pubsub/query/query.go Co-Authored-By: Ethan Buchman <ethan@coinculture.info> * Update match function to match if ANY value matches * Implement TestSubscribeDuplicateKeys * Update TestMatches to include multi-key test cases * Update events.go * Update Query interface godoc * Update match godoc * Add godoc for matchValue * DRY-up tx indexing * Return error from PublishWithEvents in EventBus#Publish * Update PublishEventNewBlockHeader to return an error * Fix build * Update events doc in ABCI * Update ABCI events godoc * Implement TestEventBusPublishEventTxDuplicateKeys * Update TestSubscribeDuplicateKeys to be table-driven * Remove mod file * Remove markdown from events godoc * Implement TestTxSearchDeprecatedIndexing test
6 years ago
  1. package query_test
  2. import (
  3. "fmt"
  4. "strings"
  5. "testing"
  6. "time"
  7. "github.com/tendermint/tendermint/abci/types"
  8. "github.com/tendermint/tendermint/internal/pubsub"
  9. "github.com/tendermint/tendermint/internal/pubsub/query"
  10. "github.com/tendermint/tendermint/internal/pubsub/query/syntax"
  11. )
  12. var _ pubsub.Query = (*query.Query)(nil)
  13. // Example events from the OpenAPI documentation:
  14. // https://github.com/tendermint/tendermint/blob/master/rpc/openapi/openapi.yaml
  15. //
  16. // Redactions:
  17. //
  18. // - Add an explicit "tm" event for the built-in attributes.
  19. // - Remove Index fields (not relevant to tests).
  20. // - Add explicit balance values (to use in tests).
  21. //
  22. var apiEvents = []types.Event{
  23. {
  24. Type: "tm",
  25. Attributes: []types.EventAttribute{
  26. {Key: "event", Value: "Tx"},
  27. {Key: "hash", Value: "XYZ"},
  28. {Key: "height", Value: "5"},
  29. },
  30. },
  31. {
  32. Type: "rewards.withdraw",
  33. Attributes: []types.EventAttribute{
  34. {Key: "address", Value: "AddrA"},
  35. {Key: "source", Value: "SrcX"},
  36. {Key: "amount", Value: "100"},
  37. {Key: "balance", Value: "1500"},
  38. },
  39. },
  40. {
  41. Type: "rewards.withdraw",
  42. Attributes: []types.EventAttribute{
  43. {Key: "address", Value: "AddrB"},
  44. {Key: "source", Value: "SrcY"},
  45. {Key: "amount", Value: "45"},
  46. {Key: "balance", Value: "999"},
  47. },
  48. },
  49. {
  50. Type: "transfer",
  51. Attributes: []types.EventAttribute{
  52. {Key: "sender", Value: "AddrC"},
  53. {Key: "recipient", Value: "AddrD"},
  54. {Key: "amount", Value: "160"},
  55. },
  56. },
  57. }
  58. func TestCompiledMatches(t *testing.T) {
  59. var (
  60. txDate = "2017-01-01"
  61. txTime = "2018-05-03T14:45:00Z"
  62. )
  63. testCases := []struct {
  64. s string
  65. events []types.Event
  66. matches bool
  67. }{
  68. {`tm.events.type='NewBlock'`,
  69. newTestEvents(`tm|events.type=NewBlock`),
  70. true},
  71. {`tx.gas > 7`,
  72. newTestEvents(`tx|gas=8`),
  73. true},
  74. {`transfer.amount > 7`,
  75. newTestEvents(`transfer|amount=8stake`),
  76. true},
  77. {`transfer.amount > 7`,
  78. newTestEvents(`transfer|amount=8.045`),
  79. true},
  80. {`transfer.amount > 7.043`,
  81. newTestEvents(`transfer|amount=8.045stake`),
  82. true},
  83. {`transfer.amount > 8.045`,
  84. newTestEvents(`transfer|amount=8.045stake`),
  85. false},
  86. {`tx.gas > 7 AND tx.gas < 9`,
  87. newTestEvents(`tx|gas=8`),
  88. true},
  89. {`body.weight >= 3.5`,
  90. newTestEvents(`body|weight=3.5`),
  91. true},
  92. {`account.balance < 1000.0`,
  93. newTestEvents(`account|balance=900`),
  94. true},
  95. {`apples.kg <= 4`,
  96. newTestEvents(`apples|kg=4.0`),
  97. true},
  98. {`body.weight >= 4.5`,
  99. newTestEvents(`body|weight=4.5`),
  100. true},
  101. {`oranges.kg < 4 AND watermellons.kg > 10`,
  102. newTestEvents(`oranges|kg=3`, `watermellons|kg=12`),
  103. true},
  104. {`peaches.kg < 4`,
  105. newTestEvents(`peaches|kg=5`),
  106. false},
  107. {`tx.date > DATE 2017-01-01`,
  108. newTestEvents(`tx|date=` + time.Now().Format(syntax.DateFormat)),
  109. true},
  110. {`tx.date = DATE 2017-01-01`,
  111. newTestEvents(`tx|date=` + txDate),
  112. true},
  113. {`tx.date = DATE 2018-01-01`,
  114. newTestEvents(`tx|date=` + txDate),
  115. false},
  116. {`tx.time >= TIME 2013-05-03T14:45:00Z`,
  117. newTestEvents(`tx|time=` + time.Now().Format(syntax.TimeFormat)),
  118. true},
  119. {`tx.time = TIME 2013-05-03T14:45:00Z`,
  120. newTestEvents(`tx|time=` + txTime),
  121. false},
  122. {`abci.owner.name CONTAINS 'Igor'`,
  123. newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`),
  124. true},
  125. {`abci.owner.name CONTAINS 'Igor'`,
  126. newTestEvents(`abci|owner.name=Pavel|owner.name=Ivan`),
  127. false},
  128. {`abci.owner.name = 'Igor'`,
  129. newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`),
  130. true},
  131. {`abci.owner.name = 'Ivan'`,
  132. newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`),
  133. true},
  134. {`abci.owner.name = 'Ivan' AND abci.owner.name = 'Igor'`,
  135. newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`),
  136. true},
  137. {`abci.owner.name = 'Ivan' AND abci.owner.name = 'John'`,
  138. newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`),
  139. false},
  140. {`tm.events.type='NewBlock'`,
  141. newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`),
  142. true},
  143. {`app.name = 'fuzzed'`,
  144. newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`),
  145. true},
  146. {`tm.events.type='NewBlock' AND app.name = 'fuzzed'`,
  147. newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`),
  148. true},
  149. {`tm.events.type='NewHeader' AND app.name = 'fuzzed'`,
  150. newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`),
  151. false},
  152. {`slash EXISTS`,
  153. newTestEvents(`slash|reason=missing_signature|power=6000`),
  154. true},
  155. {`slash EXISTS`,
  156. newTestEvents(`transfer|recipient=cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz|sender=cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5`),
  157. false},
  158. {`slash.reason EXISTS AND slash.power > 1000`,
  159. newTestEvents(`slash|reason=missing_signature|power=6000`),
  160. true},
  161. {`slash.reason EXISTS AND slash.power > 1000`,
  162. newTestEvents(`slash|reason=missing_signature|power=500`),
  163. false},
  164. {`slash.reason EXISTS`,
  165. newTestEvents(`transfer|recipient=cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz|sender=cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5`),
  166. false},
  167. // Test cases based on the OpenAPI examples.
  168. {`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrA'`,
  169. apiEvents, true},
  170. {`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrA' AND rewards.withdraw.source = 'SrcY'`,
  171. apiEvents, true},
  172. {`tm.event = 'Tx' AND transfer.sender = 'AddrA'`,
  173. apiEvents, false},
  174. {`tm.event = 'Tx' AND transfer.sender = 'AddrC'`,
  175. apiEvents, true},
  176. {`tm.event = 'Tx' AND transfer.sender = 'AddrZ'`,
  177. apiEvents, false},
  178. {`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrZ'`,
  179. apiEvents, false},
  180. {`tm.event = 'Tx' AND rewards.withdraw.source = 'W'`,
  181. apiEvents, false},
  182. }
  183. // NOTE: The original implementation allowed arbitrary prefix matches on
  184. // attribute tags, e.g., "sl" would match "slash".
  185. //
  186. // That is weird and probably wrong: "foo.ba" should not match "foo.bar",
  187. // or there is no way to distinguish the case where there were two values
  188. // for "foo.bar" or one value each for "foo.ba" and "foo.bar".
  189. //
  190. // Apart from a single test case, I could not find any attested usage of
  191. // this implementation detail. It isn't documented in the OpenAPI docs and
  192. // is not shown in any of the example inputs.
  193. //
  194. // On that basis, I removed that test case. This implementation still does
  195. // correctly handle variable type/attribute splits ("x", "y.z" / "x.y", "z")
  196. // since that was required by the original "flattened" event representation.
  197. for i, tc := range testCases {
  198. t.Run(fmt.Sprintf("%02d", i+1), func(t *testing.T) {
  199. c, err := query.New(tc.s)
  200. if err != nil {
  201. t.Fatalf("NewCompiled %#q: unexpected error: %v", tc.s, err)
  202. }
  203. got, err := c.Matches(tc.events)
  204. if err != nil {
  205. t.Errorf("Query: %#q\nInput: %+v\nMatches: got error %v",
  206. tc.s, tc.events, err)
  207. }
  208. if got != tc.matches {
  209. t.Errorf("Query: %#q\nInput: %+v\nMatches: got %v, want %v",
  210. tc.s, tc.events, got, tc.matches)
  211. }
  212. })
  213. }
  214. }
  215. func TestAllMatchesAll(t *testing.T) {
  216. events := newTestEvents(
  217. ``,
  218. `Asher|Roth=`,
  219. `Route|66=`,
  220. `Rilly|Blue=`,
  221. )
  222. for i := 0; i < len(events); i++ {
  223. match, err := query.All.Matches(events[:i])
  224. if err != nil {
  225. t.Errorf("Matches failed: %w", err)
  226. } else if !match {
  227. t.Errorf("Did not match on %+v ", events[:i])
  228. }
  229. }
  230. }
  231. // newTestEvent constructs an Event message from a template string.
  232. // The format is "type|attr1=val1|attr2=val2|...".
  233. func newTestEvent(s string) types.Event {
  234. var event types.Event
  235. parts := strings.Split(s, "|")
  236. event.Type = parts[0]
  237. if len(parts) == 1 {
  238. return event // type only, no attributes
  239. }
  240. for _, kv := range parts[1:] {
  241. key, val := splitKV(kv)
  242. event.Attributes = append(event.Attributes, types.EventAttribute{
  243. Key: key,
  244. Value: val,
  245. })
  246. }
  247. return event
  248. }
  249. // newTestEvents constructs a slice of Event messages by applying newTestEvent
  250. // to each element of ss.
  251. func newTestEvents(ss ...string) []types.Event {
  252. events := make([]types.Event, len(ss))
  253. for i, s := range ss {
  254. events[i] = newTestEvent(s)
  255. }
  256. return events
  257. }
  258. func splitKV(s string) (key, value string) {
  259. kv := strings.SplitN(s, "=", 2)
  260. return kv[0], kv[1]
  261. }