You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

555 lines
15 KiB

  1. //nolint: gosec
  2. package e2e
  3. import (
  4. "errors"
  5. "fmt"
  6. "io"
  7. "math/rand"
  8. "net"
  9. "path/filepath"
  10. "sort"
  11. "strconv"
  12. "strings"
  13. "github.com/tendermint/tendermint/crypto"
  14. "github.com/tendermint/tendermint/crypto/ed25519"
  15. "github.com/tendermint/tendermint/crypto/secp256k1"
  16. rpchttp "github.com/tendermint/tendermint/rpc/client/http"
  17. mcs "github.com/tendermint/tendermint/test/maverick/consensus"
  18. "github.com/tendermint/tendermint/types"
  19. )
  20. const (
  21. randomSeed int64 = 2308084734268
  22. proxyPortFirst uint32 = 5701
  23. networkIPv4 = "10.186.73.0/24"
  24. networkIPv6 = "fd80:b10c::/48"
  25. )
  26. type Mode string
  27. type Protocol string
  28. type Perturbation string
  29. const (
  30. ModeValidator Mode = "validator"
  31. ModeFull Mode = "full"
  32. ModeSeed Mode = "seed"
  33. ProtocolBuiltin Protocol = "builtin"
  34. ProtocolFile Protocol = "file"
  35. ProtocolGRPC Protocol = "grpc"
  36. ProtocolTCP Protocol = "tcp"
  37. ProtocolUNIX Protocol = "unix"
  38. PerturbationDisconnect Perturbation = "disconnect"
  39. PerturbationKill Perturbation = "kill"
  40. PerturbationPause Perturbation = "pause"
  41. PerturbationRestart Perturbation = "restart"
  42. )
  43. // Testnet represents a single testnet.
  44. type Testnet struct {
  45. Name string
  46. File string
  47. Dir string
  48. IP *net.IPNet
  49. InitialHeight int64
  50. InitialState map[string]string
  51. Validators map[*Node]int64
  52. ValidatorUpdates map[int64]map[*Node]int64
  53. Nodes []*Node
  54. KeyType string
  55. }
  56. // Node represents a Tendermint node in a testnet.
  57. type Node struct {
  58. Name string
  59. Testnet *Testnet
  60. Mode Mode
  61. PrivvalKey crypto.PrivKey
  62. NodeKey crypto.PrivKey
  63. IP net.IP
  64. ProxyPort uint32
  65. StartAt int64
  66. FastSync string
  67. StateSync bool
  68. Database string
  69. ABCIProtocol Protocol
  70. PrivvalProtocol Protocol
  71. PersistInterval uint64
  72. SnapshotInterval uint64
  73. RetainBlocks uint64
  74. Seeds []*Node
  75. PersistentPeers []*Node
  76. Perturbations []Perturbation
  77. Misbehaviors map[int64]string
  78. }
  79. // LoadTestnet loads a testnet from a manifest file, using the filename to
  80. // determine the testnet name and directory (from the basename of the file).
  81. // The testnet generation must be deterministic, since it is generated
  82. // separately by the runner and the test cases. For this reason, testnets use a
  83. // random seed to generate e.g. keys.
  84. func LoadTestnet(file string) (*Testnet, error) {
  85. manifest, err := LoadManifest(file)
  86. if err != nil {
  87. return nil, err
  88. }
  89. dir := strings.TrimSuffix(file, filepath.Ext(file))
  90. // Set up resource generators. These must be deterministic.
  91. netAddress := networkIPv4
  92. if manifest.IPv6 {
  93. netAddress = networkIPv6
  94. }
  95. _, ipNet, err := net.ParseCIDR(netAddress)
  96. if err != nil {
  97. return nil, fmt.Errorf("invalid IP network address %q: %w", netAddress, err)
  98. }
  99. ipGen := newIPGenerator(ipNet)
  100. keyGen := newKeyGenerator(randomSeed)
  101. proxyPortGen := newPortGenerator(proxyPortFirst)
  102. testnet := &Testnet{
  103. Name: filepath.Base(dir),
  104. File: file,
  105. Dir: dir,
  106. IP: ipGen.Network(),
  107. InitialHeight: 1,
  108. InitialState: manifest.InitialState,
  109. Validators: map[*Node]int64{},
  110. ValidatorUpdates: map[int64]map[*Node]int64{},
  111. Nodes: []*Node{},
  112. KeyType: "ed25519",
  113. }
  114. if len(manifest.KeyType) != 0 {
  115. testnet.KeyType = manifest.KeyType
  116. }
  117. if manifest.InitialHeight > 0 {
  118. testnet.InitialHeight = manifest.InitialHeight
  119. }
  120. // Set up nodes, in alphabetical order (IPs and ports get same order).
  121. nodeNames := []string{}
  122. for name := range manifest.Nodes {
  123. nodeNames = append(nodeNames, name)
  124. }
  125. sort.Strings(nodeNames)
  126. for _, name := range nodeNames {
  127. nodeManifest := manifest.Nodes[name]
  128. node := &Node{
  129. Name: name,
  130. Testnet: testnet,
  131. PrivvalKey: keyGen.Generate(manifest.KeyType),
  132. NodeKey: keyGen.Generate("ed25519"),
  133. IP: ipGen.Next(),
  134. ProxyPort: proxyPortGen.Next(),
  135. Mode: ModeValidator,
  136. Database: "goleveldb",
  137. ABCIProtocol: ProtocolUNIX,
  138. PrivvalProtocol: ProtocolFile,
  139. StartAt: nodeManifest.StartAt,
  140. FastSync: nodeManifest.FastSync,
  141. StateSync: nodeManifest.StateSync,
  142. PersistInterval: 1,
  143. SnapshotInterval: nodeManifest.SnapshotInterval,
  144. RetainBlocks: nodeManifest.RetainBlocks,
  145. Perturbations: []Perturbation{},
  146. Misbehaviors: make(map[int64]string),
  147. }
  148. if node.StartAt == testnet.InitialHeight {
  149. node.StartAt = 0 // normalize to 0 for initial nodes, since code expects this
  150. }
  151. if nodeManifest.Mode != "" {
  152. node.Mode = Mode(nodeManifest.Mode)
  153. }
  154. if nodeManifest.Database != "" {
  155. node.Database = nodeManifest.Database
  156. }
  157. if nodeManifest.ABCIProtocol != "" {
  158. node.ABCIProtocol = Protocol(nodeManifest.ABCIProtocol)
  159. }
  160. if nodeManifest.PrivvalProtocol != "" {
  161. node.PrivvalProtocol = Protocol(nodeManifest.PrivvalProtocol)
  162. }
  163. if nodeManifest.PersistInterval != nil {
  164. node.PersistInterval = *nodeManifest.PersistInterval
  165. }
  166. for _, p := range nodeManifest.Perturb {
  167. node.Perturbations = append(node.Perturbations, Perturbation(p))
  168. }
  169. for heightString, misbehavior := range nodeManifest.Misbehaviors {
  170. height, err := strconv.ParseInt(heightString, 10, 64)
  171. if err != nil {
  172. return nil, fmt.Errorf("unable to parse height %s to int64: %w", heightString, err)
  173. }
  174. node.Misbehaviors[height] = misbehavior
  175. }
  176. testnet.Nodes = append(testnet.Nodes, node)
  177. }
  178. // We do a second pass to set up seeds and persistent peers, which allows graph cycles.
  179. for _, node := range testnet.Nodes {
  180. nodeManifest, ok := manifest.Nodes[node.Name]
  181. if !ok {
  182. return nil, fmt.Errorf("failed to look up manifest for node %q", node.Name)
  183. }
  184. for _, seedName := range nodeManifest.Seeds {
  185. seed := testnet.LookupNode(seedName)
  186. if seed == nil {
  187. return nil, fmt.Errorf("unknown seed %q for node %q", seedName, node.Name)
  188. }
  189. node.Seeds = append(node.Seeds, seed)
  190. }
  191. for _, peerName := range nodeManifest.PersistentPeers {
  192. peer := testnet.LookupNode(peerName)
  193. if peer == nil {
  194. return nil, fmt.Errorf("unknown persistent peer %q for node %q", peerName, node.Name)
  195. }
  196. node.PersistentPeers = append(node.PersistentPeers, peer)
  197. }
  198. // If there are no seeds or persistent peers specified, default to persistent
  199. // connections to all other nodes.
  200. if len(node.PersistentPeers) == 0 && len(node.Seeds) == 0 {
  201. for _, peer := range testnet.Nodes {
  202. if peer.Name == node.Name {
  203. continue
  204. }
  205. node.PersistentPeers = append(node.PersistentPeers, peer)
  206. }
  207. }
  208. }
  209. // Set up genesis validators. If not specified explicitly, use all validator nodes.
  210. if manifest.Validators != nil {
  211. for validatorName, power := range *manifest.Validators {
  212. validator := testnet.LookupNode(validatorName)
  213. if validator == nil {
  214. return nil, fmt.Errorf("unknown validator %q", validatorName)
  215. }
  216. testnet.Validators[validator] = power
  217. }
  218. } else {
  219. for _, node := range testnet.Nodes {
  220. if node.Mode == ModeValidator {
  221. testnet.Validators[node] = 100
  222. }
  223. }
  224. }
  225. // Set up validator updates.
  226. for heightStr, validators := range manifest.ValidatorUpdates {
  227. height, err := strconv.Atoi(heightStr)
  228. if err != nil {
  229. return nil, fmt.Errorf("invalid validator update height %q: %w", height, err)
  230. }
  231. valUpdate := map[*Node]int64{}
  232. for name, power := range validators {
  233. node := testnet.LookupNode(name)
  234. if node == nil {
  235. return nil, fmt.Errorf("unknown validator %q for update at height %v", name, height)
  236. }
  237. valUpdate[node] = power
  238. }
  239. testnet.ValidatorUpdates[int64(height)] = valUpdate
  240. }
  241. return testnet, testnet.Validate()
  242. }
  243. // Validate validates a testnet.
  244. func (t Testnet) Validate() error {
  245. if t.Name == "" {
  246. return errors.New("network has no name")
  247. }
  248. if t.IP == nil {
  249. return errors.New("network has no IP")
  250. }
  251. if len(t.Nodes) == 0 {
  252. return errors.New("network has no nodes")
  253. }
  254. switch t.KeyType {
  255. case "", types.ABCIPubKeyTypeEd25519, types.ABCIPubKeyTypeSecp256k1:
  256. default:
  257. return errors.New("unsupported KeyType")
  258. }
  259. for _, node := range t.Nodes {
  260. if err := node.Validate(t); err != nil {
  261. return fmt.Errorf("invalid node %q: %w", node.Name, err)
  262. }
  263. }
  264. return nil
  265. }
  266. // Validate validates a node.
  267. func (n Node) Validate(testnet Testnet) error {
  268. if n.Name == "" {
  269. return errors.New("node has no name")
  270. }
  271. if n.IP == nil {
  272. return errors.New("node has no IP address")
  273. }
  274. if !testnet.IP.Contains(n.IP) {
  275. return fmt.Errorf("node IP %v is not in testnet network %v", n.IP, testnet.IP)
  276. }
  277. if n.ProxyPort > 0 {
  278. if n.ProxyPort <= 1024 {
  279. return fmt.Errorf("local port %v must be >1024", n.ProxyPort)
  280. }
  281. for _, peer := range testnet.Nodes {
  282. if peer.Name != n.Name && peer.ProxyPort == n.ProxyPort {
  283. return fmt.Errorf("peer %q also has local port %v", peer.Name, n.ProxyPort)
  284. }
  285. }
  286. }
  287. switch n.FastSync {
  288. case "", "v0", "v2":
  289. default:
  290. return fmt.Errorf("invalid fast sync setting %q", n.FastSync)
  291. }
  292. switch n.Database {
  293. case "goleveldb", "cleveldb", "boltdb", "rocksdb", "badgerdb":
  294. default:
  295. return fmt.Errorf("invalid database setting %q", n.Database)
  296. }
  297. switch n.ABCIProtocol {
  298. case ProtocolBuiltin, ProtocolUNIX, ProtocolTCP, ProtocolGRPC:
  299. default:
  300. return fmt.Errorf("invalid ABCI protocol setting %q", n.ABCIProtocol)
  301. }
  302. switch n.PrivvalProtocol {
  303. case ProtocolFile, ProtocolUNIX, ProtocolTCP:
  304. default:
  305. return fmt.Errorf("invalid privval protocol setting %q", n.PrivvalProtocol)
  306. }
  307. if n.StartAt > 0 && n.StartAt < n.Testnet.InitialHeight {
  308. return fmt.Errorf("cannot start at height %v lower than initial height %v",
  309. n.StartAt, n.Testnet.InitialHeight)
  310. }
  311. if n.StateSync && n.StartAt == 0 {
  312. return errors.New("state synced nodes cannot start at the initial height")
  313. }
  314. if n.PersistInterval == 0 && n.RetainBlocks > 0 {
  315. return errors.New("persist_interval=0 requires retain_blocks=0")
  316. }
  317. if n.PersistInterval > 1 && n.RetainBlocks > 0 && n.RetainBlocks < n.PersistInterval {
  318. return errors.New("persist_interval must be less than or equal to retain_blocks")
  319. }
  320. if n.SnapshotInterval > 0 && n.RetainBlocks > 0 && n.RetainBlocks < n.SnapshotInterval {
  321. return errors.New("snapshot_interval must be less than er equal to retain_blocks")
  322. }
  323. for _, perturbation := range n.Perturbations {
  324. switch perturbation {
  325. case PerturbationDisconnect, PerturbationKill, PerturbationPause, PerturbationRestart:
  326. default:
  327. return fmt.Errorf("invalid perturbation %q", perturbation)
  328. }
  329. }
  330. if (n.PrivvalProtocol != "file" || n.Mode != "validator") && len(n.Misbehaviors) != 0 {
  331. return errors.New("must be using \"file\" privval protocol to implement misbehaviors")
  332. }
  333. for height, misbehavior := range n.Misbehaviors {
  334. if height < n.StartAt {
  335. return fmt.Errorf("misbehavior height %d is below node start height %d",
  336. height, n.StartAt)
  337. }
  338. if height < testnet.InitialHeight {
  339. return fmt.Errorf("misbehavior height %d is below network initial height %d",
  340. height, testnet.InitialHeight)
  341. }
  342. exists := false
  343. for possibleBehaviors := range mcs.MisbehaviorList {
  344. if possibleBehaviors == misbehavior {
  345. exists = true
  346. }
  347. }
  348. if !exists {
  349. return fmt.Errorf("misbehavior %s does not exist", misbehavior)
  350. }
  351. }
  352. return nil
  353. }
  354. // LookupNode looks up a node by name. For now, simply do a linear search.
  355. func (t Testnet) LookupNode(name string) *Node {
  356. for _, node := range t.Nodes {
  357. if node.Name == name {
  358. return node
  359. }
  360. }
  361. return nil
  362. }
  363. // ArchiveNodes returns a list of archive nodes that start at the initial height
  364. // and contain the entire blockchain history. They are used e.g. as light client
  365. // RPC servers.
  366. func (t Testnet) ArchiveNodes() []*Node {
  367. nodes := []*Node{}
  368. for _, node := range t.Nodes {
  369. if node.Mode != ModeSeed && node.StartAt == 0 && node.RetainBlocks == 0 {
  370. nodes = append(nodes, node)
  371. }
  372. }
  373. return nodes
  374. }
  375. // RandomNode returns a random non-seed node.
  376. func (t Testnet) RandomNode() *Node {
  377. for {
  378. node := t.Nodes[rand.Intn(len(t.Nodes))]
  379. if node.Mode != ModeSeed {
  380. return node
  381. }
  382. }
  383. }
  384. // IPv6 returns true if the testnet is an IPv6 network.
  385. func (t Testnet) IPv6() bool {
  386. return t.IP.IP.To4() == nil
  387. }
  388. // HasPerturbations returns whether the network has any perturbations.
  389. func (t Testnet) HasPerturbations() bool {
  390. for _, node := range t.Nodes {
  391. if len(node.Perturbations) > 0 {
  392. return true
  393. }
  394. }
  395. return false
  396. }
  397. // LastMisbehaviorHeight returns the height of the last misbehavior.
  398. func (t Testnet) LastMisbehaviorHeight() int64 {
  399. lastHeight := int64(0)
  400. for _, node := range t.Nodes {
  401. for height := range node.Misbehaviors {
  402. if height > lastHeight {
  403. lastHeight = height
  404. }
  405. }
  406. }
  407. return lastHeight
  408. }
  409. // Address returns a P2P endpoint address for the node.
  410. func (n Node) AddressP2P(withID bool) string {
  411. ip := n.IP.String()
  412. if n.IP.To4() == nil {
  413. // IPv6 addresses must be wrapped in [] to avoid conflict with : port separator
  414. ip = fmt.Sprintf("[%v]", ip)
  415. }
  416. addr := fmt.Sprintf("%v:26656", ip)
  417. if withID {
  418. addr = fmt.Sprintf("%x@%v", n.NodeKey.PubKey().Address().Bytes(), addr)
  419. }
  420. return addr
  421. }
  422. // Address returns an RPC endpoint address for the node.
  423. func (n Node) AddressRPC() string {
  424. ip := n.IP.String()
  425. if n.IP.To4() == nil {
  426. // IPv6 addresses must be wrapped in [] to avoid conflict with : port separator
  427. ip = fmt.Sprintf("[%v]", ip)
  428. }
  429. return fmt.Sprintf("%v:26657", ip)
  430. }
  431. // Client returns an RPC client for a node.
  432. func (n Node) Client() (*rpchttp.HTTP, error) {
  433. return rpchttp.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort), "/websocket")
  434. }
  435. // keyGenerator generates pseudorandom Ed25519 keys based on a seed.
  436. type keyGenerator struct {
  437. random *rand.Rand
  438. }
  439. func newKeyGenerator(seed int64) *keyGenerator {
  440. return &keyGenerator{
  441. random: rand.New(rand.NewSource(seed)),
  442. }
  443. }
  444. func (g *keyGenerator) Generate(keyType string) crypto.PrivKey {
  445. seed := make([]byte, ed25519.SeedSize)
  446. _, err := io.ReadFull(g.random, seed)
  447. if err != nil {
  448. panic(err) // this shouldn't happen
  449. }
  450. switch keyType {
  451. case "secp256k1":
  452. return secp256k1.GenPrivKeySecp256k1(seed)
  453. case "", "ed25519":
  454. return ed25519.GenPrivKeyFromSecret(seed)
  455. default:
  456. panic("KeyType not supported") // should not make it this far
  457. }
  458. }
  459. // portGenerator generates local Docker proxy ports for each node.
  460. type portGenerator struct {
  461. nextPort uint32
  462. }
  463. func newPortGenerator(firstPort uint32) *portGenerator {
  464. return &portGenerator{nextPort: firstPort}
  465. }
  466. func (g *portGenerator) Next() uint32 {
  467. port := g.nextPort
  468. g.nextPort++
  469. if g.nextPort == 0 {
  470. panic("port overflow")
  471. }
  472. return port
  473. }
  474. // ipGenerator generates sequential IP addresses for each node, using a random
  475. // network address.
  476. type ipGenerator struct {
  477. network *net.IPNet
  478. nextIP net.IP
  479. }
  480. func newIPGenerator(network *net.IPNet) *ipGenerator {
  481. nextIP := make([]byte, len(network.IP))
  482. copy(nextIP, network.IP)
  483. gen := &ipGenerator{network: network, nextIP: nextIP}
  484. // Skip network and gateway addresses
  485. gen.Next()
  486. gen.Next()
  487. return gen
  488. }
  489. func (g *ipGenerator) Network() *net.IPNet {
  490. n := &net.IPNet{
  491. IP: make([]byte, len(g.network.IP)),
  492. Mask: make([]byte, len(g.network.Mask)),
  493. }
  494. copy(n.IP, g.network.IP)
  495. copy(n.Mask, g.network.Mask)
  496. return n
  497. }
  498. func (g *ipGenerator) Next() net.IP {
  499. ip := make([]byte, len(g.nextIP))
  500. copy(ip, g.nextIP)
  501. for i := len(g.nextIP) - 1; i >= 0; i-- {
  502. g.nextIP[i]++
  503. if g.nextIP[i] != 0 {
  504. break
  505. }
  506. }
  507. return ip
  508. }