You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

562 lines
16 KiB

  1. //nolint: gosec
  2. package e2e
  3. import (
  4. "errors"
  5. "fmt"
  6. "io"
  7. "math/rand"
  8. "net"
  9. "path/filepath"
  10. "sort"
  11. "strconv"
  12. "strings"
  13. "time"
  14. "github.com/tendermint/tendermint/crypto"
  15. "github.com/tendermint/tendermint/crypto/ed25519"
  16. "github.com/tendermint/tendermint/crypto/secp256k1"
  17. rpchttp "github.com/tendermint/tendermint/rpc/client/http"
  18. "github.com/tendermint/tendermint/types"
  19. )
  20. const (
  21. randomSeed int64 = 2308084734268
  22. proxyPortFirst uint32 = 5701
  23. networkIPv4 = "10.186.73.0/24"
  24. networkIPv6 = "fd80:b10c::/48"
  25. )
  26. type Mode string
  27. type Protocol string
  28. type Perturbation string
  29. const (
  30. ModeValidator Mode = "validator"
  31. ModeFull Mode = "full"
  32. ModeLight Mode = "light"
  33. ModeSeed Mode = "seed"
  34. ProtocolBuiltin Protocol = "builtin"
  35. ProtocolFile Protocol = "file"
  36. ProtocolGRPC Protocol = "grpc"
  37. ProtocolTCP Protocol = "tcp"
  38. ProtocolUNIX Protocol = "unix"
  39. PerturbationDisconnect Perturbation = "disconnect"
  40. PerturbationKill Perturbation = "kill"
  41. PerturbationPause Perturbation = "pause"
  42. PerturbationRestart Perturbation = "restart"
  43. EvidenceAgeHeight int64 = 7
  44. EvidenceAgeTime time.Duration = 500 * time.Millisecond
  45. StateSyncP2P = "p2p"
  46. StateSyncRPC = "rpc"
  47. StateSyncDisabled = ""
  48. )
  49. // Testnet represents a single testnet.
  50. type Testnet struct {
  51. Name string
  52. File string
  53. Dir string
  54. IP *net.IPNet
  55. InitialHeight int64
  56. InitialState map[string]string
  57. Validators map[*Node]int64
  58. ValidatorUpdates map[int64]map[*Node]int64
  59. Nodes []*Node
  60. KeyType string
  61. Evidence int
  62. LogLevel string
  63. TxSize int64
  64. }
  65. // Node represents a Tendermint node in a testnet.
  66. type Node struct {
  67. Name string
  68. Testnet *Testnet
  69. Mode Mode
  70. PrivvalKey crypto.PrivKey
  71. NodeKey crypto.PrivKey
  72. IP net.IP
  73. ProxyPort uint32
  74. StartAt int64
  75. BlockSync string
  76. Mempool string
  77. StateSync string
  78. Database string
  79. ABCIProtocol Protocol
  80. PrivvalProtocol Protocol
  81. PersistInterval uint64
  82. SnapshotInterval uint64
  83. RetainBlocks uint64
  84. Seeds []*Node
  85. PersistentPeers []*Node
  86. Perturbations []Perturbation
  87. LogLevel string
  88. UseLegacyP2P bool
  89. QueueType string
  90. HasStarted bool
  91. }
  92. // LoadTestnet loads a testnet from a manifest file, using the filename to
  93. // determine the testnet name and directory (from the basename of the file).
  94. // The testnet generation must be deterministic, since it is generated
  95. // separately by the runner and the test cases. For this reason, testnets use a
  96. // random seed to generate e.g. keys.
  97. func LoadTestnet(file string) (*Testnet, error) {
  98. manifest, err := LoadManifest(file)
  99. if err != nil {
  100. return nil, err
  101. }
  102. dir := strings.TrimSuffix(file, filepath.Ext(file))
  103. // Set up resource generators. These must be deterministic.
  104. netAddress := networkIPv4
  105. if manifest.IPv6 {
  106. netAddress = networkIPv6
  107. }
  108. _, ipNet, err := net.ParseCIDR(netAddress)
  109. if err != nil {
  110. return nil, fmt.Errorf("invalid IP network address %q: %w", netAddress, err)
  111. }
  112. ipGen := newIPGenerator(ipNet)
  113. keyGen := newKeyGenerator(randomSeed)
  114. proxyPortGen := newPortGenerator(proxyPortFirst)
  115. testnet := &Testnet{
  116. Name: filepath.Base(dir),
  117. File: file,
  118. Dir: dir,
  119. IP: ipGen.Network(),
  120. InitialHeight: 1,
  121. InitialState: manifest.InitialState,
  122. Validators: map[*Node]int64{},
  123. ValidatorUpdates: map[int64]map[*Node]int64{},
  124. Nodes: []*Node{},
  125. Evidence: manifest.Evidence,
  126. KeyType: "ed25519",
  127. LogLevel: manifest.LogLevel,
  128. TxSize: manifest.TxSize,
  129. }
  130. if len(manifest.KeyType) != 0 {
  131. testnet.KeyType = manifest.KeyType
  132. }
  133. if testnet.TxSize <= 0 {
  134. testnet.TxSize = 1024
  135. }
  136. if manifest.InitialHeight > 0 {
  137. testnet.InitialHeight = manifest.InitialHeight
  138. }
  139. // Set up nodes, in alphabetical order (IPs and ports get same order).
  140. nodeNames := []string{}
  141. for name := range manifest.Nodes {
  142. nodeNames = append(nodeNames, name)
  143. }
  144. sort.Strings(nodeNames)
  145. for _, name := range nodeNames {
  146. nodeManifest := manifest.Nodes[name]
  147. node := &Node{
  148. Name: name,
  149. Testnet: testnet,
  150. PrivvalKey: keyGen.Generate(manifest.KeyType),
  151. NodeKey: keyGen.Generate("ed25519"),
  152. IP: ipGen.Next(),
  153. ProxyPort: proxyPortGen.Next(),
  154. Mode: ModeValidator,
  155. Database: "goleveldb",
  156. ABCIProtocol: ProtocolBuiltin,
  157. PrivvalProtocol: ProtocolFile,
  158. StartAt: nodeManifest.StartAt,
  159. BlockSync: nodeManifest.BlockSync,
  160. Mempool: nodeManifest.Mempool,
  161. StateSync: nodeManifest.StateSync,
  162. PersistInterval: 1,
  163. SnapshotInterval: nodeManifest.SnapshotInterval,
  164. RetainBlocks: nodeManifest.RetainBlocks,
  165. Perturbations: []Perturbation{},
  166. LogLevel: manifest.LogLevel,
  167. QueueType: manifest.QueueType,
  168. UseLegacyP2P: nodeManifest.UseLegacyP2P,
  169. }
  170. if node.StartAt == testnet.InitialHeight {
  171. node.StartAt = 0 // normalize to 0 for initial nodes, since code expects this
  172. }
  173. if nodeManifest.Mode != "" {
  174. node.Mode = Mode(nodeManifest.Mode)
  175. }
  176. if nodeManifest.Database != "" {
  177. node.Database = nodeManifest.Database
  178. }
  179. if nodeManifest.ABCIProtocol != "" {
  180. node.ABCIProtocol = Protocol(nodeManifest.ABCIProtocol)
  181. }
  182. if nodeManifest.PrivvalProtocol != "" {
  183. node.PrivvalProtocol = Protocol(nodeManifest.PrivvalProtocol)
  184. }
  185. if nodeManifest.PersistInterval != nil {
  186. node.PersistInterval = *nodeManifest.PersistInterval
  187. }
  188. for _, p := range nodeManifest.Perturb {
  189. node.Perturbations = append(node.Perturbations, Perturbation(p))
  190. }
  191. if nodeManifest.LogLevel != "" {
  192. node.LogLevel = nodeManifest.LogLevel
  193. }
  194. testnet.Nodes = append(testnet.Nodes, node)
  195. }
  196. // We do a second pass to set up seeds and persistent peers, which allows graph cycles.
  197. for _, node := range testnet.Nodes {
  198. nodeManifest, ok := manifest.Nodes[node.Name]
  199. if !ok {
  200. return nil, fmt.Errorf("failed to look up manifest for node %q", node.Name)
  201. }
  202. for _, seedName := range nodeManifest.Seeds {
  203. seed := testnet.LookupNode(seedName)
  204. if seed == nil {
  205. return nil, fmt.Errorf("unknown seed %q for node %q", seedName, node.Name)
  206. }
  207. node.Seeds = append(node.Seeds, seed)
  208. }
  209. for _, peerName := range nodeManifest.PersistentPeers {
  210. peer := testnet.LookupNode(peerName)
  211. if peer == nil {
  212. return nil, fmt.Errorf("unknown persistent peer %q for node %q", peerName, node.Name)
  213. }
  214. if peer.Mode == ModeLight {
  215. return nil, fmt.Errorf("can not have a light client as a persistent peer (for %q)", node.Name)
  216. }
  217. node.PersistentPeers = append(node.PersistentPeers, peer)
  218. }
  219. // If there are no seeds or persistent peers specified, default to persistent
  220. // connections to all other full nodes.
  221. if len(node.PersistentPeers) == 0 && len(node.Seeds) == 0 {
  222. for _, peer := range testnet.Nodes {
  223. if peer.Name == node.Name {
  224. continue
  225. }
  226. if peer.Mode == ModeLight {
  227. continue
  228. }
  229. node.PersistentPeers = append(node.PersistentPeers, peer)
  230. }
  231. }
  232. }
  233. // Set up genesis validators. If not specified explicitly, use all validator nodes.
  234. if manifest.Validators != nil {
  235. for validatorName, power := range *manifest.Validators {
  236. validator := testnet.LookupNode(validatorName)
  237. if validator == nil {
  238. return nil, fmt.Errorf("unknown validator %q", validatorName)
  239. }
  240. testnet.Validators[validator] = power
  241. }
  242. } else {
  243. for _, node := range testnet.Nodes {
  244. if node.Mode == ModeValidator {
  245. testnet.Validators[node] = 100
  246. }
  247. }
  248. }
  249. // Set up validator updates.
  250. for heightStr, validators := range manifest.ValidatorUpdates {
  251. height, err := strconv.Atoi(heightStr)
  252. if err != nil {
  253. return nil, fmt.Errorf("invalid validator update height %q: %w", height, err)
  254. }
  255. valUpdate := map[*Node]int64{}
  256. for name, power := range validators {
  257. node := testnet.LookupNode(name)
  258. if node == nil {
  259. return nil, fmt.Errorf("unknown validator %q for update at height %v", name, height)
  260. }
  261. valUpdate[node] = power
  262. }
  263. testnet.ValidatorUpdates[int64(height)] = valUpdate
  264. }
  265. return testnet, testnet.Validate()
  266. }
  267. // Validate validates a testnet.
  268. func (t Testnet) Validate() error {
  269. if t.Name == "" {
  270. return errors.New("network has no name")
  271. }
  272. if t.IP == nil {
  273. return errors.New("network has no IP")
  274. }
  275. if len(t.Nodes) == 0 {
  276. return errors.New("network has no nodes")
  277. }
  278. switch t.KeyType {
  279. case "", types.ABCIPubKeyTypeEd25519, types.ABCIPubKeyTypeSecp256k1:
  280. default:
  281. return errors.New("unsupported KeyType")
  282. }
  283. for _, node := range t.Nodes {
  284. if err := node.Validate(t); err != nil {
  285. return fmt.Errorf("invalid node %q: %w", node.Name, err)
  286. }
  287. }
  288. return nil
  289. }
  290. // Validate validates a node.
  291. func (n Node) Validate(testnet Testnet) error {
  292. if n.Name == "" {
  293. return errors.New("node has no name")
  294. }
  295. if n.IP == nil {
  296. return errors.New("node has no IP address")
  297. }
  298. if !testnet.IP.Contains(n.IP) {
  299. return fmt.Errorf("node IP %v is not in testnet network %v", n.IP, testnet.IP)
  300. }
  301. if n.ProxyPort > 0 {
  302. if n.ProxyPort <= 1024 {
  303. return fmt.Errorf("local port %v must be >1024", n.ProxyPort)
  304. }
  305. for _, peer := range testnet.Nodes {
  306. if peer.Name != n.Name && peer.ProxyPort == n.ProxyPort {
  307. return fmt.Errorf("peer %q also has local port %v", peer.Name, n.ProxyPort)
  308. }
  309. }
  310. }
  311. switch n.BlockSync {
  312. case "", "v0", "v2":
  313. default:
  314. return fmt.Errorf("invalid block sync setting %q", n.BlockSync)
  315. }
  316. switch n.StateSync {
  317. case StateSyncDisabled, StateSyncP2P, StateSyncRPC:
  318. default:
  319. return fmt.Errorf("invalid state sync setting %q", n.StateSync)
  320. }
  321. switch n.Mempool {
  322. case "", "v0", "v1":
  323. default:
  324. return fmt.Errorf("invalid mempool version %q", n.Mempool)
  325. }
  326. switch n.QueueType {
  327. case "", "priority", "wdrr", "fifo":
  328. default:
  329. return fmt.Errorf("unsupported p2p queue type: %s", n.QueueType)
  330. }
  331. switch n.Database {
  332. case "goleveldb", "cleveldb", "boltdb", "rocksdb", "badgerdb":
  333. default:
  334. return fmt.Errorf("invalid database setting %q", n.Database)
  335. }
  336. switch n.ABCIProtocol {
  337. case ProtocolBuiltin, ProtocolUNIX, ProtocolTCP, ProtocolGRPC:
  338. default:
  339. return fmt.Errorf("invalid ABCI protocol setting %q", n.ABCIProtocol)
  340. }
  341. if n.Mode == ModeLight && n.ABCIProtocol != ProtocolBuiltin {
  342. return errors.New("light client must use builtin protocol")
  343. }
  344. switch n.PrivvalProtocol {
  345. case ProtocolFile, ProtocolTCP, ProtocolGRPC, ProtocolUNIX:
  346. default:
  347. return fmt.Errorf("invalid privval protocol setting %q", n.PrivvalProtocol)
  348. }
  349. if n.StartAt > 0 && n.StartAt < n.Testnet.InitialHeight {
  350. return fmt.Errorf("cannot start at height %v lower than initial height %v",
  351. n.StartAt, n.Testnet.InitialHeight)
  352. }
  353. if n.StateSync != StateSyncDisabled && n.StartAt == 0 {
  354. return errors.New("state synced nodes cannot start at the initial height")
  355. }
  356. if n.RetainBlocks != 0 && n.RetainBlocks < uint64(EvidenceAgeHeight) {
  357. return fmt.Errorf("retain_blocks must be greater or equal to max evidence age (%d)",
  358. EvidenceAgeHeight)
  359. }
  360. if n.PersistInterval == 0 && n.RetainBlocks > 0 {
  361. return errors.New("persist_interval=0 requires retain_blocks=0")
  362. }
  363. if n.PersistInterval > 1 && n.RetainBlocks > 0 && n.RetainBlocks < n.PersistInterval {
  364. return errors.New("persist_interval must be less than or equal to retain_blocks")
  365. }
  366. if n.SnapshotInterval > 0 && n.RetainBlocks > 0 && n.RetainBlocks < n.SnapshotInterval {
  367. return errors.New("snapshot_interval must be less than er equal to retain_blocks")
  368. }
  369. for _, perturbation := range n.Perturbations {
  370. switch perturbation {
  371. case PerturbationDisconnect, PerturbationKill, PerturbationPause, PerturbationRestart:
  372. default:
  373. return fmt.Errorf("invalid perturbation %q", perturbation)
  374. }
  375. }
  376. return nil
  377. }
  378. // LookupNode looks up a node by name. For now, simply do a linear search.
  379. func (t Testnet) LookupNode(name string) *Node {
  380. for _, node := range t.Nodes {
  381. if node.Name == name {
  382. return node
  383. }
  384. }
  385. return nil
  386. }
  387. // ArchiveNodes returns a list of archive nodes that start at the initial height
  388. // and contain the entire blockchain history. They are used e.g. as light client
  389. // RPC servers.
  390. func (t Testnet) ArchiveNodes() []*Node {
  391. nodes := []*Node{}
  392. for _, node := range t.Nodes {
  393. if !node.Stateless() && node.StartAt == 0 && node.RetainBlocks == 0 {
  394. nodes = append(nodes, node)
  395. }
  396. }
  397. return nodes
  398. }
  399. // IPv6 returns true if the testnet is an IPv6 network.
  400. func (t Testnet) IPv6() bool {
  401. return t.IP.IP.To4() == nil
  402. }
  403. // HasPerturbations returns whether the network has any perturbations.
  404. func (t Testnet) HasPerturbations() bool {
  405. for _, node := range t.Nodes {
  406. if len(node.Perturbations) > 0 {
  407. return true
  408. }
  409. }
  410. return false
  411. }
  412. // Address returns a P2P endpoint address for the node.
  413. func (n Node) AddressP2P(withID bool) string {
  414. ip := n.IP.String()
  415. if n.IP.To4() == nil {
  416. // IPv6 addresses must be wrapped in [] to avoid conflict with : port separator
  417. ip = fmt.Sprintf("[%v]", ip)
  418. }
  419. addr := fmt.Sprintf("%v:26656", ip)
  420. if withID {
  421. addr = fmt.Sprintf("%x@%v", n.NodeKey.PubKey().Address().Bytes(), addr)
  422. }
  423. return addr
  424. }
  425. // Address returns an RPC endpoint address for the node.
  426. func (n Node) AddressRPC() string {
  427. ip := n.IP.String()
  428. if n.IP.To4() == nil {
  429. // IPv6 addresses must be wrapped in [] to avoid conflict with : port separator
  430. ip = fmt.Sprintf("[%v]", ip)
  431. }
  432. return fmt.Sprintf("%v:26657", ip)
  433. }
  434. // Client returns an RPC client for a node.
  435. func (n Node) Client() (*rpchttp.HTTP, error) {
  436. return rpchttp.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort))
  437. }
  438. // Stateless returns true if the node is either a seed node or a light node
  439. func (n Node) Stateless() bool {
  440. return n.Mode == ModeLight || n.Mode == ModeSeed
  441. }
  442. // keyGenerator generates pseudorandom Ed25519 keys based on a seed.
  443. type keyGenerator struct {
  444. random *rand.Rand
  445. }
  446. func newKeyGenerator(seed int64) *keyGenerator {
  447. return &keyGenerator{
  448. random: rand.New(rand.NewSource(seed)),
  449. }
  450. }
  451. func (g *keyGenerator) Generate(keyType string) crypto.PrivKey {
  452. seed := make([]byte, ed25519.SeedSize)
  453. _, err := io.ReadFull(g.random, seed)
  454. if err != nil {
  455. panic(err) // this shouldn't happen
  456. }
  457. switch keyType {
  458. case "secp256k1":
  459. return secp256k1.GenPrivKeySecp256k1(seed)
  460. case "", "ed25519":
  461. return ed25519.GenPrivKeyFromSecret(seed)
  462. default:
  463. panic("KeyType not supported") // should not make it this far
  464. }
  465. }
  466. // portGenerator generates local Docker proxy ports for each node.
  467. type portGenerator struct {
  468. nextPort uint32
  469. }
  470. func newPortGenerator(firstPort uint32) *portGenerator {
  471. return &portGenerator{nextPort: firstPort}
  472. }
  473. func (g *portGenerator) Next() uint32 {
  474. port := g.nextPort
  475. g.nextPort++
  476. if g.nextPort == 0 {
  477. panic("port overflow")
  478. }
  479. return port
  480. }
  481. // ipGenerator generates sequential IP addresses for each node, using a random
  482. // network address.
  483. type ipGenerator struct {
  484. network *net.IPNet
  485. nextIP net.IP
  486. }
  487. func newIPGenerator(network *net.IPNet) *ipGenerator {
  488. nextIP := make([]byte, len(network.IP))
  489. copy(nextIP, network.IP)
  490. gen := &ipGenerator{network: network, nextIP: nextIP}
  491. // Skip network and gateway addresses
  492. gen.Next()
  493. gen.Next()
  494. return gen
  495. }
  496. func (g *ipGenerator) Network() *net.IPNet {
  497. n := &net.IPNet{
  498. IP: make([]byte, len(g.network.IP)),
  499. Mask: make([]byte, len(g.network.Mask)),
  500. }
  501. copy(n.IP, g.network.IP)
  502. copy(n.Mask, g.network.Mask)
  503. return n
  504. }
  505. func (g *ipGenerator) Next() net.IP {
  506. ip := make([]byte, len(g.nextIP))
  507. copy(ip, g.nextIP)
  508. for i := len(g.nextIP) - 1; i >= 0; i-- {
  509. g.nextIP[i]++
  510. if g.nextIP[i] != 0 {
  511. break
  512. }
  513. }
  514. return ip
  515. }