You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

427 lines
12 KiB

  1. // nolint: gosec
  2. package main
  3. import (
  4. "bytes"
  5. "encoding/base64"
  6. "encoding/json"
  7. "errors"
  8. "fmt"
  9. "io/ioutil"
  10. "os"
  11. "path/filepath"
  12. "regexp"
  13. "sort"
  14. "strings"
  15. "text/template"
  16. "time"
  17. "github.com/BurntSushi/toml"
  18. "github.com/tendermint/tendermint/config"
  19. "github.com/tendermint/tendermint/crypto/ed25519"
  20. "github.com/tendermint/tendermint/privval"
  21. e2e "github.com/tendermint/tendermint/test/e2e/pkg"
  22. "github.com/tendermint/tendermint/types"
  23. )
  24. const (
  25. AppAddressTCP = "tcp://127.0.0.1:30000"
  26. AppAddressUNIX = "unix:///var/run/app.sock"
  27. PrivvalAddressTCP = "tcp://0.0.0.0:27559"
  28. PrivvalAddressGRPC = "grpc://0.0.0.0:27559"
  29. PrivvalAddressUNIX = "unix:///var/run/privval.sock"
  30. PrivvalKeyFile = "config/priv_validator_key.json"
  31. PrivvalStateFile = "data/priv_validator_state.json"
  32. PrivvalDummyKeyFile = "config/dummy_validator_key.json"
  33. PrivvalDummyStateFile = "data/dummy_validator_state.json"
  34. )
  35. // Setup sets up the testnet configuration.
  36. func Setup(testnet *e2e.Testnet) error {
  37. logger.Info(fmt.Sprintf("Generating testnet files in %q", testnet.Dir))
  38. err := os.MkdirAll(testnet.Dir, os.ModePerm)
  39. if err != nil {
  40. return err
  41. }
  42. compose, err := MakeDockerCompose(testnet)
  43. if err != nil {
  44. return err
  45. }
  46. err = ioutil.WriteFile(filepath.Join(testnet.Dir, "docker-compose.yml"), compose, 0644)
  47. if err != nil {
  48. return err
  49. }
  50. genesis, err := MakeGenesis(testnet)
  51. if err != nil {
  52. return err
  53. }
  54. for _, node := range testnet.Nodes {
  55. nodeDir := filepath.Join(testnet.Dir, node.Name)
  56. dirs := []string{
  57. filepath.Join(nodeDir, "config"),
  58. filepath.Join(nodeDir, "data"),
  59. filepath.Join(nodeDir, "data", "app"),
  60. }
  61. for _, dir := range dirs {
  62. // light clients don't need an app directory
  63. if node.Mode == e2e.ModeLight && strings.Contains(dir, "app") {
  64. continue
  65. }
  66. err := os.MkdirAll(dir, 0755)
  67. if err != nil {
  68. return err
  69. }
  70. }
  71. cfg, err := MakeConfig(node)
  72. if err != nil {
  73. return err
  74. }
  75. config.WriteConfigFile(nodeDir, cfg) // panics
  76. appCfg, err := MakeAppConfig(node)
  77. if err != nil {
  78. return err
  79. }
  80. err = ioutil.WriteFile(filepath.Join(nodeDir, "config", "app.toml"), appCfg, 0644)
  81. if err != nil {
  82. return err
  83. }
  84. if node.Mode == e2e.ModeLight {
  85. // stop early if a light client
  86. continue
  87. }
  88. err = genesis.SaveAs(filepath.Join(nodeDir, "config", "genesis.json"))
  89. if err != nil {
  90. return err
  91. }
  92. err = (&types.NodeKey{PrivKey: node.NodeKey}).SaveAs(filepath.Join(nodeDir, "config", "node_key.json"))
  93. if err != nil {
  94. return err
  95. }
  96. (privval.NewFilePV(node.PrivvalKey,
  97. filepath.Join(nodeDir, PrivvalKeyFile),
  98. filepath.Join(nodeDir, PrivvalStateFile),
  99. )).Save()
  100. // Set up a dummy validator. Tendermint requires a file PV even when not used, so we
  101. // give it a dummy such that it will fail if it actually tries to use it.
  102. (privval.NewFilePV(ed25519.GenPrivKey(),
  103. filepath.Join(nodeDir, PrivvalDummyKeyFile),
  104. filepath.Join(nodeDir, PrivvalDummyStateFile),
  105. )).Save()
  106. }
  107. return nil
  108. }
  109. // MakeDockerCompose generates a Docker Compose config for a testnet.
  110. func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) {
  111. // Must use version 2 Docker Compose format, to support IPv6.
  112. tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{
  113. "addUint32": func(x, y uint32) uint32 {
  114. return x + y
  115. },
  116. }).Parse(`version: '2.4'
  117. networks:
  118. {{ .Name }}:
  119. labels:
  120. e2e: true
  121. driver: bridge
  122. {{- if .IPv6 }}
  123. enable_ipv6: true
  124. {{- end }}
  125. ipam:
  126. driver: default
  127. config:
  128. - subnet: {{ .IP }}
  129. services:
  130. {{- range .Nodes }}
  131. {{ .Name }}:
  132. labels:
  133. e2e: true
  134. container_name: {{ .Name }}
  135. image: tendermint/e2e-node
  136. {{- if eq .ABCIProtocol "builtin" }}
  137. entrypoint: /usr/bin/entrypoint-builtin
  138. {{- else if .LogLevel }}
  139. command: start --log-level {{ .LogLevel }}
  140. {{- end }}
  141. init: true
  142. ports:
  143. - 26656
  144. - {{ if .ProxyPort }}{{ addUint32 .ProxyPort 1000 }}:{{ end }}26660
  145. - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657
  146. - 6060
  147. volumes:
  148. - ./{{ .Name }}:/tendermint
  149. networks:
  150. {{ $.Name }}:
  151. ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }}
  152. {{end}}`)
  153. if err != nil {
  154. return nil, err
  155. }
  156. var buf bytes.Buffer
  157. err = tmpl.Execute(&buf, testnet)
  158. if err != nil {
  159. return nil, err
  160. }
  161. return buf.Bytes(), nil
  162. }
  163. // MakeGenesis generates a genesis document.
  164. func MakeGenesis(testnet *e2e.Testnet) (types.GenesisDoc, error) {
  165. genesis := types.GenesisDoc{
  166. GenesisTime: time.Now(),
  167. ChainID: testnet.Name,
  168. ConsensusParams: types.DefaultConsensusParams(),
  169. InitialHeight: testnet.InitialHeight,
  170. }
  171. switch testnet.KeyType {
  172. case "", types.ABCIPubKeyTypeEd25519, types.ABCIPubKeyTypeSecp256k1:
  173. genesis.ConsensusParams.Validator.PubKeyTypes =
  174. append(genesis.ConsensusParams.Validator.PubKeyTypes, types.ABCIPubKeyTypeSecp256k1)
  175. default:
  176. return genesis, errors.New("unsupported KeyType")
  177. }
  178. genesis.ConsensusParams.Evidence.MaxAgeNumBlocks = e2e.EvidenceAgeHeight
  179. genesis.ConsensusParams.Evidence.MaxAgeDuration = e2e.EvidenceAgeTime
  180. for validator, power := range testnet.Validators {
  181. genesis.Validators = append(genesis.Validators, types.GenesisValidator{
  182. Name: validator.Name,
  183. Address: validator.PrivvalKey.PubKey().Address(),
  184. PubKey: validator.PrivvalKey.PubKey(),
  185. Power: power,
  186. })
  187. }
  188. // The validator set will be sorted internally by Tendermint ranked by power,
  189. // but we sort it here as well so that all genesis files are identical.
  190. sort.Slice(genesis.Validators, func(i, j int) bool {
  191. return strings.Compare(genesis.Validators[i].Name, genesis.Validators[j].Name) == -1
  192. })
  193. if len(testnet.InitialState) > 0 {
  194. appState, err := json.Marshal(testnet.InitialState)
  195. if err != nil {
  196. return genesis, err
  197. }
  198. genesis.AppState = appState
  199. }
  200. return genesis, genesis.ValidateAndComplete()
  201. }
  202. // MakeConfig generates a Tendermint config for a node.
  203. func MakeConfig(node *e2e.Node) (*config.Config, error) {
  204. cfg := config.DefaultConfig()
  205. cfg.Moniker = node.Name
  206. cfg.ProxyApp = AppAddressTCP
  207. if node.LogLevel != "" {
  208. cfg.LogLevel = node.LogLevel
  209. }
  210. cfg.RPC.ListenAddress = "tcp://0.0.0.0:26657"
  211. cfg.RPC.PprofListenAddress = ":6060"
  212. cfg.P2P.ExternalAddress = fmt.Sprintf("tcp://%v", node.AddressP2P(false))
  213. cfg.P2P.AddrBookStrict = false
  214. cfg.P2P.UseLegacy = node.UseLegacyP2P
  215. cfg.P2P.QueueType = node.QueueType
  216. cfg.DBBackend = node.Database
  217. cfg.StateSync.DiscoveryTime = 5 * time.Second
  218. if node.Mode != e2e.ModeLight {
  219. cfg.Mode = string(node.Mode)
  220. }
  221. switch node.ABCIProtocol {
  222. case e2e.ProtocolUNIX:
  223. cfg.ProxyApp = AppAddressUNIX
  224. case e2e.ProtocolTCP:
  225. cfg.ProxyApp = AppAddressTCP
  226. case e2e.ProtocolGRPC:
  227. cfg.ProxyApp = AppAddressTCP
  228. cfg.ABCI = "grpc"
  229. case e2e.ProtocolBuiltin:
  230. cfg.ProxyApp = ""
  231. cfg.ABCI = ""
  232. default:
  233. return nil, fmt.Errorf("unexpected ABCI protocol setting %q", node.ABCIProtocol)
  234. }
  235. // Tendermint errors if it does not have a privval key set up, regardless of whether
  236. // it's actually needed (e.g. for remote KMS or non-validators). We set up a dummy
  237. // key here by default, and use the real key for actual validators that should use
  238. // the file privval.
  239. cfg.PrivValidator.ListenAddr = ""
  240. cfg.PrivValidator.Key = PrivvalDummyKeyFile
  241. cfg.PrivValidator.State = PrivvalDummyStateFile
  242. switch node.Mode {
  243. case e2e.ModeValidator:
  244. switch node.PrivvalProtocol {
  245. case e2e.ProtocolFile:
  246. cfg.PrivValidator.Key = PrivvalKeyFile
  247. cfg.PrivValidator.State = PrivvalStateFile
  248. case e2e.ProtocolUNIX:
  249. cfg.PrivValidator.ListenAddr = PrivvalAddressUNIX
  250. case e2e.ProtocolTCP:
  251. cfg.PrivValidator.ListenAddr = PrivvalAddressTCP
  252. case e2e.ProtocolGRPC:
  253. cfg.PrivValidator.ListenAddr = PrivvalAddressGRPC
  254. default:
  255. return nil, fmt.Errorf("invalid privval protocol setting %q", node.PrivvalProtocol)
  256. }
  257. case e2e.ModeSeed:
  258. cfg.P2P.PexReactor = true
  259. case e2e.ModeFull, e2e.ModeLight:
  260. // Don't need to do anything, since we're using a dummy privval key by default.
  261. default:
  262. return nil, fmt.Errorf("unexpected mode %q", node.Mode)
  263. }
  264. if node.Mempool != "" {
  265. cfg.Mempool.Version = node.Mempool
  266. }
  267. if node.BlockSync == "" {
  268. cfg.BlockSync.Enable = false
  269. } else {
  270. cfg.BlockSync.Version = node.BlockSync
  271. }
  272. switch node.StateSync {
  273. case e2e.StateSyncP2P:
  274. cfg.StateSync.Enable = true
  275. cfg.StateSync.UseP2P = true
  276. case e2e.StateSyncRPC:
  277. cfg.StateSync.Enable = true
  278. cfg.StateSync.RPCServers = []string{}
  279. for _, peer := range node.Testnet.ArchiveNodes() {
  280. if peer.Name == node.Name {
  281. continue
  282. }
  283. cfg.StateSync.RPCServers = append(cfg.StateSync.RPCServers, peer.AddressRPC())
  284. }
  285. if len(cfg.StateSync.RPCServers) < 2 {
  286. return nil, errors.New("unable to find 2 suitable state sync RPC servers")
  287. }
  288. }
  289. cfg.P2P.Seeds = ""
  290. for _, seed := range node.Seeds {
  291. if len(cfg.P2P.Seeds) > 0 {
  292. cfg.P2P.Seeds += ","
  293. }
  294. cfg.P2P.Seeds += seed.AddressP2P(true)
  295. }
  296. cfg.P2P.PersistentPeers = ""
  297. for _, peer := range node.PersistentPeers {
  298. if len(cfg.P2P.PersistentPeers) > 0 {
  299. cfg.P2P.PersistentPeers += ","
  300. }
  301. cfg.P2P.PersistentPeers += peer.AddressP2P(true)
  302. }
  303. cfg.Instrumentation.Prometheus = true
  304. return cfg, nil
  305. }
  306. // MakeAppConfig generates an ABCI application config for a node.
  307. func MakeAppConfig(node *e2e.Node) ([]byte, error) {
  308. cfg := map[string]interface{}{
  309. "chain_id": node.Testnet.Name,
  310. "dir": "data/app",
  311. "listen": AppAddressUNIX,
  312. "mode": node.Mode,
  313. "proxy_port": node.ProxyPort,
  314. "protocol": "socket",
  315. "persist_interval": node.PersistInterval,
  316. "snapshot_interval": node.SnapshotInterval,
  317. "retain_blocks": node.RetainBlocks,
  318. "key_type": node.PrivvalKey.Type(),
  319. "use_legacy_p2p": node.UseLegacyP2P,
  320. }
  321. switch node.ABCIProtocol {
  322. case e2e.ProtocolUNIX:
  323. cfg["listen"] = AppAddressUNIX
  324. case e2e.ProtocolTCP:
  325. cfg["listen"] = AppAddressTCP
  326. case e2e.ProtocolGRPC:
  327. cfg["listen"] = AppAddressTCP
  328. cfg["protocol"] = "grpc"
  329. case e2e.ProtocolBuiltin:
  330. delete(cfg, "listen")
  331. cfg["protocol"] = "builtin"
  332. default:
  333. return nil, fmt.Errorf("unexpected ABCI protocol setting %q", node.ABCIProtocol)
  334. }
  335. if node.Mode == e2e.ModeValidator {
  336. switch node.PrivvalProtocol {
  337. case e2e.ProtocolFile:
  338. case e2e.ProtocolTCP:
  339. cfg["privval_server"] = PrivvalAddressTCP
  340. cfg["privval_key"] = PrivvalKeyFile
  341. cfg["privval_state"] = PrivvalStateFile
  342. case e2e.ProtocolUNIX:
  343. cfg["privval_server"] = PrivvalAddressUNIX
  344. cfg["privval_key"] = PrivvalKeyFile
  345. cfg["privval_state"] = PrivvalStateFile
  346. case e2e.ProtocolGRPC:
  347. cfg["privval_server"] = PrivvalAddressGRPC
  348. cfg["privval_key"] = PrivvalKeyFile
  349. cfg["privval_state"] = PrivvalStateFile
  350. default:
  351. return nil, fmt.Errorf("unexpected privval protocol setting %q", node.PrivvalProtocol)
  352. }
  353. }
  354. if len(node.Testnet.ValidatorUpdates) > 0 {
  355. validatorUpdates := map[string]map[string]int64{}
  356. for height, validators := range node.Testnet.ValidatorUpdates {
  357. updateVals := map[string]int64{}
  358. for node, power := range validators {
  359. updateVals[base64.StdEncoding.EncodeToString(node.PrivvalKey.PubKey().Bytes())] = power
  360. }
  361. validatorUpdates[fmt.Sprintf("%v", height)] = updateVals
  362. }
  363. cfg["validator_update"] = validatorUpdates
  364. }
  365. var buf bytes.Buffer
  366. err := toml.NewEncoder(&buf).Encode(cfg)
  367. if err != nil {
  368. return nil, fmt.Errorf("failed to generate app config: %w", err)
  369. }
  370. return buf.Bytes(), nil
  371. }
  372. // UpdateConfigStateSync updates the state sync config for a node.
  373. func UpdateConfigStateSync(node *e2e.Node, height int64, hash []byte) error {
  374. cfgPath := filepath.Join(node.Testnet.Dir, node.Name, "config", "config.toml")
  375. // FIXME Apparently there's no function to simply load a config file without
  376. // involving the entire Viper apparatus, so we'll just resort to regexps.
  377. bz, err := ioutil.ReadFile(cfgPath)
  378. if err != nil {
  379. return err
  380. }
  381. bz = regexp.MustCompile(`(?m)^trust-height =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust-height = %v`, height)))
  382. bz = regexp.MustCompile(`(?m)^trust-hash =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust-hash = "%X"`, hash)))
  383. return ioutil.WriteFile(cfgPath, bz, 0644)
  384. }