You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

421 lines
12 KiB

  1. // nolint: gosec
  2. package main
  3. import (
  4. "bytes"
  5. "encoding/base64"
  6. "encoding/json"
  7. "errors"
  8. "fmt"
  9. "io/ioutil"
  10. "os"
  11. "path/filepath"
  12. "regexp"
  13. "sort"
  14. "strings"
  15. "text/template"
  16. "time"
  17. "github.com/BurntSushi/toml"
  18. "github.com/tendermint/tendermint/config"
  19. "github.com/tendermint/tendermint/crypto/ed25519"
  20. "github.com/tendermint/tendermint/internal/p2p"
  21. "github.com/tendermint/tendermint/privval"
  22. e2e "github.com/tendermint/tendermint/test/e2e/pkg"
  23. "github.com/tendermint/tendermint/types"
  24. )
  25. const (
  26. AppAddressTCP = "tcp://127.0.0.1:30000"
  27. AppAddressUNIX = "unix:///var/run/app.sock"
  28. PrivvalAddressTCP = "tcp://0.0.0.0:27559"
  29. PrivvalAddressGRPC = "grpc://0.0.0.0:27559"
  30. PrivvalAddressUNIX = "unix:///var/run/privval.sock"
  31. PrivvalKeyFile = "config/priv_validator_key.json"
  32. PrivvalStateFile = "data/priv_validator_state.json"
  33. PrivvalDummyKeyFile = "config/dummy_validator_key.json"
  34. PrivvalDummyStateFile = "data/dummy_validator_state.json"
  35. )
  36. // Setup sets up the testnet configuration.
  37. func Setup(testnet *e2e.Testnet) error {
  38. logger.Info(fmt.Sprintf("Generating testnet files in %q", testnet.Dir))
  39. err := os.MkdirAll(testnet.Dir, os.ModePerm)
  40. if err != nil {
  41. return err
  42. }
  43. compose, err := MakeDockerCompose(testnet)
  44. if err != nil {
  45. return err
  46. }
  47. err = ioutil.WriteFile(filepath.Join(testnet.Dir, "docker-compose.yml"), compose, 0644)
  48. if err != nil {
  49. return err
  50. }
  51. genesis, err := MakeGenesis(testnet)
  52. if err != nil {
  53. return err
  54. }
  55. for _, node := range testnet.Nodes {
  56. nodeDir := filepath.Join(testnet.Dir, node.Name)
  57. dirs := []string{
  58. filepath.Join(nodeDir, "config"),
  59. filepath.Join(nodeDir, "data"),
  60. filepath.Join(nodeDir, "data", "app"),
  61. }
  62. for _, dir := range dirs {
  63. // light clients don't need an app directory
  64. if node.Mode == e2e.ModeLight && strings.Contains(dir, "app") {
  65. continue
  66. }
  67. err := os.MkdirAll(dir, 0755)
  68. if err != nil {
  69. return err
  70. }
  71. }
  72. cfg, err := MakeConfig(node)
  73. if err != nil {
  74. return err
  75. }
  76. config.WriteConfigFile(nodeDir, cfg) // panics
  77. appCfg, err := MakeAppConfig(node)
  78. if err != nil {
  79. return err
  80. }
  81. err = ioutil.WriteFile(filepath.Join(nodeDir, "config", "app.toml"), appCfg, 0644)
  82. if err != nil {
  83. return err
  84. }
  85. if node.Mode == e2e.ModeLight {
  86. // stop early if a light client
  87. continue
  88. }
  89. err = genesis.SaveAs(filepath.Join(nodeDir, "config", "genesis.json"))
  90. if err != nil {
  91. return err
  92. }
  93. err = (&p2p.NodeKey{PrivKey: node.NodeKey}).SaveAs(filepath.Join(nodeDir, "config", "node_key.json"))
  94. if err != nil {
  95. return err
  96. }
  97. (privval.NewFilePV(node.PrivvalKey,
  98. filepath.Join(nodeDir, PrivvalKeyFile),
  99. filepath.Join(nodeDir, PrivvalStateFile),
  100. )).Save()
  101. // Set up a dummy validator. Tendermint requires a file PV even when not used, so we
  102. // give it a dummy such that it will fail if it actually tries to use it.
  103. (privval.NewFilePV(ed25519.GenPrivKey(),
  104. filepath.Join(nodeDir, PrivvalDummyKeyFile),
  105. filepath.Join(nodeDir, PrivvalDummyStateFile),
  106. )).Save()
  107. }
  108. return nil
  109. }
  110. // MakeDockerCompose generates a Docker Compose config for a testnet.
  111. func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) {
  112. // Must use version 2 Docker Compose format, to support IPv6.
  113. tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{
  114. "addUint32": func(x, y uint32) uint32 {
  115. return x + y
  116. },
  117. }).Parse(`version: '2.4'
  118. networks:
  119. {{ .Name }}:
  120. labels:
  121. e2e: true
  122. driver: bridge
  123. {{- if .IPv6 }}
  124. enable_ipv6: true
  125. {{- end }}
  126. ipam:
  127. driver: default
  128. config:
  129. - subnet: {{ .IP }}
  130. services:
  131. {{- range .Nodes }}
  132. {{ .Name }}:
  133. labels:
  134. e2e: true
  135. container_name: {{ .Name }}
  136. image: tendermint/e2e-node
  137. {{- if eq .ABCIProtocol "builtin" }}
  138. entrypoint: /usr/bin/entrypoint-builtin
  139. {{- else if .LogLevel }}
  140. command: start --log-level {{ .LogLevel }}
  141. {{- end }}
  142. init: true
  143. ports:
  144. - 26656
  145. - {{ if .ProxyPort }}{{ addUint32 .ProxyPort 1000 }}:{{ end }}26660
  146. - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657
  147. - 6060
  148. volumes:
  149. - ./{{ .Name }}:/tendermint
  150. networks:
  151. {{ $.Name }}:
  152. ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }}
  153. {{end}}`)
  154. if err != nil {
  155. return nil, err
  156. }
  157. var buf bytes.Buffer
  158. err = tmpl.Execute(&buf, testnet)
  159. if err != nil {
  160. return nil, err
  161. }
  162. return buf.Bytes(), nil
  163. }
  164. // MakeGenesis generates a genesis document.
  165. func MakeGenesis(testnet *e2e.Testnet) (types.GenesisDoc, error) {
  166. genesis := types.GenesisDoc{
  167. GenesisTime: time.Now(),
  168. ChainID: testnet.Name,
  169. ConsensusParams: types.DefaultConsensusParams(),
  170. InitialHeight: testnet.InitialHeight,
  171. }
  172. switch testnet.KeyType {
  173. case "", types.ABCIPubKeyTypeEd25519, types.ABCIPubKeyTypeSecp256k1:
  174. genesis.ConsensusParams.Validator.PubKeyTypes =
  175. append(genesis.ConsensusParams.Validator.PubKeyTypes, types.ABCIPubKeyTypeSecp256k1)
  176. default:
  177. return genesis, errors.New("unsupported KeyType")
  178. }
  179. genesis.ConsensusParams.Evidence.MaxAgeNumBlocks = e2e.EvidenceAgeHeight
  180. genesis.ConsensusParams.Evidence.MaxAgeDuration = e2e.EvidenceAgeTime
  181. for validator, power := range testnet.Validators {
  182. genesis.Validators = append(genesis.Validators, types.GenesisValidator{
  183. Name: validator.Name,
  184. Address: validator.PrivvalKey.PubKey().Address(),
  185. PubKey: validator.PrivvalKey.PubKey(),
  186. Power: power,
  187. })
  188. }
  189. // The validator set will be sorted internally by Tendermint ranked by power,
  190. // but we sort it here as well so that all genesis files are identical.
  191. sort.Slice(genesis.Validators, func(i, j int) bool {
  192. return strings.Compare(genesis.Validators[i].Name, genesis.Validators[j].Name) == -1
  193. })
  194. if len(testnet.InitialState) > 0 {
  195. appState, err := json.Marshal(testnet.InitialState)
  196. if err != nil {
  197. return genesis, err
  198. }
  199. genesis.AppState = appState
  200. }
  201. return genesis, genesis.ValidateAndComplete()
  202. }
  203. // MakeConfig generates a Tendermint config for a node.
  204. func MakeConfig(node *e2e.Node) (*config.Config, error) {
  205. cfg := config.DefaultConfig()
  206. cfg.Moniker = node.Name
  207. cfg.ProxyApp = AppAddressTCP
  208. if node.LogLevel != "" {
  209. cfg.LogLevel = node.LogLevel
  210. }
  211. cfg.RPC.ListenAddress = "tcp://0.0.0.0:26657"
  212. cfg.RPC.PprofListenAddress = ":6060"
  213. cfg.P2P.ExternalAddress = fmt.Sprintf("tcp://%v", node.AddressP2P(false))
  214. cfg.P2P.AddrBookStrict = false
  215. cfg.P2P.DisableLegacy = node.DisableLegacyP2P
  216. cfg.P2P.QueueType = node.QueueType
  217. cfg.DBBackend = node.Database
  218. cfg.StateSync.DiscoveryTime = 5 * time.Second
  219. if node.Mode != e2e.ModeLight {
  220. cfg.Mode = string(node.Mode)
  221. }
  222. switch node.ABCIProtocol {
  223. case e2e.ProtocolUNIX:
  224. cfg.ProxyApp = AppAddressUNIX
  225. case e2e.ProtocolTCP:
  226. cfg.ProxyApp = AppAddressTCP
  227. case e2e.ProtocolGRPC:
  228. cfg.ProxyApp = AppAddressTCP
  229. cfg.ABCI = "grpc"
  230. case e2e.ProtocolBuiltin:
  231. cfg.ProxyApp = ""
  232. cfg.ABCI = ""
  233. default:
  234. return nil, fmt.Errorf("unexpected ABCI protocol setting %q", node.ABCIProtocol)
  235. }
  236. // Tendermint errors if it does not have a privval key set up, regardless of whether
  237. // it's actually needed (e.g. for remote KMS or non-validators). We set up a dummy
  238. // key here by default, and use the real key for actual validators that should use
  239. // the file privval.
  240. cfg.PrivValidator.ListenAddr = ""
  241. cfg.PrivValidator.Key = PrivvalDummyKeyFile
  242. cfg.PrivValidator.State = PrivvalDummyStateFile
  243. switch node.Mode {
  244. case e2e.ModeValidator:
  245. switch node.PrivvalProtocol {
  246. case e2e.ProtocolFile:
  247. cfg.PrivValidator.Key = PrivvalKeyFile
  248. cfg.PrivValidator.State = PrivvalStateFile
  249. case e2e.ProtocolUNIX:
  250. cfg.PrivValidator.ListenAddr = PrivvalAddressUNIX
  251. case e2e.ProtocolTCP:
  252. cfg.PrivValidator.ListenAddr = PrivvalAddressTCP
  253. case e2e.ProtocolGRPC:
  254. cfg.PrivValidator.ListenAddr = PrivvalAddressGRPC
  255. default:
  256. return nil, fmt.Errorf("invalid privval protocol setting %q", node.PrivvalProtocol)
  257. }
  258. case e2e.ModeSeed:
  259. cfg.P2P.PexReactor = true
  260. case e2e.ModeFull, e2e.ModeLight:
  261. // Don't need to do anything, since we're using a dummy privval key by default.
  262. default:
  263. return nil, fmt.Errorf("unexpected mode %q", node.Mode)
  264. }
  265. if node.FastSync == "" {
  266. cfg.FastSyncMode = false
  267. } else {
  268. cfg.FastSync.Version = node.FastSync
  269. }
  270. if node.StateSync {
  271. cfg.StateSync.Enable = true
  272. cfg.StateSync.RPCServers = []string{}
  273. for _, peer := range node.Testnet.ArchiveNodes() {
  274. if peer.Name == node.Name {
  275. continue
  276. }
  277. cfg.StateSync.RPCServers = append(cfg.StateSync.RPCServers, peer.AddressRPC())
  278. }
  279. if len(cfg.StateSync.RPCServers) < 2 {
  280. return nil, errors.New("unable to find 2 suitable state sync RPC servers")
  281. }
  282. }
  283. cfg.P2P.Seeds = ""
  284. for _, seed := range node.Seeds {
  285. if len(cfg.P2P.Seeds) > 0 {
  286. cfg.P2P.Seeds += ","
  287. }
  288. cfg.P2P.Seeds += seed.AddressP2P(true)
  289. }
  290. cfg.P2P.PersistentPeers = ""
  291. for _, peer := range node.PersistentPeers {
  292. if len(cfg.P2P.PersistentPeers) > 0 {
  293. cfg.P2P.PersistentPeers += ","
  294. }
  295. cfg.P2P.PersistentPeers += peer.AddressP2P(true)
  296. }
  297. cfg.Instrumentation.Prometheus = true
  298. return cfg, nil
  299. }
  300. // MakeAppConfig generates an ABCI application config for a node.
  301. func MakeAppConfig(node *e2e.Node) ([]byte, error) {
  302. cfg := map[string]interface{}{
  303. "chain_id": node.Testnet.Name,
  304. "dir": "data/app",
  305. "listen": AppAddressUNIX,
  306. "mode": node.Mode,
  307. "proxy_port": node.ProxyPort,
  308. "protocol": "socket",
  309. "persist_interval": node.PersistInterval,
  310. "snapshot_interval": node.SnapshotInterval,
  311. "retain_blocks": node.RetainBlocks,
  312. "key_type": node.PrivvalKey.Type(),
  313. "disable_legacy_p2p": node.DisableLegacyP2P,
  314. }
  315. switch node.ABCIProtocol {
  316. case e2e.ProtocolUNIX:
  317. cfg["listen"] = AppAddressUNIX
  318. case e2e.ProtocolTCP:
  319. cfg["listen"] = AppAddressTCP
  320. case e2e.ProtocolGRPC:
  321. cfg["listen"] = AppAddressTCP
  322. cfg["protocol"] = "grpc"
  323. case e2e.ProtocolBuiltin:
  324. delete(cfg, "listen")
  325. cfg["protocol"] = "builtin"
  326. default:
  327. return nil, fmt.Errorf("unexpected ABCI protocol setting %q", node.ABCIProtocol)
  328. }
  329. if node.Mode == e2e.ModeValidator {
  330. switch node.PrivvalProtocol {
  331. case e2e.ProtocolFile:
  332. case e2e.ProtocolTCP:
  333. cfg["privval_server"] = PrivvalAddressTCP
  334. cfg["privval_key"] = PrivvalKeyFile
  335. cfg["privval_state"] = PrivvalStateFile
  336. case e2e.ProtocolUNIX:
  337. cfg["privval_server"] = PrivvalAddressUNIX
  338. cfg["privval_key"] = PrivvalKeyFile
  339. cfg["privval_state"] = PrivvalStateFile
  340. case e2e.ProtocolGRPC:
  341. cfg["privval_server"] = PrivvalAddressGRPC
  342. cfg["privval_key"] = PrivvalKeyFile
  343. cfg["privval_state"] = PrivvalStateFile
  344. default:
  345. return nil, fmt.Errorf("unexpected privval protocol setting %q", node.PrivvalProtocol)
  346. }
  347. }
  348. if len(node.Testnet.ValidatorUpdates) > 0 {
  349. validatorUpdates := map[string]map[string]int64{}
  350. for height, validators := range node.Testnet.ValidatorUpdates {
  351. updateVals := map[string]int64{}
  352. for node, power := range validators {
  353. updateVals[base64.StdEncoding.EncodeToString(node.PrivvalKey.PubKey().Bytes())] = power
  354. }
  355. validatorUpdates[fmt.Sprintf("%v", height)] = updateVals
  356. }
  357. cfg["validator_update"] = validatorUpdates
  358. }
  359. var buf bytes.Buffer
  360. err := toml.NewEncoder(&buf).Encode(cfg)
  361. if err != nil {
  362. return nil, fmt.Errorf("failed to generate app config: %w", err)
  363. }
  364. return buf.Bytes(), nil
  365. }
  366. // UpdateConfigStateSync updates the state sync config for a node.
  367. func UpdateConfigStateSync(node *e2e.Node, height int64, hash []byte) error {
  368. cfgPath := filepath.Join(node.Testnet.Dir, node.Name, "config", "config.toml")
  369. // FIXME Apparently there's no function to simply load a config file without
  370. // involving the entire Viper apparatus, so we'll just resort to regexps.
  371. bz, err := ioutil.ReadFile(cfgPath)
  372. if err != nil {
  373. return err
  374. }
  375. bz = regexp.MustCompile(`(?m)^trust-height =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust-height = %v`, height)))
  376. bz = regexp.MustCompile(`(?m)^trust-hash =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust-hash = "%X"`, hash)))
  377. return ioutil.WriteFile(cfgPath, bz, 0644)
  378. }