Browse Source
R4R: Split immutable and mutable parts of priv_validator.json (#2870)
* split immutable and mutable parts of priv_validator.json * fix bugs * minor changes * retrig test * delete scripts/wire2amino.go * fix test * fixes from review * privval: remove mtx * rearrange priv_validator.go * upgrade path * write tests for the upgrade * fix for unsafe_reset_all * add test * add reset testpull/3075/head
yutianwu
5 years ago
committed by
Ethan Buchman
Ethan Buchman
22 changed files with 786 additions and 426 deletions
Split View
Diff Options
-
+1 -0CHANGELOG_PENDING.md
-
+7 -5cmd/priv_val_server/main.go
-
+1 -1cmd/tendermint/commands/gen_validator.go
-
+9 -7cmd/tendermint/commands/init.go
-
+17 -11cmd/tendermint/commands/reset_priv_validator.go
-
+1 -1cmd/tendermint/commands/show_validator.go
-
+9 -2cmd/tendermint/commands/testnet.go
-
+47 -23config/config.go
-
+16 -8config/toml.go
-
+1 -1config/toml_test.go
-
+18 -13consensus/common_test.go
-
+2 -2consensus/replay_test.go
-
+3 -2consensus/wal_generator.go
-
+20 -1node/node.go
-
+80 -0privval/old_priv_validator.go
-
+77 -0privval/old_priv_validator_test.go
-
+213 -136privval/priv_validator.go
-
+99 -29privval/priv_validator_test.go
-
+3 -2rpc/test/helpers.go
-
+41 -0scripts/privValUpgrade.go
-
+121 -0scripts/privValUpgrade_test.go
-
+0 -182scripts/wire2amino.go
+ 1
- 0
CHANGELOG_PENDING.md
View File
+ 7
- 5
cmd/priv_val_server/main.go
View File
+ 1
- 1
cmd/tendermint/commands/gen_validator.go
View File
+ 9
- 7
cmd/tendermint/commands/init.go
View File
+ 17
- 11
cmd/tendermint/commands/reset_priv_validator.go
View File
+ 1
- 1
cmd/tendermint/commands/show_validator.go
View File
+ 9
- 2
cmd/tendermint/commands/testnet.go
View File
+ 47
- 23
config/config.go
View File
+ 16
- 8
config/toml.go
View File
+ 1
- 1
config/toml_test.go
View File
+ 18
- 13
consensus/common_test.go
View File
+ 2
- 2
consensus/replay_test.go
View File
+ 3
- 2
consensus/wal_generator.go
View File
+ 20
- 1
node/node.go
View File
+ 80
- 0
privval/old_priv_validator.go
View File
| @ -0,0 +1,80 @@ | |||
| package privval | |||
| import ( | |||
| "io/ioutil" | |||
| "os" | |||
| "github.com/tendermint/tendermint/crypto" | |||
| cmn "github.com/tendermint/tendermint/libs/common" | |||
| "github.com/tendermint/tendermint/types" | |||
| ) | |||
| // OldFilePV is the old version of the FilePV, pre v0.28.0. | |||
| type OldFilePV struct { | |||
| Address types.Address `json:"address"` | |||
| PubKey crypto.PubKey `json:"pub_key"` | |||
| LastHeight int64 `json:"last_height"` | |||
| LastRound int `json:"last_round"` | |||
| LastStep int8 `json:"last_step"` | |||
| LastSignature []byte `json:"last_signature,omitempty"` | |||
| LastSignBytes cmn.HexBytes `json:"last_signbytes,omitempty"` | |||
| PrivKey crypto.PrivKey `json:"priv_key"` | |||
| filePath string | |||
| } | |||
| // LoadOldFilePV loads an OldFilePV from the filePath. | |||
| func LoadOldFilePV(filePath string) (*OldFilePV, error) { | |||
| pvJSONBytes, err := ioutil.ReadFile(filePath) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| pv := &OldFilePV{} | |||
| err = cdc.UnmarshalJSON(pvJSONBytes, &pv) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| // overwrite pubkey and address for convenience | |||
| pv.PubKey = pv.PrivKey.PubKey() | |||
| pv.Address = pv.PubKey.Address() | |||
| pv.filePath = filePath | |||
| return pv, nil | |||
| } | |||
| // Upgrade convets the OldFilePV to the new FilePV, separating the immutable and mutable components, | |||
| // and persisting them to the keyFilePath and stateFilePath, respectively. | |||
| // It renames the original file by adding ".bak". | |||
| func (oldFilePV *OldFilePV) Upgrade(keyFilePath, stateFilePath string) *FilePV { | |||
| privKey := oldFilePV.PrivKey | |||
| pvKey := FilePVKey{ | |||
| PrivKey: privKey, | |||
| PubKey: privKey.PubKey(), | |||
| Address: privKey.PubKey().Address(), | |||
| filePath: keyFilePath, | |||
| } | |||
| pvState := FilePVLastSignState{ | |||
| Height: oldFilePV.LastHeight, | |||
| Round: oldFilePV.LastRound, | |||
| Step: oldFilePV.LastStep, | |||
| Signature: oldFilePV.LastSignature, | |||
| SignBytes: oldFilePV.LastSignBytes, | |||
| filePath: stateFilePath, | |||
| } | |||
| // Save the new PV files | |||
| pv := &FilePV{ | |||
| Key: pvKey, | |||
| LastSignState: pvState, | |||
| } | |||
| pv.Save() | |||
| // Rename the old PV file | |||
| err := os.Rename(oldFilePV.filePath, oldFilePV.filePath+".bak") | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| return pv | |||
| } | |||
+ 77
- 0
privval/old_priv_validator_test.go
View File
| @ -0,0 +1,77 @@ | |||
| package privval_test | |||
| import ( | |||
| "io/ioutil" | |||
| "os" | |||
| "testing" | |||
| "github.com/stretchr/testify/assert" | |||
| "github.com/stretchr/testify/require" | |||
| "github.com/tendermint/tendermint/privval" | |||
| ) | |||
| const oldPrivvalContent = `{ | |||
| "address": "1D8089FAFDFAE4A637F3D616E17B92905FA2D91D", | |||
| "pub_key": { | |||
| "type": "tendermint/PubKeyEd25519", | |||
| "value": "r3Yg2AhDZ745CNTpavsGU+mRZ8WpRXqoJuyqjN8mJq0=" | |||
| }, | |||
| "last_height": "5", | |||
| "last_round": "0", | |||
| "last_step": 3, | |||
| "last_signature": "CTr7b9ZQlrJJf+12rPl5t/YSCUc/KqV7jQogCfFJA24e7hof69X6OMT7eFLVQHyodPjD/QTA298XHV5ejxInDQ==", | |||
| "last_signbytes": "750802110500000000000000220B08B398F3E00510F48DA6402A480A20FC258973076512999C3E6839A22E9FBDB1B77CF993E8A9955412A41A59D4CAD312240A20C971B286ACB8AAA6FCA0365EB0A660B189EDC08B46B5AF2995DEFA51A28D215B10013211746573742D636861696E2D533245415533", | |||
| "priv_key": { | |||
| "type": "tendermint/PrivKeyEd25519", | |||
| "value": "7MwvTGEWWjsYwjn2IpRb+GYsWi9nnFsw8jPLLY1UtP6vdiDYCENnvjkI1Olq+wZT6ZFnxalFeqgm7KqM3yYmrQ==" | |||
| } | |||
| }` | |||
| func TestLoadAndUpgrade(t *testing.T) { | |||
| oldFilePath := initTmpOldFile(t) | |||
| defer os.Remove(oldFilePath) | |||
| newStateFile, err := ioutil.TempFile("", "priv_validator_state*.json") | |||
| defer os.Remove(newStateFile.Name()) | |||
| require.NoError(t, err) | |||
| newKeyFile, err := ioutil.TempFile("", "priv_validator_key*.json") | |||
| defer os.Remove(newKeyFile.Name()) | |||
| require.NoError(t, err) | |||
| oldPV, err := privval.LoadOldFilePV(oldFilePath) | |||
| assert.NoError(t, err) | |||
| newPV := oldPV.Upgrade(newKeyFile.Name(), newStateFile.Name()) | |||
| assertEqualPV(t, oldPV, newPV) | |||
| assert.NoError(t, err) | |||
| upgradedPV := privval.LoadFilePV(newKeyFile.Name(), newStateFile.Name()) | |||
| assertEqualPV(t, oldPV, upgradedPV) | |||
| oldPV, err = privval.LoadOldFilePV(oldFilePath + ".bak") | |||
| require.NoError(t, err) | |||
| assertEqualPV(t, oldPV, upgradedPV) | |||
| } | |||
| func assertEqualPV(t *testing.T, oldPV *privval.OldFilePV, newPV *privval.FilePV) { | |||
| assert.Equal(t, oldPV.Address, newPV.Key.Address) | |||
| assert.Equal(t, oldPV.Address, newPV.GetAddress()) | |||
| assert.Equal(t, oldPV.PubKey, newPV.Key.PubKey) | |||
| assert.Equal(t, oldPV.PubKey, newPV.GetPubKey()) | |||
| assert.Equal(t, oldPV.PrivKey, newPV.Key.PrivKey) | |||
| assert.Equal(t, oldPV.LastHeight, newPV.LastSignState.Height) | |||
| assert.Equal(t, oldPV.LastRound, newPV.LastSignState.Round) | |||
| assert.Equal(t, oldPV.LastSignature, newPV.LastSignState.Signature) | |||
| assert.Equal(t, oldPV.LastSignBytes, newPV.LastSignState.SignBytes) | |||
| assert.Equal(t, oldPV.LastStep, newPV.LastSignState.Step) | |||
| } | |||
| func initTmpOldFile(t *testing.T) string { | |||
| tmpfile, err := ioutil.TempFile("", "priv_validator_*.json") | |||
| require.NoError(t, err) | |||
| t.Logf("created test file %s", tmpfile.Name()) | |||
| _, err = tmpfile.WriteString(oldPrivvalContent) | |||
| require.NoError(t, err) | |||
| return tmpfile.Name() | |||
| } | |||
+ 213
- 136
privval/priv_validator.go
View File
+ 99
- 29
privval/priv_validator_test.go
View File
+ 3
- 2
rpc/test/helpers.go
View File
+ 41
- 0
scripts/privValUpgrade.go
View File
| @ -0,0 +1,41 @@ | |||
| package main | |||
| import ( | |||
| "fmt" | |||
| "os" | |||
| "github.com/tendermint/tendermint/libs/log" | |||
| "github.com/tendermint/tendermint/privval" | |||
| ) | |||
| var ( | |||
| logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout)) | |||
| ) | |||
| func main() { | |||
| args := os.Args[1:] | |||
| if len(args) != 3 { | |||
| fmt.Println("Expected three args: <old path> <new key path> <new state path>") | |||
| fmt.Println("Eg. ~/.tendermint/config/priv_validator.json ~/.tendermint/config/priv_validator_key.json ~/.tendermint/data/priv_validator_state.json") | |||
| os.Exit(1) | |||
| } | |||
| err := loadAndUpgrade(args[0], args[1], args[2]) | |||
| if err != nil { | |||
| fmt.Println(err) | |||
| os.Exit(1) | |||
| } | |||
| } | |||
| func loadAndUpgrade(oldPVPath, newPVKeyPath, newPVStatePath string) error { | |||
| oldPV, err := privval.LoadOldFilePV(oldPVPath) | |||
| if err != nil { | |||
| return fmt.Errorf("Error reading OldPrivValidator from %v: %v\n", oldPVPath, err) | |||
| } | |||
| logger.Info("Upgrading PrivValidator file", | |||
| "old", oldPVPath, | |||
| "newKey", newPVKeyPath, | |||
| "newState", newPVStatePath, | |||
| ) | |||
| oldPV.Upgrade(newPVKeyPath, newPVStatePath) | |||
| return nil | |||
| } | |||
+ 121
- 0
scripts/privValUpgrade_test.go
View File
| @ -0,0 +1,121 @@ | |||
| package main | |||
| import ( | |||
| "fmt" | |||
| "io/ioutil" | |||
| "os" | |||
| "testing" | |||
| "github.com/stretchr/testify/assert" | |||
| "github.com/stretchr/testify/require" | |||
| "github.com/tendermint/tendermint/privval" | |||
| ) | |||
| const oldPrivvalContent = `{ | |||
| "address": "1D8089FAFDFAE4A637F3D616E17B92905FA2D91D", | |||
| "pub_key": { | |||
| "type": "tendermint/PubKeyEd25519", | |||
| "value": "r3Yg2AhDZ745CNTpavsGU+mRZ8WpRXqoJuyqjN8mJq0=" | |||
| }, | |||
| "last_height": "5", | |||
| "last_round": "0", | |||
| "last_step": 3, | |||
| "last_signature": "CTr7b9ZQlrJJf+12rPl5t/YSCUc/KqV7jQogCfFJA24e7hof69X6OMT7eFLVQHyodPjD/QTA298XHV5ejxInDQ==", | |||
| "last_signbytes": "750802110500000000000000220B08B398F3E00510F48DA6402A480A20FC258973076512999C3E6839A22E9FBDB1B77CF993E8A9955412A41A59D4CAD312240A20C971B286ACB8AAA6FCA0365EB0A660B189EDC08B46B5AF2995DEFA51A28D215B10013211746573742D636861696E2D533245415533", | |||
| "priv_key": { | |||
| "type": "tendermint/PrivKeyEd25519", | |||
| "value": "7MwvTGEWWjsYwjn2IpRb+GYsWi9nnFsw8jPLLY1UtP6vdiDYCENnvjkI1Olq+wZT6ZFnxalFeqgm7KqM3yYmrQ==" | |||
| } | |||
| }` | |||
| func TestLoadAndUpgrade(t *testing.T) { | |||
| oldFilePath := initTmpOldFile(t) | |||
| defer os.Remove(oldFilePath) | |||
| newStateFile, err := ioutil.TempFile("", "priv_validator_state*.json") | |||
| defer os.Remove(newStateFile.Name()) | |||
| require.NoError(t, err) | |||
| newKeyFile, err := ioutil.TempFile("", "priv_validator_key*.json") | |||
| defer os.Remove(newKeyFile.Name()) | |||
| require.NoError(t, err) | |||
| emptyOldFile, err := ioutil.TempFile("", "priv_validator_empty*.json") | |||
| require.NoError(t, err) | |||
| defer os.Remove(emptyOldFile.Name()) | |||
| type args struct { | |||
| oldPVPath string | |||
| newPVKeyPath string | |||
| newPVStatePath string | |||
| } | |||
| tests := []struct { | |||
| name string | |||
| args args | |||
| wantErr bool | |||
| wantPanic bool | |||
| }{ | |||
| {"successful upgrade", | |||
| args{oldPVPath: oldFilePath, newPVKeyPath: newKeyFile.Name(), newPVStatePath: newStateFile.Name()}, | |||
| false, false, | |||
| }, | |||
| {"unsuccessful upgrade: empty old privval file", | |||
| args{oldPVPath: emptyOldFile.Name(), newPVKeyPath: newKeyFile.Name(), newPVStatePath: newStateFile.Name()}, | |||
| true, false, | |||
| }, | |||
| {"unsuccessful upgrade: invalid new paths (1/3)", | |||
| args{oldPVPath: oldFilePath, newPVKeyPath: "", newPVStatePath: newStateFile.Name()}, | |||
| false, true, | |||
| }, | |||
| {"unsuccessful upgrade: invalid new paths (2/3)", | |||
| args{oldPVPath: oldFilePath, newPVKeyPath: newKeyFile.Name(), newPVStatePath: ""}, | |||
| false, true, | |||
| }, | |||
| {"unsuccessful upgrade: invalid new paths (3/3)", | |||
| args{oldPVPath: oldFilePath, newPVKeyPath: "", newPVStatePath: ""}, | |||
| false, true, | |||
| }, | |||
| } | |||
| for _, tt := range tests { | |||
| t.Run(tt.name, func(t *testing.T) { | |||
| // need to re-write the file everytime because upgrading renames it | |||
| err := ioutil.WriteFile(oldFilePath, []byte(oldPrivvalContent), 0600) | |||
| require.NoError(t, err) | |||
| if tt.wantPanic { | |||
| require.Panics(t, func() { loadAndUpgrade(tt.args.oldPVPath, tt.args.newPVKeyPath, tt.args.newPVStatePath) }) | |||
| } else { | |||
| err = loadAndUpgrade(tt.args.oldPVPath, tt.args.newPVKeyPath, tt.args.newPVStatePath) | |||
| if tt.wantErr { | |||
| assert.Error(t, err) | |||
| fmt.Println("ERR", err) | |||
| } else { | |||
| assert.NoError(t, err) | |||
| upgradedPV := privval.LoadFilePV(tt.args.newPVKeyPath, tt.args.newPVStatePath) | |||
| oldPV, err := privval.LoadOldFilePV(tt.args.oldPVPath + ".bak") | |||
| require.NoError(t, err) | |||
| assert.Equal(t, oldPV.Address, upgradedPV.Key.Address) | |||
| assert.Equal(t, oldPV.Address, upgradedPV.GetAddress()) | |||
| assert.Equal(t, oldPV.PubKey, upgradedPV.Key.PubKey) | |||
| assert.Equal(t, oldPV.PubKey, upgradedPV.GetPubKey()) | |||
| assert.Equal(t, oldPV.PrivKey, upgradedPV.Key.PrivKey) | |||
| assert.Equal(t, oldPV.LastHeight, upgradedPV.LastSignState.Height) | |||
| assert.Equal(t, oldPV.LastRound, upgradedPV.LastSignState.Round) | |||
| assert.Equal(t, oldPV.LastSignature, upgradedPV.LastSignState.Signature) | |||
| assert.Equal(t, oldPV.LastSignBytes, upgradedPV.LastSignState.SignBytes) | |||
| assert.Equal(t, oldPV.LastStep, upgradedPV.LastSignState.Step) | |||
| } | |||
| } | |||
| }) | |||
| } | |||
| } | |||
| func initTmpOldFile(t *testing.T) string { | |||
| tmpfile, err := ioutil.TempFile("", "priv_validator_*.json") | |||
| require.NoError(t, err) | |||
| t.Logf("created test file %s", tmpfile.Name()) | |||
| _, err = tmpfile.WriteString(oldPrivvalContent) | |||
| require.NoError(t, err) | |||
| return tmpfile.Name() | |||
| } | |||
+ 0
- 182
scripts/wire2amino.go
View File
| @ -1,182 +0,0 @@ | |||
| package main | |||
| import ( | |||
| "encoding/json" | |||
| "fmt" | |||
| "io/ioutil" | |||
| "os" | |||
| "path/filepath" | |||
| "time" | |||
| "github.com/tendermint/go-amino" | |||
| "github.com/tendermint/tendermint/crypto/ed25519" | |||
| cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino" | |||
| cmn "github.com/tendermint/tendermint/libs/common" | |||
| "github.com/tendermint/tendermint/p2p" | |||
| "github.com/tendermint/tendermint/privval" | |||
| "github.com/tendermint/tendermint/types" | |||
| ) | |||
| type GenesisValidator struct { | |||
| PubKey Data `json:"pub_key"` | |||
| Power int64 `json:"power"` | |||
| Name string `json:"name"` | |||
| } | |||
| type Genesis struct { | |||
| GenesisTime time.Time `json:"genesis_time"` | |||
| ChainID string `json:"chain_id"` | |||
| ConsensusParams *types.ConsensusParams `json:"consensus_params,omitempty"` | |||
| Validators []GenesisValidator `json:"validators"` | |||
| AppHash cmn.HexBytes `json:"app_hash"` | |||
| AppState json.RawMessage `json:"app_state,omitempty"` | |||
| AppOptions json.RawMessage `json:"app_options,omitempty"` // DEPRECATED | |||
| } | |||
| type NodeKey struct { | |||
| PrivKey Data `json:"priv_key"` | |||
| } | |||
| type PrivVal struct { | |||
| Address cmn.HexBytes `json:"address"` | |||
| LastHeight int64 `json:"last_height"` | |||
| LastRound int `json:"last_round"` | |||
| LastStep int8 `json:"last_step"` | |||
| PubKey Data `json:"pub_key"` | |||
| PrivKey Data `json:"priv_key"` | |||
| } | |||
| type Data struct { | |||
| Type string `json:"type"` | |||
| Data cmn.HexBytes `json:"data"` | |||
| } | |||
| func convertNodeKey(cdc *amino.Codec, jsonBytes []byte) ([]byte, error) { | |||
| var nodeKey NodeKey | |||
| err := json.Unmarshal(jsonBytes, &nodeKey) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| var privKey ed25519.PrivKeyEd25519 | |||
| copy(privKey[:], nodeKey.PrivKey.Data) | |||
| nodeKeyNew := p2p.NodeKey{privKey} | |||
| bz, err := cdc.MarshalJSON(nodeKeyNew) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| return bz, nil | |||
| } | |||
| func convertPrivVal(cdc *amino.Codec, jsonBytes []byte) ([]byte, error) { | |||
| var privVal PrivVal | |||
| err := json.Unmarshal(jsonBytes, &privVal) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| var privKey ed25519.PrivKeyEd25519 | |||
| copy(privKey[:], privVal.PrivKey.Data) | |||
| var pubKey ed25519.PubKeyEd25519 | |||
| copy(pubKey[:], privVal.PubKey.Data) | |||
| privValNew := privval.FilePV{ | |||
| Address: pubKey.Address(), | |||
| PubKey: pubKey, | |||
| LastHeight: privVal.LastHeight, | |||
| LastRound: privVal.LastRound, | |||
| LastStep: privVal.LastStep, | |||
| PrivKey: privKey, | |||
| } | |||
| bz, err := cdc.MarshalJSON(privValNew) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| return bz, nil | |||
| } | |||
| func convertGenesis(cdc *amino.Codec, jsonBytes []byte) ([]byte, error) { | |||
| var genesis Genesis | |||
| err := json.Unmarshal(jsonBytes, &genesis) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| genesisNew := types.GenesisDoc{ | |||
| GenesisTime: genesis.GenesisTime, | |||
| ChainID: genesis.ChainID, | |||
| ConsensusParams: genesis.ConsensusParams, | |||
| // Validators | |||
| AppHash: genesis.AppHash, | |||
| AppState: genesis.AppState, | |||
| } | |||
| if genesis.AppOptions != nil { | |||
| genesisNew.AppState = genesis.AppOptions | |||
| } | |||
| for _, v := range genesis.Validators { | |||
| var pubKey ed25519.PubKeyEd25519 | |||
| copy(pubKey[:], v.PubKey.Data) | |||
| genesisNew.Validators = append( | |||
| genesisNew.Validators, | |||
| types.GenesisValidator{ | |||
| PubKey: pubKey, | |||
| Power: v.Power, | |||
| Name: v.Name, | |||
| }, | |||
| ) | |||
| } | |||
| bz, err := cdc.MarshalJSON(genesisNew) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| return bz, nil | |||
| } | |||
| func main() { | |||
| cdc := amino.NewCodec() | |||
| cryptoAmino.RegisterAmino(cdc) | |||
| args := os.Args[1:] | |||
| if len(args) != 1 { | |||
| fmt.Println("Please specify a file to convert") | |||
| os.Exit(1) | |||
| } | |||
| filePath := args[0] | |||
| fileName := filepath.Base(filePath) | |||
| fileBytes, err := ioutil.ReadFile(filePath) | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| var bz []byte | |||
| switch fileName { | |||
| case "node_key.json": | |||
| bz, err = convertNodeKey(cdc, fileBytes) | |||
| case "priv_validator.json": | |||
| bz, err = convertPrivVal(cdc, fileBytes) | |||
| case "genesis.json": | |||
| bz, err = convertGenesis(cdc, fileBytes) | |||
| default: | |||
| fmt.Println("Expected file name to be in (node_key.json, priv_validator.json, genesis.json)") | |||
| os.Exit(1) | |||
| } | |||
| if err != nil { | |||
| panic(err) | |||
| } | |||
| fmt.Println(string(bz)) | |||
| } | |||