|
@ -13,78 +13,102 @@ import ( |
|
|
|
|
|
|
|
|
"github.com/tendermint/tendermint/config" |
|
|
"github.com/tendermint/tendermint/config" |
|
|
"github.com/tendermint/tendermint/libs/cli" |
|
|
"github.com/tendermint/tendermint/libs/cli" |
|
|
|
|
|
"github.com/tendermint/tendermint/libs/log" |
|
|
rpchttp "github.com/tendermint/tendermint/rpc/client/http" |
|
|
rpchttp "github.com/tendermint/tendermint/rpc/client/http" |
|
|
) |
|
|
) |
|
|
|
|
|
|
|
|
var dumpCmd = &cobra.Command{ |
|
|
|
|
|
Use: "dump [output-directory]", |
|
|
|
|
|
Short: "Continuously poll a Tendermint process and dump debugging data into a single location", |
|
|
|
|
|
Long: `Continuously poll a Tendermint process and dump debugging data into a single |
|
|
|
|
|
|
|
|
func getDumpCmd(logger log.Logger) *cobra.Command { |
|
|
|
|
|
cmd := &cobra.Command{ |
|
|
|
|
|
Use: "dump [output-directory]", |
|
|
|
|
|
Short: "Continuously poll a Tendermint process and dump debugging data into a single location", |
|
|
|
|
|
Long: `Continuously poll a Tendermint process and dump debugging data into a single |
|
|
location at a specified frequency. At each frequency interval, an archived and compressed |
|
|
location at a specified frequency. At each frequency interval, an archived and compressed |
|
|
file will contain node debugging information including the goroutine and heap profiles |
|
|
file will contain node debugging information including the goroutine and heap profiles |
|
|
if enabled.`, |
|
|
if enabled.`, |
|
|
Args: cobra.ExactArgs(1), |
|
|
|
|
|
RunE: dumpCmdHandler, |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
func init() { |
|
|
|
|
|
dumpCmd.Flags().UintVar( |
|
|
|
|
|
&frequency, |
|
|
|
|
|
|
|
|
Args: cobra.ExactArgs(1), |
|
|
|
|
|
RunE: func(cmd *cobra.Command, args []string) error { |
|
|
|
|
|
outDir := args[0] |
|
|
|
|
|
if outDir == "" { |
|
|
|
|
|
return errors.New("invalid output directory") |
|
|
|
|
|
} |
|
|
|
|
|
frequency, err := cmd.Flags().GetUint(flagFrequency) |
|
|
|
|
|
if err != nil { |
|
|
|
|
|
return fmt.Errorf("flag %q not defined: %w", flagFrequency, err) |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if frequency == 0 { |
|
|
|
|
|
return errors.New("frequency must be positive") |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
nodeRPCAddr, err := cmd.Flags().GetString(flagNodeRPCAddr) |
|
|
|
|
|
if err != nil { |
|
|
|
|
|
return fmt.Errorf("flag %q not defined: %w", flagNodeRPCAddr, err) |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
profAddr, err := cmd.Flags().GetString(flagProfAddr) |
|
|
|
|
|
if err != nil { |
|
|
|
|
|
return fmt.Errorf("flag %q not defined: %w", flagProfAddr, err) |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if _, err := os.Stat(outDir); os.IsNotExist(err) { |
|
|
|
|
|
if err := os.Mkdir(outDir, os.ModePerm); err != nil { |
|
|
|
|
|
return fmt.Errorf("failed to create output directory: %w", err) |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
rpc, err := rpchttp.New(nodeRPCAddr) |
|
|
|
|
|
if err != nil { |
|
|
|
|
|
return fmt.Errorf("failed to create new http client: %w", err) |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
ctx := cmd.Context() |
|
|
|
|
|
|
|
|
|
|
|
home := viper.GetString(cli.HomeFlag) |
|
|
|
|
|
conf := config.DefaultConfig() |
|
|
|
|
|
conf = conf.SetRoot(home) |
|
|
|
|
|
config.EnsureRoot(conf.RootDir) |
|
|
|
|
|
|
|
|
|
|
|
dumpArgs := dumpDebugDataArgs{ |
|
|
|
|
|
conf: conf, |
|
|
|
|
|
outDir: outDir, |
|
|
|
|
|
profAddr: profAddr, |
|
|
|
|
|
} |
|
|
|
|
|
dumpDebugData(ctx, logger, rpc, dumpArgs) |
|
|
|
|
|
|
|
|
|
|
|
ticker := time.NewTicker(time.Duration(frequency) * time.Second) |
|
|
|
|
|
for range ticker.C { |
|
|
|
|
|
dumpDebugData(ctx, logger, rpc, dumpArgs) |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
return nil |
|
|
|
|
|
}, |
|
|
|
|
|
} |
|
|
|
|
|
cmd.Flags().Uint( |
|
|
flagFrequency, |
|
|
flagFrequency, |
|
|
30, |
|
|
30, |
|
|
"the frequency (seconds) in which to poll, aggregate and dump Tendermint debug data", |
|
|
"the frequency (seconds) in which to poll, aggregate and dump Tendermint debug data", |
|
|
) |
|
|
) |
|
|
|
|
|
|
|
|
dumpCmd.Flags().StringVar( |
|
|
|
|
|
&profAddr, |
|
|
|
|
|
|
|
|
cmd.Flags().String( |
|
|
flagProfAddr, |
|
|
flagProfAddr, |
|
|
"", |
|
|
"", |
|
|
"the profiling server address (<host>:<port>)", |
|
|
"the profiling server address (<host>:<port>)", |
|
|
) |
|
|
) |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
func dumpCmdHandler(cmd *cobra.Command, args []string) error { |
|
|
|
|
|
outDir := args[0] |
|
|
|
|
|
if outDir == "" { |
|
|
|
|
|
return errors.New("invalid output directory") |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if frequency == 0 { |
|
|
|
|
|
return errors.New("frequency must be positive") |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if _, err := os.Stat(outDir); os.IsNotExist(err) { |
|
|
|
|
|
if err := os.Mkdir(outDir, os.ModePerm); err != nil { |
|
|
|
|
|
return fmt.Errorf("failed to create output directory: %w", err) |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
rpc, err := rpchttp.New(nodeRPCAddr) |
|
|
|
|
|
if err != nil { |
|
|
|
|
|
return fmt.Errorf("failed to create new http client: %w", err) |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
ctx := cmd.Context() |
|
|
|
|
|
|
|
|
return cmd |
|
|
|
|
|
|
|
|
home := viper.GetString(cli.HomeFlag) |
|
|
|
|
|
conf := config.DefaultConfig() |
|
|
|
|
|
conf = conf.SetRoot(home) |
|
|
|
|
|
config.EnsureRoot(conf.RootDir) |
|
|
|
|
|
|
|
|
|
|
|
dumpDebugData(ctx, outDir, conf, rpc) |
|
|
|
|
|
|
|
|
|
|
|
ticker := time.NewTicker(time.Duration(frequency) * time.Second) |
|
|
|
|
|
for range ticker.C { |
|
|
|
|
|
dumpDebugData(ctx, outDir, conf, rpc) |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
return nil |
|
|
|
|
|
|
|
|
type dumpDebugDataArgs struct { |
|
|
|
|
|
conf *config.Config |
|
|
|
|
|
outDir string |
|
|
|
|
|
profAddr string |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
func dumpDebugData(ctx context.Context, outDir string, conf *config.Config, rpc *rpchttp.HTTP) { |
|
|
|
|
|
|
|
|
func dumpDebugData(ctx context.Context, logger log.Logger, rpc *rpchttp.HTTP, args dumpDebugDataArgs) { |
|
|
start := time.Now().UTC() |
|
|
start := time.Now().UTC() |
|
|
|
|
|
|
|
|
tmpDir, err := os.MkdirTemp(outDir, "tendermint_debug_tmp") |
|
|
|
|
|
|
|
|
tmpDir, err := os.MkdirTemp(args.outDir, "tendermint_debug_tmp") |
|
|
if err != nil { |
|
|
if err != nil { |
|
|
logger.Error("failed to create temporary directory", "dir", tmpDir, "error", err) |
|
|
logger.Error("failed to create temporary directory", "dir", tmpDir, "error", err) |
|
|
return |
|
|
return |
|
@ -110,26 +134,26 @@ func dumpDebugData(ctx context.Context, outDir string, conf *config.Config, rpc |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
logger.Info("copying node WAL...") |
|
|
logger.Info("copying node WAL...") |
|
|
if err := copyWAL(conf, tmpDir); err != nil { |
|
|
|
|
|
|
|
|
if err := copyWAL(args.conf, tmpDir); err != nil { |
|
|
logger.Error("failed to copy node WAL", "error", err) |
|
|
logger.Error("failed to copy node WAL", "error", err) |
|
|
return |
|
|
return |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
if profAddr != "" { |
|
|
|
|
|
|
|
|
if args.profAddr != "" { |
|
|
logger.Info("getting node goroutine profile...") |
|
|
logger.Info("getting node goroutine profile...") |
|
|
if err := dumpProfile(tmpDir, profAddr, "goroutine", 2); err != nil { |
|
|
|
|
|
|
|
|
if err := dumpProfile(tmpDir, args.profAddr, "goroutine", 2); err != nil { |
|
|
logger.Error("failed to dump goroutine profile", "error", err) |
|
|
logger.Error("failed to dump goroutine profile", "error", err) |
|
|
return |
|
|
return |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
logger.Info("getting node heap profile...") |
|
|
logger.Info("getting node heap profile...") |
|
|
if err := dumpProfile(tmpDir, profAddr, "heap", 2); err != nil { |
|
|
|
|
|
|
|
|
if err := dumpProfile(tmpDir, args.profAddr, "heap", 2); err != nil { |
|
|
logger.Error("failed to dump heap profile", "error", err) |
|
|
logger.Error("failed to dump heap profile", "error", err) |
|
|
return |
|
|
return |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
outFile := filepath.Join(outDir, fmt.Sprintf("%s.zip", start.Format(time.RFC3339))) |
|
|
|
|
|
|
|
|
outFile := filepath.Join(args.outDir, fmt.Sprintf("%s.zip", start.Format(time.RFC3339))) |
|
|
if err := zipDir(tmpDir, outFile); err != nil { |
|
|
if err := zipDir(tmpDir, outFile); err != nil { |
|
|
logger.Error("failed to create and compress archive", "file", outFile, "error", err) |
|
|
logger.Error("failed to create and compress archive", "file", outFile, "error", err) |
|
|
} |
|
|
} |
|
|