From c1866c0be66037b34c544eabe835fcc5043aedb8 Mon Sep 17 00:00:00 2001 From: huofei <68298506@qq.com> Date: Sat, 16 Jul 2022 02:34:56 +0800 Subject: [PATCH 01/14] fix potential file inclusion via variable (#3549) --- e2etest/util.go | 3 ++- ioctl/client.go | 3 ++- ioctl/cmd/account/account.go | 2 +- ioctl/cmd/contract/contract.go | 3 ++- ioctl/cmd/contract/contractshare.go | 4 ++-- ioctl/cmd/hdwallet/hdwalletderive.go | 3 ++- ioctl/config/config.go | 3 ++- ioctl/doc/doc.go | 2 +- ioctl/util/util.go | 3 ++- pkg/recovery/recovery.go | 2 +- state/factory/patchstore.go | 9 +++++---- tools/actioninjector.v2/internal/cmd/inject.go | 3 ++- tools/util/injectorutil.go | 3 ++- 13 files changed, 26 insertions(+), 17 deletions(-) diff --git a/e2etest/util.go b/e2etest/util.go index 6592871cf9..b6d6433d1b 100644 --- a/e2etest/util.go +++ b/e2etest/util.go @@ -11,6 +11,7 @@ import ( "encoding/hex" "math/big" "os" + "path/filepath" "github.com/iotexproject/go-pkgs/hash" "github.com/pkg/errors" @@ -194,7 +195,7 @@ func addTestingTsfBlocks(bc blockchain.Blockchain, ap actpool.ActPool) error { } func copyDB(srcDB, dstDB string) error { - input, err := os.ReadFile(srcDB) + input, err := os.ReadFile(filepath.Clean(srcDB)) if err != nil { return errors.Wrap(err, "failed to read source db file") } diff --git a/ioctl/client.go b/ioctl/client.go index b7921efebc..3d2e739f9a 100644 --- a/ioctl/client.go +++ b/ioctl/client.go @@ -17,6 +17,7 @@ import ( "net/http" "os" "os/exec" + "path/filepath" "strings" "github.com/ethereum/go-ethereum/accounts/keystore" @@ -257,7 +258,7 @@ func (c *client) NewKeyStore() *keystore.KeyStore { } func (c *client) DecryptPrivateKey(passwordOfKeyStore, keyStorePath string) (*ecdsa.PrivateKey, error) { - keyJSON, err := os.ReadFile(keyStorePath) + keyJSON, err := os.ReadFile(filepath.Clean(keyStorePath)) if err != nil { return nil, fmt.Errorf("keystore file \"%s\" read error", keyStorePath) } diff --git a/ioctl/cmd/account/account.go b/ioctl/cmd/account/account.go index 31c35a772b..6462d6c668 100644 --- a/ioctl/cmd/account/account.go +++ b/ioctl/cmd/account/account.go @@ -306,7 +306,7 @@ func newAccountByKey(alias string, privateKey string, walletDir string) (string, } func newAccountByKeyStore(alias, passwordOfKeyStore, keyStorePath string, walletDir string) (string, error) { - keyJSON, err := os.ReadFile(keyStorePath) + keyJSON, err := os.ReadFile(filepath.Clean(keyStorePath)) if err != nil { return "", output.NewError(output.ReadFileError, fmt.Sprintf("keystore file \"%s\" read error", keyStorePath), nil) diff --git a/ioctl/cmd/contract/contract.go b/ioctl/cmd/contract/contract.go index 17f042f85c..8b71f96cbe 100644 --- a/ioctl/cmd/contract/contract.go +++ b/ioctl/cmd/contract/contract.go @@ -10,6 +10,7 @@ import ( "encoding/hex" "fmt" "os" + "path/filepath" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common/compiler" @@ -101,7 +102,7 @@ func checkCompilerVersion(solc *compiler.Solidity) bool { } func readAbiFile(abiFile string) (*abi.ABI, error) { - abiBytes, err := os.ReadFile(abiFile) + abiBytes, err := os.ReadFile(filepath.Clean(abiFile)) if err != nil { return nil, output.NewError(output.ReadFileError, "failed to read abi file", err) } diff --git a/ioctl/cmd/contract/contractshare.go b/ioctl/cmd/contract/contractshare.go index 5fcbc769ad..704251410c 100644 --- a/ioctl/cmd/contract/contractshare.go +++ b/ioctl/cmd/contract/contractshare.go @@ -92,7 +92,7 @@ func isDir(path string) bool { func isReadOnly(path string) bool { var readOnly = false - file, err := os.OpenFile(path, os.O_WRONLY, 0666) + file, err := os.OpenFile(filepath.Clean(path), os.O_WRONLY, 0666) if err != nil { if os.IsPermission(err) { log.Println("Error: Write permission denied.") @@ -198,7 +198,7 @@ func share(args []string) error { t := request.Payload getPayload := reflect.ValueOf(t).Index(0).Interface().(map[string]interface{}) getPayloadPath := getPayload["path"].(string) - upload, err := os.ReadFile(_givenPath + "/" + getPayloadPath) + upload, err := os.ReadFile(filepath.Clean(_givenPath + "/" + getPayloadPath)) if err != nil { log.Println("read file failed: ", err) } diff --git a/ioctl/cmd/hdwallet/hdwalletderive.go b/ioctl/cmd/hdwallet/hdwalletderive.go index 44d0ec23c3..4c5b3a2211 100644 --- a/ioctl/cmd/hdwallet/hdwalletderive.go +++ b/ioctl/cmd/hdwallet/hdwalletderive.go @@ -10,6 +10,7 @@ import ( "bytes" "fmt" "os" + "path/filepath" ecrypt "github.com/ethereum/go-ethereum/crypto" hdwallet "github.com/miguelmota/go-ethereum-hdwallet" @@ -71,7 +72,7 @@ func DeriveKey(account, change, index uint32, password string) (string, crypto.P return "", nil, output.NewError(output.InputError, "Run 'ioctl hdwallet create' to create your HDWallet first.", nil) } - enctxt, err := os.ReadFile(hdWalletConfigFile) + enctxt, err := os.ReadFile(filepath.Clean(hdWalletConfigFile)) if err != nil { return "", nil, output.NewError(output.InputError, "failed to read config", err) } diff --git a/ioctl/config/config.go b/ioctl/config/config.go index 228a0a66e7..d1e3ba2bde 100644 --- a/ioctl/config/config.go +++ b/ioctl/config/config.go @@ -9,6 +9,7 @@ package config import ( "fmt" "os" + "path/filepath" "github.com/spf13/cobra" "gopkg.in/yaml.v2" @@ -137,7 +138,7 @@ func LoadConfig() (Config, error) { ReadConfig := Config{ Aliases: make(map[string]string), } - in, err := os.ReadFile(DefaultConfigFile) + in, err := os.ReadFile(filepath.Clean(DefaultConfigFile)) if err == nil { if err := yaml.Unmarshal(in, &ReadConfig); err != nil { return ReadConfig, err diff --git a/ioctl/doc/doc.go b/ioctl/doc/doc.go index 2a7cebed66..d887ccf4bc 100644 --- a/ioctl/doc/doc.go +++ b/ioctl/doc/doc.go @@ -32,7 +32,7 @@ func GenMarkdownTreeCustom(c *cobra.Command, dir string, name string, path strin filename = filepath.Join(path, "README.md") } - f, err := os.Create(filename) + f, err := os.Create(filepath.Clean(filename)) if err != nil { return err } diff --git a/ioctl/util/util.go b/ioctl/util/util.go index e2251ebbf2..fcf1f394a8 100644 --- a/ioctl/util/util.go +++ b/ioctl/util/util.go @@ -13,6 +13,7 @@ import ( "math/big" "os" "os/signal" + "path/filepath" "strconv" "strings" "syscall" @@ -169,7 +170,7 @@ func Address(in string) (string, error) { // JwtAuth used for ioctl set auth and send for every grpc request func JwtAuth() (jwt metadata.MD, err error) { jwtFile := os.Getenv("HOME") + "/.config/ioctl/default/auth.jwt" - jwtString, err := os.ReadFile(jwtFile) + jwtString, err := os.ReadFile(filepath.Clean(jwtFile)) if err != nil { return nil, err } diff --git a/pkg/recovery/recovery.go b/pkg/recovery/recovery.go index a2612817d3..3d2d4fe957 100644 --- a/pkg/recovery/recovery.go +++ b/pkg/recovery/recovery.go @@ -79,7 +79,7 @@ func LogCrash(r interface{}) { } func writeHeapProfile(path string) { - f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0644) + f, err := os.OpenFile(filepath.Clean(path), os.O_CREATE|os.O_RDWR, 0644) if err != nil { log.S().Errorf("crashlog: open heap profile error: %v", err) return diff --git a/state/factory/patchstore.go b/state/factory/patchstore.go index 064d884a88..a48d665090 100644 --- a/state/factory/patchstore.go +++ b/state/factory/patchstore.go @@ -11,6 +11,7 @@ import ( "encoding/hex" "io" "os" + "path/filepath" "strconv" "github.com/pkg/errors" @@ -43,16 +44,16 @@ type ( * key: hex string * value: hex string */ -func newPatchStore(filepath string) (*patchStore, error) { +func newPatchStore(fpath string) (*patchStore, error) { store := &patchStore{ patchs: map[uint64][]*patch{}, } - if filepath == "" { + if fpath == "" { return store, nil } - file, err := os.Open(filepath) + file, err := os.Open(filepath.Clean(fpath)) if err != nil { - return nil, errors.Wrapf(err, "failed to open kvstore patch, %s", filepath) + return nil, errors.Wrapf(err, "failed to open kvstore patch, %s", fpath) } reader := csv.NewReader(file) reader.FieldsPerRecord = -1 diff --git a/tools/actioninjector.v2/internal/cmd/inject.go b/tools/actioninjector.v2/internal/cmd/inject.go index 3ac37d0733..1a03076938 100644 --- a/tools/actioninjector.v2/internal/cmd/inject.go +++ b/tools/actioninjector.v2/internal/cmd/inject.go @@ -16,6 +16,7 @@ import ( "math/big" "math/rand" "os" + "path/filepath" "strings" "sync" "time" @@ -112,7 +113,7 @@ func (p *injectProcessor) randAccounts(num int) error { } func (p *injectProcessor) loadAccounts(keypairsPath string) error { - keyPairBytes, err := os.ReadFile(keypairsPath) + keyPairBytes, err := os.ReadFile(filepath.Clean(keypairsPath)) if err != nil { return errors.Wrap(err, "failed to read key pairs file") } diff --git a/tools/util/injectorutil.go b/tools/util/injectorutil.go index 59c9825193..ba650c6a79 100644 --- a/tools/util/injectorutil.go +++ b/tools/util/injectorutil.go @@ -12,6 +12,7 @@ import ( "math/big" "math/rand" "os" + "path/filepath" "sync" "sync/atomic" "time" @@ -81,7 +82,7 @@ func GetTotalTsfFailed() uint64 { // LoadAddresses loads key pairs from key pair path and construct addresses func LoadAddresses(keypairsPath string, chainID uint32) ([]*AddressKey, error) { // Load Senders' public/private key pairs - keyPairBytes, err := os.ReadFile(keypairsPath) + keyPairBytes, err := os.ReadFile(filepath.Clean(keypairsPath)) if err != nil { return nil, errors.Wrap(err, "failed to read key pairs file") } From 7783ced927bf5008c47d059252ab525c561f8f16 Mon Sep 17 00:00:00 2001 From: Jeremy Chou Date: Sat, 16 Jul 2022 09:08:49 +0800 Subject: [PATCH 02/14] [ioctl] Build action command line into new ioctl (#3472) * [ioctl] build action command line into new ioctl --- ioctl/newcmd/action/action.go | 500 +++++++++++++++++++++++++++++ ioctl/newcmd/action/action_test.go | 127 ++++++++ 2 files changed, 627 insertions(+) create mode 100644 ioctl/newcmd/action/action.go create mode 100644 ioctl/newcmd/action/action_test.go diff --git a/ioctl/newcmd/action/action.go b/ioctl/newcmd/action/action.go new file mode 100644 index 0000000000..802aa14752 --- /dev/null +++ b/ioctl/newcmd/action/action.go @@ -0,0 +1,500 @@ +// Copyright (c) 2022 IoTeX Foundation +// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no +// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent +// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache +// License 2.0 that can be found in the LICENSE file. + +package action + +import ( + "context" + "encoding/hex" + "math/big" + "strings" + + "github.com/grpc-ecosystem/go-grpc-middleware/util/metautils" + "github.com/iotexproject/go-pkgs/hash" + "github.com/iotexproject/iotex-address/address" + "github.com/iotexproject/iotex-proto/golang/iotexapi" + "github.com/iotexproject/iotex-proto/golang/iotextypes" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "go.uber.org/zap" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + "github.com/iotexproject/iotex-core/action" + "github.com/iotexproject/iotex-core/ioctl" + "github.com/iotexproject/iotex-core/ioctl/config" + "github.com/iotexproject/iotex-core/ioctl/flag" + "github.com/iotexproject/iotex-core/ioctl/newcmd/account" + "github.com/iotexproject/iotex-core/ioctl/newcmd/bc" + "github.com/iotexproject/iotex-core/ioctl/util" + "github.com/iotexproject/iotex-core/pkg/log" + "github.com/iotexproject/iotex-core/pkg/util/byteutil" +) + +// Multi-language support +var ( + _actionCmdShorts = map[config.Language]string{ + config.English: "Manage actions of IoTeX blockchain", + config.Chinese: "管理IoTex区块链的行为", // this translation + } + _infoWarn = map[config.Language]string{ + config.English: "** This is an irreversible action!\n" + + "Once an account is deleted, all the assets under this account may be lost!\n" + + "Type 'YES' to continue, quit for anything else.", + config.Chinese: "** 这是一个不可逆转的操作!\n" + + "一旦一个账户被删除, 该账户下的所有资源都可能会丢失!\n" + + "输入 'YES' 以继续, 否则退出", + } + _infoQuit = map[config.Language]string{ + config.English: "quit", + config.Chinese: "退出", + } + _flagGasLimitUsages = map[config.Language]string{ + config.English: "set gas limit", + config.Chinese: "设置燃气上限", + } + _flagGasPriceUsages = map[config.Language]string{ + config.English: `set gas price (unit: 10^(-6)IOTX), use suggested gas price if input is "0"`, + config.Chinese: `设置燃气费（单位：10^(-6)IOTX），如果输入为「0」，则使用默认燃气费`, + } + _flagNonceUsages = map[config.Language]string{ + config.English: "set nonce (default using pending nonce)", + config.Chinese: "设置 nonce (默认使用 pending nonce)", + } + _flagSignerUsages = map[config.Language]string{ + config.English: "choose a signing account", + config.Chinese: "选择要签名的帐户", + } + _flagBytecodeUsages = map[config.Language]string{ + config.English: "set the byte code", + config.Chinese: "设置字节码", + } + _flagAssumeYesUsages = map[config.Language]string{ + config.English: "answer yes for all confirmations", + config.Chinese: "为所有确认设置 yes", + } + _flagPasswordUsages = map[config.Language]string{ + config.English: "input password for account", + config.Chinese: "设置密码", + } +) + +// Flag label, short label and defaults +const ( + gasLimitFlagLabel = "gas-limit" + gasLimitFlagShortLabel = "l" + gasLimitFlagDefault = uint64(20000000) + gasPriceFlagLabel = "gas-price" + gasPriceFlagShortLabel = "p" + gasPriceFlagDefault = "1" + nonceFlagLabel = "nonce" + nonceFlagShortLabel = "n" + nonceFlagDefault = uint64(0) + signerFlagLabel = "signer" + signerFlagShortLabel = "s" + signerFlagDefault = "" + bytecodeFlagLabel = "bytecode" + bytecodeFlagShortLabel = "b" + bytecodeFlagDefault = "" + assumeYesFlagLabel = "assume-yes" + assumeYesFlagShortLabel = "y" + assumeYesFlagDefault = false + passwordFlagLabel = "password" + passwordFlagShortLabel = "P" + passwordFlagDefault = "" +) + +func registerGasLimitFlag(client ioctl.Client, cmd *cobra.Command) { + usage, _ := client.SelectTranslation(_flagGasLimitUsages) + flag.NewUint64VarP(gasLimitFlagLabel, gasLimitFlagShortLabel, gasLimitFlagDefault, usage).RegisterCommand(cmd) +} + +func registerGasPriceFlag(client ioctl.Client, cmd *cobra.Command) { + usage, _ := client.SelectTranslation(_flagGasPriceUsages) + flag.NewStringVarP(gasPriceFlagLabel, gasPriceFlagShortLabel, gasPriceFlagDefault, usage).RegisterCommand(cmd) +} + +func registerNonceFlag(client ioctl.Client, cmd *cobra.Command) { + usage, _ := client.SelectTranslation(_flagNonceUsages) + flag.NewUint64VarP(nonceFlagLabel, nonceFlagShortLabel, nonceFlagDefault, usage).RegisterCommand(cmd) +} + +func registerSignerFlag(client ioctl.Client, cmd *cobra.Command) { + usage, _ := client.SelectTranslation(_flagSignerUsages) + flag.NewStringVarP(signerFlagLabel, signerFlagShortLabel, signerFlagDefault, usage).RegisterCommand(cmd) +} + +func registerBytecodeFlag(client ioctl.Client, cmd *cobra.Command) { + usage, _ := client.SelectTranslation(_flagBytecodeUsages) + flag.NewStringVarP(bytecodeFlagLabel, bytecodeFlagShortLabel, bytecodeFlagDefault, usage).RegisterCommand(cmd) +} + +func registerAssumeYesFlag(client ioctl.Client, cmd *cobra.Command) { + usage, _ := client.SelectTranslation(_flagAssumeYesUsages) + flag.BoolVarP(assumeYesFlagLabel, assumeYesFlagShortLabel, assumeYesFlagDefault, usage).RegisterCommand(cmd) +} + +func registerPasswordFlag(client ioctl.Client, cmd *cobra.Command) { + usage, _ := client.SelectTranslation(_flagPasswordUsages) + flag.NewStringVarP(passwordFlagLabel, passwordFlagShortLabel, passwordFlagDefault, usage).RegisterCommand(cmd) +} + +func mustString(v string, err error) string { + if err != nil { + log.L().Panic("input flag must be string", zap.Error(err)) + } + return v +} + +func mustUint64(v uint64, err error) uint64 { + if err != nil { + log.L().Panic("input flag must be uint64", zap.Error(err)) + } + return v +} + +func mustBoolean(v bool, err error) bool { + if err != nil { + log.L().Panic("input flag must be boolean", zap.Error(err)) + } + return v +} + +func gasLimitFlagValue(cmd *cobra.Command) (v uint64) { + return mustUint64(cmd.Flags().GetUint64(gasLimitFlagLabel)) +} + +func gasPriceFlagValue(cmd *cobra.Command) (v string) { + return mustString(cmd.Flags().GetString(gasPriceFlagLabel)) +} + +func getNonceFlagValue(cmd *cobra.Command) (v uint64) { + return mustUint64(cmd.Flags().GetUint64(nonceFlagLabel)) +} + +func getSignerFlagValue(cmd *cobra.Command) (v string) { + return mustString(cmd.Flags().GetString(signerFlagLabel)) +} + +func getBytecodeFlagValue(cmd *cobra.Command) (v string) { + return mustString(cmd.Flags().GetString(bytecodeFlagLabel)) +} + +func getDecodeBytecode(cmd *cobra.Command) ([]byte, error) { + return hex.DecodeString(util.TrimHexPrefix(getBytecodeFlagValue(cmd))) +} + +func getAssumeYesFlagValue(cmd *cobra.Command) (v bool) { + return mustBoolean(cmd.Flags().GetBool(assumeYesFlagLabel)) +} + +func getPasswordFlagValue(cmd *cobra.Command) (v string) { + return mustString(cmd.Flags().GetString(passwordFlagLabel)) +} + +func selectTranslation(client ioctl.Client, trls map[config.Language]string) string { + txt, _ := client.SelectTranslation(trls) + return txt +} + +// NewActionCmd represents the action command +func NewActionCmd(client ioctl.Client) *cobra.Command { + ac := &cobra.Command{ + Use: "action", + Short: selectTranslation(client, _actionCmdShorts), + } + + // TODO add sub commands + // cmd.AddCommand(NewActionHash(client)) + // cmd.AddCommand(NewActionTransfer(client)) + // cmd.AddCommand(NewActionDeploy(client)) + // cmd.AddCommand(NewActionInvoke(client)) + // cmd.AddCommand(NewActionRead(client)) + // cmd.AddCommand(NewActionClaim(client)) + // cmd.AddCommand(NewActionDeposit(client)) + // cmd.AddCommand(NewActionSendRaw(client)) + + client.SetEndpointWithFlag(ac.PersistentFlags().StringVar) + client.SetInsecureWithFlag(ac.PersistentFlags().BoolVar) + + return ac +} + +// RegisterWriteCommand registers action flags for command +func RegisterWriteCommand(client ioctl.Client, cmd *cobra.Command) { + registerGasLimitFlag(client, cmd) + registerGasPriceFlag(client, cmd) + registerSignerFlag(client, cmd) + registerNonceFlag(client, cmd) + registerAssumeYesFlag(client, cmd) + registerPasswordFlag(client, cmd) +} + +func handleClientRequestError(err error, apiName string) error { + sta, ok := status.FromError(err) + if ok { + return errors.New(sta.Message()) + } + return errors.Wrapf(err, "failed to invoke %s api", apiName) +} + +// Signer returns signer's address +func Signer(client ioctl.Client, cmd *cobra.Command) (address string, err error) { + addressOrAlias := getSignerFlagValue(cmd) + if util.AliasIsHdwalletKey(addressOrAlias) { + return addressOrAlias, nil + } + return client.AddressWithDefaultIfNotExist(addressOrAlias) +} + +func nonce(client ioctl.Client, cmd *cobra.Command, executor string) (uint64, error) { + if util.AliasIsHdwalletKey(executor) { + // for hdwallet key, get the nonce in SendAction() + return 0, nil + } + if nonce := getNonceFlagValue(cmd); nonce != 0 { + return nonce, nil + } + accountMeta, err := account.Meta(client, executor) + if err != nil { + return 0, errors.Wrap(err, "failed to get account meta") + } + return accountMeta.PendingNonce, nil +} + +// gasPriceInRau returns the suggest gas price +func gasPriceInRau(client ioctl.Client, cmd *cobra.Command) (*big.Int, error) { + if client.IsCryptoSm2() { + return big.NewInt(0), nil + } + gasPrice := gasPriceFlagValue(cmd) + if len(gasPrice) != 0 { + return util.StringToRau(gasPrice, util.GasPriceDecimalNum) + } + + cli, err := client.APIServiceClient() + if err != nil { + return nil, errors.Wrap(err, "failed to connect to endpoint") + } + + ctx := context.Background() + if jwtMD, err := util.JwtAuth(); err == nil { + ctx = metautils.NiceMD(jwtMD).ToOutgoing(ctx) + } + + rsp, err := cli.SuggestGasPrice(ctx, &iotexapi.SuggestGasPriceRequest{}) + if err != nil { + return nil, handleClientRequestError(err, "SuggestGasPrice") + } + return new(big.Int).SetUint64(rsp.GasPrice), nil +} + +func fixGasLimit(client ioctl.Client, caller string, execution *action.Execution) (*action.Execution, error) { + cli, err := client.APIServiceClient() + if err != nil { + return nil, errors.Wrap(err, "failed to connect to endpoint") + } + + ctx := context.Background() + if jwtMD, err := util.JwtAuth(); err == nil { + ctx = metautils.NiceMD(jwtMD).ToOutgoing(ctx) + } + + res, err := cli.EstimateActionGasConsumption(ctx, + &iotexapi.EstimateActionGasConsumptionRequest{ + Action: &iotexapi.EstimateActionGasConsumptionRequest_Execution{ + Execution: execution.Proto(), + }, + CallerAddress: caller, + }) + if err != nil { + return nil, handleClientRequestError(err, "EstimateActionGasConsumption") + } + return action.NewExecution(execution.Contract(), execution.Nonce(), execution.Amount(), res.Gas, execution.GasPrice(), execution.Data()) +} + +// SendRaw sends raw action to blockchain +func SendRaw(client ioctl.Client, cmd *cobra.Command, selp *iotextypes.Action) error { + cli, err := client.APIServiceClient() + if err != nil { + return errors.Wrap(err, "failed to connect to endpoint") + } + + ctx := context.Background() + if jwtMD, err := util.JwtAuth(); err == nil { + ctx = metautils.NiceMD(jwtMD).ToOutgoing(ctx) + } + + _, err = cli.SendAction(ctx, &iotexapi.SendActionRequest{Action: selp}) + if err != nil { + return handleClientRequestError(err, "SendAction") + } + + shash := hash.Hash256b(byteutil.Must(proto.Marshal(selp))) + txhash := hex.EncodeToString(shash[:]) + URL := "https://" + endpoint := client.Config().Endpoint + explorer := client.Config().Explorer + switch explorer { + case "iotexscan": + if strings.Contains(endpoint, "testnet") { + URL += "testnet." + } + URL += "iotexscan.io/action/" + txhash + case "iotxplorer": + URL = "iotxplorer.io/actions/" + txhash + default: + URL = explorer + txhash + } + cmd.Printf("Action has been sent to blockchain.\nWait for several seconds and query this action by hash: %s\n", URL) + return nil +} + +// SendAction sends signed action to blockchain +func SendAction(client ioctl.Client, cmd *cobra.Command, elp action.Envelope, signer string) error { + sk, err := account.PrivateKeyFromSigner(client, cmd, signer, getPasswordFlagValue(cmd)) + if err != nil { + return err + } + + chainMeta, err := bc.GetChainMeta(client) + if err != nil { + return errors.Wrap(err, "failed to get chain meta") + } + elp.SetChainID(chainMeta.GetChainID()) + + if util.AliasIsHdwalletKey(signer) { + addr := sk.PublicKey().Address() + signer = addr.String() + nonce, err := nonce(client, cmd, signer) + if err != nil { + return errors.Wrap(err, "failed to get nonce ") + } + elp.SetNonce(nonce) + } + + sealed, err := action.Sign(elp, sk) + if err != nil { + return errors.Wrap(err, "failed to sign action") + } + if err := isBalanceEnough(client, signer, sealed); err != nil { + return errors.Wrap(err, "failed to pass balance check") + } + + selp := sealed.Proto() + sk.Zero() + // TODO wait newcmd/action/actionhash impl pr #3425 + // actionInfo, err := printActionProto(selp) + // if err != nil { + // return errors.Wrap(err, "failed to print action proto message") + // } + // cmd.Println(actionInfo) + + if !getAssumeYesFlagValue(cmd) { + infoWarn := selectTranslation(client, _infoWarn) + infoQuit := selectTranslation(client, _infoQuit) + if !client.AskToConfirm(infoWarn) { + cmd.Println(infoQuit) + } + return nil + } + + return SendRaw(client, cmd, selp) +} + +// Execute sends signed execution transaction to blockchain +func Execute(client ioctl.Client, cmd *cobra.Command, contract string, amount *big.Int, bytecode []byte) error { + if len(contract) == 0 && len(bytecode) == 0 { + return errors.New("failed to deploy contract with empty bytecode") + } + gasPriceRau, err := gasPriceInRau(client, cmd) + if err != nil { + return errors.Wrap(err, "failed to get gas price") + } + signer, err := Signer(client, cmd) + if err != nil { + return errors.Wrap(err, "failed to get signer address") + } + nonce, err := nonce(client, cmd, signer) + if err != nil { + return errors.Wrap(err, "failed to get nonce") + } + gasLimit := gasLimitFlagValue(cmd) + tx, err := action.NewExecution(contract, nonce, amount, gasLimit, gasPriceRau, bytecode) + if err != nil || tx == nil { + return errors.Wrap(err, "failed to make a Execution instance") + } + if gasLimit == 0 { + tx, err = fixGasLimit(client, signer, tx) + if err != nil || tx == nil { + return errors.Wrap(err, "failed to fix Execution gas limit") + } + gasLimit = tx.GasLimit() + } + return SendAction( + client, + cmd, + (&action.EnvelopeBuilder{}). + SetNonce(nonce). + SetGasPrice(gasPriceRau). + SetGasLimit(gasLimit). + SetAction(tx).Build(), + signer, + ) +} + +// Read reads smart contract on IoTeX blockchain +func Read(client ioctl.Client, cmd *cobra.Command, contract address.Address, amount string, bytecode []byte) (string, error) { + cli, err := client.APIServiceClient() + if err != nil { + return "", errors.Wrap(err, "failed to connect to endpoint") + } + + ctx := context.Background() + if jwtMD, err := util.JwtAuth(); err == nil { + ctx = metautils.NiceMD(jwtMD).ToOutgoing(ctx) + } + + callerAddr, _ := Signer(client, cmd) + if callerAddr == "" { + callerAddr = address.ZeroAddress + } + + res, err := cli.ReadContract(ctx, + &iotexapi.ReadContractRequest{ + Execution: &iotextypes.Execution{ + Amount: amount, + Contract: contract.String(), + Data: bytecode, + }, + CallerAddress: callerAddr, + GasLimit: gasLimitFlagValue(cmd), + }, + ) + if err != nil { + return "", handleClientRequestError(err, "ReadContract") + } + return res.Data, nil +} + +func isBalanceEnough(client ioctl.Client, address string, act action.SealedEnvelope) error { + accountMeta, err := account.Meta(client, address) + if err != nil { + return errors.Wrap(err, "failed to get account meta") + } + balance, ok := new(big.Int).SetString(accountMeta.Balance, 10) + if !ok { + return errors.New("failed to convert balance into big int") + } + cost, err := act.Cost() + if err != nil { + return errors.Wrap(err, "failed to check cost of an action") + } + if balance.Cmp(cost) < 0 { + return errors.New("balance is not enough") + } + return nil +} diff --git a/ioctl/newcmd/action/action_test.go b/ioctl/newcmd/action/action_test.go new file mode 100644 index 0000000000..8a6bc1eb7c --- /dev/null +++ b/ioctl/newcmd/action/action_test.go @@ -0,0 +1,127 @@ +// Copyright (c) 2022 IoTeX Foundation +// This is an alpha (internal) release and is not suitable for production. This source code is provided 'as is' and no +// warranties are given as to title or non-infringement, merchantability or fitness for purpose and, to the extent +// permitted by law, all liability for your use of the code is disclaimed. This source code is governed by Apache +// License 2.0 that can be found in the LICENSE file. + +package action + +import ( + "testing" + + "github.com/golang/mock/gomock" + "github.com/iotexproject/iotex-proto/golang/iotexapi" + "github.com/iotexproject/iotex-proto/golang/iotexapi/mock_iotexapi" + "github.com/iotexproject/iotex-proto/golang/iotextypes" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" + + "github.com/iotexproject/iotex-core/ioctl/config" + "github.com/iotexproject/iotex-core/ioctl/util" + "github.com/iotexproject/iotex-core/test/mock/mock_ioctlclient" +) + +func TestSigner(t *testing.T) { + require := require.New(t) + ctrl := gomock.NewController(t) + client := mock_ioctlclient.NewMockClient(ctrl) + client.EXPECT().SelectTranslation(gomock.Any()).Return("mockTranslationString", config.English).Times(3) + client.EXPECT().SetEndpointWithFlag(gomock.Any()).Do(func(_ func(*string, string, string, string)) {}) + client.EXPECT().SetInsecureWithFlag(gomock.Any()).Do(func(_ func(*bool, string, bool, string)) {}) + + t.Run("returns signer's address", func(t *testing.T) { + client.EXPECT().AddressWithDefaultIfNotExist(gomock.Any()).Return("test", nil).AnyTimes() + + cmd := NewActionCmd(client) + registerSignerFlag(client, cmd) + _, err := util.ExecuteCmd(cmd, "--signer", "test") + require.NoError(err) + result, err := Signer(client, cmd) + require.NoError(err) + require.Equal(result, "test") + }) +} + +func TestSendRaw(t *testing.T) { + require := require.New(t) + ctrl := gomock.NewController(t) + client := mock_ioctlclient.NewMockClient(ctrl) + apiServiceClient := mock_iotexapi.NewMockAPIServiceClient(ctrl) + selp := &iotextypes.Action{} + + client.EXPECT().SelectTranslation(gomock.Any()).Return("mockTranslationString", config.English).Times(12) + client.EXPECT().APIServiceClient().Return(apiServiceClient, nil).Times(7) + + for _, test := range []struct { + endpoint string + insecure bool + }{ + { + endpoint: "111:222:333:444:5678", + insecure: false, + }, + { + endpoint: "", + insecure: true, + }, + } { + callbackEndpoint := func(cb func(*string, string, string, string)) { + cb(&test.endpoint, "endpoint", test.endpoint, "endpoint usage") + } + callbackInsecure := func(cb func(*bool, string, bool, string)) { + cb(&test.insecure, "insecure", !test.insecure, "insecure usage") + } + client.EXPECT().SetEndpointWithFlag(gomock.Any()).Do(callbackEndpoint).Times(3) + client.EXPECT().SetInsecureWithFlag(gomock.Any()).Do(callbackInsecure).Times(3) + + t.Run("sends raw action to blockchain", func(t *testing.T) { + response := &iotexapi.SendActionResponse{} + + apiServiceClient.EXPECT().SendAction(gomock.Any(), gomock.Any()).Return(response, nil).Times(3) + + cmd := NewActionCmd(client) + _, err := util.ExecuteCmd(cmd) + require.NoError(err) + + t.Run("endpoint iotexscan", func(t *testing.T) { + client.EXPECT().Config().Return(config.Config{ + Explorer: "iotexscan", + Endpoint: "testnet1", + }).Times(2) + + err = SendRaw(client, cmd, selp) + require.NoError(err) + }) + + t.Run("endpoint iotxplorer", func(t *testing.T) { + client.EXPECT().Config().Return(config.Config{ + Explorer: "iotxplorer", + }).Times(2) + + err := SendRaw(client, cmd, selp) + require.NoError(err) + }) + + t.Run("endpoint default", func(t *testing.T) { + client.EXPECT().Config().Return(config.Config{ + Explorer: "test", + }).Times(2) + + err := SendRaw(client, cmd, selp) + require.NoError(err) + }) + }) + } + + t.Run("failed to invoke SendAction api", func(t *testing.T) { + expectedErr := errors.New("failed to invoke SendAction api") + + apiServiceClient.EXPECT().SendAction(gomock.Any(), gomock.Any()).Return(nil, expectedErr) + + cmd := NewActionCmd(client) + _, err := util.ExecuteCmd(cmd) + require.NoError(err) + err = SendRaw(client, cmd, selp) + require.Contains(err.Error(), expectedErr.Error()) + }) +} From 8d1e8364a3be5835f53e407cbbe1ae80b321f821 Mon Sep 17 00:00:00 2001 From: Daulet Dulanov Date: Tue, 19 Jul 2022 02:55:25 +0200 Subject: [PATCH 03/14] [api] impl. TestGrpcServer_GetServerMeta (#3559) --- api/grpcserver_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/api/grpcserver_test.go b/api/grpcserver_test.go index 91b0c280f9..684304a456 100644 --- a/api/grpcserver_test.go +++ b/api/grpcserver_test.go @@ -104,7 +104,20 @@ func TestGrpcServer_GetReceiptByAction(t *testing.T) { } func TestGrpcServer_GetServerMeta(t *testing.T) { + require := require.New(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + core := mock_apicoreservice.NewMockCoreService(ctrl) + grpcSvr := newGRPCHandler(core) + core.EXPECT().ServerMeta().Return("packageVersion", "packageCommitID", "gitStatus", "goVersion", "buildTime") + res, err := grpcSvr.GetServerMeta(context.Background(), &iotexapi.GetServerMetaRequest{}) + require.NoError(err) + require.Equal("packageVersion", res.ServerMeta.PackageVersion) + require.Equal("packageCommitID", res.ServerMeta.PackageCommitID) + require.Equal("gitStatus", res.ServerMeta.GitStatus) + require.Equal("goVersion", res.ServerMeta.GoVersion) + require.Equal("buildTime", res.ServerMeta.BuildTime) } func TestGrpcServer_ReadContract(t *testing.T) { From 378185ab78a6e64117d5ad466c4e71808f7b63f3 Mon Sep 17 00:00:00 2001 From: huofei <68298506@qq.com> Date: Tue, 19 Jul 2022 12:52:09 +0800 Subject: [PATCH 04/14] fix uncontrolled data used in path expression (#3547) --- ioctl/cmd/contract/contractshare.go | 61 ++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/ioctl/cmd/contract/contractshare.go b/ioctl/cmd/contract/contractshare.go index 704251410c..098c93a437 100644 --- a/ioctl/cmd/contract/contractshare.go +++ b/ioctl/cmd/contract/contractshare.go @@ -117,7 +117,7 @@ func isExist(path string) bool { } func rename(oldPath string, newPath string, c chan bool) { - if isExist(_givenPath + "/" + oldPath) { + if isExist(oldPath) { if err := os.Rename(oldPath, newPath); err != nil { log.Println("Rename file failed: ", err) } @@ -127,7 +127,7 @@ func rename(oldPath string, newPath string, c chan bool) { } func share(args []string) error { - _givenPath = args[0] + _givenPath = filepath.Clean(args[0]) if len(_givenPath) == 0 { return output.NewError(output.ReadFileError, "failed to get directory", nil) } @@ -186,7 +186,7 @@ func share(args []string) error { case "list": payload := make(map[string]bool) for _, ele := range _fileList { - payload[ele] = isReadOnly(_givenPath + "/" + ele) + payload[ele] = isReadOnly(filepath.Join(_givenPath, ele)) } response.Payload = payload if err := conn.WriteJSON(&response); err != nil { @@ -197,48 +197,73 @@ func share(args []string) error { t := request.Payload getPayload := reflect.ValueOf(t).Index(0).Interface().(map[string]interface{}) - getPayloadPath := getPayload["path"].(string) - upload, err := os.ReadFile(filepath.Clean(_givenPath + "/" + getPayloadPath)) + getPayloadPath, err := cleanPath(getPayload["path"].(string)) + if err != nil { + log.Println("clean file path failed: ", err) + break + } + getPayloadPath = filepath.Clean(filepath.Join(_givenPath, getPayloadPath)) + upload, err := os.ReadFile(getPayloadPath) if err != nil { log.Println("read file failed: ", err) + break } payload["content"] = string(upload) - payload["readonly"] = isReadOnly(_givenPath + "/" + getPayloadPath) + payload["readonly"] = isReadOnly(getPayloadPath) response.Payload = payload if err := conn.WriteJSON(&response); err != nil { log.Println("send get response: ", err) break } - log.Println("share: " + _givenPath + "/" + getPayloadPath) + log.Println("share: " + getPayloadPath) case "rename": c := make(chan bool) t := request.Payload renamePayload := reflect.ValueOf(t).Index(0).Interface().(map[string]interface{}) - oldPath := renamePayload["oldPath"].(string) - newPath := renamePayload["newPath"].(string) - go rename(oldPath, newPath, c) + oldRenamePath, err := cleanPath(renamePayload["oldPath"].(string)) + if err != nil { + log.Println("clean file path failed: ", err) + break + } + newRenamePath, err := cleanPath(renamePayload["newPath"].(string)) + if err != nil { + log.Println("clean file path failed: ", err) + break + } + oldRenamePath = filepath.Join(_givenPath, oldRenamePath) + newRenamePath = filepath.Join(_givenPath, newRenamePath) + go rename(oldRenamePath, newRenamePath, c) response.Payload = <-c if err := conn.WriteJSON(&response); err != nil { log.Println("send get response: ", err) break } - log.Println("rename: " + _givenPath + "/" + oldPath + " to " + _givenPath + "/" + newPath) + log.Println("rename: " + oldRenamePath + " to " + newRenamePath) case "set": t := request.Payload setPayload := reflect.ValueOf(t).Index(0).Interface().(map[string]interface{}) - setPath := setPayload["path"].(string) content := setPayload["content"].(string) - err := os.WriteFile(_givenPath+"/"+setPath, []byte(content), 0777) + setPath, err := cleanPath(setPayload["path"].(string)) if err != nil { + log.Println("clean file path failed: ", err) + break + } + setPath = filepath.Join(_givenPath, setPath) + if err := os.MkdirAll(filepath.Dir(setPath), 0755); err != nil { + log.Println("mkdir failed: ", err) + break + } + if err := os.WriteFile(setPath, []byte(content), 0644); err != nil { log.Println("set file failed: ", err) + break } if err := conn.WriteJSON(&response); err != nil { log.Println("send set response: ", err) break } - log.Println("set: " + _givenPath + "/" + setPath) + log.Println("set: " + setPath) default: log.Println("Don't support this IDE yet. Can not handle websocket method: " + request.Key) @@ -249,5 +274,13 @@ func share(args []string) error { log.Fatal(http.ListenAndServe(*_addr, nil)) return nil +} +func cleanPath(path string) (string, error) { + path = filepath.Clean(filepath.Join("/", path)) + real, err := filepath.Rel("/", path) + if err != nil { + return "", err + } + return real, nil } From 5a9ab76985eed6722b9d0a9b8ece344d54b04af9 Mon Sep 17 00:00:00 2001 From: Haaai <55118568+Liuhaai@users.noreply.github.com> Date: Mon, 18 Jul 2022 22:22:53 -0700 Subject: [PATCH 05/14] add log in rolldposctx (#3553) Co-authored-by: huofei <68298506@qq.com> --- consensus/consensus.go | 2 +- consensus/scheme/rolldpos/rolldposctx.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 860b3ce6e1..d19f369431 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -161,7 +161,7 @@ func NewConsensus( commitBlockCB := func(blk *block.Block) error { err := bc.CommitBlock(blk) if err != nil { - log.Logger("consensus").Info("Failed to commit the block.", zap.Error(err), zap.Uint64("height", blk.Height())) + log.Logger("consensus").Error("Failed to commit the block.", zap.Error(err), zap.Uint64("height", blk.Height())) } return err } diff --git a/consensus/scheme/rolldpos/rolldposctx.go b/consensus/scheme/rolldpos/rolldposctx.go index a86f9f1547..06a3c875ab 100644 --- a/consensus/scheme/rolldpos/rolldposctx.go +++ b/consensus/scheme/rolldpos/rolldposctx.go @@ -486,6 +486,7 @@ func (ctx *rollDPoSCtx) Commit(msg interface{}) (bool, error) { case nil: break default: + log.L().Error("error when committing the block", zap.Error(err)) return false, errors.Wrap(err, "error when committing a block") } // Broadcast the committed block to the network From db12104b264e3632bd8b8289725061866eeda5cb Mon Sep 17 00:00:00 2001 From: huofei <68298506@qq.com> Date: Tue, 19 Jul 2022 14:08:17 +0800 Subject: [PATCH 06/14] [ioctl] fix log entries created from user input (#3546) * fix log entries created from user input * resolve conflict --- ioctl/cmd/contract/contractshare.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ioctl/cmd/contract/contractshare.go b/ioctl/cmd/contract/contractshare.go index 098c93a437..0827cf5d0a 100644 --- a/ioctl/cmd/contract/contractshare.go +++ b/ioctl/cmd/contract/contractshare.go @@ -154,7 +154,7 @@ func share(args []string) error { return nil }) - log.Println("Listening on 127.0.0.1:65520, Please open your IDE ( " + _iotexIDE + " ) to connect to local files") + log.Printf("Listening on 127.0.0.1:65520, Please open your IDE ( %s ) to connect to local files", _iotexIDE) http.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) { conn, err := _upgrade.Upgrade(writer, request, nil) @@ -202,7 +202,7 @@ func share(args []string) error { log.Println("clean file path failed: ", err) break } - getPayloadPath = filepath.Clean(filepath.Join(_givenPath, getPayloadPath)) + getPayloadPath = filepath.Join(_givenPath, getPayloadPath) upload, err := os.ReadFile(getPayloadPath) if err != nil { log.Println("read file failed: ", err) @@ -215,7 +215,7 @@ func share(args []string) error { log.Println("send get response: ", err) break } - log.Println("share: " + getPayloadPath) + log.Printf("share: %s\n", easpcapeString(getPayloadPath)) case "rename": c := make(chan bool) @@ -239,7 +239,7 @@ func share(args []string) error { log.Println("send get response: ", err) break } - log.Println("rename: " + oldRenamePath + " to " + newRenamePath) + log.Printf("rename: %s to %s\n", easpcapeString(oldRenamePath), easpcapeString(newRenamePath)) case "set": t := request.Payload @@ -263,10 +263,10 @@ func share(args []string) error { log.Println("send set response: ", err) break } - log.Println("set: " + setPath) + log.Printf("set: %s\n", easpcapeString(setPath)) default: - log.Println("Don't support this IDE yet. Can not handle websocket method: " + request.Key) + log.Printf("Don't support this IDE yet. Can not handle websocket method: %s\n", easpcapeString(request.Key)) } } @@ -276,6 +276,11 @@ func share(args []string) error { return nil } +func easpcapeString(str string) string { + escaped := strings.Replace(str, "\n", "", -1) + return strings.Replace(escaped, "\r", "", -1) +} + func cleanPath(path string) (string, error) { path = filepath.Clean(filepath.Join("/", path)) real, err := filepath.Rel("/", path) From 2ed5b0387338860ed93331c19f9dc589f21da94e Mon Sep 17 00:00:00 2001 From: millken Date: Tue, 19 Jul 2022 15:29:44 +0800 Subject: [PATCH 07/14] move chanid metrics to chainservice (#3544) Co-authored-by: dustinxie --- api/grpcserver.go | 38 --------------------------- api/grpcserver_test.go | 1 - chainservice/chainservice.go | 50 ++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 39 deletions(-) diff --git a/api/grpcserver.go b/api/grpcserver.go index 1a7ca92703..ff242ec30f 100644 --- a/api/grpcserver.go +++ b/api/grpcserver.go @@ -26,7 +26,6 @@ import ( "github.com/iotexproject/iotex-proto/golang/iotexapi" "github.com/iotexproject/iotex-proto/golang/iotextypes" "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel/attribute" "go.uber.org/zap" @@ -35,7 +34,6 @@ import ( "google.golang.org/grpc/health" "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/keepalive" - "google.golang.org/grpc/peer" "google.golang.org/grpc/reflection" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" @@ -72,28 +70,8 @@ var ( Time: 60 * time.Second, // Ping the client if it is idle for 60 seconds to ensure the connection is still active Timeout: 10 * time.Second, // Wait 10 seconds for the ping ack before assuming the connection is dead } - - _apiCallSourceWithChainIDMtc = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "iotex_apicallsource_chainid_metrics", - Help: "API call Source ChainID Statistics", - }, - []string{"chain_id"}, - ) - _apiCallSourceWithOutChainIDMtc = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "iotex_apicallsource_nochainid_metrics", - Help: "API call Source Without ChainID Statistics", - }, - []string{"client_ip", "sender"}, - ) ) -func init() { - prometheus.MustRegister(_apiCallSourceWithChainIDMtc) - prometheus.MustRegister(_apiCallSourceWithOutChainIDMtc) -} - // RecoveryInterceptor handles panic to a custom error func RecoveryInterceptor() grpc_recovery.Option { return grpc_recovery.WithRecoveryHandler(func(p interface{}) (err error) { @@ -342,22 +320,6 @@ func (svr *gRPCHandler) SendAction(ctx context.Context, in *iotexapi.SendActionR // tags output span.SetAttributes(attribute.String("actType", fmt.Sprintf("%T", in.GetAction().GetCore()))) defer span.End() - chainID := strconv.FormatUint(uint64(in.GetAction().GetCore().GetChainID()), 10) - if in.GetAction().GetCore().GetChainID() > 0 { - _apiCallSourceWithChainIDMtc.WithLabelValues(chainID).Inc() - } else { - selp, err := (&action.Deserializer{}).SetEvmNetworkID(svr.coreService.EVMNetworkID()).ActionToSealedEnvelope(in.GetAction()) - if err != nil { - return nil, err - } - var clientIP string - if p, ok := peer.FromContext(ctx); ok { - clientIP, _, _ = net.SplitHostPort(p.Addr.String()) - } else { - clientIP = "unknownIP" - } - _apiCallSourceWithOutChainIDMtc.WithLabelValues(clientIP, selp.SenderAddress().String()).Inc() - } actHash, err := svr.coreService.SendAction(ctx, in.GetAction()) if err != nil { return nil, err diff --git a/api/grpcserver_test.go b/api/grpcserver_test.go index 684304a456..5145aab049 100644 --- a/api/grpcserver_test.go +++ b/api/grpcserver_test.go @@ -86,7 +86,6 @@ func TestGrpcServer_SendAction(t *testing.T) { grpcSvr := newGRPCHandler(core) for _, test := range _sendActionTests { - core.EXPECT().EVMNetworkID().Return(uint32(1)) core.EXPECT().SendAction(context.Background(), test.actionPb).Return(test.actionHash, nil) request := &iotexapi.SendActionRequest{Action: test.actionPb} res, err := grpcSvr.SendAction(context.Background(), request) diff --git a/chainservice/chainservice.go b/chainservice/chainservice.go index 823c226156..515429d73f 100644 --- a/chainservice/chainservice.go +++ b/chainservice/chainservice.go @@ -8,11 +8,14 @@ package chainservice import ( "context" + "strconv" "github.com/libp2p/go-libp2p-core/peer" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "google.golang.org/protobuf/proto" + "github.com/iotexproject/iotex-address/address" "github.com/iotexproject/iotex-election/committee" "github.com/iotexproject/iotex-proto/golang/iotexrpc" "github.com/iotexproject/iotex-proto/golang/iotextypes" @@ -36,6 +39,28 @@ import ( "github.com/iotexproject/iotex-core/state/factory" ) +var ( + _apiCallWithChainIDMtc = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "iotex_apicall_chainid_metrics", + Help: "API call ChainID Statistics", + }, + []string{"chain_id"}, + ) + _apiCallWithOutChainIDMtc = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "iotex_apicall_nochainid_metrics", + Help: "API call Without ChainID Statistics", + }, + []string{"sender", "recipient"}, + ) +) + +func init() { + prometheus.MustRegister(_apiCallWithChainIDMtc) + prometheus.MustRegister(_apiCallWithOutChainIDMtc) +} + // ChainService is a blockchain service with all blockchain components. type ChainService struct { lifecycle lifecycle.Lifecycle @@ -80,9 +105,34 @@ func (cs *ChainService) HandleAction(ctx context.Context, actPb *iotextypes.Acti if err != nil { log.L().Debug(err.Error()) } + chainIDmetrics(act) return err } +func chainIDmetrics(act action.SealedEnvelope) { + chainID := strconv.FormatUint(uint64(act.ChainID()), 10) + if act.ChainID() > 0 { + _apiCallWithChainIDMtc.WithLabelValues(chainID).Inc() + } else { + recipient, _ := act.Destination() + //it will be empty for staking action, change string to staking in such case + if recipient == "" { + act, ok := act.Action().(action.EthCompatibleAction) + if ok { + if ethTx, err := act.ToEthTx(); err == nil && ethTx.To() != nil { + if add, err := address.FromHex(ethTx.To().Hex()); err == nil { + recipient = add.String() + } + } + } + if recipient == "" { + recipient = "staking" + } + } + _apiCallWithOutChainIDMtc.WithLabelValues(act.SenderAddress().String(), recipient).Inc() + } +} + // HandleBlock handles incoming block request. func (cs *ChainService) HandleBlock(ctx context.Context, peer string, pbBlock *iotextypes.Block) error { blk, err := block.NewDeserializer(cs.chain.EvmNetworkID()).FromBlockProto(pbBlock) From c216396b39b655afdb97ac8a63c46185f1bf76fb Mon Sep 17 00:00:00 2001 From: dustinxie Date: Tue, 19 Jul 2022 08:14:56 -0700 Subject: [PATCH 08/14] [staking] unexport namespace (#3551) --- action/protocol/rewarding/protocol.go | 10 ++++---- action/protocol/staking/bucket_pool.go | 6 ++--- action/protocol/staking/bucket_pool_test.go | 4 ++-- .../staking/candidate_statemanager.go | 24 +++++++++---------- .../protocol/staking/candidate_statereader.go | 16 ++++++------- action/protocol/staking/handlers_test.go | 4 ++-- action/protocol/staking/protocol.go | 10 ++++---- action/protocol/staking/protocol_test.go | 8 +++---- action/protocol/staking/read_state.go | 4 ++-- action/protocol/staking/vote_bucket_test.go | 2 +- action/protocol/staking/vote_reviser_test.go | 2 +- e2etest/native_staking_test.go | 12 ++++++---- 12 files changed, 52 insertions(+), 50 deletions(-) diff --git a/action/protocol/rewarding/protocol.go b/action/protocol/rewarding/protocol.go index 15df94adc4..531b3abea1 100644 --- a/action/protocol/rewarding/protocol.go +++ b/action/protocol/rewarding/protocol.go @@ -28,8 +28,8 @@ import ( const ( // TODO: it works only for one instance per protocol definition now - _protocolID = "rewarding" - _v2Namespace = "Rewarding" + _protocolID = "rewarding" + _v2RewardingNamespace = "Rewarding" ) var ( @@ -332,7 +332,7 @@ func (p *Protocol) stateV1(sm protocol.StateReader, key []byte, value interface{ func (p *Protocol) stateV2(sm protocol.StateReader, key []byte, value interface{}) (uint64, error) { k := append(p.keyPrefix, key...) - return sm.State(value, protocol.KeyOption(k), protocol.NamespaceOption(_v2Namespace)) + return sm.State(value, protocol.KeyOption(k), protocol.NamespaceOption(_v2RewardingNamespace)) } func (p *Protocol) putState(ctx context.Context, sm protocol.StateManager, key []byte, value interface{}) error { @@ -350,7 +350,7 @@ func (p *Protocol) putStateV1(sm protocol.StateManager, key []byte, value interf func (p *Protocol) putStateV2(sm protocol.StateManager, key []byte, value interface{}) error { k := append(p.keyPrefix, key...) - _, err := sm.PutState(value, protocol.KeyOption(k), protocol.NamespaceOption(_v2Namespace)) + _, err := sm.PutState(value, protocol.KeyOption(k), protocol.NamespaceOption(_v2RewardingNamespace)) return err } @@ -373,7 +373,7 @@ func (p *Protocol) deleteStateV1(sm protocol.StateManager, key []byte) error { func (p *Protocol) deleteStateV2(sm protocol.StateManager, key []byte) error { k := append(p.keyPrefix, key...) - _, err := sm.DelState(protocol.KeyOption(k), protocol.NamespaceOption(_v2Namespace)) + _, err := sm.DelState(protocol.KeyOption(k), protocol.NamespaceOption(_v2RewardingNamespace)) if errors.Cause(err) == state.ErrStateNotExist { // don't care if not exist return nil diff --git a/action/protocol/staking/bucket_pool.go b/action/protocol/staking/bucket_pool.go index f62ec577ac..a7df9023fc 100644 --- a/action/protocol/staking/bucket_pool.go +++ b/action/protocol/staking/bucket_pool.go @@ -115,7 +115,7 @@ func (bp *BucketPool) Copy(enableSMStorage bool) *BucketPool { // Sync syncs the data from state manager func (bp *BucketPool) Sync(sm protocol.StateManager) error { if bp.enableSMStorage { - _, err := sm.State(bp.total, protocol.NamespaceOption(StakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) + _, err := sm.State(bp.total, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) return err } // get stashed total amount @@ -138,7 +138,7 @@ func (bp *BucketPool) CreditPool(sm protocol.StateManager, amount *big.Int) erro } if bp.enableSMStorage { - _, err := sm.PutState(bp.total, protocol.NamespaceOption(StakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) + _, err := sm.PutState(bp.total, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) return err } return sm.Load(_protocolID, _stakingBucketPool, bp.total) @@ -148,7 +148,7 @@ func (bp *BucketPool) CreditPool(sm protocol.StateManager, amount *big.Int) erro func (bp *BucketPool) DebitPool(sm protocol.StateManager, amount *big.Int, newBucket bool) error { bp.total.AddBalance(amount, newBucket) if bp.enableSMStorage { - _, err := sm.PutState(bp.total, protocol.NamespaceOption(StakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) + _, err := sm.PutState(bp.total, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) return err } return sm.Load(_protocolID, _stakingBucketPool, bp.total) diff --git a/action/protocol/staking/bucket_pool_test.go b/action/protocol/staking/bucket_pool_test.go index b24de208c7..fffc7cb7a7 100644 --- a/action/protocol/staking/bucket_pool_test.go +++ b/action/protocol/staking/bucket_pool_test.go @@ -157,7 +157,7 @@ func TestBucketPool(t *testing.T) { } if !testGreenland && v.postGreenland { - _, err = sm.PutState(c.BaseView().bucketPool.total, protocol.NamespaceOption(StakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) + _, err = sm.PutState(c.BaseView().bucketPool.total, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) r.NoError(err) testGreenland = true } @@ -165,7 +165,7 @@ func TestBucketPool(t *testing.T) { // verify state has been created successfully var b totalAmount - _, err = sm.State(&b, protocol.NamespaceOption(StakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) + _, err = sm.State(&b, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) r.NoError(err) r.Equal(total, b.amount) r.Equal(count, b.count) diff --git a/action/protocol/staking/candidate_statemanager.go b/action/protocol/staking/candidate_statemanager.go index a0ded16209..950e680a18 100644 --- a/action/protocol/staking/candidate_statemanager.go +++ b/action/protocol/staking/candidate_statemanager.go @@ -196,7 +196,7 @@ func (csm *candSM) updateBucket(index uint64, bucket *VoteBucket) error { _, err := csm.PutState( bucket, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(bucketKey(index))) return err } @@ -205,7 +205,7 @@ func (csm *candSM) putBucket(bucket *VoteBucket) (uint64, error) { var tc totalBucketCount if _, err := csm.State( &tc, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(TotalBucketKey)); err != nil && errors.Cause(err) != state.ErrStateNotExist { return 0, err } @@ -215,21 +215,21 @@ func (csm *candSM) putBucket(bucket *VoteBucket) (uint64, error) { bucket.Index = index if _, err := csm.PutState( bucket, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(bucketKey(index))); err != nil { return 0, err } tc.count++ _, err := csm.PutState( &tc, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(TotalBucketKey)) return index, err } func (csm *candSM) delBucket(index uint64) error { _, err := csm.DelState( - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(bucketKey(index))) return err } @@ -272,14 +272,14 @@ func (csm *candSM) putBucketIndex(addr address.Address, prefix byte, index uint6 ) if _, err := csm.State( &bis, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(key)); err != nil && errors.Cause(err) != state.ErrStateNotExist { return err } bis.addBucketIndex(index) _, err := csm.PutState( &bis, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(key)) return err } @@ -295,7 +295,7 @@ func (csm *candSM) delBucketIndex(addr address.Address, prefix byte, index uint6 ) if _, err := csm.State( &bis, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(key)); err != nil { return err } @@ -304,12 +304,12 @@ func (csm *candSM) delBucketIndex(addr address.Address, prefix byte, index uint6 var err error if len(bis) == 0 { _, err = csm.DelState( - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(key)) } else { _, err = csm.PutState( &bis, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(key)) } return err @@ -320,7 +320,7 @@ func (csm *candSM) delVoterBucketIndex(addr address.Address, index uint64) error } func (csm *candSM) putCandidate(d *Candidate) error { - _, err := csm.PutState(d, protocol.NamespaceOption(CandidateNameSpace), protocol.KeyOption(d.Owner.Bytes())) + _, err := csm.PutState(d, protocol.NamespaceOption(_candidateNameSpace), protocol.KeyOption(d.Owner.Bytes())) return err } @@ -329,7 +329,7 @@ func (csm *candSM) putCandBucketIndex(addr address.Address, index uint64) error } func (csm *candSM) delCandidate(name address.Address) error { - _, err := csm.DelState(protocol.NamespaceOption(CandidateNameSpace), protocol.KeyOption(name.Bytes())) + _, err := csm.DelState(protocol.NamespaceOption(_candidateNameSpace), protocol.KeyOption(name.Bytes())) return err } diff --git a/action/protocol/staking/candidate_statereader.go b/action/protocol/staking/candidate_statereader.go index dad079aab2..a5b7d4ba48 100644 --- a/action/protocol/staking/candidate_statereader.go +++ b/action/protocol/staking/candidate_statereader.go @@ -203,7 +203,7 @@ func (c *candSR) getTotalBucketCount() (uint64, error) { var tc totalBucketCount _, err := c.State( &tc, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(TotalBucketKey)) return tc.count, err } @@ -215,14 +215,14 @@ func (c *candSR) getBucket(index uint64) (*VoteBucket, error) { ) if _, err = c.State( &vb, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(bucketKey(index))); err != nil { return nil, err } var tc totalBucketCount if _, err := c.State( &tc, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(TotalBucketKey)); err != nil && errors.Cause(err) != state.ErrStateNotExist { return nil, err } @@ -234,7 +234,7 @@ func (c *candSR) getBucket(index uint64) (*VoteBucket, error) { func (c *candSR) getAllBuckets() ([]*VoteBucket, uint64, error) { height, iter, err := c.States( - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeysOption(func() ([][]byte, error) { // TODO (zhi): fix potential racing issue count, err := c.getTotalBucketCount() @@ -285,7 +285,7 @@ func (c *candSR) getBucketIndices(addr address.Address, prefix byte) (*BucketInd ) height, err := c.State( &bis, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(key)) if err != nil { return nil, height, err @@ -306,12 +306,12 @@ func (c *candSR) getCandidate(name address.Address) (*Candidate, uint64, error) return nil, 0, ErrNilParameters } var d Candidate - height, err := c.State(&d, protocol.NamespaceOption(CandidateNameSpace), protocol.KeyOption(name.Bytes())) + height, err := c.State(&d, protocol.NamespaceOption(_candidateNameSpace), protocol.KeyOption(name.Bytes())) return &d, height, err } func (c *candSR) getAllCandidates() (CandidateList, uint64, error) { - height, iter, err := c.States(protocol.NamespaceOption(CandidateNameSpace)) + height, iter, err := c.States(protocol.NamespaceOption(_candidateNameSpace)) if err != nil { return nil, height, err } @@ -336,7 +336,7 @@ func (c *candSR) NewBucketPool(enableSMStorage bool) (*BucketPool, error) { } if bp.enableSMStorage { - switch _, err := c.State(bp.total, protocol.NamespaceOption(StakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)); errors.Cause(err) { + switch _, err := c.State(bp.total, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)); errors.Cause(err) { case nil: return &bp, nil case state.ErrStateNotExist: diff --git a/action/protocol/staking/handlers_test.go b/action/protocol/staking/handlers_test.go index 1b9727afe5..51bb33dffa 100644 --- a/action/protocol/staking/handlers_test.go +++ b/action/protocol/staking/handlers_test.go @@ -65,7 +65,7 @@ func TestProtocol_HandleCreateStake(t *testing.T) { csr := newCandidateStateReader(sm) _, err := sm.PutState( &totalBucketCount{count: 0}, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(TotalBucketKey), ) require.NoError(err) @@ -2658,7 +2658,7 @@ func initAll(t *testing.T, ctrl *gomock.Controller) (protocol.StateManager, *Pro csm := newCandidateStateManager(sm) _, err := sm.PutState( &totalBucketCount{count: 0}, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(TotalBucketKey), ) require.NoError(err) diff --git a/action/protocol/staking/protocol.go b/action/protocol/staking/protocol.go index 69c10fa304..49ee758c1e 100644 --- a/action/protocol/staking/protocol.go +++ b/action/protocol/staking/protocol.go @@ -34,11 +34,11 @@ const ( // _protocolID is the protocol ID _protocolID = "staking" - // StakingNameSpace is the bucket name for staking state - StakingNameSpace = "Staking" + // _stakingNameSpace is the bucket name for staking state + _stakingNameSpace = "Staking" - // CandidateNameSpace is the bucket name for candidate state - CandidateNameSpace = "Candidate" + // _candidateNameSpace is the bucket name for candidate state + _candidateNameSpace = "Candidate" ) const ( @@ -239,7 +239,7 @@ func (p *Protocol) CreatePreStates(ctx context.Context, sm protocol.StateManager if err != nil { return err } - if _, err = sm.PutState(csr.BaseView().bucketPool.total, protocol.NamespaceOption(StakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)); err != nil { + if _, err = sm.PutState(csr.BaseView().bucketPool.total, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)); err != nil { return err } } diff --git a/action/protocol/staking/protocol_test.go b/action/protocol/staking/protocol_test.go index 7bb25c6951..365e6a9165 100644 --- a/action/protocol/staking/protocol_test.go +++ b/action/protocol/staking/protocol_test.go @@ -43,7 +43,7 @@ func TestProtocol(t *testing.T) { csmTemp := newCandidateStateManager(sm) _, err := sm.PutState( &totalBucketCount{count: 0}, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(TotalBucketKey), ) r.NoError(err) @@ -207,7 +207,7 @@ func TestCreatePreStates(t *testing.T) { _, err = NewCandidateStateManager(sm, true) require.Error(err) require.NoError(p.CreatePreStates(ctx, sm)) - _, err = sm.State(nil, protocol.NamespaceOption(StakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) + _, err = sm.State(nil, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) require.EqualError(errors.Cause(err), state.ErrStateNotExist.Error()) ctx = protocol.WithBlockCtx( ctx, @@ -216,7 +216,7 @@ func TestCreatePreStates(t *testing.T) { }, ) require.NoError(p.CreatePreStates(ctx, sm)) - _, err = sm.State(nil, protocol.NamespaceOption(StakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) + _, err = sm.State(nil, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) require.EqualError(errors.Cause(err), state.ErrStateNotExist.Error()) ctx = protocol.WithBlockCtx( ctx, @@ -226,7 +226,7 @@ func TestCreatePreStates(t *testing.T) { ) require.NoError(p.CreatePreStates(ctx, sm)) total := &totalAmount{} - _, err = sm.State(total, protocol.NamespaceOption(StakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) + _, err = sm.State(total, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) require.NoError(err) } diff --git a/action/protocol/staking/read_state.go b/action/protocol/staking/read_state.go index 49fe163aaf..0df2234849 100644 --- a/action/protocol/staking/read_state.go +++ b/action/protocol/staking/read_state.go @@ -76,7 +76,7 @@ func getTotalStakedAmount(ctx context.Context, csr CandidateStateReader) (*big.I if featureCtx.ReadStateFromDB(csr.Height()) { // after Greenland, read state from db var total totalAmount - h, err := csr.SR().State(&total, protocol.NamespaceOption(StakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) + h, err := csr.SR().State(&total, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) if err != nil { return nil, h, err } @@ -92,7 +92,7 @@ func getActiveBucketsCount(ctx context.Context, csr CandidateStateReader) (uint6 if featureCtx.ReadStateFromDB(csr.Height()) { // after Greenland, read state from db var total totalAmount - h, err := csr.SR().State(&total, protocol.NamespaceOption(StakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) + h, err := csr.SR().State(&total, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(_bucketPoolAddrKey)) if err != nil { return 0, h, err } diff --git a/action/protocol/staking/vote_bucket_test.go b/action/protocol/staking/vote_bucket_test.go index 3bd6f7cf9b..6cf6eaacf7 100644 --- a/action/protocol/staking/vote_bucket_test.go +++ b/action/protocol/staking/vote_bucket_test.go @@ -36,7 +36,7 @@ func TestGetPutStaking(t *testing.T) { csr := newCandidateStateReader(sm) sm.PutState( &totalBucketCount{count: 0}, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(TotalBucketKey), ) diff --git a/action/protocol/staking/vote_reviser_test.go b/action/protocol/staking/vote_reviser_test.go index eb54488bed..c7e2bf55e3 100644 --- a/action/protocol/staking/vote_reviser_test.go +++ b/action/protocol/staking/vote_reviser_test.go @@ -26,7 +26,7 @@ func TestVoteReviser(t *testing.T) { csr := newCandidateStateReader(sm) _, err := sm.PutState( &totalBucketCount{count: 0}, - protocol.NamespaceOption(StakingNameSpace), + protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(TotalBucketKey), ) r.NoError(err) diff --git a/e2etest/native_staking_test.go b/e2etest/native_staking_test.go index dd55f01f31..fd12745b9e 100644 --- a/e2etest/native_staking_test.go +++ b/e2etest/native_staking_test.go @@ -34,6 +34,8 @@ const ( _bucket _voterIndex _candIndex + _stakingNameSpace = "Staking" + _candidateNameSpace = "Candidate" candidate1Name = "candidate1" candidate2Name = "candidate2" @@ -226,12 +228,12 @@ func TestNativeStaking(t *testing.T) { // check buckets var bis staking.BucketIndices - _, err = sf.State(&bis, protocol.NamespaceOption(staking.StakingNameSpace), + _, err = sf.State(&bis, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(staking.AddrKeyWithPrefix(voter1Addr, _voterIndex))) require.Error(err) require.Equal(state.ErrStateNotExist, errors.Cause(err)) - _, err = sf.State(&bis, protocol.NamespaceOption(staking.StakingNameSpace), + _, err = sf.State(&bis, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(staking.AddrKeyWithPrefix(voter2Addr, _voterIndex))) require.NoError(err) require.Equal(2, len(bis)) @@ -378,12 +380,12 @@ func TestNativeStaking(t *testing.T) { require.Equal(hash.BytesToHash256(cand1Addr.Bytes()), logs[0].Topics[2]) // check buckets - _, err = sf.State(&bis, protocol.NamespaceOption(staking.StakingNameSpace), + _, err = sf.State(&bis, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(staking.AddrKeyWithPrefix(cand1Addr, _voterIndex))) require.NoError(err) require.Equal(1, len(bis)) - _, err = sf.State(&bis, protocol.NamespaceOption(staking.StakingNameSpace), + _, err = sf.State(&bis, protocol.NamespaceOption(_stakingNameSpace), protocol.KeyOption(staking.AddrKeyWithPrefix(cand1Addr, _candIndex))) require.NoError(err) require.Equal(2, len(bis)) @@ -435,7 +437,7 @@ func checkCandidateState( candidateAddr address.Address, ) error { var cand staking.Candidate - if _, err := sr.State(&cand, protocol.NamespaceOption(staking.CandidateNameSpace), protocol.KeyOption(candidateAddr.Bytes())); err != nil { + if _, err := sr.State(&cand, protocol.NamespaceOption(_candidateNameSpace), protocol.KeyOption(candidateAddr.Bytes())); err != nil { return err } if expectedName != cand.Name { From 40143257786aa8e41f804f58b6844282b99926a4 Mon Sep 17 00:00:00 2001 From: dustinxie Date: Tue, 19 Jul 2022 09:53:24 -0700 Subject: [PATCH 09/14] [httputil] add ReadHeaderTimeout (#3550) --- api/http.go | 16 ++++------ pkg/probe/probe.go | 2 +- pkg/util/httputil/httputil.go | 50 ++++++++++++++++++++++++------ pkg/util/httputil/httputil_test.go | 13 ++++---- server/itx/server.go | 2 +- 5 files changed, 55 insertions(+), 28 deletions(-) diff --git a/api/http.go b/api/http.go index 66579ba1bd..f652fa336c 100644 --- a/api/http.go +++ b/api/http.go @@ -11,6 +11,7 @@ import ( apitypes "github.com/iotexproject/iotex-core/api/types" "github.com/iotexproject/iotex-core/pkg/log" + "github.com/iotexproject/iotex-core/pkg/util/httputil" ) type ( @@ -26,22 +27,17 @@ type ( ) // NewHTTPServer creates a new http server -// TODO: move timeout into config func NewHTTPServer(route string, port int, handler http.Handler) *HTTPServer { if port == 0 { return nil } - svr := &HTTPServer{ - svr: &http.Server{ - Addr: ":" + strconv.Itoa(port), - WriteTimeout: 30 * time.Second, - ReadHeaderTimeout: 10 * time.Second, - }, - } mux := http.NewServeMux() mux.Handle("/"+route, handler) - svr.svr.Handler = mux - return svr + + svr := httputil.NewServer(":"+strconv.Itoa(port), mux, httputil.ReadHeaderTimeout(10*time.Second)) + return &HTTPServer{ + svr: &svr, + } } // Start starts the http server diff --git a/pkg/probe/probe.go b/pkg/probe/probe.go index d88f9a514a..6484e174af 100644 --- a/pkg/probe/probe.go +++ b/pkg/probe/probe.go @@ -55,7 +55,7 @@ func New(port int, opts ...Option) *Server { mux.HandleFunc("/health", readiness) mux.Handle("/metrics", promhttp.Handler()) - s.server = httputil.Server(fmt.Sprintf(":%d", port), mux) + s.server = httputil.NewServer(fmt.Sprintf(":%d", port), mux) return s } diff --git a/pkg/util/httputil/httputil.go b/pkg/util/httputil/httputil.go index d5dbea833b..5d3aa63194 100644 --- a/pkg/util/httputil/httputil.go +++ b/pkg/util/httputil/httputil.go @@ -10,19 +10,49 @@ import ( const ( _connectionCount = 400 - _readTimeout = 35 * time.Second - _writeTimeout = 35 * time.Second - _idleTimeout = 120 * time.Second ) -// Server creates a HTTP server with time out settings. -func Server(addr string, handler http.Handler) http.Server { +type ( + // ServerOption is a server option + ServerOption func(*serverConfig) + + serverConfig struct { + ReadHeaderTimeout time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration + IdleTimeout time.Duration + } +) + +// DefaultServerConfig is the default server config +var DefaultServerConfig = serverConfig{ + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 120 * time.Second, +} + +// ReadHeaderTimeout sets header timeout +func ReadHeaderTimeout(h time.Duration) ServerOption { + return func(cfg *serverConfig) { + cfg.ReadHeaderTimeout = h + } +} + +// NewServer creates a HTTP server with time out settings. +func NewServer(addr string, handler http.Handler, opts ...ServerOption) http.Server { + cfg := DefaultServerConfig + for _, opt := range opts { + opt(&cfg) + } + return http.Server{ - ReadTimeout: _readTimeout, - WriteTimeout: _writeTimeout, - IdleTimeout: _idleTimeout, - Addr: addr, - Handler: handler, + ReadHeaderTimeout: cfg.ReadHeaderTimeout, + ReadTimeout: cfg.ReadTimeout, + WriteTimeout: cfg.WriteTimeout, + IdleTimeout: cfg.IdleTimeout, + Addr: addr, + Handler: handler, } } diff --git a/pkg/util/httputil/httputil_test.go b/pkg/util/httputil/httputil_test.go index 29189ed983..2c98dc9b78 100644 --- a/pkg/util/httputil/httputil_test.go +++ b/pkg/util/httputil/httputil_test.go @@ -15,13 +15,14 @@ func TestServer(t *testing.T) { addr := "myAddress" expectValue := http.Server{ - ReadTimeout: 35 * time.Second, - WriteTimeout: 35 * time.Second, - IdleTimeout: 120 * time.Second, - Addr: addr, - Handler: handler, + ReadHeaderTimeout: 2 * time.Second, + ReadTimeout: DefaultServerConfig.ReadTimeout, + WriteTimeout: DefaultServerConfig.WriteTimeout, + IdleTimeout: DefaultServerConfig.IdleTimeout, + Addr: addr, + Handler: handler, } - result := Server(addr, handler) + result := NewServer(addr, handler, ReadHeaderTimeout(2*time.Second)) require.Equal(t, expectValue, result) }) } diff --git a/server/itx/server.go b/server/itx/server.go index 1c550b6886..69633b6ad3 100644 --- a/server/itx/server.go +++ b/server/itx/server.go @@ -243,7 +243,7 @@ func StartServer(ctx context.Context, svr *Server, probeSvr *probe.Server, cfg c mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) port := fmt.Sprintf(":%d", cfg.System.HTTPAdminPort) - adminserv = httputil.Server(port, mux) + adminserv = httputil.NewServer(port, mux) defer func() { if err := adminserv.Shutdown(ctx); err != nil { log.L().Error("Error when serving metrics data.", zap.Error(err)) From 4a7b0b06a8167c806faa43f0558fcd048eb38fb3 Mon Sep 17 00:00:00 2001 From: huofei <68298506@qq.com> Date: Wed, 20 Jul 2022 02:08:13 +0800 Subject: [PATCH 10/14] [ioctl] Modify file permission as 0600 (#3563) --- ioctl/cmd/contract/contractshare.go | 4 ++-- pkg/log/log.go | 2 +- pkg/recovery/recovery.go | 2 +- tools/addrgen/internal/cmd/createconfig.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ioctl/cmd/contract/contractshare.go b/ioctl/cmd/contract/contractshare.go index 0827cf5d0a..f47de65a49 100644 --- a/ioctl/cmd/contract/contractshare.go +++ b/ioctl/cmd/contract/contractshare.go @@ -92,7 +92,7 @@ func isDir(path string) bool { func isReadOnly(path string) bool { var readOnly = false - file, err := os.OpenFile(filepath.Clean(path), os.O_WRONLY, 0666) + file, err := os.OpenFile(filepath.Clean(path), os.O_WRONLY, 0600) if err != nil { if os.IsPermission(err) { log.Println("Error: Write permission denied.") @@ -255,7 +255,7 @@ func share(args []string) error { log.Println("mkdir failed: ", err) break } - if err := os.WriteFile(setPath, []byte(content), 0644); err != nil { + if err := os.WriteFile(setPath, []byte(content), 0600); err != nil { log.Println("set file failed: ", err) break } diff --git a/pkg/log/log.go b/pkg/log/log.go index 74b533e7f5..bb992f3c38 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -90,7 +90,7 @@ func InitLoggers(globalCfg GlobalConfig, subCfgs map[string]GlobalConfig, opts . return err } if cfg.StderrRedirectFile != nil { - stderrF, err := os.OpenFile(*cfg.StderrRedirectFile, os.O_WRONLY|os.O_CREATE|os.O_SYNC|os.O_APPEND, 0644) + stderrF, err := os.OpenFile(*cfg.StderrRedirectFile, os.O_WRONLY|os.O_CREATE|os.O_SYNC|os.O_APPEND, 0600) if err != nil { return err } diff --git a/pkg/recovery/recovery.go b/pkg/recovery/recovery.go index 3d2d4fe957..6d82330dbb 100644 --- a/pkg/recovery/recovery.go +++ b/pkg/recovery/recovery.go @@ -79,7 +79,7 @@ func LogCrash(r interface{}) { } func writeHeapProfile(path string) { - f, err := os.OpenFile(filepath.Clean(path), os.O_CREATE|os.O_RDWR, 0644) + f, err := os.OpenFile(filepath.Clean(path), os.O_CREATE|os.O_RDWR, 0600) if err != nil { log.S().Errorf("crashlog: open heap profile error: %v", err) return diff --git a/tools/addrgen/internal/cmd/createconfig.go b/tools/addrgen/internal/cmd/createconfig.go index 2aea23de02..3d642b1756 100644 --- a/tools/addrgen/internal/cmd/createconfig.go +++ b/tools/addrgen/internal/cmd/createconfig.go @@ -37,7 +37,7 @@ var createConfigCmd = &cobra.Command{ priKeyBytes, pubKeyBytes, ) - if err := os.WriteFile(_outputFile, []byte(cfgStr), 0666); err != nil { + if err := os.WriteFile(_outputFile, []byte(cfgStr), 0600); err != nil { log.L().Fatal("failed to write file", zap.Error(err)) } }, From 4e1a2b997afe4ef691498f13a45356da12092add Mon Sep 17 00:00:00 2001 From: huofei <68298506@qq.com> Date: Wed, 20 Jul 2022 02:52:08 +0800 Subject: [PATCH 11/14] Add MinVersion in tls.Config (#3562) --- ioctl/client.go | 2 +- ioctl/util/util.go | 2 +- tools/actioninjector.v2/internal/client/client.go | 2 +- tools/actioninjector.v2/internal/cmd/inject.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ioctl/client.go b/ioctl/client.go index 3d2e739f9a..ed85ab6da8 100644 --- a/ioctl/client.go +++ b/ioctl/client.go @@ -214,7 +214,7 @@ func (c *client) APIServiceClient() (iotexapi.APIServiceClient, error) { if c.insecure { c.conn, err = grpc.Dial(c.endpoint, grpc.WithInsecure()) } else { - c.conn, err = grpc.Dial(c.endpoint, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{}))) + c.conn, err = grpc.Dial(c.endpoint, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12}))) } if err != nil { return nil, err diff --git a/ioctl/util/util.go b/ioctl/util/util.go index fcf1f394a8..f34b219c92 100644 --- a/ioctl/util/util.go +++ b/ioctl/util/util.go @@ -57,7 +57,7 @@ func ConnectToEndpoint(secure bool) (*grpc.ClientConn, error) { if !secure { return grpc.Dial(endpoint, grpc.WithInsecure()) } - return grpc.Dial(endpoint, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{}))) + return grpc.Dial(endpoint, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12}))) } // StringToRau converts different unit string into Rau big int diff --git a/tools/actioninjector.v2/internal/client/client.go b/tools/actioninjector.v2/internal/client/client.go index d328ab1fd6..547ef1b115 100644 --- a/tools/actioninjector.v2/internal/client/client.go +++ b/tools/actioninjector.v2/internal/client/client.go @@ -31,7 +31,7 @@ func New(serverAddr string, insecure bool) (*Client, error) { conn, err = grpc.DialContext(grpcctx, serverAddr, grpc.WithBlock(), grpc.WithInsecure()) } else { log.L().Info("secure connection") - conn, err = grpc.DialContext(grpcctx, serverAddr, grpc.WithBlock(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{}))) + conn, err = grpc.DialContext(grpcctx, serverAddr, grpc.WithBlock(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12}))) } if err != nil { return nil, err diff --git a/tools/actioninjector.v2/internal/cmd/inject.go b/tools/actioninjector.v2/internal/cmd/inject.go index 1a03076938..145e861bfe 100644 --- a/tools/actioninjector.v2/internal/cmd/inject.go +++ b/tools/actioninjector.v2/internal/cmd/inject.go @@ -73,7 +73,7 @@ func newInjectionProcessor() (*injectProcessor, error) { conn, err = grpc.DialContext(grpcctx, injectCfg.serverAddr, grpc.WithBlock(), grpc.WithInsecure()) } else { log.L().Info("secure connection") - conn, err = grpc.DialContext(grpcctx, injectCfg.serverAddr, grpc.WithBlock(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{}))) + conn, err = grpc.DialContext(grpcctx, injectCfg.serverAddr, grpc.WithBlock(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12}))) } if err != nil { return nil, err From a969b74c201ef9e707124b858e7e20d05971573e Mon Sep 17 00:00:00 2001 From: Haaai <55118568+Liuhaai@users.noreply.github.com> Date: Tue, 19 Jul 2022 12:21:30 -0700 Subject: [PATCH 12/14] [action] Refactor handleTransfer() (#3557) --- action/protocol/account/transfer.go | 36 +++++++++++++---------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/action/protocol/account/transfer.go b/action/protocol/account/transfer.go index 929088c83d..8c267e8e79 100644 --- a/action/protocol/account/transfer.go +++ b/action/protocol/account/transfer.go @@ -29,11 +29,15 @@ func (p *Protocol) handleTransfer(ctx context.Context, act action.Action, sm pro if !ok { return nil, nil } + var ( + fCtx = protocol.MustGetFeatureCtx(ctx) + actionCtx = protocol.MustGetActionCtx(ctx) + blkCtx = protocol.MustGetBlockCtx(ctx) + ) accountCreationOpts := []state.AccountCreationOption{} - if protocol.MustGetFeatureCtx(ctx).CreateLegacyNonceAccount { + if fCtx.CreateLegacyNonceAccount { accountCreationOpts = append(accountCreationOpts, state.LegacyNonceAccountTypeOption()) } - actionCtx := protocol.MustGetActionCtx(ctx) // check sender sender, err := accountutil.LoadOrCreateAccount(sm, actionCtx.Caller, accountCreationOpts...) if err != nil { @@ -51,10 +55,7 @@ func (p *Protocol) handleTransfer(ctx context.Context, act action.Action, sm pro ) } - var ( - depositLog *action.TransactionLog - fCtx = protocol.MustGetFeatureCtx(ctx) - ) + var depositLog *action.TransactionLog if !fCtx.FixDoubleChargeGas { // charge sender gas if err := sender.SubBalance(gasFee); err != nil { @@ -68,6 +69,13 @@ func (p *Protocol) handleTransfer(ctx context.Context, act action.Action, sm pro } } + if fCtx.FixGasAndNonceUpdate || tsf.Nonce() != 0 { + // update sender Nonce + if err := sender.SetPendingNonce(tsf.Nonce() + 1); err != nil { + return nil, errors.Wrapf(err, "failed to update pending nonce of sender %s", actionCtx.Caller.String()) + } + } + var recipientAddr address.Address if fCtx.TolerateLegacyAddress { recipientAddr, err = address.FromStringLegacy(tsf.Recipient()) @@ -81,15 +89,7 @@ func (p *Protocol) handleTransfer(ctx context.Context, act action.Action, sm pro if err != nil { return nil, errors.Wrapf(err, "failed to load address %s", tsf.Recipient()) } - fixNonce := protocol.MustGetFeatureCtx(ctx).FixGasAndNonceUpdate - blkCtx := protocol.MustGetBlockCtx(ctx) if recipientAcct.IsContract() { - if fixNonce || tsf.Nonce() != 0 { - // update sender Nonce - if err := sender.SetPendingNonce(tsf.Nonce() + 1); err != nil { - return nil, errors.Wrapf(err, "failed to update pending nonce of sender %s", actionCtx.Caller.String()) - } - } // put updated sender's state to trie if err := accountutil.StoreAccount(sm, actionCtx.Caller, sender); err != nil { return nil, errors.Wrap(err, "failed to update pending account changes to trie") @@ -117,16 +117,12 @@ func (p *Protocol) handleTransfer(ctx context.Context, act action.Action, sm pro if err := sender.SubBalance(tsf.Amount()); err != nil { return nil, errors.Wrapf(err, "failed to update the Balance of sender %s", actionCtx.Caller.String()) } - if fixNonce || tsf.Nonce() != 0 { - // update sender Nonce - if err := sender.SetPendingNonce(tsf.Nonce() + 1); err != nil { - return nil, errors.Wrapf(err, "failed to update pending nonce of sender %s", actionCtx.Caller.String()) - } - } + // put updated sender's state to trie if err := accountutil.StoreAccount(sm, actionCtx.Caller, sender); err != nil { return nil, errors.Wrap(err, "failed to update pending account changes to trie") } + // check recipient recipient, err := accountutil.LoadOrCreateAccount(sm, recipientAddr, accountCreationOpts...) if err != nil { From e209d4ebb728abc9189b8acd360c289a222e7d3c Mon Sep 17 00:00:00 2001 From: huofei <68298506@qq.com> Date: Wed, 20 Jul 2022 09:10:42 +0800 Subject: [PATCH 13/14] [pkg] fix deferring unsafe method "Close" on type "*os.File" (#3548) --- ioctl/doc/doc.go | 18 ++++++++++-------- pkg/recovery/recovery.go | 8 +++++++- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/ioctl/doc/doc.go b/ioctl/doc/doc.go index d887ccf4bc..37573ca7ae 100644 --- a/ioctl/doc/doc.go +++ b/ioctl/doc/doc.go @@ -16,12 +16,12 @@ import ( // GenMarkdownTreeCustom is the the same as GenMarkdownTree, but // with custom filePrepender and linkHandler. func GenMarkdownTreeCustom(c *cobra.Command, dir string, name string, path string, filePrepender func(string) string, - linkHandler func(*cobra.Command, string) string) error { + linkHandler func(*cobra.Command, string) string) (err error) { for _, child := range c.Commands() { if !child.IsAvailableCommand() || child.IsAdditionalHelpTopicCommand() { continue } - if err := GenMarkdownTreeCustom(child, dir, name, path, filePrepender, linkHandler); err != nil { + if err = GenMarkdownTreeCustom(child, dir, name, path, filePrepender, linkHandler); err != nil { return err } } @@ -32,19 +32,21 @@ func GenMarkdownTreeCustom(c *cobra.Command, dir string, name string, path strin filename = filepath.Join(path, "README.md") } - f, err := os.Create(filepath.Clean(filename)) + var f *os.File + f, err = os.Create(filepath.Clean(filename)) if err != nil { return err } - defer f.Close() - - if _, err := io.WriteString(f, filePrepender(filename)); err != nil { + defer func() { + err = f.Close() + }() + if _, err = io.WriteString(f, filePrepender(filename)); err != nil { return err } - if err := GenMarkdownCustom(c, f, linkHandler); err != nil { + if err = GenMarkdownCustom(c, f, linkHandler); err != nil { return err } - return nil + return err } // GenMarkdownCustom creates custom markdown output. diff --git a/pkg/recovery/recovery.go b/pkg/recovery/recovery.go index 6d82330dbb..b9c012e494 100644 --- a/pkg/recovery/recovery.go +++ b/pkg/recovery/recovery.go @@ -84,9 +84,15 @@ func writeHeapProfile(path string) { log.S().Errorf("crashlog: open heap profile error: %v", err) return } - defer f.Close() + defer func() { + if err = f.Close(); err != nil { + log.S().Errorf("crashlog: close heap profile error: %v", err) + return + } + }() if err := pprof.WriteHeapProfile(f); err != nil { log.S().Errorf("crashlog: write heap profile error: %v", err) + return } } From cc09a6c8e963abaa4d5059d9fa61ea9bffa373af Mon Sep 17 00:00:00 2001 From: Haaai <55118568+Liuhaai@users.noreply.github.com> Date: Tue, 19 Jul 2022 22:39:19 -0700 Subject: [PATCH 14/14] [test] Disable workingset cache in the benchmark test (#3558) --- blockchain/blockdao/blockdao.go | 12 ++++++------ blockchain/filedao/filedao_legacy.go | 2 +- blockchain/filedao/filedao_v2.go | 2 +- blockchain/integrity/benchmark_test.go | 4 ++-- db/kvstore_impl.go | 4 ++-- db/kvstorewithcache.go | 4 ++-- go.mod | 2 +- go.sum | 3 ++- state/factory/factory.go | 2 +- state/factory/statedb.go | 10 +++++++++- 10 files changed, 27 insertions(+), 18 deletions(-) diff --git a/blockchain/blockdao/blockdao.go b/blockchain/blockdao/blockdao.go index 0e1995bd3a..68fc45f1d8 100644 --- a/blockchain/blockdao/blockdao.go +++ b/blockchain/blockdao/blockdao.go @@ -52,9 +52,9 @@ type ( indexers []BlockIndexer timerFactory *prometheustimer.TimerFactory lifecycle lifecycle.Lifecycle - headerCache *cache.ThreadSafeLruCache - bodyCache *cache.ThreadSafeLruCache - footerCache *cache.ThreadSafeLruCache + headerCache cache.LRUCache + bodyCache cache.LRUCache + footerCache cache.LRUCache tipHeight uint64 } ) @@ -336,20 +336,20 @@ func createBlockDAO(blkStore filedao.FileDAO, indexers []BlockIndexer, cfg db.Co return blockDAO } -func lruCacheGet(c *cache.ThreadSafeLruCache, key interface{}) (interface{}, bool) { +func lruCacheGet(c cache.LRUCache, key interface{}) (interface{}, bool) { if c != nil { return c.Get(key) } return nil, false } -func lruCachePut(c *cache.ThreadSafeLruCache, k, v interface{}) { +func lruCachePut(c cache.LRUCache, k, v interface{}) { if c != nil { c.Add(k, v) } } -func lruCacheDel(c *cache.ThreadSafeLruCache, k interface{}) { +func lruCacheDel(c cache.LRUCache, k interface{}) { if c != nil { c.Remove(k) } diff --git a/blockchain/filedao/filedao_legacy.go b/blockchain/filedao/filedao_legacy.go index 1279157af8..8bc388119f 100644 --- a/blockchain/filedao/filedao_legacy.go +++ b/blockchain/filedao/filedao_legacy.go @@ -55,7 +55,7 @@ type ( topIndex atomic.Value htf db.RangeIndex kvStore db.KVStore - kvStores *cache.ThreadSafeLruCache //store like map[index]db.KVStore,index from 1...N + kvStores cache.LRUCache //store like map[index]db.KVStore,index from 1...N deser *block.Deserializer } ) diff --git a/blockchain/filedao/filedao_v2.go b/blockchain/filedao/filedao_v2.go index 243f6c014a..8673039dc7 100644 --- a/blockchain/filedao/filedao_v2.go +++ b/blockchain/filedao/filedao_v2.go @@ -42,7 +42,7 @@ type ( header *FileHeader tip *FileTip blkBuffer *stagingBuffer - blkCache *cache.ThreadSafeLruCache + blkCache cache.LRUCache kvStore db.KVStore batch batch.KVStoreBatch hashStore db.CountingIndex // store block hash diff --git a/blockchain/integrity/benchmark_test.go b/blockchain/integrity/benchmark_test.go index 2258439dc2..28a6958fe8 100644 --- a/blockchain/integrity/benchmark_test.go +++ b/blockchain/integrity/benchmark_test.go @@ -223,12 +223,12 @@ func newChainInDB() (blockchain.Blockchain, actpool.ActPool, error) { cfg.Consensus.Scheme = config.RollDPoSScheme cfg.Genesis.BlockGasLimit = genesis.Default.BlockGasLimit * 100 cfg.ActPool.MinGasPriceStr = "0" - cfg.ActPool.MaxNumActsPerAcct = 1000000000 + cfg.ActPool.MaxNumActsPerAcct = 10000 cfg.Genesis.EnableGravityChainVoting = false registry := protocol.NewRegistry() var sf factory.Factory kv := db.NewBoltDB(cfg.DB) - sf, err = factory.NewStateDB(cfg, factory.PrecreatedStateDBOption(kv), factory.RegistryStateDBOption(registry)) + sf, err = factory.NewStateDB(cfg, factory.PrecreatedStateDBOption(kv), factory.RegistryStateDBOption(registry), factory.DisableWorkingSetCacheOption()) if err != nil { return nil, nil, err } diff --git a/db/kvstore_impl.go b/db/kvstore_impl.go index 8946a9d4c4..980fa5f360 100644 --- a/db/kvstore_impl.go +++ b/db/kvstore_impl.go @@ -33,8 +33,8 @@ const ( // memKVStore is the in-memory implementation of KVStore for testing purpose type memKVStore struct { - data *cache.ThreadSafeLruCache - bucket *cache.ThreadSafeLruCache + data cache.LRUCache + bucket cache.LRUCache } // NewMemKVStore instantiates an in-memory KV store diff --git a/db/kvstorewithcache.go b/db/kvstorewithcache.go index 370511c96f..b2bc469fac 100644 --- a/db/kvstorewithcache.go +++ b/db/kvstorewithcache.go @@ -14,7 +14,7 @@ import ( type kvStoreWithCache struct { mutex sync.RWMutex // lock for stateCaches store KVStore - stateCaches map[string]*cache.ThreadSafeLruCache // map having lru cache to store current states for speed up + stateCaches map[string]cache.LRUCache // map having lru cache to store current states for speed up cacheSize int } @@ -22,7 +22,7 @@ type kvStoreWithCache struct { func NewKvStoreWithCache(kvstore KVStore, cacheSize int) KVStore { return &kvStoreWithCache{ store: kvstore, - stateCaches: make(map[string]*cache.ThreadSafeLruCache), + stateCaches: make(map[string]cache.LRUCache), cacheSize: cacheSize, } } diff --git a/go.mod b/go.mod index a29fc3cf17..84baece15a 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/iotexproject/go-fsm v1.0.0 github.com/iotexproject/go-p2p v0.3.4 - github.com/iotexproject/go-pkgs v0.1.12 + github.com/iotexproject/go-pkgs v0.1.13 github.com/iotexproject/iotex-address v0.2.8 github.com/iotexproject/iotex-antenna-go/v2 v2.5.1 github.com/iotexproject/iotex-election v0.3.5-0.20210611041425-20ddf674363d diff --git a/go.sum b/go.sum index 3f857a2437..092ae5e847 100644 --- a/go.sum +++ b/go.sum @@ -446,8 +446,9 @@ github.com/iotexproject/go-p2p v0.3.4 h1:V+Hq4K4lbEI9gAjHqKLpzS4lqCPqjeosRY/LnIp github.com/iotexproject/go-p2p v0.3.4/go.mod h1:P32N/mwLUWMQ/okMcNLlfBO7iD5ITyBOvN41roKdPdg= github.com/iotexproject/go-pkgs v0.1.5-0.20210604060651-be5ee19f2575/go.mod h1:ttXhcwrtODyh7JozpJlCml09CjP0pcKqTe2B0MbTGc8= github.com/iotexproject/go-pkgs v0.1.6/go.mod h1:E+Fl0XQZz56EzXFajFET2yFoRGsIbL6wQooVIkSim3o= -github.com/iotexproject/go-pkgs v0.1.12 h1:WQcGDHxC9d2gv7r+8aBK7IgechL0XflimRkh+xN+ybI= github.com/iotexproject/go-pkgs v0.1.12/go.mod h1:t5X9kQ1VL5H+L+DC5GmohXnFKlcxaTcRnIBBuydcsTQ= +github.com/iotexproject/go-pkgs v0.1.13 h1:bK48DVenkfYkC4TRoqL77RLFRBE1MUfscCW495kzcC8= +github.com/iotexproject/go-pkgs v0.1.13/go.mod h1:t5X9kQ1VL5H+L+DC5GmohXnFKlcxaTcRnIBBuydcsTQ= github.com/iotexproject/iotex-address v0.2.4/go.mod h1:K78yPSMB4K7gF/iQ7djT1amph0RBrP3rSkFfH7gNG70= github.com/iotexproject/iotex-address v0.2.5/go.mod h1:K78yPSMB4K7gF/iQ7djT1amph0RBrP3rSkFfH7gNG70= github.com/iotexproject/iotex-address v0.2.7/go.mod h1:K78yPSMB4K7gF/iQ7djT1amph0RBrP3rSkFfH7gNG70= diff --git a/state/factory/factory.go b/state/factory/factory.go index fd2fb9ad74..6a4bb5867b 100644 --- a/state/factory/factory.go +++ b/state/factory/factory.go @@ -99,7 +99,7 @@ type ( twoLayerTrie trie.TwoLayerTrie // global state trie, this is a read only trie dao db.KVStore // the underlying DB for account/contract storage timerFactory *prometheustimer.TimerFactory - workingsets *cache.ThreadSafeLruCache // lru cache for workingsets + workingsets cache.LRUCache // lru cache for workingsets protocolView protocol.View skipBlockValidationOnPut bool ps *patchStore diff --git a/state/factory/statedb.go b/state/factory/statedb.go index cd1312a86c..2a4fe0b69e 100644 --- a/state/factory/statedb.go +++ b/state/factory/statedb.go @@ -44,7 +44,7 @@ type stateDB struct { registry *protocol.Registry dao db.KVStore // the underlying DB for account/contract storage timerFactory *prometheustimer.TimerFactory - workingsets *cache.ThreadSafeLruCache // lru cache for workingsets + workingsets cache.LRUCache // lru cache for workingsets protocolView protocol.View skipBlockValidationOnPut bool ps *patchStore @@ -124,6 +124,14 @@ func SkipBlockValidationStateDBOption() StateDBOption { } } +// DisableWorkingSetCacheOption disable workingset cache +func DisableWorkingSetCacheOption() StateDBOption { + return func(sdb *stateDB, cfg config.Config) error { + sdb.workingsets = cache.NewDummyLruCache() + return nil + } +} + // NewStateDB creates a new state db func NewStateDB(cfg config.Config, opts ...StateDBOption) (Factory, error) { sdb := stateDB{