Skip to content

Commit

Permalink
Merge pull request #1 from dirkmc/feat/rcvd-cids-bottleneck
Browse files Browse the repository at this point in the history
  • Loading branch information
@raulk
raulk committed Jun 8, 2021
2 parents 73a7d63 + 4fb5491 commit 3599b24
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 11 deletions.
5 changes: 2 additions & 3 deletions testplan/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,9 @@ go 1.15
require (
github.com/davecgh/go-spew v1.1.1
github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434
github.com/filecoin-project/go-data-transfer v1.2.7
github.com/filecoin-project/go-fil-markets v1.1.9
github.com/filecoin-project/go-data-transfer v1.6.1-0.20210604151507-da2f4ddd460b
github.com/filecoin-project/go-fil-markets v1.5.0
github.com/filecoin-project/go-state-types v0.1.0
github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
github.com/filecoin-project/lotus v1.5.2
github.com/ipfs/go-blockservice v0.1.4
github.com/ipfs/go-cid v0.0.7
Expand Down
12 changes: 12 additions & 0 deletions testplan/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bep/debounce v1.2.0 h1:wXds8Kq8qRfwAOpAxHrJDbCXgC5aHSzgQb/0gKsHQqo=
github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
github.com/briandowns/spinner v1.11.1/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ=
Expand Down Expand Up @@ -236,13 +238,19 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod
github.com/filecoin-project/go-data-transfer v1.0.1/go.mod h1:UxvfUAY9v3ub0a21BSK9u3pB2aq30Y0KMsG+w9/ysyo=
github.com/filecoin-project/go-data-transfer v1.2.7 h1:WE5Cpp9eMt5BDoWOVR64QegSn6bwHQaDzyyjVU377Y0=
github.com/filecoin-project/go-data-transfer v1.2.7/go.mod h1:mvjZ+C3NkBX10JP4JMu27DCjUouHFjHwUGh+Xc4yvDA=
github.com/filecoin-project/go-data-transfer v1.6.0 h1:DHIzEc23ydRCCBwtFet3MfgO8gMpZEnw60Y+s71oX6o=
github.com/filecoin-project/go-data-transfer v1.6.0/go.mod h1:E3WW4mCEYwU2y65swPEajSZoFWFmfXt7uwGduoACZQc=
github.com/filecoin-project/go-data-transfer v1.6.1-0.20210604151507-da2f4ddd460b h1:m0mPlCbr6NyBFGtMqvB8lwxcimq1JXdplEf/dECGPic=
github.com/filecoin-project/go-data-transfer v1.6.1-0.20210604151507-da2f4ddd460b/go.mod h1:E3WW4mCEYwU2y65swPEajSZoFWFmfXt7uwGduoACZQc=
github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ=
github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s=
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg=
github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c=
github.com/filecoin-project/go-fil-markets v1.1.9 h1:sA0NIEOpy7brZaeXeNgdXg5pvHaBtD5OTRlraOUbI0w=
github.com/filecoin-project/go-fil-markets v1.1.9/go.mod h1:0yQu5gvrjFoAIyzPSSJ+xUdCG83vjInAFbTswIB5/hk=
github.com/filecoin-project/go-fil-markets v1.5.0 h1:3KEs01L8XFCEgujZ6ggFjr1XWjpjTQcmSSeo3I99I0k=
github.com/filecoin-project/go-fil-markets v1.5.0/go.mod h1:7be6zzFwaN8kxVeYZf/UUj/JilHC0ogPvWqE1TW8Ptk=
github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24=
github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM=
Expand Down Expand Up @@ -544,6 +552,8 @@ github.com/ipfs/go-graphsync v0.4.2/go.mod h1:/VmbZTUdUMTbNkgzAiCEucIIAU3BkLE2cZ
github.com/ipfs/go-graphsync v0.4.3/go.mod h1:mPOwDYv128gf8gxPFgXnz4fNrSYPsWyqisJ7ych+XDY=
github.com/ipfs/go-graphsync v0.5.2/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk=
github.com/ipfs/go-graphsync v0.6.0/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk=
github.com/ipfs/go-graphsync v0.6.1 h1:i9wN7YkBXWwIsUjVQeuaDxFB59yWZrG1xL564Nz7aGE=
github.com/ipfs/go-graphsync v0.6.1/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk=
github.com/ipfs/go-graphsync v0.7.0 h1:ZdyU2otZYPjcvduAPwVjCdijmkPtHI1mm1VyZbRQ5KI=
github.com/ipfs/go-graphsync v0.7.0/go.mod h1:YRQg0TyvD2HzFansAZdMcUFBJ8zIJ4K+32kNdnHfHZc=
github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk=
Expand Down Expand Up @@ -838,6 +848,7 @@ github.com/libp2p/go-libp2p-core v0.8.5 h1:aEgbIcPGsKy6zYcC+5AJivYFedhYa4sW7mIpW
github.com/libp2p/go-libp2p-core v0.8.5/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8=
github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE=
github.com/libp2p/go-libp2p-crypto v0.0.2/go.mod h1:eETI5OUfBnvARGOHrJz2eWNyTUxEGZnBxMcbUjfIj4I=
github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ=
github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI=
github.com/libp2p/go-libp2p-daemon v0.2.2/go.mod h1:kyrpsLB2JeNYR2rvXSVWyY0iZuRIMhqzWR3im9BV6NQ=
github.com/libp2p/go-libp2p-discovery v0.0.1/go.mod h1:ZkkF9xIFRLA1xCc7bstYFkd80gBGK8Fc1JqGoU2i+zI=
Expand Down Expand Up @@ -887,6 +898,7 @@ github.com/libp2p/go-libp2p-noise v0.1.2 h1:IH9GRihQJTx56obm+GnpdPX4KeVIlvpXrP6x
github.com/libp2p/go-libp2p-noise v0.1.2/go.mod h1:9B10b7ueo7TIxZHHcjcDCo5Hd6kfKT2m77by82SFRfE=
github.com/libp2p/go-libp2p-peer v0.0.1/go.mod h1:nXQvOBbwVqoP+T5Y5nCjeH4sP9IX/J0AMzcDUVruVoo=
github.com/libp2p/go-libp2p-peer v0.1.1/go.mod h1:jkF12jGB4Gk/IOo+yomm+7oLWxF278F7UnrYUQ1Q8es=
github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUjer50DsY=
github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY=
github.com/libp2p/go-libp2p-peerstore v0.0.1/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20=
github.com/libp2p/go-libp2p-peerstore v0.0.6/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20=
Expand Down
24 changes: 16 additions & 8 deletions testplan/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ import (
"strings"
"time"

"github.com/davecgh/go-spew/spew"
"github.com/filecoin-project/go-commp-utils/pieceio/cario"
"github.com/filecoin-project/go-data-transfer/channelmonitor"
"github.com/filecoin-project/go-commp-utils/writer"
datatransfer "github.com/filecoin-project/go-data-transfer"
dt "github.com/filecoin-project/go-data-transfer/impl"
Expand All @@ -22,12 +22,12 @@ import (
"github.com/filecoin-project/go-fil-markets/shared"
"github.com/filecoin-project/go-fil-markets/storagemarket/impl/requestvalidation"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-storedcounter"
"github.com/filecoin-project/lotus/blockstore"
badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
"github.com/filecoin-project/lotus/node/repo"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
//logger "github.com/ipfs/go-log/v2"
"github.com/ipfs/go-datastore"
dssync "github.com/ipfs/go-datastore/sync"
graphsync "github.com/ipfs/go-graphsync/impl"
Expand Down Expand Up @@ -60,6 +60,10 @@ func main() {
}

func runTransfer(runenv *runtime.RunEnv, initCtx *run.InitContext) error {
//logger.SetLogLevel("dt-impl", "debug")
//logger.SetLogLevel("dt-chanmon", "debug")
//logger.SetLogLevel("dt_graphsync", "debug")

var (
size = runenv.SizeParam("size")

Expand Down Expand Up @@ -103,13 +107,17 @@ func runTransfer(runenv *runtime.RunEnv, initCtx *run.InitContext) error {
// instantiate the data transfer module.
// TODO make retry configurable.
retry := dtnet.RetryParameters(time.Second, 5*time.Minute, 15, 5)
dtRestartConfig := dt.ChannelRestartConfig(channelmonitor.Config{
RestartDebounce: 1 * time.Second,
RestartBackoff: 1 * time.Second,
MaxConsecutiveRestarts: 3,
})
mgr, err := dt.NewDataTransfer(
ds,
dir, // cid lists directory.
dtnet.NewFromLibp2pHost(host, retry),
gst.NewTransport(host.ID(), gs),
storedcounter.New(ds, datastore.NewKey("datatransfer")),
dt.PushChannelRestartConfig(time.Second*30, 10, 1024, 2*time.Minute, 3),
dtRestartConfig,
)
if err != nil {
return err
Expand All @@ -125,7 +133,7 @@ func runTransfer(runenv *runtime.RunEnv, initCtx *run.InitContext) error {
"sent": state.Sent(),
"received": state.Received(),
"queued": state.Queued(),
"received count": len(state.ReceivedCids()),
"received count": state.ReceivedCidsLen(),
"total size": state.TotalSize(),
"remote peer": state.OtherPeer(),
"event message": event.Message,
Expand Down Expand Up @@ -225,7 +233,7 @@ func runTransfer(runenv *runtime.RunEnv, initCtx *run.InitContext) error {
return fmt.Errorf("no in progress channel for ID %s", chID)
}

runenv.RecordMessage("channel state: %v", spew.Sdump(ch))
runenv.RecordMessage("channel state: %s (sent %d)", datatransfer.Statuses[ch.Status()], ch.Sent())
switch ch.Status() {
case datatransfer.Ongoing:
continue // all ok, keep going!
Expand Down Expand Up @@ -382,11 +390,11 @@ func CreatePayload(bs blockstore.Blockstore, size uint64) (cid.Cid, error) {

type OKValidator struct{}

func (v *OKValidator) ValidatePush(peer.ID, datatransfer.Voucher, cid.Cid, ipld.Node) (datatransfer.VoucherResult, error) {
func (v *OKValidator) ValidatePush(bool, peer.ID, datatransfer.Voucher, cid.Cid, ipld.Node) (datatransfer.VoucherResult, error) {
return nil, nil
}

func (v *OKValidator) ValidatePull(peer.ID, datatransfer.Voucher, cid.Cid, ipld.Node) (datatransfer.VoucherResult, error) {
func (v *OKValidator) ValidatePull(bool, peer.ID, datatransfer.Voucher, cid.Cid, ipld.Node) (datatransfer.VoucherResult, error) {
return nil, nil
}

Expand Down

0 comments on commit 3599b24

Please sign in to comment.