-
Notifications
You must be signed in to change notification settings - Fork 223
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
improve query performance by limiting query width to KValue peers #291
Closed
Closed
Changes from 9 commits
Commits
Show all changes
21 commits
Select commit
Hold shift + click to select a range
987c507
only add successfully queried peers to the peersQueried set
Stebalien 449b434
improve query performance by limiting query width to KValue peers
Stebalien a0bc445
run all queries to completion
Stebalien 13e5e29
findpeer: return early
Stebalien 2648b2d
findpeer: drain addresses before processing new ones
Stebalien c9f523d
implement kademlia
Stebalien 78a2de6
query(nit): reliably fill rateLimit chan
Stebalien 2a209f1
query: count already queried peers
Stebalien da40930
test: bootstrap before testing
Stebalien 6ce8383
Change bucket size to be configurable
michaelavila 1afff31
dont do all those other things
whyrusleeping bd07cbf
Merge pull request #361 from libp2p/chores/make-bucket-size-configurable
whyrusleeping 604ac88
don't add peers with only private addresses to your routing table (#360)
whyrusleeping 4b31e56
dht mode toggling (modulo dynamic switching) (#350)
whyrusleeping a4cabc7
consume identify events to evaluate routing table addition. (#365)
raulk 8fe679a
filter unworkable peers in queries + enhanced logging (#363)
raulk 138991d
remove periodic printing of routing table.
raulk 1f4263f
Merge branch 'stabilize' into fix/closest-peers
raulk eed72b8
fix distance output in logging.
raulk 836a15c
Merge branch 'stabilize' into fix/closest-peers
raulk 22017f0
start with k peers instead of alpha
Stebalien File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,121 @@ | ||
| package kpeerset | ||
|
|
||
| import ( | ||
| "container/heap" | ||
| "math/big" | ||
| "sync" | ||
|
|
||
| "github.com/libp2p/go-libp2p-peer" | ||
| ks "github.com/whyrusleeping/go-keyspace" | ||
| ) | ||
|
|
||
| // peerMetric tracks a peer and its distance to something else. | ||
| type peerMetric struct { | ||
| // the peer | ||
| peer peer.ID | ||
|
|
||
| // big.Int for XOR metric | ||
| metric *big.Int | ||
| } | ||
|
|
||
| // peerMetricHeap implements a heap of peerDistances. Taken from | ||
| // go-libp2p-peerstore/queue but inverted. This heap sorts by _furthest_. | ||
| type peerMetricHeap []*peerMetric | ||
|
|
||
| func (ph peerMetricHeap) Len() int { | ||
| return len(ph) | ||
| } | ||
|
|
||
| func (ph peerMetricHeap) Less(i, j int) bool { | ||
| return 1 == ph[i].metric.Cmp(ph[j].metric) | ||
| } | ||
|
|
||
| func (ph peerMetricHeap) Swap(i, j int) { | ||
| ph[i], ph[j] = ph[j], ph[i] | ||
| } | ||
|
|
||
| func (ph *peerMetricHeap) Push(x interface{}) { | ||
| item := x.(*peerMetric) | ||
| *ph = append(*ph, item) | ||
| } | ||
|
|
||
| func (ph *peerMetricHeap) Pop() interface{} { | ||
| old := *ph | ||
| n := len(old) | ||
| item := old[n-1] | ||
| *ph = old[0 : n-1] | ||
| return item | ||
| } | ||
|
|
||
| // KPeerSet implements heap.Interface and PeerQueue | ||
| type KPeerSet struct { | ||
| kvalue int | ||
|
|
||
| // from is the Key this PQ measures against | ||
| from ks.Key | ||
|
|
||
| // heap is a heap of peerDistance items | ||
| heap peerMetricHeap | ||
|
|
||
| lock sync.RWMutex | ||
| } | ||
|
|
||
| func (pq *KPeerSet) Len() int { | ||
| pq.lock.RLock() | ||
| defer pq.lock.RUnlock() | ||
|
|
||
| return len(pq.heap) | ||
| } | ||
|
|
||
| func (pq *KPeerSet) Check(p peer.ID) bool { | ||
| pq.lock.RLock() | ||
| defer pq.lock.RUnlock() | ||
|
|
||
| if pq.heap.Len() < pq.kvalue { | ||
| return true | ||
| } | ||
|
|
||
| distance := ks.XORKeySpace.Key([]byte(p)).Distance(pq.from) | ||
| return distance.Cmp(pq.heap[0].metric) != -1 | ||
| } | ||
|
|
||
| func (pq *KPeerSet) Add(p peer.ID) bool { | ||
| pq.lock.Lock() | ||
| defer pq.lock.Unlock() | ||
|
|
||
| distance := ks.XORKeySpace.Key([]byte(p)).Distance(pq.from) | ||
|
|
||
| if pq.heap.Len() >= pq.kvalue { | ||
| // If we're not closer than the worst peer, drop this. | ||
| if distance.Cmp(pq.heap[0].metric) != -1 { | ||
| return false | ||
| } | ||
| // Replacing something, remove it. | ||
| heap.Pop(&pq.heap) | ||
| } | ||
|
|
||
| heap.Push(&pq.heap, &peerMetric{ | ||
| peer: p, | ||
| metric: distance, | ||
| }) | ||
| return true | ||
| } | ||
|
|
||
| func (pq *KPeerSet) Peers() []peer.ID { | ||
| pq.lock.RLock() | ||
| defer pq.lock.RUnlock() | ||
|
|
||
| ret := make([]peer.ID, len(pq.heap)) | ||
| for _, pm := range pq.heap { | ||
| ret = append(ret, pm.peer) | ||
| } | ||
| return ret | ||
| } | ||
|
|
||
| func New(kvalue int, from string) *KPeerSet { | ||
| return &KPeerSet{ | ||
| from: ks.XORKeySpace.Key([]byte(from)), | ||
| kvalue: kvalue, | ||
| heap: make([]*peerMetric, 0, kvalue), | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
So, I spent a while trying to avoid having to fix this test without complicating the query logic too much. Then I realized that was just stupid.