Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

blockchain: Remove unspendable utxo set entries. #2996

Merged
merged 2 commits into from
Sep 28, 2022
Merged

blockchain: Remove unspendable utxo set entries. #2996

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7780ad9
blockchain: Update utxodb info during upgrade.
davecgh Sep 15, 2022
d1b381d
blockchain: Remove unspendable utxo set entries.
davecgh Sep 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
197 changes: 175 additions & 22 deletions internal/blockchain/upgrade.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1461,7 +1461,7 @@ func migrateUtxoSetVersion1To2(ctx context.Context, db database.DB) error {
v1BucketName := []byte("utxoset")
v2BucketName := []byte("utxosetv2")

log.Info("Migrating database utxoset. This may take a while...")
log.Info("Migrating database utxo set. This may take a while...")
start := time.Now()

// Create the new utxoset bucket as needed.
Expand Down Expand Up @@ -2104,7 +2104,7 @@ func migrateUtxoSetVersion2To3(ctx context.Context, db database.DB) error {
v2BucketName := []byte("utxosetv2")
v3BucketName := []byte("utxosetv3")

log.Info("Migrating database utxoset. This may take a while...")
log.Info("Migrating database utxo set. This may take a while...")
start := time.Now()

// Create the new utxoset bucket as needed.
Expand Down Expand Up @@ -3294,7 +3294,7 @@ func upgradeSpendJournalToVersion3(ctx context.Context, b *BlockChain) error {

// upgradeUtxoSetToVersion3 upgrades a version 2 utxo set to version 3.
func upgradeUtxoSetToVersion3(ctx context.Context, db database.DB,
utxoBackend UtxoBackend) error {
utxoBackend UtxoBackend, utxoDbInfo *UtxoBackendInfo) error {

if interruptRequested(ctx) {
return errInterruptRequested
Expand All @@ -3309,12 +3309,6 @@ func upgradeUtxoSetToVersion3(ctx context.Context, db database.DB,
return err
}

// Fetch the backend versioning info.
utxoDbInfo, err := utxoBackend.FetchInfo()
if err != nil {
return err
}

// Update and persist the UTXO set database version.
utxoDbInfo.utxoVer = 3
err = utxoBackend.PutInfo(utxoDbInfo)
Expand Down Expand Up @@ -5557,7 +5551,7 @@ func dbPutUtxoBackendInfoV1(tx UtxoBackendTx, info *UtxoBackendInfo) error {
}

// upgradeUtxoDbToVersion2 upgrades a UTXO database from version 1 to version 2.
func upgradeUtxoDbToVersion2(ctx context.Context, utxoBackend UtxoBackend) error {
func upgradeUtxoDbToVersion2(ctx context.Context, utxoBackend UtxoBackend, utxoDbInfo *UtxoBackendInfo) error {
if interruptRequested(ctx) {
return errInterruptRequested
}
Expand All @@ -5571,12 +5565,6 @@ func upgradeUtxoDbToVersion2(ctx context.Context, utxoBackend UtxoBackend) error
return err
}

// Fetch the backend versioning info.
utxoDbInfo, err := utxoBackend.FetchInfo()
if err != nil {
return err
}

// Update and persist the UTXO database version.
utxoDbInfo.version = 2
err = utxoBackend.PutInfo(utxoDbInfo)
Expand Down Expand Up @@ -5727,6 +5715,161 @@ func moveUtxoDatabase(ctx context.Context, oldPath string, newPath string) error
return nil
}

// removeTreasuryBaseUtxos removes all utxos in the utxo set of a version 2 utxo
// database that are outputs of a treasurybase transaction since they are never
// directly spendable and thus should not be part of the utxo set.
func removeTreasuryBaseUtxos(ctx context.Context, utxoBackend UtxoBackend) error {
// Hardcoded prefix so updates do not affect old upgrades.
utxoPrefixUtxoSetV3 := []byte("\x03\x03")

log.Info("Updating database utxo set. This may take a while...")
start := time.Now()

// doBatch contains the primary logic for removing unspendable treasurybase
// outputs from the version 3 utxoset in batches. This is done because
// attempting to do everything in a single database transaction could result
// in massive memory usage and could potentially crash on many systems due
// to ulimits.
const maxEntries = 50000
var totalRemoved uint64
var resumeKey []byte
doBatch := func(tx UtxoBackendTx) (bool, error) {
var logProgress bool
var numRemoved uint32
err := func() error {
// Remove treasurybase outputs so long as the max number of entries
// for this batch has not been exceeded.
iter := tx.NewIterator(utxoPrefixUtxoSetV3)
defer iter.Release()

// Iterate all entries in the utxo set while skipping entries
// already examined in previous batches.
for ok := iter.Seek(resumeKey); ok; ok = iter.Next() {
if interruptRequested(ctx) {
logProgress = true
return errInterruptRequested
}

if numRemoved >= maxEntries {
// Set the resume key so the next batch skips entries that
// have already been examined. It is necessary to make a
// copy of the iterator key since it is no longer valid once
// the db transaction is closed.
iterKey := iter.Key()
resumeKey = make([]byte, len(iterKey))
copy(resumeKey, iterKey)

logProgress = true
return errBatchFinished
}

// The version 3 utxo set consists of an entry for each unspent
// output.
//
// The serialized value format is roughly:
//
// <block height><block index><flags><rest of data>
//
// Field Type Size
// block height VLQ variable
// block index VLQ variable
// flags VLQ variable
// rest of data...
//
// The serialized flags format is:
// bit 0 - containing transaction is a coinbase
// bit 1 - containing transaction has an expiry
// bits 2-5 - transaction type
// bits 6-7 - unused
//
// Given the flags field is the only thing that needs to be
// examined, the following specifically finds the relevant byte.
// Deserialize the block height and block index to locate the
// offset of the flags that need to examined.
serialized := iter.Value()
_, bytesRead := deserializeVLQ(serialized)
offset := bytesRead
if offset >= len(serialized) {
return errDeserialize("unexpected end of data after height")
}
_, bytesRead = deserializeVLQ(serialized[offset:])
offset += bytesRead
if offset >= len(serialized) {
return errDeserialize("unexpected end of data after index")
}

// Determine the transaction type from the flags.
const (
txOutFlagTxTypeBitmask = 0x3c
txOutFlagTxTypeShift = 2
stakeTxTypeTreasuryBase = 6
)
flags, bytesRead := deserializeVLQ(serialized[offset:])
offset += bytesRead
if offset >= len(serialized) {
return errDeserialize("unexpected end of data after flags")
}
txType := (flags & txOutFlagTxTypeBitmask) >> txOutFlagTxTypeShift

// Remove treasurybase outputs.
if txType == stakeTxTypeTreasuryBase {
if err := tx.Delete(iter.Key()); err != nil {
return err
}
numRemoved++
}
}

return nil
}()
isFullyDone := err == nil
if (isFullyDone || logProgress) && numRemoved > 0 {
totalRemoved += uint64(numRemoved)
log.Infof("Removed %d unspendable outputs (%d total)", numRemoved,
totalRemoved)
}
return isFullyDone, err
}

// Remove the entries in batches for the reasons mentioned above.
if err := utxoBackendBatchedUpdate(ctx, utxoBackend, doBatch); err != nil {
return err
}

elapsed := time.Since(start).Round(time.Millisecond)
log.Infof("Done updating UTXO database. Total unspendable outputs "+
"removed: %d in %v", totalRemoved, elapsed)

return nil
}

// upgradeUtxoDbToVersion3 upgrades a UTXO database from version 2 to version 3.
// This entails removing all treasurybase outputs since they are never directly
// spendable and thus should not be part of the utxo set.
func upgradeUtxoDbToVersion3(ctx context.Context, utxoBackend UtxoBackend, utxoDbInfo *UtxoBackendInfo) error {
if interruptRequested(ctx) {
return errInterruptRequested
}

log.Info("Upgrading UTXO database to version 3...")
start := time.Now()

// Remove treasurybase outputs from the utxo set.
if err := removeTreasuryBaseUtxos(ctx, utxoBackend); err != nil {
return err
}

// Update and persist the UTXO database version.
utxoDbInfo.version = 3
if err := utxoBackend.PutInfo(utxoDbInfo); err != nil {
return err
}

elapsed := time.Since(start).Round(time.Millisecond)
log.Infof("Done upgrading database in %v.", elapsed)
return nil
}

// checkDBTooOldToUpgrade returns an ErrDBTooOldToUpgrade error if the provided
// database version can no longer be upgraded due to being too old.
func checkDBTooOldToUpgrade(dbVersion uint32) error {
Expand Down Expand Up @@ -5910,9 +6053,7 @@ func upgradeSpendJournal(ctx context.Context, b *BlockChain) error {
// NOTE: The database info housed in the passed block database instance and
// backend info housed in the passed utxo backend instance will be updated with
// the latest versions.
func upgradeUtxoDb(ctx context.Context, db database.DB,
utxoBackend UtxoBackend) error {

func upgradeUtxoDb(ctx context.Context, db database.DB, utxoBackend UtxoBackend) error {
// Fetch the backend versioning info.
utxoDbInfo, err := utxoBackend.FetchInfo()
if err != nil {
Expand All @@ -5921,14 +6062,16 @@ func upgradeUtxoDb(ctx context.Context, db database.DB,

// Update to a version 2 UTXO database as needed.
if utxoDbInfo.version == 1 {
if err := upgradeUtxoDbToVersion2(ctx, utxoBackend); err != nil {
err := upgradeUtxoDbToVersion2(ctx, utxoBackend, utxoDbInfo)
if err != nil {
return err
}
}

// Update to a version 3 utxo set as needed.
if utxoDbInfo.utxoVer == 2 {
if err := upgradeUtxoSetToVersion3(ctx, db, utxoBackend); err != nil {
if utxoDbInfo.version == 2 && utxoDbInfo.utxoVer == 2 {
err := upgradeUtxoSetToVersion3(ctx, db, utxoBackend, utxoDbInfo)
if err != nil {
return err
}
}
Expand All @@ -5951,5 +6094,15 @@ func upgradeUtxoDb(ctx context.Context, db database.DB,
}
}

// Update to a version 3 utxo database if needed. This entails removing all
// treasurybase outputs since they are never directly spendable and thus
// should not be part of the utxo set.
if utxoDbInfo.version == 2 {
err := upgradeUtxoDbToVersion3(ctx, utxoBackend, utxoDbInfo)
if err != nil {
return err
}
}

return nil
}
2 changes: 1 addition & 1 deletion internal/blockchain/utxobackend.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (

const (
// currentUtxoDatabaseVersion indicates the current UTXO database version.
currentUtxoDatabaseVersion = 2
currentUtxoDatabaseVersion = 3

// utxoDbName is the name of the UTXO database.
utxoDbName = "utxodb"
Expand Down