-
-
Notifications
You must be signed in to change notification settings - Fork 372
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Refactor database code to support standalone batches, transactions.
This is spin-off of changes from #459. Transactions are not being used yet, but batches are updated to work with the new API. `database/` package was refactored to split abstract interfaces and implementation via goleveldb. This should make it easier to implement new database types.
- Loading branch information
Showing
27 changed files
with
544 additions
and
322 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
// Package database provides KV database for meta-information | ||
package database | ||
|
||
import "errors" | ||
|
||
// Errors for Storage | ||
var ( | ||
ErrNotFound = errors.New("key not found") | ||
) | ||
|
||
// StorageProcessor is a function to process one single storage entry | ||
type StorageProcessor func(key []byte, value []byte) error | ||
|
||
// Reader provides KV read calls | ||
type Reader interface { | ||
Get(key []byte) ([]byte, error) | ||
} | ||
|
||
// PrefixReader provides prefixed operations | ||
type PrefixReader interface { | ||
HasPrefix(prefix []byte) bool | ||
ProcessByPrefix(prefix []byte, proc StorageProcessor) error | ||
KeysByPrefix(prefix []byte) [][]byte | ||
FetchByPrefix(prefix []byte) [][]byte | ||
} | ||
|
||
// Writer provides KV update/delete calls | ||
type Writer interface { | ||
Put(key []byte, value []byte) error | ||
Delete(key []byte) error | ||
} | ||
|
||
// Storage is an interface to KV storage | ||
type Storage interface { | ||
Reader | ||
Writer | ||
|
||
PrefixReader | ||
|
||
CreateBatch() Batch | ||
OpenTransaction() (Transaction, error) | ||
|
||
CreateTemporary() (Storage, error) | ||
|
||
Open() error | ||
Close() error | ||
CompactDB() error | ||
Drop() error | ||
} | ||
|
||
// Batch provides a way to pack many writes. | ||
type Batch interface { | ||
Writer | ||
|
||
// Write closes batch and send accumulated writes to the database | ||
Write() error | ||
} | ||
|
||
// Transaction provides isolated atomic way to perform updates. | ||
// | ||
// Transactions might be expensive. | ||
// Transaction should always finish with either Discard() or Commit() | ||
type Transaction interface { | ||
Reader | ||
Writer | ||
|
||
Commit() error | ||
Discard() | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
package goleveldb | ||
|
||
import ( | ||
"github.com/syndtr/goleveldb/leveldb" | ||
"github.com/syndtr/goleveldb/leveldb/opt" | ||
|
||
"github.com/aptly-dev/aptly/database" | ||
) | ||
|
||
type batch struct { | ||
db *leveldb.DB | ||
b *leveldb.Batch | ||
} | ||
|
||
func (b *batch) Put(key, value []byte) error { | ||
b.b.Put(key, value) | ||
|
||
return nil | ||
} | ||
|
||
func (b *batch) Delete(key []byte) error { | ||
b.b.Delete(key) | ||
|
||
return nil | ||
} | ||
|
||
func (b *batch) Write() error { | ||
return b.db.Write(b.b, &opt.WriteOptions{}) | ||
} | ||
|
||
// batch should implement database.Batch | ||
var ( | ||
_ database.Batch = &batch{} | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
package goleveldb | ||
|
||
import ( | ||
"github.com/syndtr/goleveldb/leveldb" | ||
"github.com/syndtr/goleveldb/leveldb/filter" | ||
"github.com/syndtr/goleveldb/leveldb/opt" | ||
leveldbstorage "github.com/syndtr/goleveldb/leveldb/storage" | ||
|
||
"github.com/aptly-dev/aptly/database" | ||
) | ||
|
||
func internalOpen(path string, throttleCompaction bool) (*leveldb.DB, error) { | ||
o := &opt.Options{ | ||
Filter: filter.NewBloomFilter(10), | ||
OpenFilesCacheCapacity: 256, | ||
} | ||
|
||
if throttleCompaction { | ||
o.CompactionL0Trigger = 32 | ||
o.WriteL0PauseTrigger = 96 | ||
o.WriteL0SlowdownTrigger = 64 | ||
} | ||
|
||
return leveldb.OpenFile(path, o) | ||
} | ||
|
||
// NewDB creates new instance of DB, but doesn't open it (yet) | ||
func NewDB(path string) (database.Storage, error) { | ||
return &storage{path: path}, nil | ||
} | ||
|
||
// NewOpenDB creates new instance of DB and opens it | ||
func NewOpenDB(path string) (database.Storage, error) { | ||
db, err := NewDB(path) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
return db, db.Open() | ||
} | ||
|
||
// RecoverDB recovers LevelDB database from corruption | ||
func RecoverDB(path string) error { | ||
stor, err := leveldbstorage.OpenFile(path, false) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
db, err := leveldb.Recover(stor, nil) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
db.Close() | ||
stor.Close() | ||
|
||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
// Package goleveldb implements database interface via goleveldb | ||
package goleveldb |
Oops, something went wrong.