diff --git a/core/chaincode/shim/chaincode.go b/core/chaincode/shim/chaincode.go index 30fde5b7545..c6869f345c8 100644 --- a/core/chaincode/shim/chaincode.go +++ b/core/chaincode/shim/chaincode.go @@ -19,15 +19,13 @@ limitations under the License. package shim import ( - "bytes" "errors" "flag" "fmt" "io" "os" - "regexp" - "strconv" "strings" + "unicode/utf8" "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes/timestamp" @@ -43,6 +41,11 @@ import ( // Logger for the shim package. var chaincodeLogger = logging.MustGetLogger("shim") +const ( + minUnicodeRuneValue = 0 //U+0000 + maxUnicodeRuneValue = utf8.MaxRune //U+10FFFF - maximum (and unallocated) code point +) + // ChaincodeStub is an object passed to chaincode for shim side handling of // APIs. type ChaincodeStub struct { @@ -359,35 +362,52 @@ func (stub *ChaincodeStub) GetQueryResult(query string) (StateQueryIteratorInter //CreateCompositeKey combines the given attributes to form a composite key. func (stub *ChaincodeStub) CreateCompositeKey(objectType string, attributes []string) (string, error) { - return createCompositeKey(stub, objectType, attributes) + return createCompositeKey(objectType, attributes) +} + +//SplitCompositeKey splits the key into attributes on which the composite key was formed. +func (stub *ChaincodeStub) SplitCompositeKey(compositeKey string) (string, []string, error) { + return splitCompositeKey(compositeKey) } -func createCompositeKey(stub ChaincodeStubInterface, objectType string, attributes []string) (string, error) { - var compositeKey bytes.Buffer - replacer := strings.NewReplacer("\x1E", "\x1E\x1E", "\x1F", "\x1E\x1F") - compositeKey.WriteString(replacer.Replace(objectType)) - for _, attribute := range attributes { - compositeKey.WriteString("\x1F" + strconv.Itoa(len(attribute)) + "\x1F") - compositeKey.WriteString(replacer.Replace(attribute)) +func createCompositeKey(objectType string, attributes []string) (string, error) { + if err := validateCompositeKeyAttribute(objectType); err != nil { + return "", err + } + ck := objectType + string(minUnicodeRuneValue) + for _, att := range attributes { + if err := validateCompositeKeyAttribute(att); err != nil { + return "", err + } + ck += att + string(minUnicodeRuneValue) } - return compositeKey.String(), nil + return ck, nil } -//SplitCompositeKey splits the key into attributes on which the composite key was formed. -func (stub *ChaincodeStub) SplitCompositeKey(compositeKey string) (string, []string, error) { - return splitCompositeKey(stub, compositeKey) +func splitCompositeKey(compositeKey string) (string, []string, error) { + componentIndex := 0 + components := []string{} + for i := 0; i < len(compositeKey); i++ { + if compositeKey[i] == minUnicodeRuneValue { + components = append(components, compositeKey[componentIndex:i]) + componentIndex = i + 1 + } + } + return components[0], components[1:], nil } -func splitCompositeKey(stub ChaincodeStubInterface, compositeKey string) (string, []string, error) { - re := regexp.MustCompile("\x1F[0-9]+\x1F") - splittedKey := re.Split(compositeKey, -1) - attributes := make([]string, 0) - replacer := strings.NewReplacer("\x1E\x1F", "\x1F", "\x1E\x1E", "\x1E") - objectType := replacer.Replace(splittedKey[0]) - for _, attr := range splittedKey[1:] { - attributes = append(attributes, replacer.Replace(attr)) +func validateCompositeKeyAttribute(str string) error { + for index, runeValue := range str { + if !utf8.ValidRune(runeValue) { + return fmt.Errorf("Not a valid utf8 string. Contains rune [%d] starting at byte position [%d]", + runeValue, index) + } + if runeValue == minUnicodeRuneValue || runeValue == maxUnicodeRuneValue { + return fmt.Errorf(`Input contain unicode %#U starting at position [%d]. %#U and %#U are not allowed in the input attribute of a composite key`, + runeValue, index, minUnicodeRuneValue, maxUnicodeRuneValue) + } } - return objectType, attributes, nil + return nil } //PartialCompositeKeyQuery function can be invoked by a chaincode to query the @@ -402,7 +422,7 @@ func (stub *ChaincodeStub) PartialCompositeKeyQuery(objectType string, attribute func partialCompositeKeyQuery(stub ChaincodeStubInterface, objectType string, attributes []string) (StateQueryIteratorInterface, error) { partialCompositeKey, _ := stub.CreateCompositeKey(objectType, attributes) - keysIter, err := stub.RangeQueryState(partialCompositeKey, partialCompositeKey+"\xFF") + keysIter, err := stub.RangeQueryState(partialCompositeKey, partialCompositeKey+string(maxUnicodeRuneValue)) if err != nil { return nil, fmt.Errorf("Error fetching rows: %s", err) } diff --git a/core/chaincode/shim/interfaces.go b/core/chaincode/shim/interfaces.go index f23223d84a6..76764afe763 100644 --- a/core/chaincode/shim/interfaces.go +++ b/core/chaincode/shim/interfaces.go @@ -77,13 +77,19 @@ type ChaincodeStubInterface interface { // iterator which can be used to iterate over all composite keys whose prefix // matches the given partial composite key. This function should be used only for // a partial composite key. For a full composite key, an iter with empty response - // would be returned. + // would be returned. The objectType and attributes are expected to have only + // valid utf8 strings and should not contain U+0000 (nil byte) and U+10FFFF (biggest and unallocated code point) PartialCompositeKeyQuery(objectType string, keys []string) (StateQueryIteratorInterface, error) // Given a list of attributes, CreateCompositeKey function combines these attributes - // to form a composite key. + // to form a composite key. The objectType and attributes are expected to have only + // valid utf8 strings and should not contain U+0000 (nil byte) and U+10FFFF (biggest and unallocated code point) CreateCompositeKey(objectType string, attributes []string) (string, error) + // Given a composite key, SplitCompositeKey function splits the key into attributes + // on which the composite key was formed. + SplitCompositeKey(compositeKey string) (string, []string, error) + // GetQueryResult function can be invoked by a chaincode to perform a // rich query against state database. Only supported by state database implementations // that support rich query. The query string is in the syntax of the underlying @@ -91,10 +97,6 @@ type ChaincodeStubInterface interface { // the query result set GetQueryResult(query string) (StateQueryIteratorInterface, error) - // Given a composite key, SplitCompositeKey function splits the key into attributes - // on which the composite key was formed. - SplitCompositeKey(compositeKey string) (string, []string, error) - // GetCallerCertificate returns caller certificate GetCallerCertificate() ([]byte, error) diff --git a/core/chaincode/shim/mockstub.go b/core/chaincode/shim/mockstub.go index ee55d04b6e1..6f48c4c075e 100644 --- a/core/chaincode/shim/mockstub.go +++ b/core/chaincode/shim/mockstub.go @@ -217,13 +217,13 @@ func (stub *MockStub) PartialCompositeKeyQuery(objectType string, attributes []s // CreateCompositeKey combines the list of attributes //to form a composite key. func (stub *MockStub) CreateCompositeKey(objectType string, attributes []string) (string, error) { - return createCompositeKey(stub, objectType, attributes) + return createCompositeKey(objectType, attributes) } // SplitCompositeKey splits the composite key into attributes // on which the composite key was formed. func (stub *MockStub) SplitCompositeKey(compositeKey string) (string, []string, error) { - return splitCompositeKey(stub, compositeKey) + return splitCompositeKey(compositeKey) } // InvokeChaincode calls a peered chaincode. diff --git a/core/chaincode/shim/mockstub_test.go b/core/chaincode/shim/mockstub_test.go index 762ea2956ae..5afbd4943e5 100644 --- a/core/chaincode/shim/mockstub_test.go +++ b/core/chaincode/shim/mockstub_test.go @@ -111,9 +111,10 @@ func TestPartialCompositeKeyQuery(t *testing.T) { stub.PutState(compositeKey3, marbleJSONBytes3) stub.MockTransactionEnd("init") - expectKeys := []string{compositeKey1, compositeKey2} - expectKeysAttributes := [][]string{{"set-1", "red"}, {"set-1", "blue"}} - expectValues := [][]byte{marbleJSONBytes1, marbleJSONBytes2} + // should return in sorted order of attributes + expectKeys := []string{compositeKey2, compositeKey1} + expectKeysAttributes := [][]string{{"set-1", "blue"}, {"set-1", "red"}} + expectValues := [][]byte{marbleJSONBytes2, marbleJSONBytes1} rqi, _ := stub.PartialCompositeKeyQuery("marble", []string{"set-1"}) fmt.Println("Running loop") diff --git a/core/ledger/util/couchdb/couchdb.go b/core/ledger/util/couchdb/couchdb.go index a09283926c5..ec17f3f04c8 100644 --- a/core/ledger/util/couchdb/couchdb.go +++ b/core/ledger/util/couchdb/couchdb.go @@ -33,6 +33,7 @@ import ( "regexp" "strconv" "strings" + "unicode/utf8" "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/version" logging "github.com/op/go-logging" @@ -335,16 +336,18 @@ func (dbclient *CouchDatabase) EnsureFullCommit() (*DBOperationResponse, error) //SaveDoc method provides a function to save a document, id and byte array func (dbclient *CouchDatabase) SaveDoc(id string, rev string, bytesDoc []byte, attachments []Attachment) (string, error) { - logger.Debugf("Entering SaveDoc()") - + if !utf8.ValidString(id) { + return "", fmt.Errorf("doc id [%x] not a valid utf8 string", id) + } saveURL, err := url.Parse(dbclient.couchInstance.conf.URL) if err != nil { logger.Errorf("URL parse error: %s", err.Error()) return "", err } - saveURL.Path = dbclient.dbName + "/" + id - + saveURL.Path = dbclient.dbName + // id can contain a '/', so encode separately + saveURL = &url.URL{Opaque: saveURL.String() + "/" + encodePathElement(id)} logger.Debugf(" id=%s, value=%s", id, string(bytesDoc)) if rev == "" { @@ -501,14 +504,17 @@ func getRevisionHeader(resp *http.Response) (string, error) { func (dbclient *CouchDatabase) ReadDoc(id string) ([]byte, string, error) { logger.Debugf("Entering ReadDoc() id=%s", id) - + if !utf8.ValidString(id) { + return nil, "", fmt.Errorf("doc id [%x] not a valid utf8 string", id) + } readURL, err := url.Parse(dbclient.couchInstance.conf.URL) if err != nil { logger.Errorf("URL parse error: %s", err.Error()) return nil, "", err } - readURL.Path = dbclient.dbName + "/" + id - + readURL.Path = dbclient.dbName + // id can contain a '/', so encode separately + readURL = &url.URL{Opaque: readURL.String() + "/" + encodePathElement(id)} query := readURL.Query() query.Add("attachments", "true") @@ -644,26 +650,19 @@ func (dbclient *CouchDatabase) ReadDocRange(startKey, endKey string, limit, skip //Append the startKey if provided if startKey != "" { - startKey = strconv.QuoteToGraphic(startKey) - startKey = strings.Replace(startKey, "\\x00", "\\u0000", -1) - startKey = strings.Replace(startKey, "\\x1e", "\\u001e", -1) - startKey = strings.Replace(startKey, "\\x1f", "\\u001f", -1) - startKey = strings.Replace(startKey, "\\xff", "\\u00ff", -1) - //TODO add general unicode support instead of special cases - + var err error + if startKey, err = encodeForJSON(startKey); err != nil { + return nil, err + } queryParms.Add("startkey", startKey) } //Append the endKey if provided if endKey != "" { - endKey = strconv.QuoteToGraphic(endKey) - endKey = strings.Replace(endKey, "\\x00", "\\u0000", -1) - endKey = strings.Replace(endKey, "\\x01", "\\u0001", -1) - endKey = strings.Replace(endKey, "\\x1e", "\\u001e", -1) - endKey = strings.Replace(endKey, "\\x1f", "\\u001f", -1) - endKey = strings.Replace(endKey, "\\xff", "\\u00ff", -1) - //TODO add general unicode support instead of special cases - + var err error + if endKey, err = encodeForJSON(endKey); err != nil { + return nil, err + } queryParms.Add("endkey", endKey) } @@ -919,3 +918,22 @@ func IsJSON(s string) bool { var js map[string]interface{} return json.Unmarshal([]byte(s), &js) == nil } + +// encodePathElement uses Golang for encoding and in addition, replaces a '/' by %2F. +// Otherwise, in the regular encoding, a '/' is treated as a path separator in the url +func encodePathElement(str string) string { + u := &url.URL{} + u.Path = str + encodedStr := u.String() + encodedStr = strings.Replace(encodedStr, "/", "%2F", -1) + return encodedStr +} + +func encodeForJSON(str string) (string, error) { + buf := &bytes.Buffer{} + encoder := json.NewEncoder(buf) + if err := encoder.Encode(str); err != nil { + return "", err + } + return buf.String(), nil +} diff --git a/core/ledger/util/couchdb/couchdb_test.go b/core/ledger/util/couchdb/couchdb_test.go index 529b3abb0c9..f88e0593675 100644 --- a/core/ledger/util/couchdb/couchdb_test.go +++ b/core/ledger/util/couchdb/couchdb_test.go @@ -19,7 +19,9 @@ package couchdb import ( "encoding/json" "fmt" + "os" "testing" + "unicode/utf8" "github.com/hyperledger/fabric/common/ledger/testutil" "github.com/hyperledger/fabric/core/ledger/ledgerconfig" @@ -52,6 +54,11 @@ type Asset struct { var assetJSON = []byte(`{"asset_name":"marble1","color":"blue","size":"35","owner":"jerry"}`) +func TestMain(m *testing.M) { + ledgertestutil.SetupCoreYAMLConfig("./../../../../peer") + os.Exit(m.Run()) +} + func TestDBConnectionDef(t *testing.T) { //call a helper method to load the core.yaml @@ -140,11 +147,11 @@ func TestDBCreateDatabaseAndPersist(t *testing.T) { testutil.AssertEquals(t, dbResp.DbName, database) //Save the test document - _, saveerr := db.SaveDoc("1", "", assetJSON, nil) + _, saveerr := db.SaveDoc("idWith/slash", "", assetJSON, nil) testutil.AssertNoError(t, saveerr, fmt.Sprintf("Error when trying to save a document")) //Retrieve the test document - dbGetResp, _, geterr := db.ReadDoc("1") + dbGetResp, _, geterr := db.ReadDoc("idWith/slash") testutil.AssertNoError(t, geterr, fmt.Sprintf("Error when trying to retrieve a document")) //Unmarshal the document to Asset structure @@ -154,6 +161,21 @@ func TestDBCreateDatabaseAndPersist(t *testing.T) { //Verify the owner retrieved matches testutil.AssertEquals(t, assetResp.Owner, "jerry") + //Save the test document + _, saveerr = db.SaveDoc("1", "", assetJSON, nil) + testutil.AssertNoError(t, saveerr, fmt.Sprintf("Error when trying to save a document")) + + //Retrieve the test document + dbGetResp, _, geterr = db.ReadDoc("1") + testutil.AssertNoError(t, geterr, fmt.Sprintf("Error when trying to retrieve a document")) + + //Unmarshal the document to Asset structure + assetResp = &Asset{} + json.Unmarshal(dbGetResp, &assetResp) + + //Verify the owner retrieved matches + testutil.AssertEquals(t, assetResp.Owner, "jerry") + //Change owner to bob assetResp.Owner = "bob" @@ -218,6 +240,60 @@ func TestDBBadJSON(t *testing.T) { } +func TestPrefixScan(t *testing.T) { + if !ledgerconfig.IsCouchDBEnabled() { + return + } + cleanup() + defer cleanup() + + //create a new instance and database object + couchInstance, err := CreateCouchInstance(connectURL, username, password) + testutil.AssertNoError(t, err, fmt.Sprintf("Error when trying to create couch instance")) + db := CouchDatabase{couchInstance: *couchInstance, dbName: database} + + //create a new database + _, errdb := db.CreateDatabaseIfNotExist() + testutil.AssertNoError(t, errdb, fmt.Sprintf("Error when trying to create database")) + + //Retrieve the info for the new database and make sure the name matches + dbResp, _, errdb := db.GetDatabaseInfo() + testutil.AssertNoError(t, errdb, fmt.Sprintf("Error when trying to retrieve database information")) + testutil.AssertEquals(t, dbResp.DbName, database) + + //Save documents + for i := 0; i < 20; i++ { + id1 := string(0) + string(i) + string(0) + id2 := string(0) + string(i) + string(1) + id3 := string(0) + string(i) + string(utf8.MaxRune-1) + _, saveerr := db.SaveDoc(id1, "", assetJSON, nil) + testutil.AssertNoError(t, saveerr, fmt.Sprintf("Error when trying to save a document")) + _, saveerr = db.SaveDoc(id2, "", assetJSON, nil) + testutil.AssertNoError(t, saveerr, fmt.Sprintf("Error when trying to save a document")) + _, saveerr = db.SaveDoc(id3, "", assetJSON, nil) + testutil.AssertNoError(t, saveerr, fmt.Sprintf("Error when trying to save a document")) + + } + startKey := string(0) + string(10) + endKey := startKey + string(utf8.MaxRune) + resultsPtr, geterr := db.ReadDocRange(startKey, endKey, 1000, 0) + testutil.AssertNoError(t, geterr, fmt.Sprintf("Error when trying to perform a range scan")) + testutil.AssertNotNil(t, resultsPtr) + results := *resultsPtr + testutil.AssertEquals(t, len(results), 3) + testutil.AssertEquals(t, results[0].ID, string(0)+string(10)+string(0)) + testutil.AssertEquals(t, results[1].ID, string(0)+string(10)+string(1)) + testutil.AssertEquals(t, results[2].ID, string(0)+string(10)+string(utf8.MaxRune-1)) + + //Drop the database + _, errdbdrop := db.DropDatabase() + testutil.AssertNoError(t, errdbdrop, fmt.Sprintf("Error dropping database")) + + //Retrieve the info for the new database and make sure the name matches + _, _, errdbinfo := db.GetDatabaseInfo() + testutil.AssertError(t, errdbinfo, fmt.Sprintf("Error should have been thrown for missing database")) +} + func TestDBSaveAttachment(t *testing.T) { if ledgerconfig.IsCouchDBEnabled() == true { diff --git a/core/ledger/util/couchdb/couchdbutil_test.go b/core/ledger/util/couchdb/couchdbutil_test.go index 06ae30f95a7..393f2fe7abc 100644 --- a/core/ledger/util/couchdb/couchdbutil_test.go +++ b/core/ledger/util/couchdb/couchdbutil_test.go @@ -22,15 +22,10 @@ import ( "github.com/hyperledger/fabric/common/ledger/testutil" "github.com/hyperledger/fabric/core/ledger/ledgerconfig" - ledgertestutil "github.com/hyperledger/fabric/core/ledger/testutil" ) //Unit test of couch db util functionality func TestCreateCouchDBConnectionAndDB(t *testing.T) { - - //call a helper method to load the core.yaml - ledgertestutil.SetupCoreYAMLConfig("./../../../../peer") - if ledgerconfig.IsCouchDBEnabled() == true { cleanup()