Skip to content

Commit

Permalink
Merge pull request #578 from semyon-dev/master
Browse files Browse the repository at this point in the history 
New implementation for HTTP service type
  • Loading branch information
@AlbinaPomogalova
AlbinaPomogalova committed Mar 5, 2024
2 parents ca16094 + 9b1d7ce commit 93c287b
Show file tree
Hide file tree
Showing 15 changed files with 368 additions and 232 deletions.
7 changes: 7 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
# idea cache
.idea/

# npm cache and configs
resources/blockchain/node_modules/
resources/blockchain/build/
snetd.db
Expand All @@ -13,6 +16,10 @@ blockchain/multi_party_escrow.go
blockchain/singularity_net_token.go
*.pb.go

# service proto
service.proto
service.proto-tmp.pb

# Binaries for programs and plugins
*.exe
*.exe~
Expand Down
60 changes: 29 additions & 31 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,17 @@ These instructions are intended to facilitate the development and testing of Sin
deploying SingularityNET services using SingularityNET Daemon should install the appropriate binary as
[released](#release).

### Prerequisites
### Prerequisites and dependencies

* [Go 1.21+](https://golang.org/dl/)
* [NodeJS 15+ w/npm](https://nodejs.org/en/download/)
* [Protoc v25.0+](https://github.com/protocolbuffers/protobuf/releases)

### Dependencies

* install [Protoc v25.0+](https://github.com/protocolbuffers/protobuf/releases)

* If you want to cross-compile you will also need Docker
Protoc, nodejs, go and go/bin should be in environment variables.

### Installing

* Clone the git repository to the following path $GOPATH/src/github.com/singnet/
* Clone the git repository (for example $GOPATH/src/github.com/singnet/)

```bash
$ git clone git@github.com:singnet/snet-daemon.git
Expand All @@ -50,12 +47,16 @@ $ cd snet-daemon
$ ./scripts/install
```

* Build snet-daemon (on Linux amd64 platform), see below section if you want to cross compile instead.
Please note using ldflags, the latest tagged version , sha1 revision and the build time are set as part of the build.
* Build snet-daemon. Please note using ldflags, the latest tagged version , sha1 revision and the build time are set as part of the build.
You need to pass the version as shown in the example below

```bash
$ ./scripts/build linux amd64 <version>
$ ./scripts/build <linux/windows/darwin> <amd64/arm/arm64> <version>
```

Example:
```bash
$ ./scripts/build linux amd64 v5.1.2
```

* Generate default config file snet-daemon (on Linux amd64 platform)
Expand All @@ -66,20 +67,12 @@ $ ./build/snetd-linux-amd64 init

**** Please update the registry address in daemon config based on the test network used

#### Cross-compiling
#### Multi-compiling

If you want to build snetd for platforms other than the one you are on, run `./scripts/build-xgo` instead
If you want to build snetd for several platforms, run `./scripts/build-all <version>` instead
of `./scripts/build`.

You can edit the script to choose a specific platform, but by default it will build for Linux, OSX, and Windows (amd64
for all, except Linux which will also build for arm6)

Please note using ldflags the latest tagged version (passed as the first parameter to the script) , sha1 revision and
the build time are set as part of the build.

```bash
$ ./scripts/build-xgo <version>
```
You can edit the script to choose a specific platforms, but by default it will build for Linux, OSX, and Windows

#### Run Deamon

Expand Down Expand Up @@ -159,24 +152,29 @@ These properties you should usually change before starting daemon for the first
time.

* **blockchain_network_selected** (required)
Name of the network to be used for Daemon possible values are one of (kovan,ropsten,main,local or rinkeby).
Name of the network to be used for Daemon possible values are one of (goerli, sepolia, main, local).
Daemon will automatically read the Registry address associated with this network For local network ( you can also
specify the registry address manually),see the blockchain_network_config.json

* **daemon_type** (required;) -
Defines the type of service. Available values :grpc,
jsonrpc, [http](https://dev.singularitynet.io/docs/ai-developers/service-type-http/), process.

* **service_credentials** (required if daemon_type is http):
* **service_credentials** (optional, for service_type http only):
Array of credentials, example:

```
[{"key": "X-Banana-API-Key",
"value": "546bd7d4-d3e1-46ba-b752-bc45e4dc5b39",
"location": "header"}]
"service_credentials": [
{
"key": "example_body_param",
"value": 12345,
"location": "body"
},
{
"key": "X-API-Key",
"value": "546bd7d4-d3e1-46ba-b752-bc45e4dc5b39",
"location": "header"
}
],
```

Location can be: query or header
Location can be: query, header or body. Query and header values must be string.

* **daemon_end_point** (required;) -
Defines the ip and the port on which the daemon listens to.
Expand Down
163 changes: 128 additions & 35 deletions blockchain/serviceMetadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,21 @@ package blockchain
import (
"encoding/json"
"fmt"
"github.com/emicklei/proto"
pproto "github.com/emicklei/proto"
"github.com/ethereum/go-ethereum/common"
"github.com/pkg/errors"
"github.com/singnet/snet-daemon/config"
"github.com/singnet/snet-daemon/ipfsutils"
log "github.com/sirupsen/logrus"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protodesc"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
"google.golang.org/protobuf/types/descriptorpb"
"math/big"
"os"
"os/exec"
"path"
"strings"
)

Expand Down Expand Up @@ -137,7 +144,10 @@ import (
]
}
*/
const IpfsPrefix = "ipfs://"
const (
IpfsPrefix = "ipfs://"
serviceProto = "service.proto"
)

type ServiceMetadata struct {
Version int `json:"version"`
Expand All @@ -156,8 +166,9 @@ type ServiceMetadata struct {
freeCallSignerAddress common.Address
isfreeCallAllowed bool
freeCallsAllowed int
dynamicPriceMethodMapping map[string]string `json:"dynamicpricing"`
trainingMethods []string `json:"training_methods"`
DynamicPriceMethodMapping map[string]string `json:"dynamicpricing"`
TrainingMethods []string `json:"training_methods"`
ProtoFile protoreflect.FileDescriptor `json:"-"`
}
type Tiers struct {
Tiers Tier `json:"tier"`
Expand Down Expand Up @@ -270,10 +281,7 @@ func ReadServiceMetaDataFromLocalFile(filename string) (*ServiceMetadata, error)
func getRegistryCaller() (reg *RegistryCaller) {
ethClient, err := GetEthereumClient()
if err != nil {

log.WithError(err).
Panic("Unable to get Blockchain client ")

log.WithError(err).Panic("Unable to get Blockchain client ")
}
defer ethClient.Close()
registryContractAddress := getRegistryAddressKey()
Expand Down Expand Up @@ -320,25 +328,24 @@ func InitServiceMetaDataFromJson(jsonData string) (*ServiceMetadata, error) {
if err := setFreeCallData(metaData); err != nil {
return nil, err
}
//If Dynamic pricing is enabled ,there will be mandatory checks on the service proto
//this is to ensure that the standards on how one defines the methods to invoke is followed
if config.GetBool(config.EnableDynamicPricing) {
if err := setServiceProto(metaData); err != nil {
return nil, err
}

if err := setServiceProto(metaData); err != nil {
return nil, err
}
e, err := json.Marshal(metaData.dynamicPriceMethodMapping)
dynamicPriceMethodMappingJson, err := json.Marshal(metaData.DynamicPriceMethodMapping)
if err != nil {
log.Println(err)
}

log.Println(string(e))
e1, err := json.Marshal(metaData.trainingMethods)
log.Debugln("dynamicPriceMethodMappingJson: ", string(dynamicPriceMethodMappingJson))

trainingMethodsJson, err := json.Marshal(metaData.TrainingMethods)
if err != nil {
log.Println(err)
}

log.Println(string(e1))
log.Debugln("trainingMethodsJson: ", string(trainingMethodsJson))

return metaData, err
}

Expand Down Expand Up @@ -445,7 +452,7 @@ func (metaData *ServiceMetadata) GetDynamicPricingMethodAssociated(methodFullNam
if !config.GetBool(config.EnableDynamicPricing) {
return
}
pricingMethod = metaData.dynamicPriceMethodMapping[methodFullName]
pricingMethod = metaData.DynamicPriceMethodMapping[methodFullName]
if strings.Compare("", pricingMethod) == 0 {
isDynamicPricingEligible = false
} else {
Expand All @@ -460,9 +467,10 @@ func (metaData *ServiceMetadata) IsModelTraining(methodFullName string) (useMode
if !config.GetBool(config.ModelTrainingEnabled) {
return false
}
useModelTrainingEndPoint = isElementInArray(methodFullName, metaData.trainingMethods)
useModelTrainingEndPoint = isElementInArray(methodFullName, metaData.TrainingMethods)
return
}

func isElementInArray(a string, list []string) bool {
for _, b := range list {
if b == a {
Expand All @@ -471,52 +479,137 @@ func isElementInArray(a string, list []string) bool {
}
return false
}

func createProtoRegistry(srcDir string, filename string) (*protoregistry.Files, error) {
// Create descriptors using the protoc binary.
// Imported dependencies are included so that the descriptors are self-contained.
tmpFile := filename + "-tmp.pb"
cmd := exec.Command("protoc",
"--include_imports",
"--descriptor_set_out="+tmpFile,
"-I"+srcDir,
path.Join(srcDir, filename))

cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
return nil, err
}
//defer os.Remove(tmpFile)

marshalledDescriptorSet, err := os.ReadFile(tmpFile)
if err != nil {
return nil, err
}
descriptorSet := descriptorpb.FileDescriptorSet{}
err = proto.Unmarshal(marshalledDescriptorSet, &descriptorSet)
if err != nil {
log.Println("can't unmarshal: ", err)
return nil, err
}

files, err := protodesc.NewFiles(&descriptorSet)
if err != nil {
log.Println(err)
return nil, err
}

return files, nil
}

func setServiceProto(metaData *ServiceMetadata) (err error) {
metaData.dynamicPriceMethodMapping = make(map[string]string, 0)
metaData.trainingMethods = make([]string, 0)
metaData.DynamicPriceMethodMapping = make(map[string]string, 0)
metaData.TrainingMethods = make([]string, 0)
//This is to handler the scenario where there could be multiple protos associated with the service proto
protoFiles, err := ipfsutils.ReadFilesCompressed(ipfsutils.GetIpfsFile(metaData.ModelIpfsHash))
if err != nil {
return err
}

if metaData.ServiceType == "http" {
if len(protoFiles) > 1 {
log.Fatalln("daemon support only one proto file for HTTP services!")
}
}

for _, file := range protoFiles {
if srvProto, err := parseServiceProto(file); err != nil {
return err
} else {
dynamicMethodMap, trainingMethodMap, err := buildDynamicPricingMethodsMap(srvProto)
log.Debugln("Protofile: ", file)

if metaData.ServiceType == "http" {
_, err = os.Create(serviceProto)
if err != nil {
log.Fatalln("Can't create proto file: ", err)
}

err = os.WriteFile(serviceProto, []byte(file), 0666)
if err != nil {
log.Fatalln("Can't write to proto file: ", err)
}
}

//If Dynamic pricing is enabled ,there will be mandatory checks on the service proto
//this is to ensure that the standards on how one defines the methods to invoke is followed
if config.GetBool(config.EnableDynamicPricing) {
if srvProto, err := parseServiceProto(file); err != nil {
return err
} else {
dynamicMethodMap, trainingMethodMap, err := buildDynamicPricingMethodsMap(srvProto)
if err != nil {
return err
}
metaData.DynamicPriceMethodMapping = dynamicMethodMap
metaData.TrainingMethods = trainingMethodMap
}
metaData.dynamicPriceMethodMapping = dynamicMethodMap
metaData.trainingMethods = trainingMethodMap
}
}

if metaData.ServiceType == "http" {
files, err := createProtoRegistry(".", serviceProto)
if err != nil {
log.Println("createProtoRegistry: ", err)
}

protoFile, err := files.FindFileByPath(serviceProto)
if err != nil {
log.Println("files.FindFileByPat: ", err)
}

err = files.RegisterFile(protoFile)
if err != nil {
log.Println("files.RegisterFile(desc): ", err)
}

metaData.ProtoFile = protoFile
}

return nil
}

func parseServiceProto(serviceProtoFile string) (*proto.Proto, error) {
func parseServiceProto(serviceProtoFile string) (*pproto.Proto, error) {
reader := strings.NewReader(serviceProtoFile)
parser := proto.NewParser(reader)
parser := pproto.NewParser(reader)
parsedProto, err := parser.Parse()
if err != nil {
return nil, err
}
return parsedProto, nil
}

func buildDynamicPricingMethodsMap(serviceProto *proto.Proto) (dynamicPricingMethodMapping map[string]string,
func buildDynamicPricingMethodsMap(serviceProto *pproto.Proto) (dynamicPricingMethodMapping map[string]string,
trainingMethodPricing []string, err error) {
dynamicPricingMethodMapping = make(map[string]string, 0)
trainingMethodPricing = make([]string, 0)
var pkgName, serviceName, methodName string
for _, elem := range serviceProto.Elements {
//package is parsed earlier than service ( per documentation)
if pkg, ok := elem.(*proto.Package); ok {
if pkg, ok := elem.(*pproto.Package); ok {
pkgName = pkg.Name
}

if service, ok := elem.(*proto.Service); ok {
if service, ok := elem.(*pproto.Service); ok {
serviceName = service.Name
for _, serviceElements := range service.Elements {
if rpcMethod, ok := serviceElements.(*proto.RPC); ok {
if rpcMethod, ok := serviceElements.(*pproto.RPC); ok {
methodName = rpcMethod.Name
for _, methodOption := range rpcMethod.Options {
if strings.Compare(methodOption.Name, "(pricing.my_method_option).estimatePriceMethod") == 0 {
Expand Down
Loading

0 comments on commit 93c287b

Please sign in to comment.