Skip to content

Commit

Permalink
PIE-176 integrate recent changes
Browse files Browse the repository at this point in the history
  • Loading branch information
puellanivis committed Sep 13, 2024
1 parent 788b935 commit 3633499
Show file tree
Hide file tree
Showing 78 changed files with 4,196 additions and 1,189 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
/.idea
bin/
71 changes: 46 additions & 25 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,35 +1,56 @@
PROTO_PATH ?= "internal/converter/testdata/proto"

default: build

.PHONY: build
build:
@echo "Generating binary (protoc-gen-jsonschema) ..."
@mkdir -p bin
@go build -o bin/protoc-gen-jsonschema cmd/protoc-gen-jsonschema/main.go
mkdir -p bin
go build -o bin/protoc-gen-jsonschema cmd/protoc-gen-jsonschema/main.go

.PHONY: fmt
fmt:
gofmt -s -w .
goimports -w -local github.com/chrusty/protoc-gen-jsonschema .

.PHONY: install
install:
@GO111MODULE=on go get -u github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema && go install github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema
go install github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema

.PHONY: build_linux
build_linux:
@echo "Generating Linux-amd64 binary (protoc-gen-jsonschema.linux-amd64) ..."
@GOOS=linux GOARCH=amd64 go build -o protoc-gen-jsonschema.linux-amd64
GOOS=linux GOARCH=amd64 go build -o protoc-gen-jsonschema.linux-amd64

PROTO_PATH ?= "internal/converter/testdata/proto"
samples:
@echo "Generating sample JSON-Schemas ..."
@mkdir -p jsonschemas
@PATH=./bin:$$PATH; protoc --jsonschema_out=allow_null_values:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/ArrayOfMessages.proto 2>/dev/null || echo "No messages found (ArrayOfMessages.proto)"
@PATH=./bin:$$PATH; protoc --jsonschema_out=allow_null_values:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/ArrayOfObjects.proto 2>/dev/null || echo "No messages found (ArrayOfObjects.proto)"
@PATH=./bin:$$PATH; protoc --jsonschema_out=allow_null_values:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/ArrayOfPrimitives.proto 2>/dev/null || echo "No messages found (ArrayOfPrimitives.proto)"
@PATH=./bin:$$PATH; protoc --jsonschema_out=disallow_additional_properties:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/Enumception.proto 2>/dev/null || echo "No messages found (Enumception.proto)"
@PATH=./bin:$$PATH; protoc --jsonschema_out=disallow_additional_properties:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/ImportedEnum.proto 2>/dev/null || echo "No messages found (ImportedEnum.proto)"
@PATH=./bin:$$PATH; protoc --jsonschema_out=disallow_additional_properties:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/NestedMessage.proto 2>/dev/null || echo "No messages found (NestedMessage.proto)"
@PATH=./bin:$$PATH; protoc --jsonschema_out=disallow_bigints_as_strings:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/NestedObject.proto 2>/dev/null || echo "No messages found (NestedObject.proto)"
@PATH=./bin:$$PATH; protoc --jsonschema_out=disallow_bigints_as_strings:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/PayloadMessage.proto 2>/dev/null || echo "No messages found (PayloadMessage.proto)"
@PATH=./bin:$$PATH; protoc --jsonschema_out=disallow_bigints_as_strings:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/SeveralEnums.proto 2>/dev/null || echo "No messages found (SeveralEnums.proto)"
@PATH=./bin:$$PATH; protoc --jsonschema_out=disallow_bigints_as_strings:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/SeveralMessages.proto 2>/dev/null || echo "No messages found (SeveralMessages.proto)"
@PATH=./bin:$$PATH; protoc --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/ArrayOfEnums.proto 2>/dev/null || echo "No messages found (SeveralMessages.proto)"
@PATH=./bin:$$PATH; protoc --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/Maps.proto 2>/dev/null || echo "No messages found (Maps.proto)"
@PATH=./bin:$$PATH; protoc --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/MessageWithComments.proto 2>/dev/null || echo "No messages found (MessageWithComments.proto)"
@PATH=./bin:$$PATH; protoc -I /usr/include --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/WellKnown.proto
.PHONY: samples
samples: build
mkdir -p jsonschemas
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=allow_null_values:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/ArrayOfMessages.proto || echo "No messages found (ArrayOfMessages.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=allow_null_values:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/ArrayOfObjects.proto || echo "No messages found (ArrayOfObjects.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=allow_null_values:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/ArrayOfPrimitives.proto || echo "No messages found (ArrayOfPrimitives.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=jsonschemas -I. --proto_path=${PROTO_PATH} ${PROTO_PATH}/Enumception.proto || echo "No messages found (Enumception.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=disallow_additional_properties:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/NestedMessage.proto || echo "No messages found (NestedMessage.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=disallow_bigints_as_strings:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/NestedObject.proto || echo "No messages found (NestedObject.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=disallow_bigints_as_strings:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/PayloadMessage.proto || echo "No messages found (PayloadMessage.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=disallow_bigints_as_strings:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/SeveralEnums.proto || echo "No messages found (SeveralEnums.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=disallow_bigints_as_strings:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/SeveralMessages.proto || echo "No messages found (SeveralMessages.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=disallow_bigints_as_strings:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/Timestamp.proto || echo "No messages found (Timestamp.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=all_fields_required:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/PayloadMessage2.proto || echo "No messages found (PayloadMessage2.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=json_fieldnames:jsonschemas -I. --proto_path=${PROTO_PATH} ${PROTO_PATH}/JSONFields.proto || echo "No messages found (JSONFields.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/ArrayOfEnums.proto || echo "No messages found (SeveralMessages.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/Maps.proto || echo "No messages found (Maps.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/MessageWithComments.proto || echo "No messages found (MessageWithComments.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/Proto2Required.proto || echo "No messages found (Proto2Required.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/Proto2NestedMessage.proto || echo "No messages found (Proto2NestedMessage.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/GoogleValue.proto || echo "No messages found (GoogleValue.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/GoogleInt64Value.proto || echo "No messages found (GoogleInt64Value.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=disallow_bigints_as_strings:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/GoogleInt64ValueDisallowString.proto || echo "No messages found (GoogleInt64ValueDisallowString.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=allow_null_values:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/GoogleInt64ValueAllowNull.proto || echo "No messages found (GoogleInt64ValueAllowNull.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=disallow_bigints_as_strings,allow_null_values:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/GoogleInt64ValueDisallowStringAllowNull.proto || echo "No messages found (GoogleInt64ValueDisallowStringAllowNull.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=enforce_oneof:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/OneOf.proto || echo "No messages found (OneOf.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=all_fields_required:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/Proto2NestedObject.proto || echo "No messages found (Proto2NestedObject.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/WellKnown.proto || echo "No messages found (WellKnown.proto)"
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/NoPackage.proto
protoc --plugin=bin/protoc-gen-jsonschema --jsonschema_out=messages=[MessageKind10+MessageKind11+MessageKind12]:jsonschemas --proto_path=${PROTO_PATH} ${PROTO_PATH}/TwelveMessages.proto || echo "No messages found (TwelveMessages.proto)"

.PHONY: test
test:
@go test ./... -cover
go test ./... -cover -v
86 changes: 61 additions & 25 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,44 +1,80 @@
Protobuf to JSON-Schema compiler
================================

This takes protobuf definitions and converts them into JSONSchemas, which can be used to dynamically validate JSON messages.

This will hopefully be useful for people who define their data using ProtoBuf, but use JSON for the "wire" format.
Useful for people who define their data using ProtoBuf, but use JSON for the "wire" format.

"Heavily influenced" by [Google's protobuf-to-BigQuery-schema compiler](https://github.com/GoogleCloudPlatform/protoc-gen-bq-schema).


Generated Schemas
-----------------

- One JSONSchema file is generated for each root-level proto message and ENUM. These are intended to be stand alone self-contained schemas which can be used to validate a payload derived from their source proto message
- Nested message schemas become [referenced "definitions"](https://cswr.github.io/JsonSchema/spec/definitions_references/). This means that you know the name of the proto message they came from, and their schema is not duplicated (within the context of one JSONSchema file at least)


Logic
-----

- For each proto file provided
- Generates schema for each ENUM
- JSONSchema filename deried from ENUM name
- Generates schema for each Message
- Builds a list of every nested message and converts them to JSONSchema
- Recursively converts attributes and nested messages within the root message
- Special handling for "OneOf"
- Special handling for arrays
- Special handling for maps
- Injects references to nested messages
- JSONSchema filename derived from Message name
- Bundles these into a protoc generator response


Installation
------------
`GO111MODULE=on go get -u github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema && go install github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema`

Links
-----
* [About JSON Schema](http://json-schema.org/)
* [Popular GoLang JSON-Schema validation library](https://github.com/xeipuuv/gojsonschema)
* [Another GoLang JSON-Schema validation library](https://github.com/lestrrat/go-jsschema)
> Note: This tool requires Go 1.11+ to be installed.
Install this plugin using Go:

```sh
go install github.com/chrusty/protoc-gen-jsonschema/cmd/protoc-gen-jsonschema@latest
```


Usage
-----
* Allow NULL values (by default, JSONSchemas will reject NULL values unless we explicitly allow them):
`protoc --jsonschema_out=allow_null_values:. --proto_path=testdata/proto testdata/proto/ArrayOfPrimitives.proto`
* Disallow additional properties (JSONSchemas won't validate JSON containing extra parameters):
`protoc --jsonschema_out=disallow_additional_properties:. --proto_path=testdata/proto testdata/proto/ArrayOfPrimitives.proto`
* Disallow permissive validation of big-integers as strings (eg scientific notation):
`protoc --jsonschema_out=disallow_bigints_as_strings:. --proto_path=testdata/proto testdata/proto/ArrayOfPrimitives.proto`
* Enable debug logging:
`protoc --jsonschema_out=debug:. --proto_path=testdata/proto testdata/proto/ArrayOfPrimitives.proto`

> Note: This plugin requires the [`protoc`](https://github.com/protocolbuffers/protobuf) CLI to be installed.
**protoc-gen-jsonschema** is designed to run like any other proto generator.

```sh
protoc \ # The protobuf compiler
--proto_path=testdata/proto testdata/proto/ArrayOfPrimitives.proto # proto input directories and folders
```

Sample protos (for testing)
---------------------------
* Proto with a simple (flat) structure: [samples.PayloadMessage](testdata/proto/PayloadMessage.proto)
* Proto containing a nested object (defined internally): [samples.NestedObject](testdata/proto/NestedObject.proto)
* Proto containing a nested message (defined in a different proto file): [samples.NestedMessage](testdata/proto/NestedMessage.proto)
* Proto containing an array of a primitive types (string, int): [samples.ArrayOfPrimitives](testdata/proto/ArrayOfPrimitives.proto)
* Proto containing an array of objects (internally defined): [samples.ArrayOfObjects](testdata/proto/ArrayOfObjects.proto)
* Proto containing an array of messages (defined in a different proto file): [samples.ArrayOfMessage](testdata/proto/ArrayOfMessage.proto)
* Proto containing multi-level enums (flat and nested and arrays): [samples.Enumception](testdata/proto/Enumception.proto)
* Proto containing a stand-alone enum: [samples.ImportedEnum](testdata/proto/ImportedEnum.proto)
* Proto containing 2 stand-alone enums: [samples.FirstEnum, samples.SecondEnum](testdata/proto/SeveralEnums.proto)
* Proto containing 2 messages: [samples.FirstMessage, samples.SecondMessage](testdata/proto/SeveralMessages.proto)

* Proto with a simple (flat) structure: [samples.PayloadMessage](internal/converter/testdata/proto/PayloadMessage.proto)
* Proto containing a nested object (defined internally): [samples.NestedObject](internal/converter/testdata/proto/NestedObject.proto)
* Proto containing a nested message (defined in a different proto file): [samples.NestedMessage](internal/converter/testdata/proto/NestedMessage.proto)
* Proto containing an array of a primitive types (string, int): [samples.ArrayOfPrimitives](internal/converter/testdata/proto/ArrayOfPrimitives.proto)
* Proto containing an array of objects (internally defined): [samples.ArrayOfObjects](internal/converter/testdata/proto/ArrayOfObjects.proto)
* Proto containing an array of messages (defined in a different proto file): [samples.ArrayOfMessage](internal/converter/testdata/proto/ArrayOfMessage.proto)
* Proto containing multi-level enums (flat and nested and arrays): [samples.Enumception](internal/converter/testdata/proto/Enumception.proto)
* Proto containing a stand-alone enum: [samples.ImportedEnum](internal/converter/testdata/proto/ImportedEnum.proto)
* Proto containing 2 stand-alone enums: [samples.FirstEnum, samples.SecondEnum](internal/converter/testdata/proto/SeveralEnums.proto)
* Proto containing 2 messages: [samples.FirstMessage, samples.SecondMessage](internal/converter/testdata/proto/SeveralMessages.proto)
* Proto containing 12 messages: [samples.MessageKind1 - samples.MessageKind12](internal/converter/testdata/proto/TwelveMessages.proto)


Links
-----

* [About JSON Schema](http://json-schema.org/)
* [Popular GoLang JSON-Schema validation library](https://github.com/xeipuuv/gojsonschema)
* [Another GoLang JSON-Schema validation library](https://github.com/lestrrat/go-jsschema)
38 changes: 25 additions & 13 deletions cmd/protoc-gen-jsonschema/main.go
Original file line number Diff line number Diff line change
@@ -1,22 +1,35 @@
// protoc plugin which converts .proto to JSON schema
// It is spawned by protoc and generates JSON-schema files.
// It is called by protoc and generates JSON-schema files.
// "Heavily influenced" by Google's "protog-gen-bq-schema"
//
// usage:
// $ bin/protoc --jsonschema_out=path/to/outdir foo.proto
//
// $ bin/protoc --jsonschema_out=path/to/outdir foo.proto
package main

import (
"flag"
"fmt"
"os"

"github.com/golang/protobuf/proto"
plugin "github.com/golang/protobuf/protoc-gen-go/plugin"
"github.com/sirupsen/logrus"
"github.com/sixt/protoc-gen-jsonschema/internal/converter"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/pluginpb"

"github.com/chrusty/protoc-gen-jsonschema/internal/converter"
)

const version = "v1.4.0"

func init() {
versionFlag := flag.Bool("version", false, "prints current version")
flag.Parse()
if *versionFlag {
fmt.Println(version)
os.Exit(0)
}
}

func main() {

// Make a Logrus logger (default to INFO):
Expand All @@ -28,15 +41,14 @@ func main() {
protoConverter := converter.New(logger)

// Convert the generator request:
var ok = true
var failed bool
logger.Debug("Processing code generator request")
res, err := protoConverter.ConvertFrom(os.Stdin)
if err != nil {
ok = false
failed = true
if res == nil {
message := fmt.Sprintf("Failed to read input: %v", err)
res = &plugin.CodeGeneratorResponse{
Error: &message,
res = &pluginpb.CodeGeneratorResponse{
Error: proto.String(fmt.Sprintf("Failed to read input: %v", err)),
}
}
}
Expand All @@ -51,10 +63,10 @@ func main() {
logger.WithError(err).Fatal("Failed to write response")
}

if ok {
logger.Debug("Succeeded to process code generator request")
} else {
if failed {
logger.Warn("Failed to process code generator but successfully sent the error to protoc")
os.Exit(1)
}

logger.Debug("Succeeded to process code generator request")
}
33 changes: 19 additions & 14 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,19 +1,24 @@
module github.com/sixt/protoc-gen-jsonschema
module github.com/chrusty/protoc-gen-jsonschema

go 1.23

require (
github.com/alecthomas/jsonschema v0.0.0-20200127222324-dd4542c1f589
github.com/golang/protobuf v1.3.2
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0
github.com/sirupsen/logrus v1.4.2
github.com/stretchr/testify v1.4.0
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.1.0
github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b
github.com/fatih/camelcase v1.0.0
github.com/google/go-cmp v0.5.5
github.com/iancoleman/orderedmap v0.3.0
github.com/iancoleman/strcase v0.3.0
github.com/sirupsen/logrus v1.9.3
github.com/stretchr/testify v1.9.0
github.com/xeipuuv/gojsonschema v1.2.0
google.golang.org/protobuf v1.34.2
)

replace (
github.com/alecthomas/jsonschema => github.com/alecthomas/jsonschema v0.0.0-20200127222324-dd4542c1f589
github.com/iancoleman/orderedmap => github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

go 1.13
Loading

0 comments on commit 3633499

Please sign in to comment.