Skip to content

Commit

Permalink
Simplify code generation logic for a faster and better UX (#12493)
Browse files Browse the repository at this point in the history
The changes here simplify the string of targets executed as part of
`make gen`. The simplifications include:
* parallelised code generation whenever possible
* re-implementation of documentation generation for a significant
  simplification and improved readability.
* unified mocks generation to avoid multiple slow calls to `go run` per
  package.

Note, the changes introduced here are purely mechanical and do not alter
the Lotus node functionality.

Fixes #8392
  • Loading branch information
masih committed Sep 20, 2024
1 parent a30d024 commit 312fa3a
Show file tree
Hide file tree
Showing 31 changed files with 1,503 additions and 1,468 deletions.
34 changes: 4 additions & 30 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,6 @@ bundle-gen:
$(GOCC) fmt ./build/...
.PHONY: bundle-gen


api-gen:
$(GOCC) run ./gen/api
goimports -w api
Expand All @@ -330,38 +329,13 @@ appimage: lotus
cp ./lotus AppDir/usr/bin/
appimage-builder

docsgen: docsgen-md docsgen-openrpc fiximports

docsgen-md-bin: api-gen actors-gen
$(GOCC) build $(GOFLAGS) -o docgen-md ./api/docgen/cmd
docsgen-openrpc-bin: api-gen actors-gen
$(GOCC) build $(GOFLAGS) -o docgen-openrpc ./api/docgen-openrpc/cmd

docsgen-md: docsgen-md-full docsgen-md-storage docsgen-md-worker

docsgen-md-full: docsgen-md-bin
./docgen-md "api/api_full.go" "FullNode" "api" "./api" > documentation/en/api-v1-unstable-methods.md
./docgen-md "api/v0api/full.go" "FullNode" "v0api" "./api/v0api" > documentation/en/api-v0-methods.md
docsgen-md-storage: docsgen-md-bin
./docgen-md "api/api_storage.go" "StorageMiner" "api" "./api" > documentation/en/api-v0-methods-miner.md
docsgen-md-worker: docsgen-md-bin
./docgen-md "api/api_worker.go" "Worker" "api" "./api" > documentation/en/api-v0-methods-worker.md

docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker docsgen-openrpc-gateway

docsgen-openrpc-full: docsgen-openrpc-bin
./docgen-openrpc "api/api_full.go" "FullNode" "api" "./api" > build/openrpc/full.json
docsgen-openrpc-storage: docsgen-openrpc-bin
./docgen-openrpc "api/api_storage.go" "StorageMiner" "api" "./api" > build/openrpc/miner.json
docsgen-openrpc-worker: docsgen-openrpc-bin
./docgen-openrpc "api/api_worker.go" "Worker" "api" "./api" > build/openrpc/worker.json
docsgen-openrpc-gateway: docsgen-openrpc-bin
./docgen-openrpc "api/api_gateway.go" "Gateway" "api" "./api" > build/openrpc/gateway.json

.PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
docsgen: fiximports
$(GOCC) run ./gen/docs
.PHONY: docsgen

fiximports:
$(GOCC) run ./scripts/fiximports
.PHONY: fiximports

gen: actors-code-gen type-gen cfgdoc-gen docsgen api-gen
$(GOCC) run ./scripts/fiximports
Expand Down
54 changes: 14 additions & 40 deletions api/docgen-openrpc/cmd/docgen_openrpc.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
package main

import (
"compress/gzip"
"encoding/json"
"io"
"log"
"fmt"
"os"

"github.com/filecoin-project/lotus/api/docgen"
Expand All @@ -29,45 +26,22 @@ Use:
*/

func main() {
Comments, GroupDocs := docgen.ParseApiASTInfo(os.Args[1], os.Args[2], os.Args[3], os.Args[4])

doc := docgen_openrpc.NewLotusOpenRPCDocument(Comments, GroupDocs)

i, _, _ := docgen.GetAPIType(os.Args[2], os.Args[3])
doc.RegisterReceiverName("Filecoin", i)

out, err := doc.Discover()
if err != nil {
log.Fatalln(err)
}

var jsonOut []byte
var writer io.WriteCloser

// Use os.Args to handle a somewhat hacky flag for the gzip option.
// Could use flags package to handle this more cleanly, but that requires changes elsewhere
// the scope of which just isn't warranted by this one use case which will usually be run
// programmatically anyways.
if len(os.Args) > 5 && os.Args[5] == "-gzip" {
jsonOut, err = json.Marshal(out)
if err != nil {
log.Fatalln(err)
}
writer = gzip.NewWriter(os.Stdout)
} else {
jsonOut, err = json.MarshalIndent(out, "", " ")
if err != nil {
log.Fatalln(err)
}
writer = os.Stdout
}

_, err = writer.Write(jsonOut)
if err != nil {
log.Fatalln(err)
}
err = writer.Close()
if err != nil {
log.Fatalln(err)
var (
apiFile = os.Args[1]
iface = os.Args[2]
pkg = os.Args[3]
dir = os.Args[4]
outGzip = len(os.Args) > 5 && os.Args[5] == "-gzip"
)
if ainfo, err := docgen.ParseApiASTInfo(apiFile, iface, pkg, dir); err != nil {
_, _ = fmt.Fprintf(os.Stderr, "Failed to parse API AST info: %v\n", err)
os.Exit(1)
} else if err := docgen_openrpc.Generate(os.Stdout, iface, pkg, ainfo, outGzip); err != nil {
_, _ = fmt.Fprintf(os.Stderr, "Failed to generate OpenRPC docs: %v\n", err)
os.Exit(1)
}
}
33 changes: 33 additions & 0 deletions api/docgen-openrpc/openrpc.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package docgenopenrpc

import (
"compress/gzip"
"encoding/json"
"go/ast"
"io"
"net"
"reflect"

Expand All @@ -29,6 +31,37 @@ const integerD = `{

const cidCidD = `{"title": "Content Identifier", "type": "string", "description": "Cid represents a self-describing content addressed identifier. It is formed by a Version, a Codec (which indicates a multicodec-packed content type) and a Multihash."}`

func Generate(out io.Writer, iface, pkg string, ainfo docgen.ApiASTInfo, outGzip bool) error {
doc := NewLotusOpenRPCDocument(ainfo.Comments, ainfo.GroupComments)
i, _, _ := docgen.GetAPIType(iface, pkg)
doc.RegisterReceiverName("Filecoin", i)

dout, err := doc.Discover()
if err != nil {
return err
}

switch {
case outGzip:
jsonOut, err := json.Marshal(dout)
if err != nil {
return err
}
writer := gzip.NewWriter(out)
if _, err = writer.Write(jsonOut); err != nil {
return err
}
return writer.Close()
default:
jsonOut, err := json.MarshalIndent(dout, "", " ")
if err != nil {
return err
}
_, err = out.Write(jsonOut)
return err
}
}

func OpenRPCSchemaTypeMapper(ty reflect.Type) *jsonschema.Type {
unmarshalJSONToJSONSchemaType := func(input string) *jsonschema.Type {
var js jsonschema.Type
Expand Down
121 changes: 12 additions & 109 deletions api/docgen/cmd/docgen.go
Original file line number Diff line number Diff line change
@@ -1,121 +1,24 @@
package main

import (
"encoding/json"
"fmt"
"os"
"reflect"
"sort"
"strings"

"github.com/filecoin-project/lotus/api/docgen"
)

func main() {
comments, groupComments := docgen.ParseApiASTInfo(os.Args[1], os.Args[2], os.Args[3], os.Args[4])

groups := make(map[string]*docgen.MethodGroup)

_, t, permStruct := docgen.GetAPIType(os.Args[2], os.Args[3])

for i := 0; i < t.NumMethod(); i++ {
m := t.Method(i)

groupName := docgen.MethodGroupFromName(m.Name)

g, ok := groups[groupName]
if !ok {
g = new(docgen.MethodGroup)
g.Header = groupComments[groupName]
g.GroupName = groupName
groups[groupName] = g
}

var args []interface{}
ft := m.Func.Type()
for j := 2; j < ft.NumIn(); j++ {
inp := ft.In(j)
args = append(args, docgen.ExampleValue(m.Name, inp, nil))
}

v, err := json.MarshalIndent(args, "", " ")
if err != nil {
panic(err)
}

outv := docgen.ExampleValue(m.Name, ft.Out(0), nil)

ov, err := json.MarshalIndent(outv, "", " ")
if err != nil {
panic(err)
}

g.Methods = append(g.Methods, &docgen.Method{
Name: m.Name,
Comment: comments[m.Name],
InputExample: string(v),
ResponseExample: string(ov),
})
}

var groupslice []*docgen.MethodGroup
for _, g := range groups {
groupslice = append(groupslice, g)
}

sort.Slice(groupslice, func(i, j int) bool {
return groupslice[i].GroupName < groupslice[j].GroupName
})

fmt.Printf("# Groups\n")

for _, g := range groupslice {
fmt.Printf("* [%s](#%s)\n", g.GroupName, g.GroupName)
for _, method := range g.Methods {
fmt.Printf(" * [%s](#%s)\n", method.Name, method.Name)
}
}

for _, g := range groupslice {
g := g
fmt.Printf("## %s\n", g.GroupName)
fmt.Printf("%s\n\n", g.Header)

sort.Slice(g.Methods, func(i, j int) bool {
return g.Methods[i].Name < g.Methods[j].Name
})

for _, m := range g.Methods {
fmt.Printf("### %s\n", m.Name)
fmt.Printf("%s\n\n", m.Comment)

var meth reflect.StructField
var ok bool
for _, ps := range permStruct {
meth, ok = ps.FieldByName(m.Name)
if ok {
break
}
}
if !ok {
panic("no perms for method: " + m.Name)
}

perms := meth.Tag.Get("perm")

fmt.Printf("Perms: %s\n\n", perms)

if strings.Count(m.InputExample, "\n") > 0 {
fmt.Printf("Inputs:\n```json\n%s\n```\n\n", m.InputExample)
} else {
fmt.Printf("Inputs: `%s`\n\n", m.InputExample)
}

if strings.Count(m.ResponseExample, "\n") > 0 {
fmt.Printf("Response:\n```json\n%s\n```\n\n", m.ResponseExample)
} else {
fmt.Printf("Response: `%s`\n\n", m.ResponseExample)
}
}
var (
apiFile = os.Args[1]
iface = os.Args[2]
pkg = os.Args[3]
dir = os.Args[4]
)
if ainfo, err := docgen.ParseApiASTInfo(apiFile, iface, pkg, dir); err != nil {
_, _ = fmt.Fprintf(os.Stderr, "Failed to parse API AST info: %v\n", err)
os.Exit(1)
} else if err := docgen.Generate(os.Stdout, iface, pkg, ainfo); err != nil {
_, _ = fmt.Fprintf(os.Stderr, "Failed to generate docs: %v\n", err)
os.Exit(1)
}
}
Loading

0 comments on commit 312fa3a

Please sign in to comment.