Merge pull request #5 from mrtamm/fix-lint-errors

Maintenance development to fix all GitHub Actions (lint, unitTest) errors

.github/workflows/tests.yaml

@@ -8,30 +8,28 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
-        with:
-          go-version: 1.21
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v4
-        with:
-          version: v1.56.2
-          args: --skip-dirs tests --modules-download-mode mod --timeout 10m --verbose
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+    - name: golangci-lint
+      uses: golangci/golangci-lint-action@v4
+      with:
+        version: v1.56.2
+        skip-pkg-cache: true
+        args: --skip-dirs tests --timeout 10m --verbose
 
   build:
     runs-on: ubuntu-latest
     steps:
+    - name: Check out code
+      uses: actions/checkout@v4
     - name: Set up Go 1.x
       uses: actions/setup-go@v5
       with:
-        go-version: ^1.21
-
-    - name: Check out code
-      uses: actions/checkout@v4
-
+        go-version-file: go.mod
     - name: Build
       run: go build -v ./
-
     - name: Store funnel
       uses: actions/upload-artifact@v4
       with:
@@ -41,12 +39,12 @@ jobs:
   unitTest:
     runs-on: ubuntu-latest
     steps:
+    - name: Check out code
+      uses: actions/checkout@v4
     - name: Set up Go 1.x
       uses: actions/setup-go@v5
       with:
-        go-version: ^1.21
-    - name: Check out code
-      uses: actions/checkout@v4
+        go-version-file: go.mod
     - name: Unit Tests
       run: make test
 
@@ -56,14 +54,12 @@ jobs:
     steps:
     - name: Check out code
       uses: actions/checkout@v4
-
-    - name: Download funnel bin
-      uses: actions/download-artifact@v4
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v5
       with:
-        name: funnelBin
+        go-version-file: go.mod
     - name: MongoTest
       run: |
-        chmod +x funnel
         make start-mongodb
         sleep 10
         make test-mongodb
@@ -74,12 +70,9 @@ jobs:
     steps:
     - name: Check out code
       uses: actions/checkout@v4
-
-    - name: Download funnel bin
-      uses: actions/download-artifact@v4
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v5
       with:
-        name: funnelBin
+        go-version-file: go.mod
     - name: Badger Test
-      run: |
-        chmod +x funnel
-        make test-badger
+      run: make test-badger

Makefile

@@ -37,22 +37,16 @@ build:
 
 # Generate the protobuf/gRPC code
 proto:
-	@cd tes && protoc \
+	@protoc \
 		$(PROTO_INC) \
-		--go_out=plugins=grpc:. \
-		--grpc-gateway_out=logtostderr=true:. \
-		tes.proto
-	@cd compute/scheduler && protoc \
-		$(PROTO_INC) \
-		--go_out=plugins=grpc:. \
-		--grpc-gateway_out=logtostderr=true:. \
-		scheduler.proto
-	@cd events && protoc \
-		$(PROTO_INC) \
-		-I ../tes \
-		--go_out=Mtes.proto=github.com/ohsu-comp-bio/funnel/tes,plugins=grpc:. \
-		--grpc-gateway_out=logtostderr=true:. \
-		events.proto
+		--go_out=. --go_opt=paths=source_relative \
+		--go-grpc_out=require_unimplemented_servers=false:. --go-grpc_opt=paths=source_relative \
+		--grpc-gateway_out=. \
+	    --grpc-gateway_opt=paths=source_relative \
+    	--grpc-gateway_opt=generate_unbound_methods=true \
+		tes/tes.proto \
+		compute/scheduler/scheduler.proto \
+		events/events.proto
 
 # Start API reference doc server
 serve-doc:

cmd/aws/batch/createall.go

@@ -44,7 +44,10 @@ var createCmd = &cobra.Command{
 
 		if funnelConfigFile != "" {
 			funnelConf := config.Config{}
-			config.ParseFile(funnelConfigFile, &funnelConf)
+			err := config.ParseFile(funnelConfigFile, &funnelConf)
+			if err != nil {
+				return err
+			}
 			conf.Funnel = funnelConf
 		}
 

cmd/aws/batch/createjobdef.go

@@ -38,7 +38,10 @@ var jobDefCmd = &cobra.Command{
 
 		if funnelConfigFile != "" {
 			funnelConf := config.Config{}
-			config.ParseFile(funnelConfigFile, &funnelConf)
+			err := config.ParseFile(funnelConfigFile, &funnelConf)
+			if err != nil {
+				return err
+			}
 			conf.Funnel = funnelConf
 		}
 

cmd/completion.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/spf13/cobra"
@@ -17,7 +18,10 @@ var bash = &cobra.Command{
 	Long: `This command generates bash CLI completion code.
 Add "source <(funnel completion bash)" to your bash profile.`,
 	Run: func(cmd *cobra.Command, args []string) {
-		RootCmd.GenBashCompletion(os.Stdout)
+		if err := RootCmd.GenBashCompletion(os.Stdout); err != nil {
+			fmt.Printf("Detected error while running GenBashCompletion "+
+				"command: %s\n", err)
+		}
 	},
 }
 

cmd/examples/examples.go

@@ -20,7 +20,7 @@ var Cmd = &cobra.Command{
 		byShortName := map[string]string{}
 		taskEx := ex.Examples()
 		for n, v := range taskEx {
-			byShortName[n] = v
+			byShortName[n] = string(v)
 		}
 
 		confEx := config.Examples()

cmd/node/run.go

@@ -2,6 +2,7 @@ package node
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/signal"
 	"syscall"
@@ -25,7 +26,10 @@ func Run(ctx context.Context, conf config.Config, log *logger.Logger) error {
 		if err != nil {
 			return err
 		}
-		w.Run(ctx)
+
+		if err := w.Run(ctx); err != nil {
+			fmt.Printf("Detected error while running a node: %s\n", err)
+		}
 		return nil
 	}
 
@@ -34,9 +38,7 @@ func Run(ctx context.Context, conf config.Config, log *logger.Logger) error {
 		return err
 	}
 
-	runctx, cancel := context.WithCancel(context.Background())
-	runctx = util.SignalContext(ctx, time.Nanosecond, syscall.SIGINT, syscall.SIGTERM)
-	defer cancel()
+	runctx := util.SignalContext(ctx, time.Nanosecond, syscall.SIGINT, syscall.SIGTERM)
 
 	hupsig := make(chan os.Signal, 1)
 	go func() {

cmd/run/cmd.go

@@ -2,6 +2,7 @@ package run
 
 import (
 	"bufio"
+	"fmt"
 	"os"
 
 	"github.com/kballard/go-shellquote"
@@ -20,11 +21,7 @@ var Cmd = &cobra.Command{
 	Use:   "run 'CMD' [flags]",
 	Short: "Run a task.",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		err := Run(args)
-		if err != nil {
-			//cmd.Usage()
-		}
-		return err
+		return Run(args)
 	},
 	DisableFlagParsing: true,
 }
@@ -116,7 +113,9 @@ func scatter(base flagVals) []flagVals {
 			// Per-scatter flags
 			sp, _ := shellquote.Split(scanner.Text())
 			tv := base
-			parseTaskArgs(&tv, sp)
+			if err := parseTaskArgs(&tv, sp); err != nil {
+				fmt.Printf("Failed to parse task args: %s\n", err)
+			}
 			// Parse scatter file flags into new flagVals
 			out = append(out, tv)
 		}

cmd/run/flags.go

@@ -158,7 +158,10 @@ func parseTopLevelArgs(vals *flagVals, args []string) error {
 		shCmd := flags.Args()[0]
 		args = append([]string{"--sh", shCmd}, args...)
 	}
-	parseTaskArgs(vals, args)
+
+	if err = parseTaskArgs(vals, args); err != nil {
+		return err
+	}
 
 	if len(vals.execs) == 0 {
 		return fmt.Errorf("you must specify a command to run")
@@ -170,10 +173,15 @@ func parseTopLevelArgs(vals *flagVals, args []string) error {
 	return nil
 }
 
-func parseTaskArgs(vals *flagVals, args []string) {
+func parseTaskArgs(vals *flagVals, args []string) error {
 	fl := newFlags(vals)
-	fl.Parse(args)
+
+	if err := fl.Parse(args); err != nil {
+		return fmt.Errorf("failed to parse task arguments: %w", err)
+	}
+
 	buildExecs(fl, vals, args)
+	return nil
 }
 
 // Visit flags to determine commands + stdin/out/err
@@ -185,7 +193,7 @@ func parseTaskArgs(vals *flagVals, args []string) {
 func buildExecs(flags *pflag.FlagSet, vals *flagVals, args []string) {
 	vals.execs = nil
 	var exec *executor
-	flags.ParseAll(args, func(f *pflag.Flag, value string) error {
+	err := flags.ParseAll(args, func(f *pflag.Flag, value string) error {
 		switch f.Name {
 		case "sh", "exec":
 			if exec != nil {
@@ -207,6 +215,9 @@ func buildExecs(flags *pflag.FlagSet, vals *flagVals, args []string) {
 		}
 		return nil
 	})
+	if err != nil {
+		fmt.Printf("Failed to parse some flags: %s\n", err)
+	}
 	if exec != nil {
 		vals.execs = append(vals.execs, *exec)
 	}

cmd/run/parse_vars.go

@@ -3,7 +3,7 @@ package run
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -13,14 +13,14 @@ import (
 )
 
 // ErrKeyFmt describes an error in input/output/env/tag flag formatting
-var ErrKeyFmt = errors.New("Arguments passed to --in, --out, --tag and --env must be of the form: KEY=VALUE")
+var ErrKeyFmt = errors.New("arguments passed to --in, --out, --tag and --env must be of the form: KEY=VALUE")
 
-// ErrStorageScheme describes an error in supported storage URL schemes.
-var ErrStorageScheme = errors.New("File paths must be prefixed with one of:\n file://\n gs://\n s3://")
+// StorageScheme describes an error in supported storage URL schemes.
+var ErrStorageScheme = errors.New("file paths must be prefixed with one of:\n file://\n gs://\n s3://")
 
 // DuplicateKeyErr returns a new error describing conflicting keys for env. vars., inputs, and outputs.
 func DuplicateKeyErr(key string) error {
-	return errors.New("Can't use the same KEY for multiple --in, --out, --tag, --env arguments: " + key)
+	return errors.New("can't use the same KEY for multiple --in, --out, --tag, --env arguments: " + key)
 }
 
 // Parse CLI variable definitions (e.g "varname=value") into usable task values.
@@ -197,7 +197,7 @@ func valsToTask(vals flagVals) (task *tes.Task, err error) {
 }
 
 func getContent(p string) string {
-	b, err := ioutil.ReadFile(p)
+	b, err := os.ReadFile(p)
 	if err != nil {
 		panic(err)
 	}
@@ -217,14 +217,6 @@ func parseCliVar(raw string) (string, string) {
 	return key, val
 }
 
-// Give a input/output URL "raw", return the path of the file
-// relative to the container.
-func containerPath(raw, base string) string {
-	url := resolvePath(raw)
-	p := stripStoragePrefix(url)
-	return base + p
-}
-
 func stripStoragePrefix(url string) string {
 	re := regexp.MustCompile("[a-z0-9]+://")
 	if !re.MatchString(url) {

cmd/run/task_group.go

@@ -49,17 +49,20 @@ func (tg *taskGroup) _run(task *tes.Task, wait bool, waitFor []string) error {
 
 	if tg.printTask {
 		// Marshal message to JSON
-		taskJSON, merr := tg.client.Marshaler.MarshalToString(task)
+		taskJSON, merr := tg.client.Marshaler.Marshal(task)
 		if merr != nil {
 			return merr
 		}
-		fmt.Println(taskJSON)
+		fmt.Println(string(taskJSON))
 		return nil
 	}
 
 	if len(waitFor) > 0 {
 		for _, tid := range waitFor {
-			tg.client.WaitForTask(context.Background(), tid)
+			err := tg.client.WaitForTask(context.Background(), tid)
+			if err != nil {
+				return err
+			}
 		}
 	}
 

cmd/server/run.go

@@ -260,7 +260,7 @@ func NewServer(ctx context.Context, conf config.Config, log *logger.Logger) (*Se
 
 	return &Server{
 		Server: &server.Server{
-			RPCAddress:       ":" + conf.Server.RPCPort,
+			RPCAddress:       conf.Server.RPCAddress(),
 			HTTPPort:         conf.Server.HTTPPort,
 			BasicAuth:        conf.Server.BasicAuth,
 			OidcAuth:         conf.Server.OidcAuth,