From b88562e9509e3bf0ab49d2cda3c5b82c6cad54a7 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 21 Sep 2021 17:17:37 -0500 Subject: [PATCH 01/14] move server.Config to separate file it is not grpc-specific --- server/config.go | 71 ++++++++++++++++++++++++++++++++++++++++++++++++ server/rpc.go | 45 ------------------------------ 2 files changed, 71 insertions(+), 45 deletions(-) create mode 100644 server/config.go diff --git a/server/config.go b/server/config.go new file mode 100644 index 0000000000..3737b0f050 --- /dev/null +++ b/server/config.go @@ -0,0 +1,71 @@ +// Copyright 2018 Drone.IO Inc. +// Copyright 2021 Informatyka Boguslawski sp. z o.o. sp.k., http://www.ib.pl/ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file has been modified by Informatyka Boguslawski sp. z o.o. sp.k. + +package server + +import ( + "time" + + "github.com/woodpecker-ci/woodpecker/cncd/logging" + "github.com/woodpecker-ci/woodpecker/cncd/pubsub" + "github.com/woodpecker-ci/woodpecker/cncd/queue" + "github.com/woodpecker-ci/woodpecker/model" +) + +var Config = struct { + Services struct { + Pubsub pubsub.Publisher + Queue queue.Queue + Logs logging.Log + Senders model.SenderService + Secrets model.SecretService + Registries model.RegistryService + Environ model.EnvironService + } + Storage struct { + // Users model.UserStore + // Repos model.RepoStore + // Builds model.BuildStore + // Logs model.LogStore + Config model.ConfigStore + Files model.FileStore + Procs model.ProcStore + // Registries model.RegistryStore + // Secrets model.SecretStore + } + Server struct { + Key string + Cert string + Host string + Port string + Pass string + RepoConfig string + SessionExpires time.Duration + // Open bool + // Orgs map[string]struct{} + // Admins map[string]struct{} + } + Prometheus struct { + AuthToken string + } + Pipeline struct { + Limits model.ResourceLimit + Volumes []string + Networks []string + Privileged []string + } +}{} diff --git a/server/rpc.go b/server/rpc.go index 42b4b22ccb..2f5f8deb4f 100644 --- a/server/rpc.go +++ b/server/rpc.go @@ -24,7 +24,6 @@ import ( "fmt" "log" "strconv" - "time" oldcontext "golang.org/x/net/context" @@ -46,50 +45,6 @@ import ( "github.com/woodpecker-ci/expr" ) -var Config = struct { - Services struct { - Pubsub pubsub.Publisher - Queue queue.Queue - Logs logging.Log - Senders model.SenderService - Secrets model.SecretService - Registries model.RegistryService - Environ model.EnvironService - } - Storage struct { - // Users model.UserStore - // Repos model.RepoStore - // Builds model.BuildStore - // Logs model.LogStore - Config model.ConfigStore - Files model.FileStore - Procs model.ProcStore - // Registries model.RegistryStore - // Secrets model.SecretStore - } - Server struct { - Key string - Cert string - Host string - Port string - Pass string - RepoConfig string - SessionExpires time.Duration - // Open bool - // Orgs map[string]struct{} - // Admins map[string]struct{} - } - Prometheus struct { - AuthToken string - } - Pipeline struct { - Limits model.ResourceLimit - Volumes []string - Networks []string - Privileged []string - } -}{} - type RPC struct { remote remote.Remote queue queue.Queue From e9bde05a0b1ad8ab39032c24c77a97a566f572c8 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 21 Sep 2021 17:20:31 -0500 Subject: [PATCH 02/14] metadata -> grpcMetadata to not shadow var --- server/rpc.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/rpc.go b/server/rpc.go index 2f5f8deb4f..ce5aa9635b 100644 --- a/server/rpc.go +++ b/server/rpc.go @@ -27,7 +27,7 @@ import ( oldcontext "golang.org/x/net/context" - "google.golang.org/grpc/metadata" + grpcMetadata "google.golang.org/grpc/metadata" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -58,7 +58,7 @@ type RPC struct { // Next implements the rpc.Next function func (s *RPC) Next(c context.Context, filter rpc.Filter) (*rpc.Pipeline, error) { - metadata, ok := metadata.FromIncomingContext(c) + metadata, ok := grpcMetadata.FromIncomingContext(c) if ok { hostname, ok := metadata["hostname"] if ok && len(hostname) != 0 { @@ -123,7 +123,7 @@ func (s *RPC) Update(c context.Context, id string, state rpc.State) error { return err } - metadata, ok := metadata.FromIncomingContext(c) + metadata, ok := grpcMetadata.FromIncomingContext(c) if ok { hostname, ok := metadata["hostname"] if ok && len(hostname) != 0 { @@ -243,7 +243,7 @@ func (s *RPC) Init(c context.Context, id string, state rpc.State) error { log.Printf("error: cannot find proc with id %d: %s", procID, err) return err } - metadata, ok := metadata.FromIncomingContext(c) + metadata, ok := grpcMetadata.FromIncomingContext(c) if ok { hostname, ok := metadata["hostname"] if ok && len(hostname) != 0 { From f55356e26a8d78ef7362edcecfeb50e2e679f44e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 21 Sep 2021 17:25:51 -0500 Subject: [PATCH 03/14] Refactor: create server/helpers package --- server/build.go | 25 +++++------ .../buildStatus.go} | 2 +- server/{ => helpers}/buildStatus_test.go | 2 +- server/{ => helpers}/configFetcher.go | 2 +- server/{ => helpers}/configFetcher_test.go | 6 +-- server/{ => helpers}/procBuilder.go | 43 ++++++++++--------- server/{ => helpers}/procBuilder_test.go | 24 +++++------ server/{ => helpers}/procStatus.go | 2 +- server/{ => helpers}/procStatus_test.go | 2 +- server/{sync.go => helpers/userSyncer.go} | 32 +++++++------- server/hook.go | 17 ++++---- server/rpc.go | 13 +++--- server/user.go | 21 ++++----- 13 files changed, 98 insertions(+), 93 deletions(-) rename server/{builsStatus.go => helpers/buildStatus.go} (99%) rename server/{ => helpers}/buildStatus_test.go (99%) rename server/{ => helpers}/configFetcher.go (99%) rename server/{ => helpers}/configFetcher_test.go (97%) rename server/{ => helpers}/procBuilder.go (85%) rename server/{ => helpers}/procBuilder_test.go (96%) rename server/{ => helpers}/procStatus.go (99%) rename server/{ => helpers}/procStatus_test.go (99%) rename server/{sync.go => helpers/userSyncer.go} (83%) diff --git a/server/build.go b/server/build.go index fda3363b01..28a007501f 100644 --- a/server/build.go +++ b/server/build.go @@ -35,6 +35,7 @@ import ( "github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/router/middleware/session" + "github.com/woodpecker-ci/woodpecker/server/helpers" ) func GetBuilds(c *gin.Context) { @@ -98,7 +99,7 @@ func GetBuildLogs(c *gin.Context) { repo := session.Repo(c) // parse the build number and job sequence number from - // the repquest parameter. + // the request parameter. num, _ := strconv.Atoi(c.Params.ByName("number")) ppid, _ := strconv.Atoi(c.Params.ByName("pid")) name := c.Params.ByName("proc") @@ -131,7 +132,7 @@ func GetProcLogs(c *gin.Context) { repo := session.Repo(c) // parse the build number and job sequence number from - // the repquest parameter. + // the request parameter. num, _ := strconv.Atoi(c.Params.ByName("number")) pid, _ := strconv.Atoi(c.Params.ByName("pid")) @@ -204,18 +205,18 @@ func DeleteBuild(c *gin.Context) { for _, proc := range procs { if proc.State == model.StatusPending { if proc.PPID != 0 { - if _, err = UpdateProcToStatusSkipped(store.FromContext(c), *proc, 0); err != nil { + if _, err = helpers.UpdateProcToStatusSkipped(store.FromContext(c), *proc, 0); err != nil { log.Printf("error: done: cannot update proc_id %d state: %s", proc.ID, err) } } else { - if _, err = UpdateProcToStatusKilled(store.FromContext(c), *proc); err != nil { + if _, err = helpers.UpdateProcToStatusKilled(store.FromContext(c), *proc); err != nil { log.Printf("error: done: cannot update proc_id %d state: %s", proc.ID, err) } } } } - killedBuild, err := UpdateToStatusKilled(store.FromContext(c), *build) + killedBuild, err := helpers.UpdateToStatusKilled(store.FromContext(c), *build) if err != nil { c.AbortWithError(500, err) return @@ -270,7 +271,7 @@ func PostApproval(c *gin.Context) { return } - if build, err = UpdateToStatusPending(store.FromContext(c), *build, user.Login); err != nil { + if build, err = helpers.UpdateToStatusPending(store.FromContext(c), *build, user.Login); err != nil { c.String(500, "error updating build. %s", err) return } @@ -301,7 +302,7 @@ func PostApproval(c *gin.Context) { yamls = append(yamls, &remote.FileMeta{Data: []byte(y.Data), Name: y.Name}) } - b := procBuilder{ + b := helpers.ProcBuilder{ Repo: repo, Curr: build, Last: last, @@ -314,12 +315,12 @@ func PostApproval(c *gin.Context) { } buildItems, err := b.Build() if err != nil { - if _, err = UpdateToStatusError(store.FromContext(c), *build, err); err != nil { + if _, err = helpers.UpdateToStatusError(store.FromContext(c), *build, err); err != nil { logrus.Errorf("Error setting error status of build for %s#%d. %s", repo.FullName, build.Number, err) } return } - build = setBuildStepsOnBuild(b.Curr, buildItems) + build = helpers.SetBuildStepsOnBuild(b.Curr, buildItems) err = store.FromContext(c).ProcCreate(build.Procs) if err != nil { @@ -364,7 +365,7 @@ func PostDecline(c *gin.Context) { return } - if _, err = UpdateToStatusDeclined(store.FromContext(c), *build, user.Login); err != nil { + if _, err = helpers.UpdateToStatusDeclined(store.FromContext(c), *build, user.Login); err != nil { c.String(500, "error updating build. %s", err) return } @@ -510,7 +511,7 @@ func PostBuild(c *gin.Context) { yamls = append(yamls, &remote.FileMeta{Data: []byte(y.Data), Name: y.Name}) } - b := procBuilder{ + b := helpers.ProcBuilder{ Repo: repo, Curr: build, Last: last, @@ -530,7 +531,7 @@ func PostBuild(c *gin.Context) { c.JSON(500, build) return } - build = setBuildStepsOnBuild(b.Curr, buildItems) + build = helpers.SetBuildStepsOnBuild(b.Curr, buildItems) err = store.FromContext(c).ProcCreate(build.Procs) if err != nil { diff --git a/server/builsStatus.go b/server/helpers/buildStatus.go similarity index 99% rename from server/builsStatus.go rename to server/helpers/buildStatus.go index 88eb31b7df..145f549fbc 100644 --- a/server/builsStatus.go +++ b/server/helpers/buildStatus.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package helpers import ( "time" diff --git a/server/buildStatus_test.go b/server/helpers/buildStatus_test.go similarity index 99% rename from server/buildStatus_test.go rename to server/helpers/buildStatus_test.go index 7318b757cb..9df92404cf 100644 --- a/server/buildStatus_test.go +++ b/server/helpers/buildStatus_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package helpers import ( "errors" diff --git a/server/configFetcher.go b/server/helpers/configFetcher.go similarity index 99% rename from server/configFetcher.go rename to server/helpers/configFetcher.go index b8c183b273..feed3f1cd1 100644 --- a/server/configFetcher.go +++ b/server/helpers/configFetcher.go @@ -1,4 +1,4 @@ -package server +package helpers import ( "strings" diff --git a/server/configFetcher_test.go b/server/helpers/configFetcher_test.go similarity index 97% rename from server/configFetcher_test.go rename to server/helpers/configFetcher_test.go index d41f53daee..8b06553baa 100644 --- a/server/configFetcher_test.go +++ b/server/helpers/configFetcher_test.go @@ -1,4 +1,4 @@ -package server_test +package helpers_test import ( "fmt" @@ -9,7 +9,7 @@ import ( "github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/remote" "github.com/woodpecker-ci/woodpecker/remote/mocks" - "github.com/woodpecker-ci/woodpecker/server" + "github.com/woodpecker-ci/woodpecker/server/helpers" ) func TestFetch(t *testing.T) { @@ -179,7 +179,7 @@ func TestFetch(t *testing.T) { r.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("File not found")) r.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("Directory not found")) - configFetcher := server.NewConfigFetcher( + configFetcher := helpers.NewConfigFetcher( r, &model.User{Token: "xxx"}, repo, diff --git a/server/procBuilder.go b/server/helpers/procBuilder.go similarity index 85% rename from server/procBuilder.go rename to server/helpers/procBuilder.go index ee64a4017d..13eddbcf5b 100644 --- a/server/procBuilder.go +++ b/server/helpers/procBuilder.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package helpers import ( "fmt" + "github.com/woodpecker-ci/woodpecker/server" "math/rand" "net/url" "path/filepath" @@ -33,8 +34,8 @@ import ( "github.com/woodpecker-ci/woodpecker/remote" ) -// Takes the hook data and the yaml and returns in internal data model -type procBuilder struct { +// ProcBuilder Takes the hook data and the yaml and returns in helpers data model +type ProcBuilder struct { Repo *model.Repo Curr *model.Build Last *model.Build @@ -46,7 +47,7 @@ type procBuilder struct { Envs map[string]string } -type buildItem struct { +type BuildItem struct { Proc *model.Proc Platform string Labels map[string]string @@ -55,8 +56,8 @@ type buildItem struct { Config *backend.Config } -func (b *procBuilder) Build() ([]*buildItem, error) { - var items []*buildItem +func (b *ProcBuilder) Build() ([]*BuildItem, error) { + var items []*BuildItem sort.Sort(remote.ByName(b.Yamls)) @@ -79,7 +80,7 @@ func (b *procBuilder) Build() ([]*buildItem, error) { PGID: pidSequence, State: model.StatusPending, Environ: axis, - Name: sanitizePath(y.Name), + Name: SanitizePath(y.Name), } metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, proc, b.Link) @@ -117,7 +118,7 @@ func (b *procBuilder) Build() ([]*buildItem, error) { continue } - item := &buildItem{ + item := &BuildItem{ Proc: proc, Config: ir, Labels: parsed.Labels, @@ -139,8 +140,8 @@ func (b *procBuilder) Build() ([]*buildItem, error) { return items, nil } -func filterItemsWithMissingDependencies(items []*buildItem) []*buildItem { - itemsToRemove := make([]*buildItem, 0) +func filterItemsWithMissingDependencies(items []*BuildItem) []*BuildItem { + itemsToRemove := make([]*BuildItem, 0) for _, item := range items { for _, dep := range item.DependsOn { @@ -151,7 +152,7 @@ func filterItemsWithMissingDependencies(items []*buildItem) []*buildItem { } if len(itemsToRemove) > 0 { - filtered := make([]*buildItem, 0) + filtered := make([]*BuildItem, 0) for _, item := range items { if !containsItemWithName(item.Proc.Name, itemsToRemove) { filtered = append(filtered, item) @@ -164,7 +165,7 @@ func filterItemsWithMissingDependencies(items []*buildItem) []*buildItem { return items } -func containsItemWithName(name string, items []*buildItem) bool { +func containsItemWithName(name string, items []*BuildItem) bool { for _, item := range items { if name == item.Proc.Name { return true @@ -173,7 +174,7 @@ func containsItemWithName(name string, items []*buildItem) bool { return false } -func (b *procBuilder) envsubst_(y string, environ map[string]string) (string, error) { +func (b *ProcBuilder) envsubst_(y string, environ map[string]string) (string, error) { return envsubst.Eval(y, func(name string) string { env := environ[name] if strings.Contains(env, "\n") { @@ -183,7 +184,7 @@ func (b *procBuilder) envsubst_(y string, environ map[string]string) (string, er }) } -func (b *procBuilder) environmentVariables(metadata frontend.Metadata, axis matrix.Axis) map[string]string { +func (b *ProcBuilder) environmentVariables(metadata frontend.Metadata, axis matrix.Axis) map[string]string { environ := metadata.Environ() for k, v := range metadata.EnvironDrone() { environ[k] = v @@ -194,7 +195,7 @@ func (b *procBuilder) environmentVariables(metadata frontend.Metadata, axis matr return environ } -func (b *procBuilder) toInternalRepresentation(parsed *yaml.Config, environ map[string]string, metadata frontend.Metadata, procID int64) *backend.Config { +func (b *ProcBuilder) toInternalRepresentation(parsed *yaml.Config, environ map[string]string, metadata frontend.Metadata, procID int64) *backend.Config { var secrets []compiler.Secret for _, sec := range b.Secs { if !sec.Match(b.Curr.Event) { @@ -220,10 +221,10 @@ func (b *procBuilder) toInternalRepresentation(parsed *yaml.Config, environ map[ return compiler.New( compiler.WithEnviron(environ), compiler.WithEnviron(b.Envs), - compiler.WithEscalated(Config.Pipeline.Privileged...), - compiler.WithResourceLimit(Config.Pipeline.Limits.MemSwapLimit, Config.Pipeline.Limits.MemLimit, Config.Pipeline.Limits.ShmSize, Config.Pipeline.Limits.CPUQuota, Config.Pipeline.Limits.CPUShares, Config.Pipeline.Limits.CPUSet), - compiler.WithVolumes(Config.Pipeline.Volumes...), - compiler.WithNetworks(Config.Pipeline.Networks...), + compiler.WithEscalated(server.Config.Pipeline.Privileged...), + compiler.WithResourceLimit(server.Config.Pipeline.Limits.MemSwapLimit, server.Config.Pipeline.Limits.MemLimit, server.Config.Pipeline.Limits.ShmSize, server.Config.Pipeline.Limits.CPUQuota, server.Config.Pipeline.Limits.CPUShares, server.Config.Pipeline.Limits.CPUSet), + compiler.WithVolumes(server.Config.Pipeline.Volumes...), + compiler.WithNetworks(server.Config.Pipeline.Networks...), compiler.WithLocal(false), compiler.WithOption( compiler.WithNetrc( @@ -248,7 +249,7 @@ func (b *procBuilder) toInternalRepresentation(parsed *yaml.Config, environ map[ ).Compile(parsed) } -func setBuildStepsOnBuild(build *model.Build, buildItems []*buildItem) *model.Build { +func SetBuildStepsOnBuild(build *model.Build, buildItems []*BuildItem) *model.Build { var pidSequence int for _, item := range buildItems { build.Procs = append(build.Procs, item.Proc) @@ -359,7 +360,7 @@ func metadataFromStruct(repo *model.Repo, build, last *model.Build, proc *model. } } -func sanitizePath(path string) string { +func SanitizePath(path string) string { path = filepath.Base(path) path = strings.TrimSuffix(path, ".yml") path = strings.TrimPrefix(path, ".") diff --git a/server/procBuilder_test.go b/server/helpers/procBuilder_test.go similarity index 96% rename from server/procBuilder_test.go rename to server/helpers/procBuilder_test.go index 534b5e5214..12d47c4e25 100644 --- a/server/procBuilder_test.go +++ b/server/helpers/procBuilder_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package helpers import ( "fmt" @@ -25,7 +25,7 @@ import ( func TestMultilineEnvsubst(t *testing.T) { t.Parallel() - b := procBuilder{ + b := ProcBuilder{ Repo: &model.Repo{}, Curr: &model.Build{ Message: `aaa @@ -61,7 +61,7 @@ pipeline: func TestMultiPipeline(t *testing.T) { t.Parallel() - b := procBuilder{ + b := ProcBuilder{ Repo: &model.Repo{}, Curr: &model.Build{}, Last: &model.Build{}, @@ -95,7 +95,7 @@ pipeline: func TestDependsOn(t *testing.T) { t.Parallel() - b := procBuilder{ + b := ProcBuilder{ Repo: &model.Repo{}, Curr: &model.Build{}, Last: &model.Build{}, @@ -141,7 +141,7 @@ depends_on: func TestRunsOn(t *testing.T) { t.Parallel() - b := procBuilder{ + b := ProcBuilder{ Repo: &model.Repo{}, Curr: &model.Build{}, Last: &model.Build{}, @@ -177,7 +177,7 @@ runs_on: func TestBranchFilter(t *testing.T) { t.Parallel() - b := procBuilder{ + b := ProcBuilder{ Repo: &model.Repo{}, Curr: &model.Build{Branch: "dev"}, Last: &model.Build{}, @@ -225,7 +225,7 @@ func TestZeroSteps(t *testing.T) { build := &model.Build{Branch: "dev"} - b := procBuilder{ + b := ProcBuilder{ Repo: &model.Repo{}, Curr: build, Last: &model.Build{}, @@ -259,7 +259,7 @@ func TestZeroStepsAsMultiPipelineDeps(t *testing.T) { build := &model.Build{Branch: "dev"} - b := procBuilder{ + b := ProcBuilder{ Repo: &model.Repo{}, Curr: build, Last: &model.Build{}, @@ -307,7 +307,7 @@ func TestZeroStepsAsMultiPipelineTransitiveDeps(t *testing.T) { build := &model.Build{Branch: "dev"} - b := procBuilder{ + b := ProcBuilder{ Repo: &model.Repo{}, Curr: build, Last: &model.Build{}, @@ -361,7 +361,7 @@ func TestTree(t *testing.T) { build := &model.Build{} - b := procBuilder{ + b := ProcBuilder{ Repo: &model.Repo{}, Curr: build, Last: &model.Build{}, @@ -378,7 +378,7 @@ pipeline: } buildItems, err := b.Build() - build = setBuildStepsOnBuild(build, buildItems) + build = SetBuildStepsOnBuild(build, buildItems) if err != nil { t.Fatal(err) } @@ -415,7 +415,7 @@ func TestSanitizePath(t *testing.T) { } for _, test := range testTable { - if test.sanitizedPath != sanitizePath(test.path) { + if test.sanitizedPath != SanitizePath(test.path) { t.Fatal("Path hasn't been sanitized correctly") } } diff --git a/server/procStatus.go b/server/helpers/procStatus.go similarity index 99% rename from server/procStatus.go rename to server/helpers/procStatus.go index f5866fa195..351ca6c4d9 100644 --- a/server/procStatus.go +++ b/server/helpers/procStatus.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package helpers import ( "time" diff --git a/server/procStatus_test.go b/server/helpers/procStatus_test.go similarity index 99% rename from server/procStatus_test.go rename to server/helpers/procStatus_test.go index 548ad84ecf..d4ccc75206 100644 --- a/server/procStatus_test.go +++ b/server/helpers/procStatus_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package helpers import ( "testing" diff --git a/server/sync.go b/server/helpers/userSyncer.go similarity index 83% rename from server/sync.go rename to server/helpers/userSyncer.go index 6ecf64e52d..982ab9655b 100644 --- a/server/sync.go +++ b/server/helpers/userSyncer.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package helpers import ( "time" @@ -22,16 +22,16 @@ import ( "github.com/woodpecker-ci/woodpecker/store" ) -// Syncer synces the user repository and permissions. -type Syncer interface { +// UserSyncer syncs the user repository and permissions. +type UserSyncer interface { Sync(user *model.User) error } -type syncer struct { - remote remote.Remote - store store.Store - perms model.PermStore - match FilterFunc +type Syncer struct { + Remote remote.Remote + Store store.Store + Perms model.PermStore + Match FilterFunc } // FilterFunc can be used to filter which repositories are @@ -58,13 +58,13 @@ func noopFilter(*model.Repo) bool { } // SetFilter sets the filter function. -func (s *syncer) SetFilter(fn FilterFunc) { - s.match = fn +func (s *Syncer) SetFilter(fn FilterFunc) { + s.Match = fn } -func (s *syncer) Sync(user *model.User) error { +func (s *Syncer) Sync(user *model.User) error { unix := time.Now().Unix() - (3601) // force immediate expiration. note 1 hour expiration is hard coded at the moment - repos, err := s.remote.Repos(user) + repos, err := s.Remote.Repos(user) if err != nil { return err } @@ -73,7 +73,7 @@ func (s *syncer) Sync(user *model.User) error { var perms []*model.Perm for _, repo := range repos { - if s.match(repo) { + if s.Match(repo) { remote = append(remote, repo) perm := model.Perm{ UserID: user.ID, @@ -89,12 +89,12 @@ func (s *syncer) Sync(user *model.User) error { } } - err = s.store.RepoBatch(remote) + err = s.Store.RepoBatch(remote) if err != nil { return err } - err = s.store.PermBatch(perms) + err = s.Store.PermBatch(perms) if err != nil { return err } @@ -110,5 +110,5 @@ func (s *syncer) Sync(user *model.User) error { return nil } - return s.perms.PermFlush(user, unix) + return s.Perms.PermFlush(user, unix) } diff --git a/server/hook.go b/server/hook.go index ea70a13164..0b95e16c88 100644 --- a/server/hook.go +++ b/server/hook.go @@ -40,6 +40,7 @@ import ( "github.com/woodpecker-ci/woodpecker/cncd/pipeline/pipeline/rpc" "github.com/woodpecker-ci/woodpecker/cncd/pubsub" "github.com/woodpecker-ci/woodpecker/cncd/queue" + "github.com/woodpecker-ci/woodpecker/server/helpers" ) var skipRe = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`) @@ -159,7 +160,7 @@ func PostHook(c *gin.Context) { } // fetch the build file from the remote - configFetcher := &configFetcher{remote_: remote_, user: user, repo: repo, build: build} + configFetcher := helpers.NewConfigFetcher(remote_, user, repo, build) remoteYamlConfigs, err := configFetcher.Fetch() if err != nil { logrus.Errorf("error: %s: cannot find %s in %s: %s", repo.FullName, repo.Config, build.Ref, err) @@ -241,7 +242,7 @@ func PostHook(c *gin.Context) { // get the previous build so that we can send status change notifications last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) - b := procBuilder{ + b := helpers.ProcBuilder{ Repo: repo, Curr: build, Last: last, @@ -254,12 +255,12 @@ func PostHook(c *gin.Context) { } buildItems, err := b.Build() if err != nil { - if _, err = UpdateToStatusError(store.FromContext(c), *build, err); err != nil { + if _, err = helpers.UpdateToStatusError(store.FromContext(c), *build, err); err != nil { logrus.Errorf("Error setting error status of build for %s#%d. %s", repo.FullName, build.Number, err) } return } - build = setBuildStepsOnBuild(b.Curr, buildItems) + build = helpers.SetBuildStepsOnBuild(b.Curr, buildItems) err = store.FromContext(c).ProcCreate(build.Procs) if err != nil { @@ -300,7 +301,7 @@ func branchFiltered(build *model.Build, remoteYamlConfigs []*remote.FileMeta) (b } func zeroSteps(build *model.Build, remoteYamlConfigs []*remote.FileMeta) bool { - b := procBuilder{ + b := helpers.ProcBuilder{ Repo: &model.Repo{}, Curr: build, Last: &model.Build{}, @@ -330,7 +331,7 @@ func findOrPersistPipelineConfig(repo *model.Repo, build *model.Build, remoteYam RepoID: build.RepoID, Data: string(remoteYamlConfig.Data), Hash: sha, - Name: sanitizePath(remoteYamlConfig.Name), + Name: helpers.SanitizePath(remoteYamlConfig.Name), } err = Config.Storage.Config.ConfigCreate(conf) if err != nil { @@ -372,7 +373,7 @@ func publishToTopic(c *gin.Context, build *model.Build, repo *model.Repo, event Config.Services.Pubsub.Publish(c, "topic/events", message) } -func queueBuild(build *model.Build, repo *model.Repo, buildItems []*buildItem) { +func queueBuild(build *model.Build, repo *model.Repo, buildItems []*helpers.BuildItem) { var tasks []*queue.Task for _, item := range buildItems { if item.Proc.State == model.StatusSkipped { @@ -402,7 +403,7 @@ func queueBuild(build *model.Build, repo *model.Repo, buildItems []*buildItem) { Config.Services.Queue.PushAtOnce(context.Background(), tasks) } -func taskIds(dependsOn []string, buildItems []*buildItem) []string { +func taskIds(dependsOn []string, buildItems []*helpers.BuildItem) []string { taskIds := []string{} for _, dep := range dependsOn { for _, buildItem := range buildItems { diff --git a/server/rpc.go b/server/rpc.go index ce5aa9635b..fffd247919 100644 --- a/server/rpc.go +++ b/server/rpc.go @@ -37,6 +37,7 @@ import ( "github.com/woodpecker-ci/woodpecker/cncd/pipeline/pipeline/rpc/proto" "github.com/woodpecker-ci/woodpecker/cncd/pubsub" "github.com/woodpecker-ci/woodpecker/cncd/queue" + "github.com/woodpecker-ci/woodpecker/server/helpers" "github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/remote" @@ -137,7 +138,7 @@ func (s *RPC) Update(c context.Context, id string, state rpc.State) error { return err } - if proc, err = UpdateProcStatus(s.store, *proc, state, build.Started); err != nil { + if proc, err = helpers.UpdateProcStatus(s.store, *proc, state, build.Started); err != nil { log.Printf("error: rpc.update: cannot update proc: %s", err) } @@ -264,7 +265,7 @@ func (s *RPC) Init(c context.Context, id string, state rpc.State) error { } if build.Status == model.StatusPending { - if build, err = UpdateToStatusRunning(s.store, *build, state.Started); err != nil { + if build, err = helpers.UpdateToStatusRunning(s.store, *build, state.Started); err != nil { log.Printf("error: init: cannot update build_id %d state: %s", build.ID, err) } } @@ -284,7 +285,7 @@ func (s *RPC) Init(c context.Context, id string, state rpc.State) error { s.pubsub.Publish(c, "topic/events", message) }() - _, err = UpdateProcToStatusStarted(s.store, *proc, state) + _, err = helpers.UpdateProcToStatusStarted(s.store, *proc, state) return err } @@ -313,7 +314,7 @@ func (s *RPC) Done(c context.Context, id string, state rpc.State) error { return err } - if proc, err = UpdateProcStatusToDone(s.store, *proc, state); err != nil { + if proc, err = helpers.UpdateProcStatusToDone(s.store, *proc, state); err != nil { log.Printf("error: done: cannot update proc_id %d state: %s", proc.ID, err) } @@ -331,7 +332,7 @@ func (s *RPC) Done(c context.Context, id string, state rpc.State) error { s.completeChildrenIfParentCompleted(procs, proc) if !isThereRunningStage(procs) { - if build, err = UpdateStatusToDone(s.store, *build, buildStatus(procs), proc.Stopped); err != nil { + if build, err = helpers.UpdateStatusToDone(s.store, *build, buildStatus(procs), proc.Stopped); err != nil { log.Printf("error: done: cannot update build_id %d final state: %s", build.ID, err) } @@ -382,7 +383,7 @@ func (s *RPC) Log(c context.Context, id string, line *rpc.Line) error { func (s *RPC) completeChildrenIfParentCompleted(procs []*model.Proc, completedProc *model.Proc) { for _, p := range procs { if p.Running() && p.PPID == completedProc.PID { - if _, err := UpdateProcToStatusSkipped(s.store, *p, completedProc.Stopped); err != nil { + if _, err := helpers.UpdateProcToStatusSkipped(s.store, *p, completedProc.Stopped); err != nil { log.Printf("error: done: cannot update proc_id %d child state: %s", p.ID, err) } } diff --git a/server/user.go b/server/user.go index 24945990c8..44b49f210f 100644 --- a/server/user.go +++ b/server/user.go @@ -27,6 +27,7 @@ import ( "github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/remote" "github.com/woodpecker-ci/woodpecker/router/middleware/session" + "github.com/woodpecker-ci/woodpecker/server/helpers" "github.com/woodpecker-ci/woodpecker/shared/token" "github.com/woodpecker-ci/woodpecker/store" ) @@ -47,11 +48,11 @@ func GetFeed(c *gin.Context) { config := ToConfig(c) - sync := syncer{ - remote: remote.FromContext(c), - store: store.FromContext(c), - perms: store.FromContext(c), - match: NamespaceFilter(config.OwnersWhitelist), + sync := helpers.Syncer{ + Remote: remote.FromContext(c), + Store: store.FromContext(c), + Perms: store.FromContext(c), + Match: helpers.NamespaceFilter(config.OwnersWhitelist), } if err := sync.Sync(user); err != nil { logrus.Debugf("sync error: %s: %s", user.Login, err) @@ -92,11 +93,11 @@ func GetRepos(c *gin.Context) { config := ToConfig(c) - sync := syncer{ - remote: remote.FromContext(c), - store: store.FromContext(c), - perms: store.FromContext(c), - match: NamespaceFilter(config.OwnersWhitelist), + sync := helpers.Syncer{ + Remote: remote.FromContext(c), + Store: store.FromContext(c), + Perms: store.FromContext(c), + Match: helpers.NamespaceFilter(config.OwnersWhitelist), } if err := sync.Sync(user); err != nil { From f45464a1575ee4cefb4e8a1e1640ca0b7683a456 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 21 Sep 2021 17:34:09 -0500 Subject: [PATCH 04/14] Refactor: create server/api package --- router/router.go | 118 +++++++++++++++++------------------ server/{ => api}/badge.go | 5 +- server/{ => api}/build.go | 39 ++++++------ server/{ => api}/file.go | 2 +- server/{ => api}/hook.go | 37 +++++------ server/{ => api}/login.go | 7 ++- server/{ => api}/registry.go | 15 ++--- server/{ => api}/repo.go | 11 ++-- server/{ => api}/secret.go | 15 ++--- server/{ => api}/stream.go | 7 ++- server/{ => api}/user.go | 2 +- server/{ => api}/users.go | 2 +- server/{ => api}/z.go | 2 +- 13 files changed, 135 insertions(+), 127 deletions(-) rename server/{ => api}/badge.go (97%) rename server/{ => api}/build.go (91%) rename server/{ => api}/file.go (99%) rename server/{ => api}/hook.go (90%) rename server/{ => api}/login.go (96%) rename server/{ => api}/registry.go (85%) rename server/{ => api}/repo.go (96%) rename server/{ => api}/secret.go (85%) rename server/{ => api}/stream.go (95%) rename server/{ => api}/user.go (99%) rename server/{ => api}/users.go (99%) rename server/{ => api}/z.go (98%) diff --git a/router/router.go b/router/router.go index a695787268..57bd195555 100644 --- a/router/router.go +++ b/router/router.go @@ -23,7 +23,7 @@ import ( "github.com/woodpecker-ci/woodpecker/router/middleware/header" "github.com/woodpecker-ci/woodpecker/router/middleware/session" "github.com/woodpecker-ci/woodpecker/router/middleware/token" - "github.com/woodpecker-ci/woodpecker/server" + "github.com/woodpecker-ci/woodpecker/server/api" "github.com/woodpecker-ci/woodpecker/server/debug" "github.com/woodpecker-ci/woodpecker/server/metrics" "github.com/woodpecker-ci/woodpecker/server/web" @@ -52,27 +52,27 @@ func Load(mux *httptreemux.ContextMux, middleware ...gin.HandlerFunc) http.Handl mux.ServeHTTP(c.Writer, req) }) - e.GET("/logout", server.GetLogout) - e.GET("/login", server.HandleLogin) + e.GET("/logout", api.GetLogout) + e.GET("/login", api.HandleLogin) user := e.Group("/api/user") { user.Use(session.MustUser()) - user.GET("", server.GetSelf) - user.GET("/feed", server.GetFeed) - user.GET("/repos", server.GetRepos) - user.POST("/token", server.PostToken) - user.DELETE("/token", server.DeleteToken) + user.GET("", api.GetSelf) + user.GET("/feed", api.GetFeed) + user.GET("/repos", api.GetRepos) + user.POST("/token", api.PostToken) + user.DELETE("/token", api.DeleteToken) } users := e.Group("/api/users") { users.Use(session.MustAdmin()) - users.GET("", server.GetUsers) - users.POST("", server.PostUser) - users.GET("/:login", server.GetUser) - users.PATCH("/:login", server.PatchUser) - users.DELETE("/:login", server.DeleteUser) + users.GET("", api.GetUsers) + users.POST("", api.PostUser) + users.GET("/:login", api.GetUser) + users.PATCH("/:login", api.PatchUser) + users.DELETE("/:login", api.DeleteUser) } repo := e.Group("/api/repos/:owner/:name") @@ -81,62 +81,62 @@ func Load(mux *httptreemux.ContextMux, middleware ...gin.HandlerFunc) http.Handl repo.Use(session.SetPerm()) repo.Use(session.MustPull) - repo.POST("", session.MustRepoAdmin(), server.PostRepo) - repo.GET("", server.GetRepo) - repo.GET("/builds", server.GetBuilds) - repo.GET("/builds/:number", server.GetBuild) - repo.GET("/logs/:number/:pid", server.GetProcLogs) - repo.GET("/logs/:number/:pid/:proc", server.GetBuildLogs) + repo.POST("", session.MustRepoAdmin(), api.PostRepo) + repo.GET("", api.GetRepo) + repo.GET("/builds", api.GetBuilds) + repo.GET("/builds/:number", api.GetBuild) + repo.GET("/logs/:number/:pid", api.GetProcLogs) + repo.GET("/logs/:number/:pid/:proc", api.GetBuildLogs) - repo.GET("/files/:number", server.FileList) - repo.GET("/files/:number/:proc/*file", server.FileGet) + repo.GET("/files/:number", api.FileList) + repo.GET("/files/:number/:proc/*file", api.FileGet) // requires push permissions - repo.GET("/secrets", session.MustPush, server.GetSecretList) - repo.POST("/secrets", session.MustPush, server.PostSecret) - repo.GET("/secrets/:secret", session.MustPush, server.GetSecret) - repo.PATCH("/secrets/:secret", session.MustPush, server.PatchSecret) - repo.DELETE("/secrets/:secret", session.MustPush, server.DeleteSecret) + repo.GET("/secrets", session.MustPush, api.GetSecretList) + repo.POST("/secrets", session.MustPush, api.PostSecret) + repo.GET("/secrets/:secret", session.MustPush, api.GetSecret) + repo.PATCH("/secrets/:secret", session.MustPush, api.PatchSecret) + repo.DELETE("/secrets/:secret", session.MustPush, api.DeleteSecret) // requires push permissions - repo.GET("/registry", session.MustPush, server.GetRegistryList) - repo.POST("/registry", session.MustPush, server.PostRegistry) - repo.GET("/registry/:registry", session.MustPush, server.GetRegistry) - repo.PATCH("/registry/:registry", session.MustPush, server.PatchRegistry) - repo.DELETE("/registry/:registry", session.MustPush, server.DeleteRegistry) + repo.GET("/registry", session.MustPush, api.GetRegistryList) + repo.POST("/registry", session.MustPush, api.PostRegistry) + repo.GET("/registry/:registry", session.MustPush, api.GetRegistry) + repo.PATCH("/registry/:registry", session.MustPush, api.PatchRegistry) + repo.DELETE("/registry/:registry", session.MustPush, api.DeleteRegistry) // requires admin permissions - repo.PATCH("", session.MustRepoAdmin(), server.PatchRepo) - repo.DELETE("", session.MustRepoAdmin(), server.DeleteRepo) - repo.POST("/chown", session.MustRepoAdmin(), server.ChownRepo) - repo.POST("/repair", session.MustRepoAdmin(), server.RepairRepo) - repo.POST("/move", session.MustRepoAdmin(), server.MoveRepo) - - repo.POST("/builds/:number", session.MustPush, server.PostBuild) - repo.DELETE("/builds/:number", session.MustPush, server.DeleteBuild) - repo.POST("/builds/:number/approve", session.MustPush, server.PostApproval) - repo.POST("/builds/:number/decline", session.MustPush, server.PostDecline) - repo.DELETE("/builds/:number/:job", session.MustPush, server.DeleteBuild) - repo.DELETE("/logs/:number", session.MustPush, server.DeleteBuildLogs) + repo.PATCH("", session.MustRepoAdmin(), api.PatchRepo) + repo.DELETE("", session.MustRepoAdmin(), api.DeleteRepo) + repo.POST("/chown", session.MustRepoAdmin(), api.ChownRepo) + repo.POST("/repair", session.MustRepoAdmin(), api.RepairRepo) + repo.POST("/move", session.MustRepoAdmin(), api.MoveRepo) + + repo.POST("/builds/:number", session.MustPush, api.PostBuild) + repo.DELETE("/builds/:number", session.MustPush, api.DeleteBuild) + repo.POST("/builds/:number/approve", session.MustPush, api.PostApproval) + repo.POST("/builds/:number/decline", session.MustPush, api.PostDecline) + repo.DELETE("/builds/:number/:job", session.MustPush, api.DeleteBuild) + repo.DELETE("/logs/:number", session.MustPush, api.DeleteBuildLogs) } badges := e.Group("/api/badges/:owner/:name") { - badges.GET("/status.svg", server.GetBadge) - badges.GET("/cc.xml", server.GetCC) + badges.GET("/status.svg", api.GetBadge) + badges.GET("/cc.xml", api.GetCC) } - e.POST("/hook", server.PostHook) - e.POST("/api/hook", server.PostHook) + e.POST("/hook", api.PostHook) + e.POST("/api/hook", api.PostHook) sse := e.Group("/stream") { - sse.GET("/events", server.EventStreamSSE) + sse.GET("/events", api.EventStreamSSE) sse.GET("/logs/:owner/:name/:build/:number", session.SetRepo(), session.SetPerm(), session.MustPull, - server.LogStreamSSE, + api.LogStreamSSE, ) } @@ -144,33 +144,33 @@ func Load(mux *httptreemux.ContextMux, middleware ...gin.HandlerFunc) http.Handl { queue.GET("/info", session.MustAdmin(), - server.GetQueueInfo, + api.GetQueueInfo, ) queue.GET("/pause", session.MustAdmin(), - server.PauseQueue, + api.PauseQueue, ) queue.GET("/resume", session.MustAdmin(), - server.ResumeQueue, + api.ResumeQueue, ) queue.GET("/norunningbuilds", session.MustAdmin(), - server.BlockTilQueueHasRunningItem, + api.BlockTilQueueHasRunningItem, ) } auth := e.Group("/authorize") { - auth.GET("", server.HandleAuth) - auth.POST("", server.HandleAuth) - auth.POST("/token", server.GetLoginToken) + auth.GET("", api.HandleAuth) + auth.POST("", api.HandleAuth) + auth.POST("/token", api.GetLoginToken) } builds := e.Group("/api/builds") { builds.Use(session.MustAdmin()) - builds.GET("", server.GetBuildQueue) + builds.GET("", api.GetBuildQueue) } debugger := e.Group("/api/debug") @@ -193,8 +193,8 @@ func Load(mux *httptreemux.ContextMux, middleware ...gin.HandlerFunc) http.Handl monitor.GET("", metrics.PromHandler()) } - e.GET("/version", server.Version) - e.GET("/healthz", server.Health) + e.GET("/version", api.Version) + e.GET("/healthz", api.Health) return e } diff --git a/server/badge.go b/server/api/badge.go similarity index 97% rename from server/badge.go rename to server/api/badge.go index 938004cba6..522e4dfbb0 100644 --- a/server/badge.go +++ b/server/api/badge.go @@ -15,7 +15,7 @@ // // This file has been modified by Informatyka Boguslawski sp. z o.o. sp.k. -package server +package api import ( "fmt" @@ -24,6 +24,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/woodpecker-ci/woodpecker/model" + "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/store" ) @@ -94,7 +95,7 @@ func GetCC(c *gin.Context) { return } - url := fmt.Sprintf("%s/%s/%d", Config.Server.Host, repo.FullName, builds[0].Number) + url := fmt.Sprintf("%s/%s/%d", server.Config.Server.Host, repo.FullName, builds[0].Number) cc := model.NewCC(repo, builds[0], url) c.XML(200, cc) } diff --git a/server/build.go b/server/api/build.go similarity index 91% rename from server/build.go rename to server/api/build.go index 28a007501f..7dbaed36e3 100644 --- a/server/build.go +++ b/server/api/build.go @@ -15,7 +15,7 @@ // // This file has been modified by Informatyka Boguslawski sp. z o.o. sp.k. -package server +package api import ( "bytes" @@ -35,6 +35,7 @@ import ( "github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/router/middleware/session" + "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server/helpers" ) @@ -196,9 +197,9 @@ func DeleteBuild(c *gin.Context) { procToEvict = append(procToEvict, fmt.Sprint(proc.ID)) } } - Config.Services.Queue.EvictAtOnce(context.Background(), procToEvict) - Config.Services.Queue.ErrorAtOnce(context.Background(), procToEvict, queue.ErrCancel) - Config.Services.Queue.ErrorAtOnce(context.Background(), procToCancel, queue.ErrCancel) + server.Config.Services.Queue.EvictAtOnce(context.Background(), procToEvict) + server.Config.Services.Queue.ErrorAtOnce(context.Background(), procToEvict, queue.ErrCancel) + server.Config.Services.Queue.ErrorAtOnce(context.Background(), procToCancel, queue.ErrCancel) // Then update the DB status for pending builds // Running ones will be set when the agents stop on the cancel signal @@ -258,7 +259,7 @@ func PostApproval(c *gin.Context) { } // fetch the build file from the database - configs, err := Config.Storage.Config.ConfigsForBuild(build.ID) + configs, err := server.Config.Storage.Config.ConfigsForBuild(build.ID) if err != nil { logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err) c.AbortWithError(404, err) @@ -281,17 +282,17 @@ func PostApproval(c *gin.Context) { // get the previous build so that we can send // on status change notifications last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) - secs, err := Config.Services.Secrets.SecretListBuild(repo, build) + secs, err := server.Config.Services.Secrets.SecretListBuild(repo, build) if err != nil { logrus.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err) } - regs, err := Config.Services.Registries.RegistryList(repo) + regs, err := server.Config.Services.Registries.RegistryList(repo) if err != nil { logrus.Debugf("Error getting registry credentials for %s#%d. %s", repo.FullName, build.Number, err) } envs := map[string]string{} - if Config.Services.Environ != nil { - globals, _ := Config.Services.Environ.EnvironList(repo) + if server.Config.Services.Environ != nil { + globals, _ := server.Config.Services.Environ.EnvironList(repo) for _, global := range globals { envs[global.Name] = global.Value } @@ -309,7 +310,7 @@ func PostApproval(c *gin.Context) { Netrc: netrc, Secs: secs, Regs: regs, - Link: Config.Server.Host, + Link: server.Config.Server.Host, Yamls: yamls, Envs: envs, } @@ -329,7 +330,7 @@ func PostApproval(c *gin.Context) { defer func() { for _, item := range buildItems { - uri := fmt.Sprintf("%s/%s/%d", Config.Server.Host, repo.FullName, build.Number) + uri := fmt.Sprintf("%s/%s/%d", server.Config.Server.Host, repo.FullName, build.Number) if len(buildItems) > 1 { err = remote_.Status(user, repo, build, uri, item.Proc) } else { @@ -370,7 +371,7 @@ func PostDecline(c *gin.Context) { return } - uri := fmt.Sprintf("%s/%s/%d", Config.Server.Host, repo.FullName, build.Number) + uri := fmt.Sprintf("%s/%s/%d", server.Config.Server.Host, repo.FullName, build.Number) err = remote_.Status(user, repo, build, uri, nil) if err != nil { logrus.Errorf("error setting commit status for %s/%d: %v", repo.FullName, build.Number, err) @@ -431,7 +432,7 @@ func PostBuild(c *gin.Context) { } // fetch the pipeline config from database - configs, err := Config.Storage.Config.ConfigsForBuild(build.ID) + configs, err := server.Config.Storage.Config.ConfigsForBuild(build.ID) if err != nil { logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err) c.AbortWithError(404, err) @@ -491,16 +492,16 @@ func PostBuild(c *gin.Context) { // get the previous build so that we can send // on status change notifications last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) - secs, err := Config.Services.Secrets.SecretListBuild(repo, build) + secs, err := server.Config.Services.Secrets.SecretListBuild(repo, build) if err != nil { logrus.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err) } - regs, err := Config.Services.Registries.RegistryList(repo) + regs, err := server.Config.Services.Registries.RegistryList(repo) if err != nil { logrus.Debugf("Error getting registry credentials for %s#%d. %s", repo.FullName, build.Number, err) } - if Config.Services.Environ != nil { - globals, _ := Config.Services.Environ.EnvironList(repo) + if server.Config.Services.Environ != nil { + globals, _ := server.Config.Services.Environ.EnvironList(repo) for _, global := range globals { buildParams[global.Name] = global.Value } @@ -518,7 +519,7 @@ func PostBuild(c *gin.Context) { Netrc: netrc, Secs: secs, Regs: regs, - Link: Config.Server.Host, + Link: server.Config.Server.Host, Yamls: yamls, Envs: buildParams, } @@ -594,7 +595,7 @@ func persistBuildConfigs(configs []*model.Config, buildID int64) error { ConfigID: conf.ID, BuildID: buildID, } - err := Config.Storage.Config.BuildConfigCreate(buildConfig) + err := server.Config.Storage.Config.BuildConfigCreate(buildConfig) if err != nil { return err } diff --git a/server/file.go b/server/api/file.go similarity index 99% rename from server/file.go rename to server/api/file.go index ee114aca51..b0cc91360a 100644 --- a/server/file.go +++ b/server/api/file.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package api import ( "io" diff --git a/server/hook.go b/server/api/hook.go similarity index 90% rename from server/hook.go rename to server/api/hook.go index 0b95e16c88..7cf407ab28 100644 --- a/server/hook.go +++ b/server/api/hook.go @@ -15,7 +15,7 @@ // // This file has been modified by Informatyka Boguslawski sp. z o.o. sp.k. -package server +package api import ( "context" @@ -40,6 +40,7 @@ import ( "github.com/woodpecker-ci/woodpecker/cncd/pipeline/pipeline/rpc" "github.com/woodpecker-ci/woodpecker/cncd/pubsub" "github.com/woodpecker-ci/woodpecker/cncd/queue" + "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server/helpers" ) @@ -51,23 +52,23 @@ func init() { func GetQueueInfo(c *gin.Context) { c.IndentedJSON(200, - Config.Services.Queue.Info(c), + server.Config.Services.Queue.Info(c), ) } func PauseQueue(c *gin.Context) { - Config.Services.Queue.Pause() + server.Config.Services.Queue.Pause() c.Status(http.StatusOK) } func ResumeQueue(c *gin.Context) { - Config.Services.Queue.Resume() + server.Config.Services.Queue.Resume() c.Status(http.StatusOK) } func BlockTilQueueHasRunningItem(c *gin.Context) { for { - info := Config.Services.Queue.Info(c) + info := server.Config.Services.Queue.Info(c) if info.Stats.Running == 0 { break } @@ -222,19 +223,19 @@ func PostHook(c *gin.Context) { } envs := map[string]string{} - if Config.Services.Environ != nil { - globals, _ := Config.Services.Environ.EnvironList(repo) + if server.Config.Services.Environ != nil { + globals, _ := server.Config.Services.Environ.EnvironList(repo) for _, global := range globals { envs[global.Name] = global.Value } } - secs, err := Config.Services.Secrets.SecretListBuild(repo, build) + secs, err := server.Config.Services.Secrets.SecretListBuild(repo, build) if err != nil { logrus.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err) } - regs, err := Config.Services.Registries.RegistryList(repo) + regs, err := server.Config.Services.Registries.RegistryList(repo) if err != nil { logrus.Debugf("Error getting registry credentials for %s#%d. %s", repo.FullName, build.Number, err) } @@ -250,7 +251,7 @@ func PostHook(c *gin.Context) { Secs: secs, Regs: regs, Envs: envs, - Link: Config.Server.Host, + Link: server.Config.Server.Host, Yamls: remoteYamlConfigs, } buildItems, err := b.Build() @@ -269,7 +270,7 @@ func PostHook(c *gin.Context) { defer func() { for _, item := range buildItems { - uri := fmt.Sprintf("%s/%s/%d", Config.Server.Host, repo.FullName, build.Number) + uri := fmt.Sprintf("%s/%s/%d", server.Config.Server.Host, repo.FullName, build.Number) if len(buildItems) > 1 { err = remote_.Status(user, repo, build, uri, item.Proc) } else { @@ -325,7 +326,7 @@ func zeroSteps(build *model.Build, remoteYamlConfigs []*remote.FileMeta) bool { func findOrPersistPipelineConfig(repo *model.Repo, build *model.Build, remoteYamlConfig *remote.FileMeta) (*model.Config, error) { sha := shasum(remoteYamlConfig.Data) - conf, err := Config.Storage.Config.ConfigFindIdentical(build.RepoID, sha) + conf, err := server.Config.Storage.Config.ConfigFindIdentical(build.RepoID, sha) if err != nil { conf = &model.Config{ RepoID: build.RepoID, @@ -333,10 +334,10 @@ func findOrPersistPipelineConfig(repo *model.Repo, build *model.Build, remoteYam Hash: sha, Name: helpers.SanitizePath(remoteYamlConfig.Name), } - err = Config.Storage.Config.ConfigCreate(conf) + err = server.Config.Storage.Config.ConfigCreate(conf) if err != nil { // retry in case we receive two hooks at the same time - conf, err = Config.Storage.Config.ConfigFindIdentical(build.RepoID, sha) + conf, err = server.Config.Storage.Config.ConfigFindIdentical(build.RepoID, sha) if err != nil { return nil, err } @@ -347,7 +348,7 @@ func findOrPersistPipelineConfig(repo *model.Repo, build *model.Build, remoteYam ConfigID: conf.ID, BuildID: build.ID, } - err = Config.Storage.Config.BuildConfigCreate(buildConfig) + err = server.Config.Storage.Config.BuildConfigCreate(buildConfig) if err != nil { return nil, err } @@ -370,7 +371,7 @@ func publishToTopic(c *gin.Context, build *model.Build, repo *model.Repo, event Repo: *repo, Build: buildCopy, }) - Config.Services.Pubsub.Publish(c, "topic/events", message) + server.Config.Services.Pubsub.Publish(c, "topic/events", message) } func queueBuild(build *model.Build, repo *model.Repo, buildItems []*helpers.BuildItem) { @@ -397,10 +398,10 @@ func queueBuild(build *model.Build, repo *model.Repo, buildItems []*helpers.Buil Timeout: repo.Timeout, }) - Config.Services.Logs.Open(context.Background(), task.ID) + server.Config.Services.Logs.Open(context.Background(), task.ID) tasks = append(tasks, task) } - Config.Services.Queue.PushAtOnce(context.Background(), tasks) + server.Config.Services.Queue.PushAtOnce(context.Background(), tasks) } func taskIds(dependsOn []string, buildItems []*helpers.BuildItem) []string { diff --git a/server/login.go b/server/api/login.go similarity index 96% rename from server/login.go rename to server/api/login.go index abe0f1d88e..f455869d9c 100644 --- a/server/login.go +++ b/server/api/login.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package api import ( "encoding/base32" @@ -22,6 +22,7 @@ import ( "github.com/gorilla/securecookie" "github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/remote" + "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/shared/httputil" "github.com/woodpecker-ci/woodpecker/shared/token" "github.com/woodpecker-ci/woodpecker/store" @@ -131,7 +132,7 @@ func HandleAuth(c *gin.Context) { return } - exp := time.Now().Add(Config.Server.SessionExpires).Unix() + exp := time.Now().Add(server.Config.Server.SessionExpires).Unix() token := token.New(token.SessToken, u.Login) tokenstr, err := token.SignExpires(u.Hash, exp) if err != nil { @@ -176,7 +177,7 @@ func GetLoginToken(c *gin.Context) { return } - exp := time.Now().Add(Config.Server.SessionExpires).Unix() + exp := time.Now().Add(server.Config.Server.SessionExpires).Unix() token := token.New(token.SessToken, user.Login) tokenstr, err := token.SignExpires(user.Hash, exp) if err != nil { diff --git a/server/registry.go b/server/api/registry.go similarity index 85% rename from server/registry.go rename to server/api/registry.go index 616f423f78..d449a19fc7 100644 --- a/server/registry.go +++ b/server/api/registry.go @@ -12,13 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package api import ( "net/http" "github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/router/middleware/session" + "github.com/woodpecker-ci/woodpecker/server" "github.com/gin-gonic/gin" ) @@ -30,7 +31,7 @@ func GetRegistry(c *gin.Context) { repo = session.Repo(c) name = c.Param("registry") ) - registry, err := Config.Services.Registries.RegistryFind(repo, name) + registry, err := server.Config.Services.Registries.RegistryFind(repo, name) if err != nil { c.String(404, "Error getting registry %q. %s", name, err) return @@ -59,7 +60,7 @@ func PostRegistry(c *gin.Context) { c.String(400, "Error inserting registry. %s", err) return } - if err := Config.Services.Registries.RegistryCreate(repo, registry); err != nil { + if err := server.Config.Services.Registries.RegistryCreate(repo, registry); err != nil { c.String(500, "Error inserting registry %q. %s", in.Address, err) return } @@ -80,7 +81,7 @@ func PatchRegistry(c *gin.Context) { return } - registry, err := Config.Services.Registries.RegistryFind(repo, name) + registry, err := server.Config.Services.Registries.RegistryFind(repo, name) if err != nil { c.String(404, "Error getting registry %q. %s", name, err) return @@ -102,7 +103,7 @@ func PatchRegistry(c *gin.Context) { c.String(400, "Error updating registry. %s", err) return } - if err := Config.Services.Registries.RegistryUpdate(repo, registry); err != nil { + if err := server.Config.Services.Registries.RegistryUpdate(repo, registry); err != nil { c.String(500, "Error updating registry %q. %s", in.Address, err) return } @@ -113,7 +114,7 @@ func PatchRegistry(c *gin.Context) { // to the response in json format. func GetRegistryList(c *gin.Context) { repo := session.Repo(c) - list, err := Config.Services.Registries.RegistryList(repo) + list, err := server.Config.Services.Registries.RegistryList(repo) if err != nil { c.String(500, "Error getting registry list. %s", err) return @@ -132,7 +133,7 @@ func DeleteRegistry(c *gin.Context) { repo = session.Repo(c) name = c.Param("registry") ) - if err := Config.Services.Registries.RegistryDelete(repo, name); err != nil { + if err := server.Config.Services.Registries.RegistryDelete(repo, name); err != nil { c.String(500, "Error deleting registry %q. %s", name, err) return } diff --git a/server/repo.go b/server/api/repo.go similarity index 96% rename from server/repo.go rename to server/api/repo.go index e0e498ef5c..ce5f823bdb 100644 --- a/server/repo.go +++ b/server/api/repo.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package api import ( "encoding/base32" @@ -26,6 +26,7 @@ import ( "github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/remote" "github.com/woodpecker-ci/woodpecker/router/middleware/session" + "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/shared/token" "github.com/woodpecker-ci/woodpecker/store" ) @@ -70,7 +71,7 @@ func PostRepo(c *gin.Context) { link := fmt.Sprintf( "%s/hook?access_token=%s", - Config.Server.Host, + server.Config.Server.Host, sig, ) @@ -186,7 +187,7 @@ func DeleteRepo(c *gin.Context) { } } - remote.Deactivate(user, repo, Config.Server.Host) + remote.Deactivate(user, repo, server.Config.Server.Host) c.JSON(200, repo) } @@ -204,7 +205,7 @@ func RepairRepo(c *gin.Context) { } // reconstruct the link - host := Config.Server.Host + host := server.Config.Server.Host link := fmt.Sprintf( "%s/hook?access_token=%s", host, @@ -290,7 +291,7 @@ func MoveRepo(c *gin.Context) { } // reconstruct the link - host := Config.Server.Host + host := server.Config.Server.Host link := fmt.Sprintf( "%s/hook?access_token=%s", host, diff --git a/server/secret.go b/server/api/secret.go similarity index 85% rename from server/secret.go rename to server/api/secret.go index ce7abde6eb..4ed8af4433 100644 --- a/server/secret.go +++ b/server/api/secret.go @@ -12,13 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package api import ( "net/http" "github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/router/middleware/session" + "github.com/woodpecker-ci/woodpecker/server" "github.com/gin-gonic/gin" ) @@ -30,7 +31,7 @@ func GetSecret(c *gin.Context) { repo = session.Repo(c) name = c.Param("secret") ) - secret, err := Config.Services.Secrets.SecretFind(repo, name) + secret, err := server.Config.Services.Secrets.SecretFind(repo, name) if err != nil { c.String(404, "Error getting secret %q. %s", name, err) return @@ -58,7 +59,7 @@ func PostSecret(c *gin.Context) { c.String(400, "Error inserting secret. %s", err) return } - if err := Config.Services.Secrets.SecretCreate(repo, secret); err != nil { + if err := server.Config.Services.Secrets.SecretCreate(repo, secret); err != nil { c.String(500, "Error inserting secret %q. %s", in.Name, err) return } @@ -79,7 +80,7 @@ func PatchSecret(c *gin.Context) { return } - secret, err := Config.Services.Secrets.SecretFind(repo, name) + secret, err := server.Config.Services.Secrets.SecretFind(repo, name) if err != nil { c.String(404, "Error getting secret %q. %s", name, err) return @@ -98,7 +99,7 @@ func PatchSecret(c *gin.Context) { c.String(400, "Error updating secret. %s", err) return } - if err := Config.Services.Secrets.SecretUpdate(repo, secret); err != nil { + if err := server.Config.Services.Secrets.SecretUpdate(repo, secret); err != nil { c.String(500, "Error updating secret %q. %s", in.Name, err) return } @@ -109,7 +110,7 @@ func PatchSecret(c *gin.Context) { // to the response in json format. func GetSecretList(c *gin.Context) { repo := session.Repo(c) - list, err := Config.Services.Secrets.SecretList(repo) + list, err := server.Config.Services.Secrets.SecretList(repo) if err != nil { c.String(500, "Error getting secret list. %s", err) return @@ -128,7 +129,7 @@ func DeleteSecret(c *gin.Context) { repo = session.Repo(c) name = c.Param("secret") ) - if err := Config.Services.Secrets.SecretDelete(repo, name); err != nil { + if err := server.Config.Services.Secrets.SecretDelete(repo, name); err != nil { c.String(500, "Error deleting secret %q. %s", name, err) return } diff --git a/server/stream.go b/server/api/stream.go similarity index 95% rename from server/stream.go rename to server/api/stream.go index 515745f45c..e6cb765c7b 100644 --- a/server/stream.go +++ b/server/api/stream.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package api import ( "context" @@ -26,6 +26,7 @@ import ( "github.com/woodpecker-ci/woodpecker/cncd/pubsub" "github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/router/middleware/session" + "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/store" "github.com/gin-gonic/gin" @@ -77,7 +78,7 @@ func EventStreamSSE(c *gin.Context) { }() go func() { - Config.Services.Pubsub.Subscribe(ctx, "topic/events", func(m pubsub.Message) { + server.Config.Services.Pubsub.Subscribe(ctx, "topic/events", func(m pubsub.Message) { defer func() { recover() // fix #2480 }() @@ -189,7 +190,7 @@ func LogStreamSSE(c *gin.Context) { go func() { // TODO remove global variable - Config.Services.Logs.Tail(ctx, fmt.Sprint(proc.ID), func(entries ...*logging.Entry) { + server.Config.Services.Logs.Tail(ctx, fmt.Sprint(proc.ID), func(entries ...*logging.Entry) { defer func() { recover() // fix #2480 }() diff --git a/server/user.go b/server/api/user.go similarity index 99% rename from server/user.go rename to server/api/user.go index 44b49f210f..826e9bf472 100644 --- a/server/user.go +++ b/server/api/user.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package api import ( "encoding/base32" diff --git a/server/users.go b/server/api/users.go similarity index 99% rename from server/users.go rename to server/api/users.go index c2335e9603..85ef49b322 100644 --- a/server/users.go +++ b/server/api/users.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package api import ( "encoding/base32" diff --git a/server/z.go b/server/api/z.go similarity index 98% rename from server/z.go rename to server/api/z.go index 93032cc5f9..e74a829dab 100644 --- a/server/z.go +++ b/server/api/z.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package server +package api import ( "github.com/gin-gonic/gin" From 5666a401a8d7081f9b51aa075fc97e6823eae507 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 21 Sep 2021 18:08:55 -0500 Subject: [PATCH 05/14] refactor: make server/grpc package fixes import cycle --- cmd/server/server.go | 3 ++- server/{ => grpc}/rpc.go | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) rename server/{ => grpc}/rpc.go (98%) diff --git a/cmd/server/server.go b/cmd/server/server.go index 8c4b56e12f..998504dfe0 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -41,6 +41,7 @@ import ( "github.com/woodpecker-ci/woodpecker/router" "github.com/woodpecker-ci/woodpecker/router/middleware" droneserver "github.com/woodpecker-ci/woodpecker/server" + woodpeckerGrpcServer "github.com/woodpecker-ci/woodpecker/server/grpc" "github.com/woodpecker-ci/woodpecker/store" "github.com/gin-gonic/contrib/ginrus" @@ -123,7 +124,7 @@ func server(c *cli.Context) error { MinTime: c.Duration("keepalive-min-time"), }), ) - droneServer := droneserver.NewDroneServer(remote_, droneserver.Config.Services.Queue, droneserver.Config.Services.Logs, droneserver.Config.Services.Pubsub, store_, droneserver.Config.Server.Host) + droneServer := woodpeckerGrpcServer.NewDroneServer(remote_, droneserver.Config.Services.Queue, droneserver.Config.Services.Logs, droneserver.Config.Services.Pubsub, store_, droneserver.Config.Server.Host) proto.RegisterDroneServer(grpcServer, droneServer) err = grpcServer.Serve(lis) diff --git a/server/rpc.go b/server/grpc/rpc.go similarity index 98% rename from server/rpc.go rename to server/grpc/rpc.go index fffd247919..3bbfa1d2dc 100644 --- a/server/rpc.go +++ b/server/grpc/rpc.go @@ -15,7 +15,7 @@ // // This file has been modified by Informatyka Boguslawski sp. z o.o. sp.k. -package server +package grpc import ( "bytes" @@ -37,6 +37,7 @@ import ( "github.com/woodpecker-ci/woodpecker/cncd/pipeline/pipeline/rpc/proto" "github.com/woodpecker-ci/woodpecker/cncd/pubsub" "github.com/woodpecker-ci/woodpecker/cncd/queue" + "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server/helpers" "github.com/woodpecker-ci/woodpecker/model" @@ -226,7 +227,7 @@ func (s *RPC) Upload(c context.Context, id string, file *rpc.File) error { } } - return Config.Storage.Files.FileCreate( + return server.Config.Storage.Files.FileCreate( report, bytes.NewBuffer(file.Data), ) @@ -424,7 +425,7 @@ func (s *RPC) updateRemoteStatus(repo *model.Repo, build *model.Build, proc *mod s.store.UpdateUser(user) } } - uri := fmt.Sprintf("%s/%s/%d", Config.Server.Host, repo.FullName, build.Number) + uri := fmt.Sprintf("%s/%s/%d", server.Config.Server.Host, repo.FullName, build.Number) err = s.remote.Status(user, repo, build, uri, proc) if err != nil { logrus.Errorf("error setting commit status for %s/%d: %v", repo.FullName, build.Number, err) From 29ab3825a119e21bcaea4704f0587e3e045b489b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 21 Sep 2021 18:16:34 -0500 Subject: [PATCH 06/14] refactor: import woodpecker/server as woodpecker instead of droneserver --- cmd/server/server.go | 62 ++++++++++++++++++++++---------------------- cmd/server/setup.go | 4 +-- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/cmd/server/server.go b/cmd/server/server.go index 998504dfe0..49c8104b01 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -40,7 +40,7 @@ import ( "github.com/woodpecker-ci/woodpecker/remote" "github.com/woodpecker-ci/woodpecker/router" "github.com/woodpecker-ci/woodpecker/router/middleware" - droneserver "github.com/woodpecker-ci/woodpecker/server" + woodpecker "github.com/woodpecker-ci/woodpecker/server" woodpeckerGrpcServer "github.com/woodpecker-ci/woodpecker/server/grpc" "github.com/woodpecker-ci/woodpecker/store" @@ -124,7 +124,7 @@ func server(c *cli.Context) error { MinTime: c.Duration("keepalive-min-time"), }), ) - droneServer := woodpeckerGrpcServer.NewDroneServer(remote_, droneserver.Config.Services.Queue, droneserver.Config.Services.Logs, droneserver.Config.Services.Pubsub, store_, droneserver.Config.Server.Host) + droneServer := woodpeckerGrpcServer.NewDroneServer(remote_, woodpecker.Config.Services.Queue, woodpecker.Config.Services.Logs, woodpecker.Config.Services.Pubsub, store_, woodpecker.Config.Server.Host) proto.RegisterDroneServer(grpcServer, droneServer) err = grpcServer.Serve(lis) @@ -202,45 +202,45 @@ func server(c *cli.Context) error { func setupEvilGlobals(c *cli.Context, v store.Store, r remote.Remote) { // storage - droneserver.Config.Storage.Files = v - droneserver.Config.Storage.Config = v + woodpecker.Config.Storage.Files = v + woodpecker.Config.Storage.Config = v // services - droneserver.Config.Services.Queue = setupQueue(c, v) - droneserver.Config.Services.Logs = logging.New() - droneserver.Config.Services.Pubsub = pubsub.New() - droneserver.Config.Services.Pubsub.Create(context.Background(), "topic/events") - droneserver.Config.Services.Registries = setupRegistryService(c, v) - droneserver.Config.Services.Secrets = setupSecretService(c, v) - droneserver.Config.Services.Senders = sender.New(v, v) - droneserver.Config.Services.Environ = setupEnvironService(c, v) + woodpecker.Config.Services.Queue = setupQueue(c, v) + woodpecker.Config.Services.Logs = logging.New() + woodpecker.Config.Services.Pubsub = pubsub.New() + woodpecker.Config.Services.Pubsub.Create(context.Background(), "topic/events") + woodpecker.Config.Services.Registries = setupRegistryService(c, v) + woodpecker.Config.Services.Secrets = setupSecretService(c, v) + woodpecker.Config.Services.Senders = sender.New(v, v) + woodpecker.Config.Services.Environ = setupEnvironService(c, v) if endpoint := c.String("gating-service"); endpoint != "" { - droneserver.Config.Services.Senders = sender.NewRemote(endpoint) + woodpecker.Config.Services.Senders = sender.NewRemote(endpoint) } // limits - droneserver.Config.Pipeline.Limits.MemSwapLimit = c.Int64("limit-mem-swap") - droneserver.Config.Pipeline.Limits.MemLimit = c.Int64("limit-mem") - droneserver.Config.Pipeline.Limits.ShmSize = c.Int64("limit-shm-size") - droneserver.Config.Pipeline.Limits.CPUQuota = c.Int64("limit-cpu-quota") - droneserver.Config.Pipeline.Limits.CPUShares = c.Int64("limit-cpu-shares") - droneserver.Config.Pipeline.Limits.CPUSet = c.String("limit-cpu-set") + woodpecker.Config.Pipeline.Limits.MemSwapLimit = c.Int64("limit-mem-swap") + woodpecker.Config.Pipeline.Limits.MemLimit = c.Int64("limit-mem") + woodpecker.Config.Pipeline.Limits.ShmSize = c.Int64("limit-shm-size") + woodpecker.Config.Pipeline.Limits.CPUQuota = c.Int64("limit-cpu-quota") + woodpecker.Config.Pipeline.Limits.CPUShares = c.Int64("limit-cpu-shares") + woodpecker.Config.Pipeline.Limits.CPUSet = c.String("limit-cpu-set") // server configuration - droneserver.Config.Server.Cert = c.String("server-cert") - droneserver.Config.Server.Key = c.String("server-key") - droneserver.Config.Server.Pass = c.String("agent-secret") - droneserver.Config.Server.Host = c.String("server-host") - droneserver.Config.Server.Port = c.String("server-addr") - droneserver.Config.Server.RepoConfig = c.String("repo-config") - droneserver.Config.Server.SessionExpires = c.Duration("session-expires") - droneserver.Config.Pipeline.Networks = c.StringSlice("network") - droneserver.Config.Pipeline.Volumes = c.StringSlice("volume") - droneserver.Config.Pipeline.Privileged = c.StringSlice("escalate") + woodpecker.Config.Server.Cert = c.String("server-cert") + woodpecker.Config.Server.Key = c.String("server-key") + woodpecker.Config.Server.Pass = c.String("agent-secret") + woodpecker.Config.Server.Host = c.String("server-host") + woodpecker.Config.Server.Port = c.String("server-addr") + woodpecker.Config.Server.RepoConfig = c.String("repo-config") + woodpecker.Config.Server.SessionExpires = c.Duration("session-expires") + woodpecker.Config.Pipeline.Networks = c.StringSlice("network") + woodpecker.Config.Pipeline.Volumes = c.StringSlice("volume") + woodpecker.Config.Pipeline.Privileged = c.StringSlice("escalate") // prometheus - droneserver.Config.Prometheus.AuthToken = c.String("prometheus-auth-token") + woodpecker.Config.Prometheus.AuthToken = c.String("prometheus-auth-token") } type authorizer struct { @@ -272,7 +272,7 @@ func (a *authorizer) authorize(ctx context.Context) error { } func redirect(w http.ResponseWriter, req *http.Request) { - var serverHost string = droneserver.Config.Server.Host + var serverHost string = woodpecker.Config.Server.Host serverHost = strings.TrimPrefix(serverHost, "http://") serverHost = strings.TrimPrefix(serverHost, "https://") req.URL.Scheme = "https" diff --git a/cmd/server/setup.go b/cmd/server/setup.go index 8650309104..80f4bea1e3 100644 --- a/cmd/server/setup.go +++ b/cmd/server/setup.go @@ -35,7 +35,7 @@ import ( "github.com/woodpecker-ci/woodpecker/remote/gitlab" "github.com/woodpecker-ci/woodpecker/remote/gitlab3" "github.com/woodpecker-ci/woodpecker/remote/gogs" - droneserver "github.com/woodpecker-ci/woodpecker/server" + woodpecker "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server/web" "github.com/woodpecker-ci/woodpecker/store" "github.com/woodpecker-ci/woodpecker/store/datastore" @@ -257,7 +257,7 @@ func setupMetrics(g *errgroup.Group, store_ store.Store) { g.Go(func() error { for { - stats := droneserver.Config.Services.Queue.Info(nil) + stats := woodpecker.Config.Services.Queue.Info(nil) pendingJobs.Set(float64(stats.Stats.Pending)) waitingJobs.Set(float64(stats.Stats.WaitingOnDeps)) runningJobs.Set(float64(stats.Stats.Running)) From 7747aa595fe43775b700b6e2cdea0a21ff381f9b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 21 Sep 2021 18:19:40 -0500 Subject: [PATCH 07/14] refactor: reformat --- cmd/server/server.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/server/server.go b/cmd/server/server.go index 49c8104b01..4bbd00cd30 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -124,7 +124,14 @@ func server(c *cli.Context) error { MinTime: c.Duration("keepalive-min-time"), }), ) - droneServer := woodpeckerGrpcServer.NewDroneServer(remote_, woodpecker.Config.Services.Queue, woodpecker.Config.Services.Logs, woodpecker.Config.Services.Pubsub, store_, woodpecker.Config.Server.Host) + droneServer := woodpeckerGrpcServer.NewDroneServer( + remote_, + woodpecker.Config.Services.Queue, + woodpecker.Config.Services.Logs, + woodpecker.Config.Services.Pubsub, + store_, + woodpecker.Config.Server.Host, + ) proto.RegisterDroneServer(grpcServer, droneServer) err = grpcServer.Serve(lis) From e4d90416381946ea1778563c4785c883ecaf7d4b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 21 Sep 2021 19:44:01 -0500 Subject: [PATCH 08/14] refactor: move server/{debug,metrics} under server/api --- router/router.go | 4 ++-- server/{ => api}/debug/debug.go | 0 server/{ => api}/metrics/prometheus.go | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename server/{ => api}/debug/debug.go (100%) rename server/{ => api}/metrics/prometheus.go (100%) diff --git a/router/router.go b/router/router.go index 57bd195555..2a4b10b0dd 100644 --- a/router/router.go +++ b/router/router.go @@ -24,8 +24,8 @@ import ( "github.com/woodpecker-ci/woodpecker/router/middleware/session" "github.com/woodpecker-ci/woodpecker/router/middleware/token" "github.com/woodpecker-ci/woodpecker/server/api" - "github.com/woodpecker-ci/woodpecker/server/debug" - "github.com/woodpecker-ci/woodpecker/server/metrics" + "github.com/woodpecker-ci/woodpecker/server/api/debug" + "github.com/woodpecker-ci/woodpecker/server/api/metrics" "github.com/woodpecker-ci/woodpecker/server/web" ) diff --git a/server/debug/debug.go b/server/api/debug/debug.go similarity index 100% rename from server/debug/debug.go rename to server/api/debug/debug.go diff --git a/server/metrics/prometheus.go b/server/api/metrics/prometheus.go similarity index 100% rename from server/metrics/prometheus.go rename to server/api/metrics/prometheus.go From 01771d03824084835e810c75cbea59b5537de654 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 21 Sep 2021 23:12:53 -0500 Subject: [PATCH 09/14] refactor: sort imports --- server/helpers/procBuilder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/helpers/procBuilder.go b/server/helpers/procBuilder.go index 13eddbcf5b..6808d6a8a5 100644 --- a/server/helpers/procBuilder.go +++ b/server/helpers/procBuilder.go @@ -16,7 +16,6 @@ package helpers import ( "fmt" - "github.com/woodpecker-ci/woodpecker/server" "math/rand" "net/url" "path/filepath" @@ -32,6 +31,7 @@ import ( "github.com/woodpecker-ci/woodpecker/cncd/pipeline/pipeline/frontend/yaml/matrix" "github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/remote" + "github.com/woodpecker-ci/woodpecker/server" ) // ProcBuilder Takes the hook data and the yaml and returns in helpers data model From d3687aec25f59e21704edac04d34a9dbd2d7f5d6 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 22 Sep 2021 12:13:43 -0500 Subject: [PATCH 10/14] typo fix in server/helpers/procBuilder.go --- server/helpers/procBuilder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/helpers/procBuilder.go b/server/helpers/procBuilder.go index 6808d6a8a5..2f54278f54 100644 --- a/server/helpers/procBuilder.go +++ b/server/helpers/procBuilder.go @@ -34,7 +34,7 @@ import ( "github.com/woodpecker-ci/woodpecker/server" ) -// ProcBuilder Takes the hook data and the yaml and returns in helpers data model +// ProcBuilder Takes the hook data and the yaml and returns in internal data model type ProcBuilder struct { Repo *model.Repo Curr *model.Build From bf82635a632deb6d9c02344f89a708d3ce844567 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 22 Sep 2021 12:17:14 -0500 Subject: [PATCH 11/14] Refactor: Import as server instead of droneserver or woodpecker --- cmd/server/server.go | 68 ++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/cmd/server/server.go b/cmd/server/server.go index 4bbd00cd30..2f12962e72 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -40,7 +40,7 @@ import ( "github.com/woodpecker-ci/woodpecker/remote" "github.com/woodpecker-ci/woodpecker/router" "github.com/woodpecker-ci/woodpecker/router/middleware" - woodpecker "github.com/woodpecker-ci/woodpecker/server" + "github.com/woodpecker-ci/woodpecker/server" woodpeckerGrpcServer "github.com/woodpecker-ci/woodpecker/server/grpc" "github.com/woodpecker-ci/woodpecker/store" @@ -126,11 +126,11 @@ func server(c *cli.Context) error { ) droneServer := woodpeckerGrpcServer.NewDroneServer( remote_, - woodpecker.Config.Services.Queue, - woodpecker.Config.Services.Logs, - woodpecker.Config.Services.Pubsub, + server.Config.Services.Queue, + server.Config.Services.Logs, + server.Config.Services.Pubsub, store_, - woodpecker.Config.Server.Host, + server.Config.Server.Host, ) proto.RegisterDroneServer(grpcServer, droneServer) @@ -209,45 +209,45 @@ func server(c *cli.Context) error { func setupEvilGlobals(c *cli.Context, v store.Store, r remote.Remote) { // storage - woodpecker.Config.Storage.Files = v - woodpecker.Config.Storage.Config = v + server.Config.Storage.Files = v + server.Config.Storage.Config = v // services - woodpecker.Config.Services.Queue = setupQueue(c, v) - woodpecker.Config.Services.Logs = logging.New() - woodpecker.Config.Services.Pubsub = pubsub.New() - woodpecker.Config.Services.Pubsub.Create(context.Background(), "topic/events") - woodpecker.Config.Services.Registries = setupRegistryService(c, v) - woodpecker.Config.Services.Secrets = setupSecretService(c, v) - woodpecker.Config.Services.Senders = sender.New(v, v) - woodpecker.Config.Services.Environ = setupEnvironService(c, v) + server.Config.Services.Queue = setupQueue(c, v) + server.Config.Services.Logs = logging.New() + server.Config.Services.Pubsub = pubsub.New() + server.Config.Services.Pubsub.Create(context.Background(), "topic/events") + server.Config.Services.Registries = setupRegistryService(c, v) + server.Config.Services.Secrets = setupSecretService(c, v) + server.Config.Services.Senders = sender.New(v, v) + server.Config.Services.Environ = setupEnvironService(c, v) if endpoint := c.String("gating-service"); endpoint != "" { - woodpecker.Config.Services.Senders = sender.NewRemote(endpoint) + server.Config.Services.Senders = sender.NewRemote(endpoint) } // limits - woodpecker.Config.Pipeline.Limits.MemSwapLimit = c.Int64("limit-mem-swap") - woodpecker.Config.Pipeline.Limits.MemLimit = c.Int64("limit-mem") - woodpecker.Config.Pipeline.Limits.ShmSize = c.Int64("limit-shm-size") - woodpecker.Config.Pipeline.Limits.CPUQuota = c.Int64("limit-cpu-quota") - woodpecker.Config.Pipeline.Limits.CPUShares = c.Int64("limit-cpu-shares") - woodpecker.Config.Pipeline.Limits.CPUSet = c.String("limit-cpu-set") + server.Config.Pipeline.Limits.MemSwapLimit = c.Int64("limit-mem-swap") + server.Config.Pipeline.Limits.MemLimit = c.Int64("limit-mem") + server.Config.Pipeline.Limits.ShmSize = c.Int64("limit-shm-size") + server.Config.Pipeline.Limits.CPUQuota = c.Int64("limit-cpu-quota") + server.Config.Pipeline.Limits.CPUShares = c.Int64("limit-cpu-shares") + server.Config.Pipeline.Limits.CPUSet = c.String("limit-cpu-set") // server configuration - woodpecker.Config.Server.Cert = c.String("server-cert") - woodpecker.Config.Server.Key = c.String("server-key") - woodpecker.Config.Server.Pass = c.String("agent-secret") - woodpecker.Config.Server.Host = c.String("server-host") - woodpecker.Config.Server.Port = c.String("server-addr") - woodpecker.Config.Server.RepoConfig = c.String("repo-config") - woodpecker.Config.Server.SessionExpires = c.Duration("session-expires") - woodpecker.Config.Pipeline.Networks = c.StringSlice("network") - woodpecker.Config.Pipeline.Volumes = c.StringSlice("volume") - woodpecker.Config.Pipeline.Privileged = c.StringSlice("escalate") + server.Config.Server.Cert = c.String("server-cert") + server.Config.Server.Key = c.String("server-key") + server.Config.Server.Pass = c.String("agent-secret") + server.Config.Server.Host = c.String("server-host") + server.Config.Server.Port = c.String("server-addr") + server.Config.Server.RepoConfig = c.String("repo-config") + server.Config.Server.SessionExpires = c.Duration("session-expires") + server.Config.Pipeline.Networks = c.StringSlice("network") + server.Config.Pipeline.Volumes = c.StringSlice("volume") + server.Config.Pipeline.Privileged = c.StringSlice("escalate") // prometheus - woodpecker.Config.Prometheus.AuthToken = c.String("prometheus-auth-token") + server.Config.Prometheus.AuthToken = c.String("prometheus-auth-token") } type authorizer struct { @@ -279,7 +279,7 @@ func (a *authorizer) authorize(ctx context.Context) error { } func redirect(w http.ResponseWriter, req *http.Request) { - var serverHost string = woodpecker.Config.Server.Host + var serverHost string = server.Config.Server.Host serverHost = strings.TrimPrefix(serverHost, "http://") serverHost = strings.TrimPrefix(serverHost, "https://") req.URL.Scheme = "https" From cceaf445e5b84c764c6aebb37c2f6e2a383cbecf Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 22 Sep 2021 12:24:37 -0500 Subject: [PATCH 12/14] Refactor: Rename server function to loop (disambiguate) --- cmd/server/main.go | 2 +- cmd/server/server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index ace1169154..48dc33389d 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -31,7 +31,7 @@ func main() { app.Name = "woodpecker-server" app.Version = version.String() app.Usage = "woodpecker server" - app.Action = server + app.Action = loop app.Flags = flags app.Before = before diff --git a/cmd/server/server.go b/cmd/server/server.go index 2f12962e72..d07c0ed1ce 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -50,7 +50,7 @@ import ( oldcontext "golang.org/x/net/context" ) -func server(c *cli.Context) error { +func loop(c *cli.Context) error { // debug level if requested by user if c.Bool("debug") { From 39e7f139c3ee84f398078df470fad8f6e3411954 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 22 Sep 2021 13:01:16 -0500 Subject: [PATCH 13/14] Refactor: Rename server/helpers -> server/shared --- server/api/build.go | 22 +++++++++---------- server/api/hook.go | 18 +++++++-------- server/api/user.go | 10 ++++----- server/grpc/rpc.go | 14 ++++++------ server/{helpers => shared}/buildStatus.go | 2 +- .../{helpers => shared}/buildStatus_test.go | 2 +- server/{helpers => shared}/configFetcher.go | 2 +- .../{helpers => shared}/configFetcher_test.go | 6 ++--- server/{helpers => shared}/procBuilder.go | 2 +- .../{helpers => shared}/procBuilder_test.go | 2 +- server/{helpers => shared}/procStatus.go | 2 +- server/{helpers => shared}/procStatus_test.go | 2 +- server/{helpers => shared}/userSyncer.go | 2 +- 13 files changed, 43 insertions(+), 43 deletions(-) rename server/{helpers => shared}/buildStatus.go (99%) rename server/{helpers => shared}/buildStatus_test.go (99%) rename server/{helpers => shared}/configFetcher.go (99%) rename server/{helpers => shared}/configFetcher_test.go (97%) rename server/{helpers => shared}/procBuilder.go (99%) rename server/{helpers => shared}/procBuilder_test.go (99%) rename server/{helpers => shared}/procStatus.go (99%) rename server/{helpers => shared}/procStatus_test.go (99%) rename server/{helpers => shared}/userSyncer.go (99%) diff --git a/server/api/build.go b/server/api/build.go index 7dbaed36e3..2439ce5a40 100644 --- a/server/api/build.go +++ b/server/api/build.go @@ -36,7 +36,7 @@ import ( "github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/router/middleware/session" "github.com/woodpecker-ci/woodpecker/server" - "github.com/woodpecker-ci/woodpecker/server/helpers" + "github.com/woodpecker-ci/woodpecker/server/shared" ) func GetBuilds(c *gin.Context) { @@ -206,18 +206,18 @@ func DeleteBuild(c *gin.Context) { for _, proc := range procs { if proc.State == model.StatusPending { if proc.PPID != 0 { - if _, err = helpers.UpdateProcToStatusSkipped(store.FromContext(c), *proc, 0); err != nil { + if _, err = shared.UpdateProcToStatusSkipped(store.FromContext(c), *proc, 0); err != nil { log.Printf("error: done: cannot update proc_id %d state: %s", proc.ID, err) } } else { - if _, err = helpers.UpdateProcToStatusKilled(store.FromContext(c), *proc); err != nil { + if _, err = shared.UpdateProcToStatusKilled(store.FromContext(c), *proc); err != nil { log.Printf("error: done: cannot update proc_id %d state: %s", proc.ID, err) } } } } - killedBuild, err := helpers.UpdateToStatusKilled(store.FromContext(c), *build) + killedBuild, err := shared.UpdateToStatusKilled(store.FromContext(c), *build) if err != nil { c.AbortWithError(500, err) return @@ -272,7 +272,7 @@ func PostApproval(c *gin.Context) { return } - if build, err = helpers.UpdateToStatusPending(store.FromContext(c), *build, user.Login); err != nil { + if build, err = shared.UpdateToStatusPending(store.FromContext(c), *build, user.Login); err != nil { c.String(500, "error updating build. %s", err) return } @@ -303,7 +303,7 @@ func PostApproval(c *gin.Context) { yamls = append(yamls, &remote.FileMeta{Data: []byte(y.Data), Name: y.Name}) } - b := helpers.ProcBuilder{ + b := shared.ProcBuilder{ Repo: repo, Curr: build, Last: last, @@ -316,12 +316,12 @@ func PostApproval(c *gin.Context) { } buildItems, err := b.Build() if err != nil { - if _, err = helpers.UpdateToStatusError(store.FromContext(c), *build, err); err != nil { + if _, err = shared.UpdateToStatusError(store.FromContext(c), *build, err); err != nil { logrus.Errorf("Error setting error status of build for %s#%d. %s", repo.FullName, build.Number, err) } return } - build = helpers.SetBuildStepsOnBuild(b.Curr, buildItems) + build = shared.SetBuildStepsOnBuild(b.Curr, buildItems) err = store.FromContext(c).ProcCreate(build.Procs) if err != nil { @@ -366,7 +366,7 @@ func PostDecline(c *gin.Context) { return } - if _, err = helpers.UpdateToStatusDeclined(store.FromContext(c), *build, user.Login); err != nil { + if _, err = shared.UpdateToStatusDeclined(store.FromContext(c), *build, user.Login); err != nil { c.String(500, "error updating build. %s", err) return } @@ -512,7 +512,7 @@ func PostBuild(c *gin.Context) { yamls = append(yamls, &remote.FileMeta{Data: []byte(y.Data), Name: y.Name}) } - b := helpers.ProcBuilder{ + b := shared.ProcBuilder{ Repo: repo, Curr: build, Last: last, @@ -532,7 +532,7 @@ func PostBuild(c *gin.Context) { c.JSON(500, build) return } - build = helpers.SetBuildStepsOnBuild(b.Curr, buildItems) + build = shared.SetBuildStepsOnBuild(b.Curr, buildItems) err = store.FromContext(c).ProcCreate(build.Procs) if err != nil { diff --git a/server/api/hook.go b/server/api/hook.go index 7cf407ab28..372e860538 100644 --- a/server/api/hook.go +++ b/server/api/hook.go @@ -41,7 +41,7 @@ import ( "github.com/woodpecker-ci/woodpecker/cncd/pubsub" "github.com/woodpecker-ci/woodpecker/cncd/queue" "github.com/woodpecker-ci/woodpecker/server" - "github.com/woodpecker-ci/woodpecker/server/helpers" + "github.com/woodpecker-ci/woodpecker/server/shared" ) var skipRe = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`) @@ -161,7 +161,7 @@ func PostHook(c *gin.Context) { } // fetch the build file from the remote - configFetcher := helpers.NewConfigFetcher(remote_, user, repo, build) + configFetcher := shared.NewConfigFetcher(remote_, user, repo, build) remoteYamlConfigs, err := configFetcher.Fetch() if err != nil { logrus.Errorf("error: %s: cannot find %s in %s: %s", repo.FullName, repo.Config, build.Ref, err) @@ -243,7 +243,7 @@ func PostHook(c *gin.Context) { // get the previous build so that we can send status change notifications last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) - b := helpers.ProcBuilder{ + b := shared.ProcBuilder{ Repo: repo, Curr: build, Last: last, @@ -256,12 +256,12 @@ func PostHook(c *gin.Context) { } buildItems, err := b.Build() if err != nil { - if _, err = helpers.UpdateToStatusError(store.FromContext(c), *build, err); err != nil { + if _, err = shared.UpdateToStatusError(store.FromContext(c), *build, err); err != nil { logrus.Errorf("Error setting error status of build for %s#%d. %s", repo.FullName, build.Number, err) } return } - build = helpers.SetBuildStepsOnBuild(b.Curr, buildItems) + build = shared.SetBuildStepsOnBuild(b.Curr, buildItems) err = store.FromContext(c).ProcCreate(build.Procs) if err != nil { @@ -302,7 +302,7 @@ func branchFiltered(build *model.Build, remoteYamlConfigs []*remote.FileMeta) (b } func zeroSteps(build *model.Build, remoteYamlConfigs []*remote.FileMeta) bool { - b := helpers.ProcBuilder{ + b := shared.ProcBuilder{ Repo: &model.Repo{}, Curr: build, Last: &model.Build{}, @@ -332,7 +332,7 @@ func findOrPersistPipelineConfig(repo *model.Repo, build *model.Build, remoteYam RepoID: build.RepoID, Data: string(remoteYamlConfig.Data), Hash: sha, - Name: helpers.SanitizePath(remoteYamlConfig.Name), + Name: shared.SanitizePath(remoteYamlConfig.Name), } err = server.Config.Storage.Config.ConfigCreate(conf) if err != nil { @@ -374,7 +374,7 @@ func publishToTopic(c *gin.Context, build *model.Build, repo *model.Repo, event server.Config.Services.Pubsub.Publish(c, "topic/events", message) } -func queueBuild(build *model.Build, repo *model.Repo, buildItems []*helpers.BuildItem) { +func queueBuild(build *model.Build, repo *model.Repo, buildItems []*shared.BuildItem) { var tasks []*queue.Task for _, item := range buildItems { if item.Proc.State == model.StatusSkipped { @@ -404,7 +404,7 @@ func queueBuild(build *model.Build, repo *model.Repo, buildItems []*helpers.Buil server.Config.Services.Queue.PushAtOnce(context.Background(), tasks) } -func taskIds(dependsOn []string, buildItems []*helpers.BuildItem) []string { +func taskIds(dependsOn []string, buildItems []*shared.BuildItem) []string { taskIds := []string{} for _, dep := range dependsOn { for _, buildItem := range buildItems { diff --git a/server/api/user.go b/server/api/user.go index 826e9bf472..438bbb45d7 100644 --- a/server/api/user.go +++ b/server/api/user.go @@ -27,7 +27,7 @@ import ( "github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/remote" "github.com/woodpecker-ci/woodpecker/router/middleware/session" - "github.com/woodpecker-ci/woodpecker/server/helpers" + "github.com/woodpecker-ci/woodpecker/server/shared" "github.com/woodpecker-ci/woodpecker/shared/token" "github.com/woodpecker-ci/woodpecker/store" ) @@ -48,11 +48,11 @@ func GetFeed(c *gin.Context) { config := ToConfig(c) - sync := helpers.Syncer{ + sync := shared.Syncer{ Remote: remote.FromContext(c), Store: store.FromContext(c), Perms: store.FromContext(c), - Match: helpers.NamespaceFilter(config.OwnersWhitelist), + Match: shared.NamespaceFilter(config.OwnersWhitelist), } if err := sync.Sync(user); err != nil { logrus.Debugf("sync error: %s: %s", user.Login, err) @@ -93,11 +93,11 @@ func GetRepos(c *gin.Context) { config := ToConfig(c) - sync := helpers.Syncer{ + sync := shared.Syncer{ Remote: remote.FromContext(c), Store: store.FromContext(c), Perms: store.FromContext(c), - Match: helpers.NamespaceFilter(config.OwnersWhitelist), + Match: shared.NamespaceFilter(config.OwnersWhitelist), } if err := sync.Sync(user); err != nil { diff --git a/server/grpc/rpc.go b/server/grpc/rpc.go index 3bbfa1d2dc..3dfceab7ee 100644 --- a/server/grpc/rpc.go +++ b/server/grpc/rpc.go @@ -38,7 +38,7 @@ import ( "github.com/woodpecker-ci/woodpecker/cncd/pubsub" "github.com/woodpecker-ci/woodpecker/cncd/queue" "github.com/woodpecker-ci/woodpecker/server" - "github.com/woodpecker-ci/woodpecker/server/helpers" + "github.com/woodpecker-ci/woodpecker/server/shared" "github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/remote" @@ -139,7 +139,7 @@ func (s *RPC) Update(c context.Context, id string, state rpc.State) error { return err } - if proc, err = helpers.UpdateProcStatus(s.store, *proc, state, build.Started); err != nil { + if proc, err = shared.UpdateProcStatus(s.store, *proc, state, build.Started); err != nil { log.Printf("error: rpc.update: cannot update proc: %s", err) } @@ -266,7 +266,7 @@ func (s *RPC) Init(c context.Context, id string, state rpc.State) error { } if build.Status == model.StatusPending { - if build, err = helpers.UpdateToStatusRunning(s.store, *build, state.Started); err != nil { + if build, err = shared.UpdateToStatusRunning(s.store, *build, state.Started); err != nil { log.Printf("error: init: cannot update build_id %d state: %s", build.ID, err) } } @@ -286,7 +286,7 @@ func (s *RPC) Init(c context.Context, id string, state rpc.State) error { s.pubsub.Publish(c, "topic/events", message) }() - _, err = helpers.UpdateProcToStatusStarted(s.store, *proc, state) + _, err = shared.UpdateProcToStatusStarted(s.store, *proc, state) return err } @@ -315,7 +315,7 @@ func (s *RPC) Done(c context.Context, id string, state rpc.State) error { return err } - if proc, err = helpers.UpdateProcStatusToDone(s.store, *proc, state); err != nil { + if proc, err = shared.UpdateProcStatusToDone(s.store, *proc, state); err != nil { log.Printf("error: done: cannot update proc_id %d state: %s", proc.ID, err) } @@ -333,7 +333,7 @@ func (s *RPC) Done(c context.Context, id string, state rpc.State) error { s.completeChildrenIfParentCompleted(procs, proc) if !isThereRunningStage(procs) { - if build, err = helpers.UpdateStatusToDone(s.store, *build, buildStatus(procs), proc.Stopped); err != nil { + if build, err = shared.UpdateStatusToDone(s.store, *build, buildStatus(procs), proc.Stopped); err != nil { log.Printf("error: done: cannot update build_id %d final state: %s", build.ID, err) } @@ -384,7 +384,7 @@ func (s *RPC) Log(c context.Context, id string, line *rpc.Line) error { func (s *RPC) completeChildrenIfParentCompleted(procs []*model.Proc, completedProc *model.Proc) { for _, p := range procs { if p.Running() && p.PPID == completedProc.PID { - if _, err := helpers.UpdateProcToStatusSkipped(s.store, *p, completedProc.Stopped); err != nil { + if _, err := shared.UpdateProcToStatusSkipped(s.store, *p, completedProc.Stopped); err != nil { log.Printf("error: done: cannot update proc_id %d child state: %s", p.ID, err) } } diff --git a/server/helpers/buildStatus.go b/server/shared/buildStatus.go similarity index 99% rename from server/helpers/buildStatus.go rename to server/shared/buildStatus.go index 145f549fbc..b7716ffccb 100644 --- a/server/helpers/buildStatus.go +++ b/server/shared/buildStatus.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package shared import ( "time" diff --git a/server/helpers/buildStatus_test.go b/server/shared/buildStatus_test.go similarity index 99% rename from server/helpers/buildStatus_test.go rename to server/shared/buildStatus_test.go index 9df92404cf..a2d5c203b0 100644 --- a/server/helpers/buildStatus_test.go +++ b/server/shared/buildStatus_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package shared import ( "errors" diff --git a/server/helpers/configFetcher.go b/server/shared/configFetcher.go similarity index 99% rename from server/helpers/configFetcher.go rename to server/shared/configFetcher.go index feed3f1cd1..4666474180 100644 --- a/server/helpers/configFetcher.go +++ b/server/shared/configFetcher.go @@ -1,4 +1,4 @@ -package helpers +package shared import ( "strings" diff --git a/server/helpers/configFetcher_test.go b/server/shared/configFetcher_test.go similarity index 97% rename from server/helpers/configFetcher_test.go rename to server/shared/configFetcher_test.go index 8b06553baa..0f1fbae4e4 100644 --- a/server/helpers/configFetcher_test.go +++ b/server/shared/configFetcher_test.go @@ -1,4 +1,4 @@ -package helpers_test +package shared_test import ( "fmt" @@ -9,7 +9,7 @@ import ( "github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/remote" "github.com/woodpecker-ci/woodpecker/remote/mocks" - "github.com/woodpecker-ci/woodpecker/server/helpers" + "github.com/woodpecker-ci/woodpecker/server/shared" ) func TestFetch(t *testing.T) { @@ -179,7 +179,7 @@ func TestFetch(t *testing.T) { r.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("File not found")) r.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("Directory not found")) - configFetcher := helpers.NewConfigFetcher( + configFetcher := shared.NewConfigFetcher( r, &model.User{Token: "xxx"}, repo, diff --git a/server/helpers/procBuilder.go b/server/shared/procBuilder.go similarity index 99% rename from server/helpers/procBuilder.go rename to server/shared/procBuilder.go index 2f54278f54..9917845c26 100644 --- a/server/helpers/procBuilder.go +++ b/server/shared/procBuilder.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package shared import ( "fmt" diff --git a/server/helpers/procBuilder_test.go b/server/shared/procBuilder_test.go similarity index 99% rename from server/helpers/procBuilder_test.go rename to server/shared/procBuilder_test.go index 12d47c4e25..30e19ed92d 100644 --- a/server/helpers/procBuilder_test.go +++ b/server/shared/procBuilder_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package shared import ( "fmt" diff --git a/server/helpers/procStatus.go b/server/shared/procStatus.go similarity index 99% rename from server/helpers/procStatus.go rename to server/shared/procStatus.go index 351ca6c4d9..8568576898 100644 --- a/server/helpers/procStatus.go +++ b/server/shared/procStatus.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package shared import ( "time" diff --git a/server/helpers/procStatus_test.go b/server/shared/procStatus_test.go similarity index 99% rename from server/helpers/procStatus_test.go rename to server/shared/procStatus_test.go index d4ccc75206..55584d8f5f 100644 --- a/server/helpers/procStatus_test.go +++ b/server/shared/procStatus_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package shared import ( "testing" diff --git a/server/helpers/userSyncer.go b/server/shared/userSyncer.go similarity index 99% rename from server/helpers/userSyncer.go rename to server/shared/userSyncer.go index 982ab9655b..c31f0cfc91 100644 --- a/server/helpers/userSyncer.go +++ b/server/shared/userSyncer.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package shared import ( "time" From 1ceda6dcc4c1e433fd90ad5e973646ce246e8047 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 22 Sep 2021 13:16:11 -0500 Subject: [PATCH 14/14] Refactor: simplify server import --- cmd/server/setup.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/server/setup.go b/cmd/server/setup.go index 80f4bea1e3..e4b06d1f3d 100644 --- a/cmd/server/setup.go +++ b/cmd/server/setup.go @@ -35,7 +35,7 @@ import ( "github.com/woodpecker-ci/woodpecker/remote/gitlab" "github.com/woodpecker-ci/woodpecker/remote/gitlab3" "github.com/woodpecker-ci/woodpecker/remote/gogs" - woodpecker "github.com/woodpecker-ci/woodpecker/server" + "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server/web" "github.com/woodpecker-ci/woodpecker/store" "github.com/woodpecker-ci/woodpecker/store/datastore" @@ -257,7 +257,7 @@ func setupMetrics(g *errgroup.Group, store_ store.Store) { g.Go(func() error { for { - stats := woodpecker.Config.Services.Queue.Info(nil) + stats := server.Config.Services.Queue.Info(nil) pendingJobs.Set(float64(stats.Stats.Pending)) waitingJobs.Set(float64(stats.Stats.WaitingOnDeps)) runningJobs.Set(float64(stats.Stats.Running))