Skip to content

Commit

Permalink
Move Server Pipeline Build code out of API into own package (#949)
Browse files Browse the repository at this point in the history 
- refactor
- create new errors to handle on them
- dedup code
- split server pipeline functionality's into dedicated functions
- add code comments to document what goes on
- add TODOs for next refactor
  • Loading branch information
@6543
6543 committed Jun 15, 2022
1 parent 0680636 commit bdcee93
Show file tree
Hide file tree
Showing 16 changed files with 1,134 additions and 716 deletions.
469 changes: 26 additions & 443 deletions server/api/build.go
View file

Large diffs are not rendered by default.

57 changes: 57 additions & 0 deletions server/api/helper.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
"net/http"

"github.com/gin-gonic/gin"
"github.com/rs/zerolog/log"

"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/pipeline"
"github.com/woodpecker-ci/woodpecker/server/remote"
"github.com/woodpecker-ci/woodpecker/server/store"
)

func handlePipelineErr(c *gin.Context, err error) {
if pipeline.IsErrNotFound(err) {
c.String(http.StatusNotFound, "%v", err)
} else if pipeline.IsErrBadRequest(err) {
c.String(http.StatusBadRequest, "%v", err)
} else if pipeline.IsErrFiltered(err) {
c.String(http.StatusNoContent, "%v", err)
} else {
_ = c.AbortWithError(http.StatusInternalServerError, err)
}
}

// if the remote has a refresh token, the current access token may be stale.
// Therefore, we should refresh prior to dispatching the job.
func refreshUserToken(c *gin.Context, user *model.User) {
_remote := server.Config.Services.Remote
_store := store.FromContext(c)
if refresher, ok := _remote.(remote.Refresher); ok {
ok, err := refresher.Refresh(c, user)
if err != nil {
log.Error().Err(err).Msgf("refresh oauth token of user '%s' failed", user.Login)
} else if ok {
if err := _store.UpdateUser(user); err != nil {
log.Error().Err(err).Msg("fail to save user to store after refresh oauth token")
}
}
}
}
285 changes: 12 additions & 273 deletions server/api/hook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,18 @@
package api

import (
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"math/rand"
"net/http"
"regexp"
"strconv"
"time"

"github.com/gin-gonic/gin"
"github.com/rs/zerolog/log"

"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
"github.com/woodpecker-ci/woodpecker/pipeline/rpc"
"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/pubsub"
"github.com/woodpecker-ci/woodpecker/server/queue"
"github.com/woodpecker-ci/woodpecker/server/remote"
"github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/woodpecker-ci/woodpecker/server/pipeline"
"github.com/woodpecker-ci/woodpecker/server/store"
"github.com/woodpecker-ci/woodpecker/shared/token"
)
Expand Down Expand Up @@ -75,17 +66,18 @@ func BlockTilQueueHasRunningItem(c *gin.Context) {
c.Status(http.StatusOK)
}

// PostHook start a pipeline triggered by a forges post webhook
func PostHook(c *gin.Context) {
_store := store.FromContext(c)

tmpRepo, build, err := server.Config.Services.Remote.Hook(c, c.Request)
tmpRepo, tmpBuild, err := server.Config.Services.Remote.Hook(c, c.Request)
if err != nil {
msg := "failure to parse hook"
log.Debug().Err(err).Msg(msg)
c.String(http.StatusBadRequest, msg)
return
}
if build == nil {
if tmpBuild == nil {
msg := "ignoring hook: hook parsing resulted in empty build"
log.Debug().Msg(msg)
c.String(http.StatusOK, msg)
Expand All @@ -98,11 +90,11 @@ func PostHook(c *gin.Context) {
return
}

// skip the build if any case-insensitive combination of the words "skip" and "ci"
// skip the tmpBuild if any case-insensitive combination of the words "skip" and "ci"
// wrapped in square brackets appear in the commit message
skipMatch := skipRe.FindString(build.Message)
skipMatch := skipRe.FindString(tmpBuild.Message)
if len(skipMatch) > 0 {
msg := fmt.Sprintf("ignoring hook: %s found in %s", skipMatch, build.Commit)
msg := fmt.Sprintf("ignoring hook: %s found in %s", skipMatch, tmpBuild.Commit)
log.Debug().Msg(msg)
c.String(http.StatusNoContent, msg)
return
Expand Down Expand Up @@ -146,270 +138,17 @@ func PostHook(c *gin.Context) {
return
}

if build.Event == model.EventPull && !repo.AllowPull {
if tmpBuild.Event == model.EventPull && !repo.AllowPull {
msg := "ignoring hook: pull requests are disabled for this repo in woodpecker"
log.Debug().Str("repo", repo.FullName).Msg(msg)
c.String(http.StatusNoContent, msg)
return
}

repoUser, err := _store.GetUser(repo.UserID)
build, err := pipeline.Create(c, _store, repo, tmpBuild)
if err != nil {
msg := fmt.Sprintf("failure to find repo owner via id '%d'", repo.UserID)
log.Error().Err(err).Str("repo", repo.FullName).Msg(msg)
c.String(http.StatusInternalServerError, msg)
return
}

// if the remote has a refresh token, the current access token
// may be stale. Therefore, we should refresh prior to dispatching
// the build.
if refresher, ok := server.Config.Services.Remote.(remote.Refresher); ok {
refreshed, err := refresher.Refresh(c, repoUser)
if err != nil {
log.Error().Err(err).Msgf("failed to refresh oauth2 token for repoUser: %s", repoUser.Login)
} else if refreshed {
if err := _store.UpdateUser(repoUser); err != nil {
log.Error().Err(err).Msgf("error while updating repoUser: %s", repoUser.Login)
// move forward
}
}
}

// fetch the build file from the remote
configFetcher := shared.NewConfigFetcher(server.Config.Services.Remote, server.Config.Services.ConfigService, repoUser, repo, build)
remoteYamlConfigs, err := configFetcher.Fetch(c)
if err != nil {
msg := fmt.Sprintf("cannot find config '%s' in '%s' with user: '%s'", repo.Config, build.Ref, repoUser.Login)
log.Debug().Err(err).Str("repo", repo.FullName).Msg(msg)
c.String(http.StatusNotFound, msg)
return
}

filtered, err := branchFiltered(build, remoteYamlConfigs)
if err != nil {
msg := "failure to parse yaml from hook"
log.Debug().Err(err).Str("repo", repo.FullName).Msg(msg)
c.String(http.StatusBadRequest, msg)
return
}
if filtered {
msg := "ignoring hook: branch does not match restrictions defined in yaml"
log.Debug().Str("repo", repo.FullName).Msg(msg)
c.String(http.StatusOK, msg)
return
}

if zeroSteps(build, remoteYamlConfigs) {
msg := "ignoring hook: step conditions yield zero runnable steps"
log.Debug().Str("repo", repo.FullName).Msg(msg)
c.String(http.StatusOK, msg)
return
}

// update some build fields
build.RepoID = repo.ID
build.Verified = true
build.Status = model.StatusPending

// TODO(336) extend gated feature with an allow/block List
if repo.IsGated {
build.Status = model.StatusBlocked
}

err = _store.CreateBuild(build, build.Procs...)
if err != nil {
msg := fmt.Sprintf("failure to save build for %s", repo.FullName)
log.Error().Err(err).Msg(msg)
c.String(http.StatusInternalServerError, msg)
return
}

// persist the build config for historical correctness, restarts, etc
for _, remoteYamlConfig := range remoteYamlConfigs {
_, err := findOrPersistPipelineConfig(_store, build, remoteYamlConfig)
if err != nil {
msg := fmt.Sprintf("failure to find or persist pipeline config for %s", repo.FullName)
log.Error().Err(err).Msg(msg)
c.String(http.StatusInternalServerError, msg)
return
}
}

build, buildItems, err := createBuildItems(c, _store, build, repoUser, repo, remoteYamlConfigs, nil)
if err != nil {
msg := fmt.Sprintf("failure to createBuildItems for %s", repo.FullName)
log.Error().Err(err).Msg(msg)
c.String(http.StatusInternalServerError, msg)
return
}

if build.Status == model.StatusBlocked {
if err := publishToTopic(c, build, repo); err != nil {
log.Error().Err(err).Msg("publishToTopic")
}

if err := updateBuildStatus(c, build, repo, repoUser); err != nil {
log.Error().Err(err).Msg("updateBuildStatus")
}

c.JSON(http.StatusOK, build)
return
}

build, err = startBuild(c, _store, build, repoUser, repo, buildItems)
if err != nil {
msg := fmt.Sprintf("failure to start build for %s", repo.FullName)
log.Error().Err(err).Msg(msg)
c.String(http.StatusInternalServerError, msg)
return
}

c.JSON(http.StatusOK, build)
}

// TODO: parse yaml once and not for each filter function
func branchFiltered(build *model.Build, remoteYamlConfigs []*remote.FileMeta) (bool, error) {
log.Trace().Msgf("hook.branchFiltered(): build branch: '%s' build event: '%s' config count: %d", build.Branch, build.Event, len(remoteYamlConfigs))

if build.Event == model.EventTag || build.Event == model.EventDeploy {
return false, nil
}

for _, remoteYamlConfig := range remoteYamlConfigs {
parsedPipelineConfig, err := yaml.ParseBytes(remoteYamlConfig.Data)
if err != nil {
log.Trace().Msgf("parse config '%s': %s", remoteYamlConfig.Name, err)
return false, err
}
log.Trace().Msgf("config '%s': %#v", remoteYamlConfig.Name, parsedPipelineConfig)

if parsedPipelineConfig.Branches.Match(build.Branch) {
return false, nil
}
}

return true, nil
}

func zeroSteps(build *model.Build, remoteYamlConfigs []*remote.FileMeta) bool {
b := shared.ProcBuilder{
Repo: &model.Repo{},
Curr: build,
Last: &model.Build{},
Netrc: &model.Netrc{},
Secs: []*model.Secret{},
Regs: []*model.Registry{},
Link: "",
Yamls: remoteYamlConfigs,
}

buildItems, err := b.Build()
if err != nil {
return false
}
if len(buildItems) == 0 {
return true
}

return false
}

func findOrPersistPipelineConfig(store store.Store, build *model.Build, remoteYamlConfig *remote.FileMeta) (*model.Config, error) {
sha := shasum(remoteYamlConfig.Data)
conf, err := store.ConfigFindIdentical(build.RepoID, sha)
if err != nil {
conf = &model.Config{
RepoID: build.RepoID,
Data: remoteYamlConfig.Data,
Hash: sha,
Name: shared.SanitizePath(remoteYamlConfig.Name),
}
err = store.ConfigCreate(conf)
if err != nil {
// retry in case we receive two hooks at the same time
conf, err = store.ConfigFindIdentical(build.RepoID, sha)
if err != nil {
return nil, err
}
}
}

buildConfig := &model.BuildConfig{
ConfigID: conf.ID,
BuildID: build.ID,
}
if err := store.BuildConfigCreate(buildConfig); err != nil {
return nil, err
handlePipelineErr(c, err)
} else {
c.JSON(200, build)
}

return conf, nil
}

// publishes message to UI clients
func publishToTopic(c context.Context, build *model.Build, repo *model.Repo) (err error) {
message := pubsub.Message{
Labels: map[string]string{
"repo": repo.FullName,
"private": strconv.FormatBool(repo.IsSCMPrivate),
},
}
buildCopy := *build
if buildCopy.Procs, err = model.Tree(buildCopy.Procs); err != nil {
return err
}

message.Data, _ = json.Marshal(model.Event{
Repo: *repo,
Build: buildCopy,
})
return server.Config.Services.Pubsub.Publish(c, "topic/events", message)
}

func queueBuild(build *model.Build, repo *model.Repo, buildItems []*shared.BuildItem) error {
var tasks []*queue.Task
for _, item := range buildItems {
if item.Proc.State == model.StatusSkipped {
continue
}
task := new(queue.Task)
task.ID = fmt.Sprint(item.Proc.ID)
task.Labels = map[string]string{}
for k, v := range item.Labels {
task.Labels[k] = v
}
task.Labels["platform"] = item.Platform
task.Labels["repo"] = repo.FullName
task.Dependencies = taskIds(item.DependsOn, buildItems)
task.RunOn = item.RunsOn
task.DepStatus = make(map[string]string)

task.Data, _ = json.Marshal(rpc.Pipeline{
ID: fmt.Sprint(item.Proc.ID),
Config: item.Config,
Timeout: repo.Timeout,
})

if err := server.Config.Services.Logs.Open(context.Background(), task.ID); err != nil {
return err
}
tasks = append(tasks, task)
}
return server.Config.Services.Queue.PushAtOnce(context.Background(), tasks)
}

func taskIds(dependsOn []string, buildItems []*shared.BuildItem) (taskIds []string) {
for _, dep := range dependsOn {
for _, buildItem := range buildItems {
if buildItem.Proc.Name == dep {
taskIds = append(taskIds, fmt.Sprint(buildItem.Proc.ID))
}
}
}
return
}

func shasum(raw []byte) string {
sum := sha256.Sum256(raw)
return fmt.Sprintf("%x", sum)
}
Loading

0 comments on commit bdcee93

Please sign in to comment.