Skip to content

Commit

Permalink
Embedding libcompose types for yaml parsing (#495)
Browse files Browse the repository at this point in the history
since github.com/docker/libcompose is deprecated, unmaintained and archived.

and license is the same as woodpecker's, we can just copy stuff into woodpecker directly.
(we only use types of that project anyway)
  • Loading branch information
6543 committed Oct 30, 2021
1 parent ed86de4 commit 0bb62be
Show file tree
Hide file tree
Showing 22 changed files with 686 additions and 549 deletions.
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ require (
github.com/docker/docker v20.10.10+incompatible
github.com/docker/docker-credential-helpers v0.6.4 // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/libcompose v0.4.0
github.com/docker/go-units v0.4.0
github.com/drone/envsubst v1.0.3
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568
github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf
github.com/ghodss/yaml v1.0.0
github.com/gin-gonic/gin v1.7.4
Expand Down
2 changes: 0 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -263,8 +263,6 @@ github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw
github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/libcompose v0.4.0 h1:zK7Ug0lCxPB8FDFNdCvR2ZjJjeJZ/607lfAYkp1hrtc=
github.com/docker/libcompose v0.4.0/go.mod h1:EyqDS+Iyca0hS44T7qIMTeO1EOYWWWNOGpufHu9R8cs=
github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
Expand Down
11 changes: 5 additions & 6 deletions pipeline/frontend/yaml/compiler/cacher.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,8 @@ import (
"path"
"strings"

libcompose "github.com/docker/libcompose/yaml"

"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
)

// Cacher defines a compiler transform that can be used
Expand All @@ -31,8 +30,8 @@ func (c *volumeCacher) Restore(repo, branch string, mounts []string) *yaml.Conta
"file": strings.Replace(branch, "/", "_", -1) + ".tar",
"fallback_to": "master.tar",
},
Volumes: libcompose.Volumes{
Volumes: []*libcompose.Volume{
Volumes: types.Volumes{
Volumes: []*types.Volume{
{
Source: path.Join(c.base, repo),
Destination: "/cache",
Expand All @@ -54,8 +53,8 @@ func (c *volumeCacher) Rebuild(repo, branch string, mounts []string) *yaml.Conta
"flush": true,
"file": strings.Replace(branch, "/", "_", -1) + ".tar",
},
Volumes: libcompose.Volumes{
Volumes: []*libcompose.Volume{
Volumes: types.Volumes{
Volumes: []*types.Volume{
{
Source: path.Join(c.base, repo),
Destination: "/cache",
Expand Down
7 changes: 4 additions & 3 deletions pipeline/frontend/yaml/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,15 @@ import (
"io/ioutil"
"os"

libcompose "github.com/docker/libcompose/yaml"
"gopkg.in/yaml.v3"

"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
)

type (
// Config defines a pipeline configuration.
Config struct {
Cache libcompose.Stringorslice
Cache types.Stringorslice
Platform string
Branches Constraint
Workspace Workspace
Expand All @@ -21,7 +22,7 @@ type (
Services Containers
Networks Networks
Volumes Volumes
Labels libcompose.SliceorMap
Labels types.SliceorMap
DependsOn []string `yaml:"depends_on,omitempty"`
RunsOn []string `yaml:"runs_on,omitempty"`
SkipClone bool `yaml:"skip_clone"`
Expand Down
7 changes: 4 additions & 3 deletions pipeline/frontend/yaml/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@ package yaml
import (
"testing"

"github.com/docker/libcompose/yaml"
"github.com/franela/goblin"

"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
)

func TestParse(t *testing.T) {
Expand All @@ -29,10 +30,10 @@ func TestParse(t *testing.T) {
g.Assert(out.Services.Containers[0].Image).Equal("mysql")
g.Assert(out.Pipeline.Containers[0].Name).Equal("test")
g.Assert(out.Pipeline.Containers[0].Image).Equal("golang")
g.Assert(out.Pipeline.Containers[0].Commands).Equal(yaml.Stringorslice{"go install", "go test"})
g.Assert(out.Pipeline.Containers[0].Commands).Equal(types.Stringorslice{"go install", "go test"})
g.Assert(out.Pipeline.Containers[1].Name).Equal("build")
g.Assert(out.Pipeline.Containers[1].Image).Equal("golang")
g.Assert(out.Pipeline.Containers[1].Commands).Equal(yaml.Stringorslice{"go build"})
g.Assert(out.Pipeline.Containers[1].Commands).Equal(types.Stringorslice{"go build"})
g.Assert(out.Pipeline.Containers[2].Name).Equal("notify")
g.Assert(out.Pipeline.Containers[2].Image).Equal("slack")
// g.Assert(out.Pipeline.Containers[2].NetworkMode).Equal("container:name")
Expand Down
15 changes: 7 additions & 8 deletions pipeline/frontend/yaml/constraint.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"strings"

"github.com/bmatcuk/doublestar/v4"
libcompose "github.com/docker/libcompose/yaml"
"gopkg.in/yaml.v3"

"github.com/woodpecker-ci/woodpecker/pipeline/frontend"
Expand Down Expand Up @@ -101,11 +100,11 @@ func (c *Constraint) Excludes(v string) bool {
// UnmarshalYAML unmarshals the constraint.
func (c *Constraint) UnmarshalYAML(value *yaml.Node) error {
var out1 = struct {
Include libcompose.Stringorslice
Exclude libcompose.Stringorslice
Include types.Stringorslice
Exclude types.Stringorslice
}{}

var out2 libcompose.Stringorslice
var out2 types.Stringorslice

err1 := value.Decode(&out1)
err2 := value.Decode(&out2)
Expand Down Expand Up @@ -179,12 +178,12 @@ func (c *ConstraintMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
// UnmarshalYAML unmarshals the constraint.
func (c *ConstraintPath) UnmarshalYAML(value *yaml.Node) error {
var out1 = struct {
Include libcompose.Stringorslice `yaml:"include,omitempty"`
Exclude libcompose.Stringorslice `yaml:"exclude,omitempty"`
IgnoreMessage string `yaml:"ignore_message,omitempty"`
Include types.Stringorslice `yaml:"include,omitempty"`
Exclude types.Stringorslice `yaml:"exclude,omitempty"`
IgnoreMessage string `yaml:"ignore_message,omitempty"`
}{}

var out2 libcompose.Stringorslice
var out2 types.Stringorslice

err1 := value.Decode(&out1)
err2 := value.Decode(&out2)
Expand Down
75 changes: 38 additions & 37 deletions pipeline/frontend/yaml/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@ package yaml
import (
"fmt"

libcompose "github.com/docker/libcompose/yaml"
"gopkg.in/yaml.v3"

"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
)

type (
Expand All @@ -22,42 +23,42 @@ type (

// Container defines a container.
Container struct {
AuthConfig AuthConfig `yaml:"auth_config,omitempty"`
CapAdd []string `yaml:"cap_add,omitempty"`
CapDrop []string `yaml:"cap_drop,omitempty"`
Command libcompose.Command `yaml:"command,omitempty"`
Commands libcompose.Stringorslice `yaml:"commands,omitempty"`
CPUQuota libcompose.StringorInt `yaml:"cpu_quota,omitempty"`
CPUSet string `yaml:"cpuset,omitempty"`
CPUShares libcompose.StringorInt `yaml:"cpu_shares,omitempty"`
Detached bool `yaml:"detach,omitempty"`
Devices []string `yaml:"devices,omitempty"`
Tmpfs []string `yaml:"tmpfs,omitempty"`
DNS libcompose.Stringorslice `yaml:"dns,omitempty"`
DNSSearch libcompose.Stringorslice `yaml:"dns_search,omitempty"`
Entrypoint libcompose.Command `yaml:"entrypoint,omitempty"`
Environment libcompose.SliceorMap `yaml:"environment,omitempty"`
ExtraHosts []string `yaml:"extra_hosts,omitempty"`
Group string `yaml:"group,omitempty"`
Image string `yaml:"image,omitempty"`
Isolation string `yaml:"isolation,omitempty"`
Labels libcompose.SliceorMap `yaml:"labels,omitempty"`
MemLimit libcompose.MemStringorInt `yaml:"mem_limit,omitempty"`
MemSwapLimit libcompose.MemStringorInt `yaml:"memswap_limit,omitempty"`
MemSwappiness libcompose.MemStringorInt `yaml:"mem_swappiness,omitempty"`
Name string `yaml:"name,omitempty"`
NetworkMode string `yaml:"network_mode,omitempty"`
IpcMode string `yaml:"ipc_mode,omitempty"`
Networks libcompose.Networks `yaml:"networks,omitempty"`
Privileged bool `yaml:"privileged,omitempty"`
Pull bool `yaml:"pull,omitempty"`
ShmSize libcompose.MemStringorInt `yaml:"shm_size,omitempty"`
Ulimits libcompose.Ulimits `yaml:"ulimits,omitempty"`
Volumes libcompose.Volumes `yaml:"volumes,omitempty"`
Secrets Secrets `yaml:"secrets,omitempty"`
Sysctls libcompose.SliceorMap `yaml:"sysctls,omitempty"`
Constraints Constraints `yaml:"when,omitempty"`
Vargs map[string]interface{} `yaml:",inline"`
AuthConfig AuthConfig `yaml:"auth_config,omitempty"`
CapAdd []string `yaml:"cap_add,omitempty"`
CapDrop []string `yaml:"cap_drop,omitempty"`
Command types.Command `yaml:"command,omitempty"`
Commands types.Stringorslice `yaml:"commands,omitempty"`
CPUQuota types.StringorInt `yaml:"cpu_quota,omitempty"`
CPUSet string `yaml:"cpuset,omitempty"`
CPUShares types.StringorInt `yaml:"cpu_shares,omitempty"`
Detached bool `yaml:"detach,omitempty"`
Devices []string `yaml:"devices,omitempty"`
Tmpfs []string `yaml:"tmpfs,omitempty"`
DNS types.Stringorslice `yaml:"dns,omitempty"`
DNSSearch types.Stringorslice `yaml:"dns_search,omitempty"`
Entrypoint types.Command `yaml:"entrypoint,omitempty"`
Environment types.SliceorMap `yaml:"environment,omitempty"`
ExtraHosts []string `yaml:"extra_hosts,omitempty"`
Group string `yaml:"group,omitempty"`
Image string `yaml:"image,omitempty"`
Isolation string `yaml:"isolation,omitempty"`
Labels types.SliceorMap `yaml:"labels,omitempty"`
MemLimit types.MemStringorInt `yaml:"mem_limit,omitempty"`
MemSwapLimit types.MemStringorInt `yaml:"memswap_limit,omitempty"`
MemSwappiness types.MemStringorInt `yaml:"mem_swappiness,omitempty"`
Name string `yaml:"name,omitempty"`
NetworkMode string `yaml:"network_mode,omitempty"`
IpcMode string `yaml:"ipc_mode,omitempty"`
Networks types.Networks `yaml:"networks,omitempty"`
Privileged bool `yaml:"privileged,omitempty"`
Pull bool `yaml:"pull,omitempty"`
ShmSize types.MemStringorInt `yaml:"shm_size,omitempty"`
Ulimits types.Ulimits `yaml:"ulimits,omitempty"`
Volumes types.Volumes `yaml:"volumes,omitempty"`
Secrets Secrets `yaml:"secrets,omitempty"`
Sysctls types.SliceorMap `yaml:"sysctls,omitempty"`
Constraints Constraints `yaml:"when,omitempty"`
Vargs map[string]interface{} `yaml:",inline"`
}
)

Expand Down
39 changes: 20 additions & 19 deletions pipeline/frontend/yaml/container_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ import (
"reflect"
"testing"

libcompose "github.com/docker/libcompose/yaml"
"github.com/kr/pretty"
"gopkg.in/yaml.v3"

"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
)

var containerYaml = []byte(`
Expand Down Expand Up @@ -68,38 +69,38 @@ func TestUnmarshalContainer(t *testing.T) {
},
CapAdd: []string{"ALL"},
CapDrop: []string{"NET_ADMIN", "SYS_ADMIN"},
Command: libcompose.Command{"bundle", "exec", "thin", "-p", "3000"},
Commands: libcompose.Stringorslice{"go build", "go test"},
CPUQuota: libcompose.StringorInt(11),
Command: types.Command{"bundle", "exec", "thin", "-p", "3000"},
Commands: types.Stringorslice{"go build", "go test"},
CPUQuota: types.StringorInt(11),
CPUSet: "1,2",
CPUShares: libcompose.StringorInt(99),
CPUShares: types.StringorInt(99),
Detached: true,
Devices: []string{"/dev/ttyUSB0:/dev/ttyUSB0"},
DNS: libcompose.Stringorslice{"8.8.8.8"},
DNSSearch: libcompose.Stringorslice{"example.com"},
Entrypoint: libcompose.Command{"/code/entrypoint.sh"},
Environment: libcompose.SliceorMap{"RACK_ENV": "development", "SHOW": "true"},
DNS: types.Stringorslice{"8.8.8.8"},
DNSSearch: types.Stringorslice{"example.com"},
Entrypoint: types.Command{"/code/entrypoint.sh"},
Environment: types.SliceorMap{"RACK_ENV": "development", "SHOW": "true"},
ExtraHosts: []string{"somehost:162.242.195.82", "otherhost:50.31.209.229"},
Image: "golang:latest",
Isolation: "hyperv",
Labels: libcompose.SliceorMap{"com.example.type": "build", "com.example.team": "frontend"},
MemLimit: libcompose.MemStringorInt(1024),
MemSwapLimit: libcompose.MemStringorInt(1024),
MemSwappiness: libcompose.MemStringorInt(1024),
Labels: types.SliceorMap{"com.example.type": "build", "com.example.team": "frontend"},
MemLimit: types.MemStringorInt(1024),
MemSwapLimit: types.MemStringorInt(1024),
MemSwappiness: types.MemStringorInt(1024),
Name: "my-build-container",
Networks: libcompose.Networks{
Networks: []*libcompose.Network{
Networks: types.Networks{
Networks: []*types.Network{
{Name: "some-network"},
{Name: "other-network"},
},
},
NetworkMode: "bridge",
Pull: true,
Privileged: true,
ShmSize: libcompose.MemStringorInt(1024),
Tmpfs: libcompose.Stringorslice{"/var/lib/test"},
Volumes: libcompose.Volumes{
Volumes: []*libcompose.Volume{
ShmSize: types.MemStringorInt(1024),
Tmpfs: types.Stringorslice{"/var/lib/test"},
Volumes: types.Volumes{
Volumes: []*types.Volume{
{Source: "", Destination: "/var/lib/mysql"},
{Source: "/opt/data", Destination: "/var/lib/mysql"},
{Source: "/etc/configs", Destination: "/etc/configs/", AccessMode: "ro"},
Expand Down
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
package yaml
package types

import (
"errors"
"fmt"

"github.com/docker/docker/api/types/strslice"
// TODO: search for maintained
"github.com/flynn/go-shlex"
)

// Command represents a docker command, can be a string or an array of strings.
type Command strslice.StrSlice

// UnmarshalYAML implements the Unmarshaller interface.
// UnmarshalYAML implements the Unmarshaler interface.
func (s *Command) UnmarshalYAML(unmarshal func(interface{}) error) error {
var stringType string
if err := unmarshal(&stringType); err == nil {
Expand Down
60 changes: 60 additions & 0 deletions pipeline/frontend/yaml/types/command_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package types

import (
"strings"
"testing"

"gopkg.in/yaml.v3"

"github.com/stretchr/testify/assert"
)

type StructCommand struct {
Entrypoint Command `yaml:"entrypoint,flow,omitempty"`
Command Command `yaml:"command,flow,omitempty"`
}

func TestUnmarshalCommand(t *testing.T) {
s := &StructCommand{}
err := yaml.Unmarshal([]byte(`command: bash`), s)

assert.Nil(t, err)
assert.Equal(t, Command{"bash"}, s.Command)
assert.Nil(t, s.Entrypoint)
bytes, err := yaml.Marshal(s)
assert.Nil(t, err)

s2 := &StructCommand{}
err = yaml.Unmarshal(bytes, s2)

assert.Nil(t, err)
assert.Equal(t, Command{"bash"}, s2.Command)
assert.Nil(t, s2.Entrypoint)

s3 := &StructCommand{}
err = yaml.Unmarshal([]byte(`command:
- echo AAA; echo "wow"
- sleep 3s`), s3)
assert.Nil(t, err)
assert.Equal(t, Command{`echo AAA; echo "wow"`, `sleep 3s`}, s3.Command)
}

var sampleEmptyCommand = `{}`

func TestUnmarshalEmptyCommand(t *testing.T) {
s := &StructCommand{}
err := yaml.Unmarshal([]byte(sampleEmptyCommand), s)

assert.Nil(t, err)
assert.Nil(t, s.Command)

bytes, err := yaml.Marshal(s)
assert.Nil(t, err)
assert.Equal(t, "{}", strings.TrimSpace(string(bytes)))

s2 := &StructCommand{}
err = yaml.Unmarshal(bytes, s2)

assert.Nil(t, err)
assert.Nil(t, s2.Command)
}
Loading

0 comments on commit 0bb62be

Please sign in to comment.