Merge pull request #4735 from moby/v0.13.0-picks

[v0.13] cherry-picks for v0.13.0

.github/workflows/validate.yml

@@ -63,6 +63,9 @@ jobs:
 
   archutil-arm64:
     runs-on: ubuntu-22.04
+    # TODO: enable when binutils-loongarch64-linux-gnu pkg is available for aarch64 arch
+    # https://github.com/moby/buildkit/pull/4392#issuecomment-1938223235
+    if: false
     steps:
       -
         name: Checkout

docs/windows.md

@@ -89,6 +89,7 @@ Now that everything is setup, let's build a [simple _hello world_ image](https:/
     ```powershell
     Set-Content Dockerfile @"
     FROM mcr.microsoft.com/windows/nanoserver:ltsc2022
+    USER ContainerAdministrator
     COPY hello.txt C:/
     RUN echo "Goodbye!" >> hello.txt
     CMD ["cmd", "/C", "type C:\\hello.txt"]

frontend/dockerfile/dockerfile2llb/convert.go

@@ -1252,18 +1252,30 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 				a = a.Copy(st, f, dest, opts...)
 			}
 		} else {
+			var patterns []string
+			if cfg.parents {
+				// detect optional pivot point
+				parent, pattern, ok := strings.Cut(src, "/./")
+				if !ok {
+					pattern = src
+					src = "/"
+				} else {
+					src = parent
+				}
+
+				pattern, err = system.NormalizePath("/", pattern, d.platform.OS, false)
+				if err != nil {
+					return errors.Wrap(err, "removing drive letter")
+				}
+
+				patterns = []string{strings.TrimPrefix(pattern, "/")}
+			}
+
 			src, err = system.NormalizePath("/", src, d.platform.OS, false)
 			if err != nil {
 				return errors.Wrap(err, "removing drive letter")
 			}
 
-			var patterns []string
-			if cfg.parents {
-				path := strings.TrimPrefix(src, "/")
-				patterns = []string{path}
-				src = "/"
-			}
-
 			opts := append([]llb.CopyOption{&llb.CopyInfo{
 				Mode:                mode,
 				FollowSymlinks:      true,

frontend/dockerfile/dockerfile_parents_test.go

@@ -18,6 +18,7 @@ import (
 
 var parentsTests = integration.TestFuncs(
 	testCopyParents,
+	testCopyRelativeParents,
 )
 
 func init() {
@@ -75,3 +76,108 @@ COPY --parents foo1/foo2/ba* .
 	require.NoError(t, err)
 	require.Equal(t, "testing2", string(dt))
 }
+
+func testCopyRelativeParents(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+
+	dockerfile := []byte(`
+FROM alpine AS base
+WORKDIR /test
+RUN <<eot
+	set -ex
+	mkdir -p a/b/c/d/e
+	mkdir -p a/b2/c/d/e
+	mkdir -p a/b/c2/d/e
+	mkdir -p a/b/c2/d/e2
+	touch a/b/c/d/foo
+	touch a/b/c/d/e/bay
+	touch a/b2/c/d/e/bar
+	touch a/b/c2/d/e/baz
+	touch a/b/c2/d/e2/baz
+eot
+
+FROM alpine AS middle
+COPY --from=base --parents /test/a/b/./c/d /out/
+RUN <<eot
+	set -ex
+	[ -d /out/c/d/e ]
+	[ -f /out/c/d/foo ]
+	[ ! -d /out/a ]
+	[ ! -d /out/e ]
+eot
+
+FROM alpine AS end
+COPY --from=base --parents /test/a/b/c/d/. /out/
+RUN <<eot
+	set -ex
+	[ -d /out/test/a/b/c/d/e ]
+	[ -f /out/test/a/b/c/d/foo ]
+eot
+
+FROM alpine AS start
+COPY --from=base --parents ./test/a/b/c/d /out/
+RUN <<eot
+	set -ex
+	[ -d /out/test/a/b/c/d/e ]
+	[ -f /out/test/a/b/c/d/foo ]
+eot
+
+FROM alpine AS double
+COPY --from=base --parents /test/a/./b/./c /out/
+RUN <<eot
+	set -ex
+	[ -d /out/b/c/d/e ]
+	[ -f /out/b/c/d/foo ]
+eot
+
+FROM alpine AS wildcard
+COPY --from=base --parents /test/a/./*/c /out/
+RUN <<eot
+	set -ex
+	[ -d /out/b/c/d/e ]
+	[ -f /out/b2/c/d/e/bar ]
+eot
+
+FROM alpine AS doublewildcard
+COPY --from=base --parents /test/a/b*/./c/**/e /out/
+RUN <<eot
+	set -ex
+	[ -d /out/c/d/e ]
+	[ -f /out/c/d/e/bay ] # via b
+	[ -f /out/c/d/e/bar ] # via b2
+eot
+
+FROM alpine AS doubleinputs
+COPY --from=base --parents /test/a/b/c*/./d/**/baz /test/a/b*/./c/**/bar /out/
+RUN <<eot
+	set -ex
+	[ -f /out/d/e/baz ]
+	[ ! -f /out/d/e/bay ]
+	[ -f /out/d/e2/baz ]
+	[ -f /out/c/d/e/bar ] # via b2
+eot
+`)
+
+	dir := integration.Tmpdir(
+		t,
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+	)
+
+	c, err := client.New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	for _, target := range []string{"middle", "end", "start", "double", "wildcard", "doublewildcard", "doubleinputs"} {
+		t.Logf("target: %s", target)
+		_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+			FrontendAttrs: map[string]string{
+				"target": target,
+			},
+			LocalMounts: map[string]fsutil.FS{
+				dockerui.DefaultLocalNameDockerfile: dir,
+				dockerui.DefaultLocalNameContext:    dir,
+			},
+		}, nil)
+		require.NoError(t, err)
+	}
+}

frontend/dockerfile/docs/reference.md

@@ -1596,8 +1596,28 @@ COPY --parents ./x/a.txt ./y/a.txt /parents/
 # /parents/y/a.txt
 ```
 
-This behavior is analogous to the [Linux `cp` utility's](https://www.man7.org/linux/man-pages/man1/cp.1.html)
-`--parents` flag.
+This behavior is similar to the [Linux `cp` utility's](https://www.man7.org/linux/man-pages/man1/cp.1.html)
+`--parents` or [`rsync`](https://man7.org/linux/man-pages/man1/rsync.1.html) `--relative` flag.
+
+As with Rsync, it is possible to limit which parent directories are preserved by
+inserting a dot and a slash (`./`) into the source path. If such point exists, only parent
+directories after it will be preserved. This may be especially useful copies between stages
+with `--from` where the source paths need to be absolute.
+
+```dockerfile
+# syntax=docker/dockerfile-upstream:master-labs
+FROM scratch
+
+COPY --parents ./x/./y/*.txt /parents/
+
+# Build context:
+# ./x/y/a.txt
+# ./x/y/b.txt
+#
+# Output:
+# /parents/y/a.txt
+# /parents/y/b.txt
+```
 
 Note that, without the `--parents` flag specified, any filename collision will
 fail the Linux `cp` operation with an explicit error message

go.mod

@@ -67,7 +67,7 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spdx/tools-golang v0.5.3
 	github.com/stretchr/testify v1.8.4
-	github.com/tonistiigi/fsutil v0.0.0-20240223190444-7a889f53dbf6
+	github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5
 	github.com/tonistiigi/go-actions-cache v0.0.0-20240227172821-a0b64f338598
 	github.com/tonistiigi/go-archvariant v1.0.0
 	github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea

go.sum

@@ -405,8 +405,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/tonistiigi/fsutil v0.0.0-20240223190444-7a889f53dbf6 h1:v9u6pmdUkarXL/1S/6LGcG9wsiBLd9N/WyJq/Y9WPcg=
-github.com/tonistiigi/fsutil v0.0.0-20240223190444-7a889f53dbf6/go.mod h1:vbbYqJlnswsbJqWUcJN8fKtBhnEgldDrcagTgnBVKKM=
+github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5 h1:oZS8KCqAg62sxJkEq/Ppzqrb6EooqzWtL8Oaex7bc5c=
+github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5/go.mod h1:vbbYqJlnswsbJqWUcJN8fKtBhnEgldDrcagTgnBVKKM=
 github.com/tonistiigi/go-actions-cache v0.0.0-20240227172821-a0b64f338598 h1:DA/NDC0YbMdnfcOSUzAnbUZE6dSM54d+0hrBqG+bOfs=
 github.com/tonistiigi/go-actions-cache v0.0.0-20240227172821-a0b64f338598/go.mod h1:anhKd3mnC1shAbQj1Q4IJ+w6xqezxnyDYlx/yKa7IXM=
 github.com/tonistiigi/go-archvariant v1.0.0 h1:5LC1eDWiBNflnTF1prCiX09yfNHIxDC/aukdhCdTyb0=

hack/dockerfiles/archutil.Dockerfile

@@ -15,7 +15,8 @@ RUN apt-get update && apt-get --no-install-recommends install -y git binutils \
   binutils-s390x-linux-gnu \
   binutils-powerpc64le-linux-gnu \
   binutils-mips64el-linux-gnuabi64 \
-  binutils-mips64-linux-gnuabi64
+  binutils-mips64-linux-gnuabi64 \
+  binutils-loongarch64-linux-gnu
 WORKDIR /src
 
 FROM base AS exit-amd64
@@ -58,6 +59,10 @@ FROM base AS exit-mips64
 COPY util/archutil/fixtures/exit.mips64.s .
 RUN mips64-linux-gnuabi64-as --noexecstack -o exit.o exit.mips64.s && mips64-linux-gnuabi64-ld -o exit -s exit.o && mips64-linux-gnuabi64-strip --strip-unneeded exit
 
+FROM base AS exit-loong64
+COPY util/archutil/fixtures/exit.loongarch64.s .
+RUN loongarch64-linux-gnu-as --noexecstack -o exit.o exit.loongarch64.s && loongarch64-linux-gnu-ld -o exit -s exit.o && loongarch64-linux-gnu-strip --strip-unneeded exit
+
 FROM scratch AS exits
 COPY --from=exit-amd64 /src/exit amd64
 COPY --from=exit-386 /src/exit 386
@@ -69,6 +74,7 @@ COPY --from=exit-ppc64 /src/exit ppc64
 COPY --from=exit-ppc64le /src/exit ppc64le
 COPY --from=exit-mips64le /src/exit mips64le
 COPY --from=exit-mips64 /src/exit mips64
+COPY --from=exit-loong64 /src/exit loong64
 
 FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS generate
 WORKDIR /go/src/github.com/moby/buildkit
@@ -86,7 +92,8 @@ RUN --mount=type=bind,target=.,rw \
     bin/archutil/ppc64 \
     bin/archutil/ppc64le \
     bin/archutil/mips64le \
-    bin/archutil/mips64
+    bin/archutil/mips64 \
+    bin/archutil/loong64
   tree -nh bin/archutil
   cp bin/archutil/*_binary.go /out
 EOT

util/archutil/detect.go

@@ -78,6 +78,11 @@ func SupportedPlatforms(noCache bool) []ocispecs.Platform {
 			arr = append(arr, linux(p))
 		}
 	}
+	if p := "loong64"; def.Architecture != p {
+		if _, err := loong64Supported(); err == nil {
+			arr = append(arr, linux(p))
+		}
+	}
 	if p := "arm"; def.Architecture != p {
 		if _, err := armSupported(); err == nil {
 			p := linux("arm")
@@ -144,6 +149,11 @@ func WarnIfUnsupported(pfs []ocispecs.Platform) {
 					printPlatformWarning(p, err)
 				}
 			}
+			if p.Architecture == "loong64" {
+				if _, err := loong64Supported(); err != nil {
+					printPlatformWarning(p, err)
+				}
+			}
 			if p.Architecture == "arm" {
 				if _, err := armSupported(); err != nil {
 					printPlatformWarning(p, err)

util/archutil/fixtures/exit.loongarch64.s

@@ -0,0 +1,6 @@
+	.global _start
+	.text
+_start:
+	li.w $a0,0
+	li.w $a7,93
+	syscall 0

util/archutil/loong64_binary.go

@@ -0,0 +1,9 @@
+//go:build !loong64
+// +build !loong64
+
+package archutil
+
+// This file is generated by running "make archutil".
+// Do not edit manually.
+
+const Binaryloong64 = "\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xaa\x77\xf5\x71\x63\x62\x64\x64\x80\x01\x26\x06\x08\x6f\x03\x03\x83\x02\x88\x76\x80\x8a\x5f\x80\xd2\xce\x60\x31\x0b\x06\x26\x06\x07\x06\x66\x06\x26\x06\x90\x1a\x56\x06\x14\xa0\xc0\x88\x44\xef\x81\x0a\xc2\x68\x98\x81\x81\x4f\x4b\x52\xd8\x18\x88\x07\x02\x50\x9a\x85\x41\x94\x81\xbb\xa4\x91\x99\x81\x41\x9b\x81\x41\xaf\x38\xa3\xb8\xa4\xa8\x24\x31\x89\x41\xaf\x24\xb5\xa2\x84\x81\x0a\x80\x9b\x81\x01\xec\x27\x98\xdb\x60\xe1\xb0\x01\xca\xe7\x41\x53\xcf\x88\x85\xcf\x8c\xc5\x5c\x98\xff\x05\x09\xe8\x07\x04\x00\x00\xff\xff\x5f\xc5\x9b\x9d\x90\x01\x00\x00"

util/archutil/loong64_check.go

@@ -0,0 +1,8 @@
+//go:build !loong64
+// +build !loong64
+
+package archutil
+
+func loong64Supported() (string, error) {
+	return check("loong64", Binaryloong64)
+}

util/archutil/loong64_check_loong64.go

@@ -0,0 +1,7 @@
+//go:build loong64
+
+package archutil
+
+func loong64Supported() (string, error) {
+	return "", nil
+}

util/system/path.go

@@ -27,7 +27,7 @@ func DefaultPathEnv(os string) string {
 
 // NormalizePath cleans the path based on the operating system the path is meant for.
 // It takes into account a potential parent path, and will join the path to the parent
-// if the path is relative. Additionally, it will apply the folliwing rules:
+// if the path is relative. Additionally, it will apply the following rules:
 //   - always return an absolute path
 //   - always strip drive letters for Windows paths
 //   - optionally keep the trailing slashes on paths

vendor/modules.txt

@@ -765,7 +765,7 @@ github.com/spdx/tools-golang/spdx/v2/v2_3
 ## explicit; go 1.20
 github.com/stretchr/testify/assert
 github.com/stretchr/testify/require
-# github.com/tonistiigi/fsutil v0.0.0-20240223190444-7a889f53dbf6
+# github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5
 ## explicit; go 1.20
 github.com/tonistiigi/fsutil
 github.com/tonistiigi/fsutil/copy