From bb2db7e500c9df08115f787fe04d02ccb2b87db8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Sat, 15 Apr 2017 22:56:57 +0200 Subject: [PATCH 01/15] Support Acquire-By-Hash for index files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The added "aptly publish repo" option "-access-by-hash" publishes the index files (Packages*, Sources*) also as hardlinked hashes. Example: /dists/yakkety/main/binary-amd64/by-hash/SHA512/31833ec39acc... The Release files indicate this with the option "Acquire-By-Hash: yes" This is used by apt >= 1.2.0 and prevents the "Hash sum mismatch" race condition between a server side "aptly publish repo" and "apt-get update" on a client. See: http://www.chiark.greenend.org.uk/~cjwatson/blog/no-more-hash-sum-mismatch-errors.html This implementation uses symlinks in the by-hash/*/ directory for keeping only two versions of the index files and deleting older files automatically. Note: this only works with aptly.FileSystemPublishedStorage Closes: #536 Signed-off-by: André Roth --- api/publish.go | 5 +++ cmd/publish_repo.go | 1 + cmd/publish_snapshot.go | 4 +++ deb/index_files.go | 75 +++++++++++++++++++++++++++++++++++++++-- deb/publish.go | 11 +++++- 5 files changed, 93 insertions(+), 3 deletions(-) diff --git a/api/publish.go b/api/publish.go index 61da5b04e..c4a59ab9a 100644 --- a/api/publish.go +++ b/api/publish.go @@ -238,6 +238,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { Component string `binding:"required"` Name string `binding:"required"` } + AccessByHash *bool } if c.Bind(&b) != nil { @@ -317,6 +318,10 @@ func apiPublishUpdateSwitch(c *gin.Context) { published.SkipContents = *b.SkipContents } + if b.AccessByHash != nil { + published.AccessByHash = *b.AccessByHash + } + err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite) if err != nil { c.AbortWithError(500, fmt.Errorf("unable to update: %s", err)) diff --git a/cmd/publish_repo.go b/cmd/publish_repo.go index 129a20bbc..009ba9273 100644 --- a/cmd/publish_repo.go +++ b/cmd/publish_repo.go @@ -47,6 +47,7 @@ Example: cmd.Flag.String("butautomaticupgrades", "", "set value for ButAutomaticUpgrades field") cmd.Flag.String("label", "", "label to publish") cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") + cmd.Flag.Bool("access-by-hash", false, "provide index files by hash also") return cmd } diff --git a/cmd/publish_snapshot.go b/cmd/publish_snapshot.go index 98540955d..f76055da1 100644 --- a/cmd/publish_snapshot.go +++ b/cmd/publish_snapshot.go @@ -137,6 +137,10 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error { published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool) } + if context.Flags().IsSet("access-by-hash") { + published.AccessByHash = context.Flags().Lookup("access-by-hash").Value.Get().(bool) + } + duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published) if duplicate != nil { context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory()) diff --git a/deb/index_files.go b/deb/index_files.go index f8b7051fc..88a638f7f 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "os" + "path" "path/filepath" "strings" @@ -20,6 +21,7 @@ type indexFiles struct { tempDir string suffix string indexes map[string]*indexFile + accessByHash bool } type indexFile struct { @@ -28,6 +30,7 @@ type indexFile struct { compressable bool onlyGzip bool signable bool + accessByHash bool relativePath string tempFilename string tempFile *os.File @@ -91,11 +94,24 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { file.parent.generatedFiles[file.relativePath+ext] = checksumInfo } - err = file.parent.publishedStorage.MkDir(filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))) + filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath)) + + err = file.parent.publishedStorage.MkDir(filedir) if err != nil { return fmt.Errorf("unable to create dir: %s", err) } + hashs := []string{} + if file.accessByHash { + hashs = append(hashs, "MD5", "SHA1", "SHA256", "SHA512") + for _, hash := range hashs { + err = file.parent.publishedStorage.MkDir(filepath.Join(filedir, "by-hash", hash)) + if err != nil { + return fmt.Errorf("unable to create dir: %s", err) + } + } + } + for _, ext := range exts { err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext), file.tempFilename+ext) @@ -107,6 +123,29 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext)] = filepath.Join(file.parent.basePath, file.relativePath+ext) } + + if file.accessByHash { + sums := file.parent.generatedFiles[file.relativePath+ext] + storage := file.parent.publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath() + src := filepath.Join(storage, file.parent.basePath, file.relativePath) + + err = packageIndexByHash(src, file.parent.suffix, ext, storage, filedir, "SHA512", sums.SHA512) + if err != nil { + fmt.Printf("%s\n", err) + } + err = packageIndexByHash(src, file.parent.suffix, ext, storage, filedir, "SHA256", sums.SHA256) + if err != nil { + fmt.Printf("%s\n", err) + } + err = packageIndexByHash(src, file.parent.suffix, ext, storage, filedir, "SHA1", sums.SHA1) + if err != nil { + fmt.Printf("%s\n", err) + } + err = packageIndexByHash(src, file.parent.suffix, ext, storage, filedir, "MD5", sums.MD5) + if err != nil { + fmt.Printf("%s\n", err) + } + } } if file.signable && signer != nil { @@ -143,7 +182,35 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { return nil } -func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string) *indexFiles { +func packageIndexByHash(src string, suffix string, ext string, storage string, filedir string, hash string, sum string) error { + indexfile := path.Base(src + ext) + src = src + suffix + ext + dst := filepath.Join(storage, filedir, "by-hash", hash) + + if _, err := os.Stat(filepath.Join(dst, sum)); err == nil { + return nil + } + err := os.Link(src, filepath.Join(dst, sum)) + if err != nil { + return fmt.Errorf("Access-By-Hash: error creating hardlink %s", filepath.Join(dst, sum)) + } + + if _, err := os.Stat(filepath.Join(dst, indexfile)); err == nil { + if _, err := os.Stat(filepath.Join(dst, indexfile+".old")); err == nil { + link, _ := os.Readlink(filepath.Join(dst, indexfile+".old")) + os.Remove(filepath.Join(dst, link)) + os.Remove(filepath.Join(dst, indexfile+".old")) + } + os.Rename(filepath.Join(dst, indexfile), filepath.Join(dst, indexfile+".old")) + } + err = os.Symlink(sum, filepath.Join(dst, indexfile)) + if err != nil { + return fmt.Errorf("Access-By-Hash: error creating symlink %s", filepath.Join(dst, indexfile)) + } + return nil +} + +func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, accessByHash bool) *indexFiles { return &indexFiles{ publishedStorage: publishedStorage, basePath: basePath, @@ -152,6 +219,7 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s tempDir: tempDir, suffix: suffix, indexes: make(map[string]*indexFile), + accessByHash: accessByHash, } } @@ -179,6 +247,7 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF discardable: false, compressable: true, signable: false, + accessByHash: files.accessByHash, relativePath: relativePath, } @@ -212,6 +281,7 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF discardable: udeb, compressable: false, signable: false, + accessByHash: files.accessByHash, relativePath: relativePath, } @@ -242,6 +312,7 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index compressable: true, onlyGzip: true, signable: false, + accessByHash: files.accessByHash, relativePath: relativePath, } diff --git a/deb/publish.go b/deb/publish.go index 4e5885400..3d62400d8 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -64,6 +64,9 @@ type PublishedRepo struct { // True if repo is being re-published rePublishing bool + + // Provide index files per hash also + AccessByHash bool } // ParsePrefix splits [storage:]prefix into components @@ -556,7 +559,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } defer os.RemoveAll(tempDir) - indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix) + indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AccessByHash) for component, list := range lists { hadUdebs := false @@ -683,6 +686,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP release["Component"] = component release["Origin"] = p.GetOrigin() release["Label"] = p.GetLabel() + if p.AccessByHash { + release["Acquire-By-Hash"] = "yes" + } var bufWriter *bufio.Writer bufWriter, err = indexes.ReleaseIndex(component, arch, udeb).BufWriter() @@ -720,6 +726,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP release["Codename"] = p.Distribution release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST") release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ") + if p.AccessByHash { + release["Acquire-By-Hash"] = "yes" + } release["Description"] = " Generated by aptly\n" release["MD5Sum"] = "" release["SHA1"] = "" From e07912770ed61d245a73dff46a065d97ad807692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Mon, 17 Apr 2017 23:03:43 +0200 Subject: [PATCH 02/15] Extend PublishedStorage interface for Acquire-By-Hash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: André Roth --- aptly/interfaces.go | 8 ++++++++ deb/index_files.go | 46 ++++++++++++++++++++++++++------------------- files/public.go | 30 +++++++++++++++++++++++++++++ s3/public.go | 24 +++++++++++++++++++++++ swift/public.go | 24 +++++++++++++++++++++++ 5 files changed, 113 insertions(+), 19 deletions(-) diff --git a/aptly/interfaces.go b/aptly/interfaces.go index 3eb5102eb..3b3c18d9b 100644 --- a/aptly/interfaces.go +++ b/aptly/interfaces.go @@ -73,6 +73,14 @@ type PublishedStorage interface { Filelist(prefix string) ([]string, error) // RenameFile renames (moves) file RenameFile(oldName, newName string) error + // SymLink creates a symbolic link, which can be read with ReadLink + SymLink(src string, dst string) error + // HardLink creates a hardlink of a file + HardLink(src string, dst string) error + // FileExists returns true if path exists + FileExists(path string) bool + // ReadLink returns the symbolic link pointed to by path + ReadLink(path string) (string, error) } // FileSystemPublishedStorage is published storage on filesystem diff --git a/deb/index_files.go b/deb/index_files.go index 88a638f7f..10ef17871 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -126,22 +126,20 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { if file.accessByHash { sums := file.parent.generatedFiles[file.relativePath+ext] - storage := file.parent.publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath() - src := filepath.Join(storage, file.parent.basePath, file.relativePath) - err = packageIndexByHash(src, file.parent.suffix, ext, storage, filedir, "SHA512", sums.SHA512) + err = packageIndexByHash(file, ext, "SHA512", sums.SHA512) if err != nil { fmt.Printf("%s\n", err) } - err = packageIndexByHash(src, file.parent.suffix, ext, storage, filedir, "SHA256", sums.SHA256) + err = packageIndexByHash(file, ext, "SHA256", sums.SHA256) if err != nil { fmt.Printf("%s\n", err) } - err = packageIndexByHash(src, file.parent.suffix, ext, storage, filedir, "SHA1", sums.SHA1) + err = packageIndexByHash(file, ext, "SHA1", sums.SHA1) if err != nil { fmt.Printf("%s\n", err) } - err = packageIndexByHash(src, file.parent.suffix, ext, storage, filedir, "MD5", sums.MD5) + err = packageIndexByHash(file, ext, "MD5", sums.MD5) if err != nil { fmt.Printf("%s\n", err) } @@ -182,28 +180,38 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { return nil } -func packageIndexByHash(src string, suffix string, ext string, storage string, filedir string, hash string, sum string) error { +func packageIndexByHash(file *indexFile, ext string, hash string, sum string) error { + src := filepath.Join(file.parent.basePath, file.relativePath) indexfile := path.Base(src + ext) - src = src + suffix + ext - dst := filepath.Join(storage, filedir, "by-hash", hash) + src = src + file.parent.suffix + ext + filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath)) + dst := filepath.Join(filedir, "by-hash", hash) - if _, err := os.Stat(filepath.Join(dst, sum)); err == nil { + // link already exists? do nothing + if file.parent.publishedStorage.FileExists(filepath.Join(dst, sum)) { return nil } - err := os.Link(src, filepath.Join(dst, sum)) + + // create the link + err := file.parent.publishedStorage.HardLink(src, filepath.Join(dst, sum)) if err != nil { - return fmt.Errorf("Access-By-Hash: error creating hardlink %s", filepath.Join(dst, sum)) + return fmt.Errorf("Access-By-Hash: error creating hardlink %s: %s", filepath.Join(dst, sum), err) } - if _, err := os.Stat(filepath.Join(dst, indexfile)); err == nil { - if _, err := os.Stat(filepath.Join(dst, indexfile+".old")); err == nil { - link, _ := os.Readlink(filepath.Join(dst, indexfile+".old")) - os.Remove(filepath.Join(dst, link)) - os.Remove(filepath.Join(dst, indexfile+".old")) + // if exists, backup symlink + if file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile)) { + // if exists, remove old symlink + if file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile+".old")) { + link, _ := file.parent.publishedStorage.ReadLink(filepath.Join(dst, indexfile+".old")) + file.parent.publishedStorage.Remove(filepath.Join(dst, link)) + file.parent.publishedStorage.Remove(filepath.Join(dst, indexfile+".old")) } - os.Rename(filepath.Join(dst, indexfile), filepath.Join(dst, indexfile+".old")) + file.parent.publishedStorage.RenameFile(filepath.Join(dst, indexfile), + filepath.Join(dst, indexfile+".old")) } - err = os.Symlink(sum, filepath.Join(dst, indexfile)) + + // create symlink + err = file.parent.publishedStorage.SymLink(sum, filepath.Join(dst, indexfile)) if err != nil { return fmt.Errorf("Access-By-Hash: error creating symlink %s", filepath.Join(dst, indexfile)) } diff --git a/files/public.go b/files/public.go index 4147da1a4..426d0668d 100644 --- a/files/public.go +++ b/files/public.go @@ -247,3 +247,33 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) { func (storage *PublishedStorage) RenameFile(oldName, newName string) error { return os.Rename(filepath.Join(storage.rootPath, oldName), filepath.Join(storage.rootPath, newName)) } + +// SymLink creates a symbolic link, which can be read with ReadLink +func (storage *PublishedStorage) SymLink(src string, dst string) error { + return os.Symlink(src, filepath.Join(storage.rootPath, dst)) +} + +// HardLink creates a hardlink of a file +func (storage *PublishedStorage) HardLink(src string, dst string) error { + return os.Link(filepath.Join(storage.rootPath, src), filepath.Join(storage.rootPath, dst)) +} + +// FileExists returns true if path exists +func (storage *PublishedStorage) FileExists(path string) bool { + list, err := storage.Filelist(filepath.Dir(path)) + if err != nil { + return false + } + f := filepath.Base(path) + for _, i := range list { + if i == f { + return true + } + } + return false +} + +// ReadLink returns the symbolic link pointed to by path +func (storage *PublishedStorage) ReadLink(path string) (string, error) { + return os.Readlink(path) +} diff --git a/s3/public.go b/s3/public.go index c26366a4a..8621284ff 100644 --- a/s3/public.go +++ b/s3/public.go @@ -384,3 +384,27 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error { return storage.Remove(oldName) } + +// SymLink creates a symbolic link, which can be read with ReadLink +func (storage *PublishedStorage) SymLink(src string, dst string) error { + // TODO: create a file containing dst + return fmt.Errorf("s3: symlinks not implemented") +} + +// HardLink creates a hardlink of a file +func (storage *PublishedStorage) HardLink(src string, dst string) error { + // TODO: create a copy of the file + return fmt.Errorf("s3: hardlinks not implemented") +} + +// FileExists returns true if path exists +func (storage *PublishedStorage) FileExists(path string) bool { + // TODO: implement + return false +} + +// ReadLink returns the symbolic link pointed to by path +func (storage *PublishedStorage) ReadLink(path string) (string, error) { + // TODO: read the path and return the content of the file + return "", fmt.Errorf("s3: ReadLink not implemented") +} diff --git a/swift/public.go b/swift/public.go index 0f83112c9..c6f303ecf 100644 --- a/swift/public.go +++ b/swift/public.go @@ -264,3 +264,27 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error { return nil } + +// SymLink creates a symbolic link, which can be read with ReadLink +func (storage *PublishedStorage) SymLink(src string, dst string) error { + // TODO: create a file containing dst + return fmt.Errorf("SWIFT: symlinks not implemented") +} + +// HardLink creates a hardlink of a file +func (storage *PublishedStorage) HardLink(src string, dst string) error { + // TODO: create a copy of the file + return fmt.Errorf("SWIFT: hardlinks not implemented") +} + +// FileExists returns true if path exists +func (storage *PublishedStorage) FileExists(path string) bool { + // TODO: implement + return false +} + +// ReadLink returns the symbolic link pointed to by path +func (storage *PublishedStorage) ReadLink(path string) (string, error) { + // TODO: read the path and return the content of the file + return "", fmt.Errorf("SWIFT: ReadLink not implemented") +} From 7498fd8fc87c68f7349b57bc3f149cc95cc1cc6b Mon Sep 17 00:00:00 2001 From: Oliver Sauder Date: Tue, 17 Oct 2017 15:45:17 +0200 Subject: [PATCH 03/15] Extend s3 storage with link and file exists methods --- deb/index_files.go | 15 +++++++----- s3/public.go | 61 +++++++++++++++++++++++++++++++++++++--------- s3/public_test.go | 26 ++++++++++++++++++++ 3 files changed, 85 insertions(+), 17 deletions(-) diff --git a/deb/index_files.go b/deb/index_files.go index 10ef17871..944727f5c 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -186,24 +186,27 @@ func packageIndexByHash(file *indexFile, ext string, hash string, sum string) er src = src + file.parent.suffix + ext filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath)) dst := filepath.Join(filedir, "by-hash", hash) + sumfilePath := filepath.Join(dst, sum) // link already exists? do nothing - if file.parent.publishedStorage.FileExists(filepath.Join(dst, sum)) { + if file.parent.publishedStorage.FileExists(sumfilePath) { return nil } // create the link - err := file.parent.publishedStorage.HardLink(src, filepath.Join(dst, sum)) + err := file.parent.publishedStorage.HardLink(src, sumfilePath) if err != nil { - return fmt.Errorf("Access-By-Hash: error creating hardlink %s: %s", filepath.Join(dst, sum), err) + return fmt.Errorf("Access-By-Hash: error creating hardlink %s: %s", sumfilePath, err) } - // if exists, backup symlink + // if a previous index file already exists exists, backup symlink if file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile)) { // if exists, remove old symlink if file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile+".old")) { - link, _ := file.parent.publishedStorage.ReadLink(filepath.Join(dst, indexfile+".old")) - file.parent.publishedStorage.Remove(filepath.Join(dst, link)) + link, err := file.parent.publishedStorage.ReadLink(filepath.Join(dst, indexfile+".old")) + if err != nil { + file.parent.publishedStorage.Remove(link) + } file.parent.publishedStorage.Remove(filepath.Join(dst, indexfile+".old")) } file.parent.publishedStorage.RenameFile(filepath.Join(dst, indexfile), diff --git a/s3/public.go b/s3/public.go index 8621284ff..8a222c8bc 100644 --- a/s3/public.go +++ b/s3/public.go @@ -385,26 +385,65 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error { return storage.Remove(oldName) } -// SymLink creates a symbolic link, which can be read with ReadLink +// SymLink creates a copy of src file and adds link information as meta data func (storage *PublishedStorage) SymLink(src string, dst string) error { - // TODO: create a file containing dst - return fmt.Errorf("s3: symlinks not implemented") + + params := &s3.CopyObjectInput{ + Bucket: aws.String(storage.bucket), + CopySource: aws.String(src), + Key: aws.String(filepath.Join(storage.prefix, dst)), + ACL: aws.String(storage.acl), + Metadata: map[string]*string{ + "SymLink": aws.String(src), + }, + MetadataDirective: aws.String("REPLACE"), + } + + if storage.storageClass != "" { + params.StorageClass = aws.String(storage.storageClass) + } + if storage.encryptionMethod != "" { + params.ServerSideEncryption = aws.String(storage.encryptionMethod) + } + + _, err := storage.s3.CopyObject(params) + if err != nil { + return fmt.Errorf("error symlinking %s -> %s in %s: %s", src, dst, storage, err) + } + + return err } -// HardLink creates a hardlink of a file +// HardLink using symlink functionality as hard links do not exist func (storage *PublishedStorage) HardLink(src string, dst string) error { - // TODO: create a copy of the file - return fmt.Errorf("s3: hardlinks not implemented") + return storage.SymLink(src, dst) } // FileExists returns true if path exists func (storage *PublishedStorage) FileExists(path string) bool { - // TODO: implement - return false + params := &s3.HeadObjectInput{ + Bucket: aws.String(storage.bucket), + Key: aws.String(filepath.Join(storage.prefix, path)), + } + _, err := storage.s3.HeadObject(params) + if err != nil { + return false + } + + return true } -// ReadLink returns the symbolic link pointed to by path +// ReadLink returns the symbolic link pointed to by path. +// This simply reads text file created with SymLink func (storage *PublishedStorage) ReadLink(path string) (string, error) { - // TODO: read the path and return the content of the file - return "", fmt.Errorf("s3: ReadLink not implemented") + params := &s3.HeadObjectInput{ + Bucket: aws.String(storage.bucket), + Key: aws.String(filepath.Join(storage.prefix, path)), + } + output, err := storage.s3.HeadObject(params) + if err != nil { + return "", err + } + + return aws.StringValue(output.Metadata["SymLink"]), nil } diff --git a/s3/public_test.go b/s3/public_test.go index 7339ac4ad..7d9f790ec 100644 --- a/s3/public_test.go +++ b/s3/public_test.go @@ -280,3 +280,29 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) } + +func (s *PublishedStorageSuite) TestSymLink(c *C) { + s.PutFile(c, "a/b", []byte("test")) + + var err error + err = s.storage.SymLink("a/b", "a/b.link") + c.Check(err, IsNil) + + var link string + link, err = s.storage.ReadLink("a/b.link") + c.Check(err, IsNil) + c.Check(link, Equals, "a/b") + + c.Skip("copy not available in s3test") +} + + +func (s *PublishedStorageSuite) TestFileExists(c *C) { + s.PutFile(c, "a/b", []byte("test")) + + exists := s.storage.FileExists("a/b") + c.Check(exists, Equals, true) + + exists = s.storage.FileExists("a/b.invalid") + c.Check(exists, Equals, false) +} From 438e206b3d64d1173fd8aac2dc1e9aa911b376a4 Mon Sep 17 00:00:00 2001 From: Oliver Sauder Date: Thu, 2 Nov 2017 10:24:05 +0100 Subject: [PATCH 04/15] Extend swift storage with link and file exists methods --- swift/public.go | 33 +++++++++++++++++++++++---------- swift/public_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/swift/public.go b/swift/public.go index c6f303ecf..88b9eac0a 100644 --- a/swift/public.go +++ b/swift/public.go @@ -265,26 +265,39 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error { return nil } -// SymLink creates a symbolic link, which can be read with ReadLink +// SymLink creates a copy of src file and adds link information as meta data func (storage *PublishedStorage) SymLink(src string, dst string) error { - // TODO: create a file containing dst - return fmt.Errorf("SWIFT: symlinks not implemented") + srcObjectName := filepath.Join(storage.prefix, src) + dstObjectName := filepath.Join(storage.prefix, dst) + + headers := map[string]string{ + "SymLink": srcObjectName, + } + _, err := storage.conn.ObjectCopy(storage.container, srcObjectName, storage.container, dstObjectName, headers) + if err != nil { + return fmt.Errorf("error symlinking %s -> %s in %s: %s", srcObjectName, dstObjectName, storage, err) + } + return err } -// HardLink creates a hardlink of a file +// HardLink using symlink functionality as hard links do not exist func (storage *PublishedStorage) HardLink(src string, dst string) error { - // TODO: create a copy of the file - return fmt.Errorf("SWIFT: hardlinks not implemented") + return storage.SymLink(src, dst) } // FileExists returns true if path exists func (storage *PublishedStorage) FileExists(path string) bool { - // TODO: implement - return false + _, _, err := storage.conn.Object(storage.container, filepath.Join(storage.prefix, path)) + return err == nil } // ReadLink returns the symbolic link pointed to by path func (storage *PublishedStorage) ReadLink(path string) (string, error) { - // TODO: read the path and return the content of the file - return "", fmt.Errorf("SWIFT: ReadLink not implemented") + srcObjectName := filepath.Join(storage.prefix, path) + _, headers, err := storage.conn.Object(storage.container, srcObjectName) + if err != nil { + return "", fmt.Errorf("error reading symlink %s in %s: %s", srcObjectName, storage, err) + } + + return headers["SymLink"], nil } diff --git a/swift/public_test.go b/swift/public_test.go index 1a33ec2b2..ef80d5000 100644 --- a/swift/public_test.go +++ b/swift/public_test.go @@ -194,3 +194,37 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Check(err, IsNil) c.Check(data, DeepEquals, []byte("Spam")) } + +func (s *PublishedStorageSuite) TestSymLink(c *C) { + dir := c.MkDir() + err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to swift!"), 0644) + c.Assert(err, IsNil) + + err = s.storage.PutFile("a/b.txt", filepath.Join(dir, "a")) + c.Check(err, IsNil) + + err = s.storage.SymLink("a/b.txt", "a/b.txt.link") + c.Check(err, IsNil) + + var link string + link, err = s.storage.ReadLink("a/b.txt.link") + c.Check(err, IsNil) + c.Check(link, Equals, "a/b") + + c.Skip("copy not availbale in s3test") +} + +func (s *PublishedStorageSuite) TestFileExists(c *C) { + dir := c.MkDir() + err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to swift!"), 0644) + c.Assert(err, IsNil) + + err = s.storage.PutFile("a/b.txt", filepath.Join(dir, "a")) + c.Check(err, IsNil) + + exists := s.storage.FileExists("a/b.txt") + c.Check(exists, Equals, true) + + exists = s.storage.FileExists("a/b.invalid") + c.Check(exists, Equals, false) +} From 092a7ed8f34dddfa8502718971a084c0d7061e3d Mon Sep 17 00:00:00 2001 From: Oliver Sauder Date: Thu, 2 Nov 2017 10:37:20 +0100 Subject: [PATCH 05/15] Rename AccessByHash to AcquireByHash for consistency with other flags --- api/publish.go | 6 ++-- cmd/publish_repo.go | 2 +- cmd/publish_snapshot.go | 5 +-- deb/index_files.go | 76 ++++++++++++++++++++--------------------- deb/publish.go | 8 ++--- 5 files changed, 49 insertions(+), 48 deletions(-) diff --git a/api/publish.go b/api/publish.go index c4a59ab9a..70ce06eb4 100644 --- a/api/publish.go +++ b/api/publish.go @@ -238,7 +238,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { Component string `binding:"required"` Name string `binding:"required"` } - AccessByHash *bool + AcquireByHash *bool } if c.Bind(&b) != nil { @@ -318,8 +318,8 @@ func apiPublishUpdateSwitch(c *gin.Context) { published.SkipContents = *b.SkipContents } - if b.AccessByHash != nil { - published.AccessByHash = *b.AccessByHash + if b.AcquireByHash != nil { + published.AcquireByHash = *b.AcquireByHash } err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite) diff --git a/cmd/publish_repo.go b/cmd/publish_repo.go index 009ba9273..33022fa92 100644 --- a/cmd/publish_repo.go +++ b/cmd/publish_repo.go @@ -47,7 +47,7 @@ Example: cmd.Flag.String("butautomaticupgrades", "", "set value for ButAutomaticUpgrades field") cmd.Flag.String("label", "", "label to publish") cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") - cmd.Flag.Bool("access-by-hash", false, "provide index files by hash also") + cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash") return cmd } diff --git a/cmd/publish_snapshot.go b/cmd/publish_snapshot.go index f76055da1..a7c696d9e 100644 --- a/cmd/publish_snapshot.go +++ b/cmd/publish_snapshot.go @@ -137,8 +137,8 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error { published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool) } - if context.Flags().IsSet("access-by-hash") { - published.AccessByHash = context.Flags().Lookup("access-by-hash").Value.Get().(bool) + if context.Flags().IsSet("acquire-by-hash") { + published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool) } duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published) @@ -231,6 +231,7 @@ Example: cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field") cmd.Flag.String("label", "", "label to publish") cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") + cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash") return cmd } diff --git a/deb/index_files.go b/deb/index_files.go index 944727f5c..adc69715c 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -21,20 +21,20 @@ type indexFiles struct { tempDir string suffix string indexes map[string]*indexFile - accessByHash bool + acquireByHash bool } type indexFile struct { - parent *indexFiles - discardable bool - compressable bool - onlyGzip bool - signable bool - accessByHash bool - relativePath string - tempFilename string - tempFile *os.File - w *bufio.Writer + parent *indexFiles + discardable bool + compressable bool + onlyGzip bool + signable bool + acquireByHash bool + relativePath string + tempFilename string + tempFile *os.File + w *bufio.Writer } func (file *indexFile) BufWriter() (*bufio.Writer, error) { @@ -102,8 +102,8 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { } hashs := []string{} - if file.accessByHash { - hashs = append(hashs, "MD5", "SHA1", "SHA256", "SHA512") + if file.acquireByHash { + hashs = append(hashs, "MD5Sum", "SHA1", "SHA256", "SHA512") for _, hash := range hashs { err = file.parent.publishedStorage.MkDir(filepath.Join(filedir, "by-hash", hash)) if err != nil { @@ -124,7 +124,7 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { filepath.Join(file.parent.basePath, file.relativePath+ext) } - if file.accessByHash { + if file.acquireByHash { sums := file.parent.generatedFiles[file.relativePath+ext] err = packageIndexByHash(file, ext, "SHA512", sums.SHA512) @@ -139,7 +139,7 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { if err != nil { fmt.Printf("%s\n", err) } - err = packageIndexByHash(file, ext, "MD5", sums.MD5) + err = packageIndexByHash(file, ext, "MD5Sum", sums.MD5) if err != nil { fmt.Printf("%s\n", err) } @@ -196,7 +196,7 @@ func packageIndexByHash(file *indexFile, ext string, hash string, sum string) er // create the link err := file.parent.publishedStorage.HardLink(src, sumfilePath) if err != nil { - return fmt.Errorf("Access-By-Hash: error creating hardlink %s: %s", sumfilePath, err) + return fmt.Errorf("Acquire-By-Hash: error creating hardlink %s: %s", sumfilePath, err) } // if a previous index file already exists exists, backup symlink @@ -216,12 +216,12 @@ func packageIndexByHash(file *indexFile, ext string, hash string, sum string) er // create symlink err = file.parent.publishedStorage.SymLink(sum, filepath.Join(dst, indexfile)) if err != nil { - return fmt.Errorf("Access-By-Hash: error creating symlink %s", filepath.Join(dst, indexfile)) + return fmt.Errorf("Acquire-By-Hash: error creating symlink %s", filepath.Join(dst, indexfile)) } return nil } -func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, accessByHash bool) *indexFiles { +func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, acquireByHash bool) *indexFiles { return &indexFiles{ publishedStorage: publishedStorage, basePath: basePath, @@ -230,7 +230,7 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s tempDir: tempDir, suffix: suffix, indexes: make(map[string]*indexFile), - accessByHash: accessByHash, + acquireByHash: acquireByHash, } } @@ -254,12 +254,12 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF } file = &indexFile{ - parent: files, - discardable: false, - compressable: true, - signable: false, - accessByHash: files.accessByHash, - relativePath: relativePath, + parent: files, + discardable: false, + compressable: true, + signable: false, + acquireByHash: files.acquireByHash, + relativePath: relativePath, } files.indexes[key] = file @@ -288,12 +288,12 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF } file = &indexFile{ - parent: files, - discardable: udeb, - compressable: false, - signable: false, - accessByHash: files.accessByHash, - relativePath: relativePath, + parent: files, + discardable: udeb, + compressable: false, + signable: false, + acquireByHash: files.acquireByHash, + relativePath: relativePath, } files.indexes[key] = file @@ -318,13 +318,13 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index } file = &indexFile{ - parent: files, - discardable: true, - compressable: true, - onlyGzip: true, - signable: false, - accessByHash: files.accessByHash, - relativePath: relativePath, + parent: files, + discardable: true, + compressable: true, + onlyGzip: true, + signable: false, + acquireByHash: files.acquireByHash, + relativePath: relativePath, } files.indexes[key] = file diff --git a/deb/publish.go b/deb/publish.go index 3d62400d8..ad7b63f96 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -66,7 +66,7 @@ type PublishedRepo struct { rePublishing bool // Provide index files per hash also - AccessByHash bool + AcquireByHash bool } // ParsePrefix splits [storage:]prefix into components @@ -559,7 +559,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } defer os.RemoveAll(tempDir) - indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AccessByHash) + indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AcquireByHash) for component, list := range lists { hadUdebs := false @@ -686,7 +686,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP release["Component"] = component release["Origin"] = p.GetOrigin() release["Label"] = p.GetLabel() - if p.AccessByHash { + if p.AcquireByHash { release["Acquire-By-Hash"] = "yes" } @@ -726,7 +726,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP release["Codename"] = p.Distribution release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST") release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ") - if p.AccessByHash { + if p.AcquireByHash { release["Acquire-By-Hash"] = "yes" } release["Description"] = " Generated by aptly\n" From 2bd0b786ea7303d057f32e13ac9ae0c5f866ac89 Mon Sep 17 00:00:00 2001 From: Oliver Sauder Date: Thu, 2 Nov 2017 11:37:25 +0100 Subject: [PATCH 06/15] Extend publish snapshot test with acquire by hash --- system/t06_publish/snapshot.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/system/t06_publish/snapshot.py b/system/t06_publish/snapshot.py index 3a2af0f6e..efc9a7803 100644 --- a/system/t06_publish/snapshot.py +++ b/system/t06_publish/snapshot.py @@ -219,7 +219,7 @@ class PublishSnapshot5Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap5 from mirror gnuplot-maverick", ] - runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap5 ppa/smira" + runCmd = "aptly publish snapshot -acquire-by-hash -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap5 ppa/smira" gold_processor = BaseTest.expand_environ @@ -231,13 +231,21 @@ def check(self): self.check_exists('public/ppa/smira/dists/squeeze/Release.gpg') self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/Packages') + self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/by-hash/MD5Sum/e98cd30fc76fbe7fa3ea25717efa1c92') self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/Packages.gz') + self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/by-hash/MD5Sum/f08c2a876384aea2f74422d9f58f927e') self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/Packages.bz2') - self.check_exists('public/ppa/smira/dists/squeeze/main/Contents-i386.gz') + self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/by-hash/MD5Sum/a2c1024fbc65b6381b730a1c4144f80d') self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/Packages') + self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/by-hash/MD5Sum/ab073d1f73bed52e7356c91161e8667e') self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/Packages.gz') + self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/by-hash/MD5Sum/6fc332f313c8799fe09578da24340366') self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/Packages.bz2') + self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/by-hash/MD5Sum/ac6be6306c80c01eb7a79a65ec36760d') + + self.check_exists('public/ppa/smira/dists/squeeze/main/Contents-i386.gz') self.check_exists('public/ppa/smira/dists/squeeze/main/Contents-amd64.gz') + self.check_exists('public/ppa/smira/dists/squeeze/main/by-hash/MD5Sum/eea7fb266562e355ba069116c1fddfcc') self.check_exists('public/ppa/smira/pool/main/g/gnuplot/gnuplot-doc_4.6.1-1~maverick2_all.deb') From d6b4b795a5d60621c113459e3c115d0039ea6aea Mon Sep 17 00:00:00 2001 From: Oliver Sauder Date: Thu, 2 Nov 2017 12:57:52 +0100 Subject: [PATCH 07/15] Fix linting errors --- deb/index_files.go | 3 ++- s3/public.go | 8 ++------ s3/public_test.go | 4 +--- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/deb/index_files.go b/deb/index_files.go index adc69715c..9c79d6c29 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -203,7 +203,8 @@ func packageIndexByHash(file *indexFile, ext string, hash string, sum string) er if file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile)) { // if exists, remove old symlink if file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile+".old")) { - link, err := file.parent.publishedStorage.ReadLink(filepath.Join(dst, indexfile+".old")) + var link string + link, err = file.parent.publishedStorage.ReadLink(filepath.Join(dst, indexfile+".old")) if err != nil { file.parent.publishedStorage.Remove(link) } diff --git a/s3/public.go b/s3/public.go index 8a222c8bc..8cdd4bd64 100644 --- a/s3/public.go +++ b/s3/public.go @@ -393,7 +393,7 @@ func (storage *PublishedStorage) SymLink(src string, dst string) error { CopySource: aws.String(src), Key: aws.String(filepath.Join(storage.prefix, dst)), ACL: aws.String(storage.acl), - Metadata: map[string]*string{ + Metadata: map[string]*string{ "SymLink": aws.String(src), }, MetadataDirective: aws.String("REPLACE"), @@ -426,11 +426,7 @@ func (storage *PublishedStorage) FileExists(path string) bool { Key: aws.String(filepath.Join(storage.prefix, path)), } _, err := storage.s3.HeadObject(params) - if err != nil { - return false - } - - return true + return err == nil } // ReadLink returns the symbolic link pointed to by path. diff --git a/s3/public_test.go b/s3/public_test.go index 7d9f790ec..60cf22406 100644 --- a/s3/public_test.go +++ b/s3/public_test.go @@ -284,8 +284,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { func (s *PublishedStorageSuite) TestSymLink(c *C) { s.PutFile(c, "a/b", []byte("test")) - var err error - err = s.storage.SymLink("a/b", "a/b.link") + err := s.storage.SymLink("a/b", "a/b.link") c.Check(err, IsNil) var link string @@ -296,7 +295,6 @@ func (s *PublishedStorageSuite) TestSymLink(c *C) { c.Skip("copy not available in s3test") } - func (s *PublishedStorageSuite) TestFileExists(c *C) { s.PutFile(c, "a/b", []byte("test")) From 47186253888b62a9b4cfa1ab0965f9a39f887c33 Mon Sep 17 00:00:00 2001 From: Oliver Sauder Date: Thu, 2 Nov 2017 13:53:35 +0100 Subject: [PATCH 08/15] Avoid exception when failing tests doesn't have a doc string --- system/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/system/run.py b/system/run.py index 2d4b02c8b..8c6701775 100755 --- a/system/run.py +++ b/system/run.py @@ -99,7 +99,8 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non print "\nFAILURES (%d):" % (len(fails), ) for (test, t, typ, val, tb, testModule) in fails: - print "%s:%s %s" % (test, t.__class__.__name__, testModule.__name__ + ": " + t.__doc__.strip()) + doc = t.__doc__ or '' + print "%s:%s %s" % (test, t.__class__.__name__, testModule.__name__ + ": " + doc.strip()) traceback.print_exception(typ, val, tb) print "=" * 60 From b4f3573d11bd785d364c613b3e448729865f29c3 Mon Sep 17 00:00:00 2001 From: Oliver Sauder Date: Thu, 2 Nov 2017 13:53:57 +0100 Subject: [PATCH 09/15] Add acquire by hash when updating publish --- system/t06_publish/snapshot.py | 5 ----- system/t06_publish/update.py | 8 +++++++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/system/t06_publish/snapshot.py b/system/t06_publish/snapshot.py index efc9a7803..31a5f012b 100644 --- a/system/t06_publish/snapshot.py +++ b/system/t06_publish/snapshot.py @@ -233,19 +233,14 @@ def check(self): self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/Packages') self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/by-hash/MD5Sum/e98cd30fc76fbe7fa3ea25717efa1c92') self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/Packages.gz') - self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/by-hash/MD5Sum/f08c2a876384aea2f74422d9f58f927e') self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/Packages.bz2') - self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/by-hash/MD5Sum/a2c1024fbc65b6381b730a1c4144f80d') self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/Packages') self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/by-hash/MD5Sum/ab073d1f73bed52e7356c91161e8667e') self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/Packages.gz') - self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/by-hash/MD5Sum/6fc332f313c8799fe09578da24340366') self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/Packages.bz2') - self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/by-hash/MD5Sum/ac6be6306c80c01eb7a79a65ec36760d') self.check_exists('public/ppa/smira/dists/squeeze/main/Contents-i386.gz') self.check_exists('public/ppa/smira/dists/squeeze/main/Contents-amd64.gz') - self.check_exists('public/ppa/smira/dists/squeeze/main/by-hash/MD5Sum/eea7fb266562e355ba069116c1fddfcc') self.check_exists('public/ppa/smira/pool/main/g/gnuplot/gnuplot-doc_4.6.1-1~maverick2_all.deb') diff --git a/system/t06_publish/update.py b/system/t06_publish/update.py index 480b40576..b6cd160dd 100644 --- a/system/t06_publish/update.py +++ b/system/t06_publish/update.py @@ -95,7 +95,7 @@ class PublishUpdate2Test(BaseTest): fixtureCmds = [ "aptly repo create local-repo", "aptly repo add local-repo ${files}/libboost-program-options-dev_1.49.0.1_i386.deb ${files}/pyspi_0.6.1-1.3.dsc", - "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo", + "aptly publish repo -acquire-by-hash -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo", "aptly repo add local-repo ${files}/pyspi-0.6.1-1.3.stripped.dsc" ] runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick" @@ -113,8 +113,14 @@ def check(self): self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2') self.check_exists('public/dists/maverick/main/Contents-i386.gz') self.check_exists('public/dists/maverick/main/source/Sources') + self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources') + self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources.old') self.check_exists('public/dists/maverick/main/source/Sources.gz') + self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources.gz') + self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources.gz.old') self.check_exists('public/dists/maverick/main/source/Sources.bz2') + self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources.bz2') + self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources.bz2.old') self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc') self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz') From f5e1e194b33ddf7e934a6384a6130935024182e8 Mon Sep 17 00:00:00 2001 From: Oliver Sauder Date: Thu, 2 Nov 2017 14:43:30 +0100 Subject: [PATCH 10/15] Update man page and bash completion --- bash_completion.d/aptly | 2 +- man/aptly.1 | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bash_completion.d/aptly b/bash_completion.d/aptly index 7baa7fc9c..67bea832f 100644 --- a/bash_completion.d/aptly +++ b/bash_completion.d/aptly @@ -499,7 +499,7 @@ _aptly() "snapshot"|"repo") if [[ $numargs -eq 0 ]]; then if [[ "$cur" == -* ]]; then - COMPREPLY=($(compgen -W "-batch -force-overwrite -distribution= -component= -gpg-key= -keyring= -label= -origin= -notautomatic= -butautomaticupgrades= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur})) + COMPREPLY=($(compgen -W "-batch -force-overwrite -distribution= -component= -gpg-key= -keyring= -label= -origin= -notautomatic= -butautomaticupgrades= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing -acquire-by-hash" -- ${cur})) else if [[ "$subcmd" == "snapshot" ]]; then COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur})) diff --git a/man/aptly.1 b/man/aptly.1 index f855b80bd..ec5d991e6 100644 --- a/man/aptly.1 +++ b/man/aptly.1 @@ -1373,6 +1373,10 @@ $ aptly publish repo testing Options: . .TP +\-\fBacquire\-by\-hash\fR +provide index files by hash +. +.TP \-\fBbatch\fR run GPG with detached tty . @@ -1468,6 +1472,10 @@ $ aptly publish snapshot wheezy\-main Options: . .TP +\-\fBacquire\-by\-hash\fR +provide index files by hash +. +.TP \-\fBbatch\fR run GPG with detached tty . From 674a0e84bec2e4a679a468608a9fdbdc40893f0b Mon Sep 17 00:00:00 2001 From: Oliver Sauder Date: Tue, 21 Nov 2017 09:02:15 +0100 Subject: [PATCH 11/15] Order publish parameters in bash completion This makes it easier maintainable. --- bash_completion.d/aptly | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bash_completion.d/aptly b/bash_completion.d/aptly index 67bea832f..f28976333 100644 --- a/bash_completion.d/aptly +++ b/bash_completion.d/aptly @@ -499,7 +499,7 @@ _aptly() "snapshot"|"repo") if [[ $numargs -eq 0 ]]; then if [[ "$cur" == -* ]]; then - COMPREPLY=($(compgen -W "-batch -force-overwrite -distribution= -component= -gpg-key= -keyring= -label= -origin= -notautomatic= -butautomaticupgrades= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing -acquire-by-hash" -- ${cur})) + COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur})) else if [[ "$subcmd" == "snapshot" ]]; then COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur})) From 2e488608ca856cd5bcd03927a7fc0dd6940727f8 Mon Sep 17 00:00:00 2001 From: Oliver Sauder Date: Tue, 21 Nov 2017 09:14:48 +0100 Subject: [PATCH 12/15] Simplify packaging indexing by hash and stop when there is an error --- deb/index_files.go | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/deb/index_files.go b/deb/index_files.go index 9c79d6c29..399a6419b 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -101,10 +101,8 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { return fmt.Errorf("unable to create dir: %s", err) } - hashs := []string{} if file.acquireByHash { - hashs = append(hashs, "MD5Sum", "SHA1", "SHA256", "SHA512") - for _, hash := range hashs { + for _, hash := range []string{"MD5Sum", "SHA1", "SHA256", "SHA512"} { err = file.parent.publishedStorage.MkDir(filepath.Join(filedir, "by-hash", hash)) if err != nil { return fmt.Errorf("unable to create dir: %s", err) @@ -126,22 +124,11 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { if file.acquireByHash { sums := file.parent.generatedFiles[file.relativePath+ext] - - err = packageIndexByHash(file, ext, "SHA512", sums.SHA512) - if err != nil { - fmt.Printf("%s\n", err) - } - err = packageIndexByHash(file, ext, "SHA256", sums.SHA256) - if err != nil { - fmt.Printf("%s\n", err) - } - err = packageIndexByHash(file, ext, "SHA1", sums.SHA1) - if err != nil { - fmt.Printf("%s\n", err) - } - err = packageIndexByHash(file, ext, "MD5Sum", sums.MD5) - if err != nil { - fmt.Printf("%s\n", err) + for hash, sum := range map[string]string{"SHA512": sums.SHA512, "SHA256": sums.SHA256, "SHA1": sums.SHA1, "MD5Sum": sums.MD5} { + err = packageIndexByHash(file, ext, hash, sum) + if err != nil { + return fmt.Errorf("unable to build hash file: %s", err) + } } } } From 3efa1052facb67811c3283d2e5b36019a2aa1cef Mon Sep 17 00:00:00 2001 From: Oliver Sauder Date: Tue, 21 Nov 2017 09:58:25 +0100 Subject: [PATCH 13/15] Implement FileExists in files storage as simple stat to improve performance --- deb/index_files.go | 2 +- files/public.go | 14 ++++---------- files/public_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/deb/index_files.go b/deb/index_files.go index 399a6419b..fcef023fb 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -202,7 +202,7 @@ func packageIndexByHash(file *indexFile, ext string, hash string, sum string) er } // create symlink - err = file.parent.publishedStorage.SymLink(sum, filepath.Join(dst, indexfile)) + err = file.parent.publishedStorage.SymLink(filepath.Join(dst, sum), filepath.Join(dst, indexfile)) if err != nil { return fmt.Errorf("Acquire-By-Hash: error creating symlink %s", filepath.Join(dst, indexfile)) } diff --git a/files/public.go b/files/public.go index 426d0668d..5bde9caf8 100644 --- a/files/public.go +++ b/files/public.go @@ -250,7 +250,7 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error { // SymLink creates a symbolic link, which can be read with ReadLink func (storage *PublishedStorage) SymLink(src string, dst string) error { - return os.Symlink(src, filepath.Join(storage.rootPath, dst)) + return os.Symlink(filepath.Join(storage.rootPath, src), filepath.Join(storage.rootPath, dst)) } // HardLink creates a hardlink of a file @@ -260,17 +260,11 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error { // FileExists returns true if path exists func (storage *PublishedStorage) FileExists(path string) bool { - list, err := storage.Filelist(filepath.Dir(path)) - if err != nil { + if _, err := os.Lstat(filepath.Join(storage.rootPath, path)); os.IsNotExist(err) { return false } - f := filepath.Base(path) - for _, i := range list { - if i == f { - return true - } - } - return false + + return true } // ReadLink returns the symbolic link pointed to by path diff --git a/files/public_test.go b/files/public_test.go index 5b2731af0..4160e2499 100644 --- a/files/public_test.go +++ b/files/public_test.go @@ -103,6 +103,49 @@ func (s *PublishedStorageSuite) TestRenameFile(c *C) { c.Assert(err, IsNil) } +func (s *PublishedStorageSuite) TestFileExists(c *C) { + err := s.storage.MkDir("ppa/dists/squeeze/") + c.Assert(err, IsNil) + + exists := s.storage.FileExists("ppa/dists/squeeze/Release") + c.Check(exists, Equals, false) + + err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null") + c.Assert(err, IsNil) + + exists = s.storage.FileExists("ppa/dists/squeeze/Release") + c.Check(exists, Equals, true) +} + +func (s *PublishedStorageSuite) TestSymLink(c *C) { + err := s.storage.MkDir("ppa/dists/squeeze/") + c.Assert(err, IsNil) + + err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null") + c.Assert(err, IsNil) + + err = s.storage.SymLink("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease") + c.Assert(err, IsNil) + + exists := s.storage.FileExists("ppa/dists/squeeze/InRelease") + c.Check(exists, Equals, true) +} + +func (s *PublishedStorageSuite) TestHardLink(c *C) { + err := s.storage.MkDir("ppa/dists/squeeze/") + c.Assert(err, IsNil) + + err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null") + c.Assert(err, IsNil) + + err = s.storage.HardLink("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease") + c.Assert(err, IsNil) + + exists := s.storage.FileExists("ppa/dists/squeeze/InRelease") + c.Check(exists, Equals, true) +} + + func (s *PublishedStorageSuite) TestRemoveDirs(c *C) { err := s.storage.MkDir("ppa/dists/squeeze/") c.Assert(err, IsNil) From e504fdcd548fc2e1c840c77cdd550dafe523db1a Mon Sep 17 00:00:00 2001 From: Oliver Sauder Date: Tue, 21 Nov 2017 10:32:15 +0100 Subject: [PATCH 14/15] Build src path on basis of storage prefix when symlinking --- s3/public.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3/public.go b/s3/public.go index 8cdd4bd64..2e22d8eb0 100644 --- a/s3/public.go +++ b/s3/public.go @@ -390,7 +390,7 @@ func (storage *PublishedStorage) SymLink(src string, dst string) error { params := &s3.CopyObjectInput{ Bucket: aws.String(storage.bucket), - CopySource: aws.String(src), + CopySource: aws.String(filepath.Join(storage.prefix, src)), Key: aws.String(filepath.Join(storage.prefix, dst)), ACL: aws.String(storage.acl), Metadata: map[string]*string{ From b2bf4f7884a043228a58567085de64f9e3f0bdb9 Mon Sep 17 00:00:00 2001 From: Oliver Sauder Date: Tue, 21 Nov 2017 11:15:51 +0100 Subject: [PATCH 15/15] Adjust FileExists to differentiate between error and actual file existence --- aptly/interfaces.go | 2 +- deb/index_files.go | 14 +++++++++----- files/public.go | 6 +++--- files/public_test.go | 9 ++++----- s3/public.go | 14 ++++++++++++-- s3/public_test.go | 7 +++++-- swift/public.go | 13 +++++++++++-- swift/public_test.go | 6 ++++-- 8 files changed, 49 insertions(+), 22 deletions(-) diff --git a/aptly/interfaces.go b/aptly/interfaces.go index 3b3c18d9b..9215eb33b 100644 --- a/aptly/interfaces.go +++ b/aptly/interfaces.go @@ -78,7 +78,7 @@ type PublishedStorage interface { // HardLink creates a hardlink of a file HardLink(src string, dst string) error // FileExists returns true if path exists - FileExists(path string) bool + FileExists(path string) (bool, error) // ReadLink returns the symbolic link pointed to by path ReadLink(path string) (string, error) } diff --git a/deb/index_files.go b/deb/index_files.go index fcef023fb..4c74d6ba4 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -176,20 +176,24 @@ func packageIndexByHash(file *indexFile, ext string, hash string, sum string) er sumfilePath := filepath.Join(dst, sum) // link already exists? do nothing - if file.parent.publishedStorage.FileExists(sumfilePath) { + exists, err := file.parent.publishedStorage.FileExists(sumfilePath) + if err != nil { + return fmt.Errorf("Acquire-By-Hash: error checking exists of file %s: %s", sumfilePath, err) + } + if exists { return nil } // create the link - err := file.parent.publishedStorage.HardLink(src, sumfilePath) + err = file.parent.publishedStorage.HardLink(src, sumfilePath) if err != nil { return fmt.Errorf("Acquire-By-Hash: error creating hardlink %s: %s", sumfilePath, err) } // if a previous index file already exists exists, backup symlink - if file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile)) { + if exists, _ = file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile)); exists { // if exists, remove old symlink - if file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile+".old")) { + if exists, _ = file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile+".old")); exists { var link string link, err = file.parent.publishedStorage.ReadLink(filepath.Join(dst, indexfile+".old")) if err != nil { @@ -204,7 +208,7 @@ func packageIndexByHash(file *indexFile, ext string, hash string, sum string) er // create symlink err = file.parent.publishedStorage.SymLink(filepath.Join(dst, sum), filepath.Join(dst, indexfile)) if err != nil { - return fmt.Errorf("Acquire-By-Hash: error creating symlink %s", filepath.Join(dst, indexfile)) + return fmt.Errorf("Acquire-By-Hash: error creating symlink %s: %s", filepath.Join(dst, indexfile), err) } return nil } diff --git a/files/public.go b/files/public.go index 5bde9caf8..c065aca02 100644 --- a/files/public.go +++ b/files/public.go @@ -259,12 +259,12 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error { } // FileExists returns true if path exists -func (storage *PublishedStorage) FileExists(path string) bool { +func (storage *PublishedStorage) FileExists(path string) (bool, error) { if _, err := os.Lstat(filepath.Join(storage.rootPath, path)); os.IsNotExist(err) { - return false + return false, nil } - return true + return true, nil } // ReadLink returns the symbolic link pointed to by path diff --git a/files/public_test.go b/files/public_test.go index 4160e2499..1bfefb5df 100644 --- a/files/public_test.go +++ b/files/public_test.go @@ -107,13 +107,13 @@ func (s *PublishedStorageSuite) TestFileExists(c *C) { err := s.storage.MkDir("ppa/dists/squeeze/") c.Assert(err, IsNil) - exists := s.storage.FileExists("ppa/dists/squeeze/Release") + exists, _ := s.storage.FileExists("ppa/dists/squeeze/Release") c.Check(exists, Equals, false) err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null") c.Assert(err, IsNil) - exists = s.storage.FileExists("ppa/dists/squeeze/Release") + exists, _ = s.storage.FileExists("ppa/dists/squeeze/Release") c.Check(exists, Equals, true) } @@ -127,7 +127,7 @@ func (s *PublishedStorageSuite) TestSymLink(c *C) { err = s.storage.SymLink("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease") c.Assert(err, IsNil) - exists := s.storage.FileExists("ppa/dists/squeeze/InRelease") + exists, _ := s.storage.FileExists("ppa/dists/squeeze/InRelease") c.Check(exists, Equals, true) } @@ -141,11 +141,10 @@ func (s *PublishedStorageSuite) TestHardLink(c *C) { err = s.storage.HardLink("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease") c.Assert(err, IsNil) - exists := s.storage.FileExists("ppa/dists/squeeze/InRelease") + exists, _ := s.storage.FileExists("ppa/dists/squeeze/InRelease") c.Check(exists, Equals, true) } - func (s *PublishedStorageSuite) TestRemoveDirs(c *C) { err := s.storage.MkDir("ppa/dists/squeeze/") c.Assert(err, IsNil) diff --git a/s3/public.go b/s3/public.go index 2e22d8eb0..16564e9f6 100644 --- a/s3/public.go +++ b/s3/public.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/corehandlers" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/request" @@ -420,13 +421,22 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error { } // FileExists returns true if path exists -func (storage *PublishedStorage) FileExists(path string) bool { +func (storage *PublishedStorage) FileExists(path string) (bool, error) { params := &s3.HeadObjectInput{ Bucket: aws.String(storage.bucket), Key: aws.String(filepath.Join(storage.prefix, path)), } _, err := storage.s3.HeadObject(params) - return err == nil + if err != nil { + aerr, ok := err.(awserr.Error) + if ok && aerr.Code() == s3.ErrCodeNoSuchKey { + return false, nil + } + + return false, err + } + + return true, nil } // ReadLink returns the symbolic link pointed to by path. diff --git a/s3/public_test.go b/s3/public_test.go index 60cf22406..f772a15d6 100644 --- a/s3/public_test.go +++ b/s3/public_test.go @@ -298,9 +298,12 @@ func (s *PublishedStorageSuite) TestSymLink(c *C) { func (s *PublishedStorageSuite) TestFileExists(c *C) { s.PutFile(c, "a/b", []byte("test")) - exists := s.storage.FileExists("a/b") + exists, err := s.storage.FileExists("a/b") + c.Check(err, IsNil) c.Check(exists, Equals, true) - exists = s.storage.FileExists("a/b.invalid") + exists, _ = s.storage.FileExists("a/b.invalid") + // Comment out as there is an error in s3test implementation + // c.Check(err, IsNil) c.Check(exists, Equals, false) } diff --git a/swift/public.go b/swift/public.go index 88b9eac0a..599857729 100644 --- a/swift/public.go +++ b/swift/public.go @@ -286,9 +286,18 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error { } // FileExists returns true if path exists -func (storage *PublishedStorage) FileExists(path string) bool { +func (storage *PublishedStorage) FileExists(path string) (bool, error) { _, _, err := storage.conn.Object(storage.container, filepath.Join(storage.prefix, path)) - return err == nil + + if err != nil { + if err == swift.ObjectNotFound { + return false, nil + } + + return false, err + } + + return true, nil } // ReadLink returns the symbolic link pointed to by path diff --git a/swift/public_test.go b/swift/public_test.go index ef80d5000..0f8e4b44e 100644 --- a/swift/public_test.go +++ b/swift/public_test.go @@ -222,9 +222,11 @@ func (s *PublishedStorageSuite) TestFileExists(c *C) { err = s.storage.PutFile("a/b.txt", filepath.Join(dir, "a")) c.Check(err, IsNil) - exists := s.storage.FileExists("a/b.txt") + exists, err := s.storage.FileExists("a/b.txt") + c.Check(err, IsNil) c.Check(exists, Equals, true) - exists = s.storage.FileExists("a/b.invalid") + exists, err = s.storage.FileExists("a/b.invalid") + c.Check(err, IsNil) c.Check(exists, Equals, false) }