Merge pull request #664 from sliverc/acquire-by-hash

Support Acquire-By-Hash for index files

api/publish.go

@@ -238,6 +238,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 			Component string `binding:"required"`
 			Name      string `binding:"required"`
 		}
+		AcquireByHash *bool
 	}
 
 	if c.Bind(&b) != nil {
@@ -317,6 +318,10 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		published.SkipContents = *b.SkipContents
 	}
 
+	if b.AcquireByHash != nil {
+		published.AcquireByHash = *b.AcquireByHash
+	}
+
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
 	if err != nil {
 		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))

aptly/interfaces.go

@@ -73,6 +73,14 @@ type PublishedStorage interface {
 	Filelist(prefix string) ([]string, error)
 	// RenameFile renames (moves) file
 	RenameFile(oldName, newName string) error
+	// SymLink creates a symbolic link, which can be read with ReadLink
+	SymLink(src string, dst string) error
+	// HardLink creates a hardlink of a file
+	HardLink(src string, dst string) error
+	// FileExists returns true if path exists
+	FileExists(path string) (bool, error)
+	// ReadLink returns the symbolic link pointed to by path
+	ReadLink(path string) (string, error)
 }
 
 // FileSystemPublishedStorage is published storage on filesystem

bash_completion.d/aptly

@@ -499,7 +499,7 @@ _aptly()
           "snapshot"|"repo")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-batch -force-overwrite -distribution= -component= -gpg-key= -keyring= -label= -origin= -notautomatic= -butautomaticupgrades= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
               else
                 if [[ "$subcmd" == "snapshot" ]]; then
                   COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))

cmd/publish_repo.go

@@ -47,6 +47,7 @@ Example:
 	cmd.Flag.String("butautomaticupgrades", "", "set  value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 
 	return cmd
 }

cmd/publish_snapshot.go

@@ -137,6 +137,10 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
 	}
 
+	if context.Flags().IsSet("acquire-by-hash") {
+		published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
+	}
+
 	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
 		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
@@ -227,6 +231,7 @@ Example:
 	cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 
 	return cmd
 }

deb/index_files.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -20,18 +21,20 @@ type indexFiles struct {
 	tempDir          string
 	suffix           string
 	indexes          map[string]*indexFile
+	acquireByHash    bool
 }
 
 type indexFile struct {
-	parent       *indexFiles
-	discardable  bool
-	compressable bool
-	onlyGzip     bool
-	signable     bool
-	relativePath string
-	tempFilename string
-	tempFile     *os.File
-	w            *bufio.Writer
+	parent        *indexFiles
+	discardable   bool
+	compressable  bool
+	onlyGzip      bool
+	signable      bool
+	acquireByHash bool
+	relativePath  string
+	tempFilename  string
+	tempFile      *os.File
+	w             *bufio.Writer
 }
 
 func (file *indexFile) BufWriter() (*bufio.Writer, error) {
@@ -91,11 +94,22 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 		file.parent.generatedFiles[file.relativePath+ext] = checksumInfo
 	}
 
-	err = file.parent.publishedStorage.MkDir(filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath)))
+	filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))
+
+	err = file.parent.publishedStorage.MkDir(filedir)
 	if err != nil {
 		return fmt.Errorf("unable to create dir: %s", err)
 	}
 
+	if file.acquireByHash {
+		for _, hash := range []string{"MD5Sum", "SHA1", "SHA256", "SHA512"} {
+			err = file.parent.publishedStorage.MkDir(filepath.Join(filedir, "by-hash", hash))
+			if err != nil {
+				return fmt.Errorf("unable to create dir: %s", err)
+			}
+		}
+	}
+
 	for _, ext := range exts {
 		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext),
 			file.tempFilename+ext)
@@ -107,6 +121,16 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext)] =
 				filepath.Join(file.parent.basePath, file.relativePath+ext)
 		}
+
+		if file.acquireByHash {
+			sums := file.parent.generatedFiles[file.relativePath+ext]
+			for hash, sum := range map[string]string{"SHA512": sums.SHA512, "SHA256": sums.SHA256, "SHA1": sums.SHA1, "MD5Sum": sums.MD5} {
+				err = packageIndexByHash(file, ext, hash, sum)
+				if err != nil {
+					return fmt.Errorf("unable to build hash file: %s", err)
+				}
+			}
+		}
 	}
 
 	if file.signable && signer != nil {
@@ -143,7 +167,53 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 	return nil
 }
 
-func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string) *indexFiles {
+func packageIndexByHash(file *indexFile, ext string, hash string, sum string) error {
+	src := filepath.Join(file.parent.basePath, file.relativePath)
+	indexfile := path.Base(src + ext)
+	src = src + file.parent.suffix + ext
+	filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))
+	dst := filepath.Join(filedir, "by-hash", hash)
+	sumfilePath := filepath.Join(dst, sum)
+
+	// link already exists? do nothing
+	exists, err := file.parent.publishedStorage.FileExists(sumfilePath)
+	if err != nil {
+		return fmt.Errorf("Acquire-By-Hash: error checking exists of file %s: %s", sumfilePath, err)
+	}
+	if exists {
+		return nil
+	}
+
+	// create the link
+	err = file.parent.publishedStorage.HardLink(src, sumfilePath)
+	if err != nil {
+		return fmt.Errorf("Acquire-By-Hash: error creating hardlink %s: %s", sumfilePath, err)
+	}
+
+	// if a previous index file already exists exists, backup symlink
+	if exists, _ = file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile)); exists {
+		// if exists, remove old symlink
+		if exists, _ = file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile+".old")); exists {
+			var link string
+			link, err = file.parent.publishedStorage.ReadLink(filepath.Join(dst, indexfile+".old"))
+			if err != nil {
+				file.parent.publishedStorage.Remove(link)
+			}
+			file.parent.publishedStorage.Remove(filepath.Join(dst, indexfile+".old"))
+		}
+		file.parent.publishedStorage.RenameFile(filepath.Join(dst, indexfile),
+			filepath.Join(dst, indexfile+".old"))
+	}
+
+	// create symlink
+	err = file.parent.publishedStorage.SymLink(filepath.Join(dst, sum), filepath.Join(dst, indexfile))
+	if err != nil {
+		return fmt.Errorf("Acquire-By-Hash: error creating symlink %s: %s", filepath.Join(dst, indexfile), err)
+	}
+	return nil
+}
+
+func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, acquireByHash bool) *indexFiles {
 	return &indexFiles{
 		publishedStorage: publishedStorage,
 		basePath:         basePath,
@@ -152,6 +222,7 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
 		tempDir:          tempDir,
 		suffix:           suffix,
 		indexes:          make(map[string]*indexFile),
+		acquireByHash:    acquireByHash,
 	}
 }
 
@@ -175,11 +246,12 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 		}
 
 		file = &indexFile{
-			parent:       files,
-			discardable:  false,
-			compressable: true,
-			signable:     false,
-			relativePath: relativePath,
+			parent:        files,
+			discardable:   false,
+			compressable:  true,
+			signable:      false,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
 		}
 
 		files.indexes[key] = file
@@ -208,11 +280,12 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 		}
 
 		file = &indexFile{
-			parent:       files,
-			discardable:  udeb,
-			compressable: false,
-			signable:     false,
-			relativePath: relativePath,
+			parent:        files,
+			discardable:   udeb,
+			compressable:  false,
+			signable:      false,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
 		}
 
 		files.indexes[key] = file
@@ -237,12 +310,13 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index
 		}
 
 		file = &indexFile{
-			parent:       files,
-			discardable:  true,
-			compressable: true,
-			onlyGzip:     true,
-			signable:     false,
-			relativePath: relativePath,
+			parent:        files,
+			discardable:   true,
+			compressable:  true,
+			onlyGzip:      true,
+			signable:      false,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
 		}
 
 		files.indexes[key] = file

deb/publish.go

@@ -64,6 +64,9 @@ type PublishedRepo struct {
 
 	// True if repo is being re-published
 	rePublishing bool
+
+	// Provide index files per hash also
+	AcquireByHash bool
 }
 
 // ParsePrefix splits [storage:]prefix into components
@@ -556,7 +559,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	}
 	defer os.RemoveAll(tempDir)
 
-	indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix)
+	indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AcquireByHash)
 
 	for component, list := range lists {
 		hadUdebs := false
@@ -683,6 +686,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				release["Component"] = component
 				release["Origin"] = p.GetOrigin()
 				release["Label"] = p.GetLabel()
+				if p.AcquireByHash {
+					release["Acquire-By-Hash"] = "yes"
+				}
 
 				var bufWriter *bufio.Writer
 				bufWriter, err = indexes.ReleaseIndex(component, arch, udeb).BufWriter()
@@ -720,6 +726,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	release["Codename"] = p.Distribution
 	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
 	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ")
+	if p.AcquireByHash {
+		release["Acquire-By-Hash"] = "yes"
+	}
 	release["Description"] = " Generated by aptly\n"
 	release["MD5Sum"] = ""
 	release["SHA1"] = ""

files/public.go

@@ -247,3 +247,27 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
 	return os.Rename(filepath.Join(storage.rootPath, oldName), filepath.Join(storage.rootPath, newName))
 }
+
+// SymLink creates a symbolic link, which can be read with ReadLink
+func (storage *PublishedStorage) SymLink(src string, dst string) error {
+	return os.Symlink(filepath.Join(storage.rootPath, src), filepath.Join(storage.rootPath, dst))
+}
+
+// HardLink creates a hardlink of a file
+func (storage *PublishedStorage) HardLink(src string, dst string) error {
+	return os.Link(filepath.Join(storage.rootPath, src), filepath.Join(storage.rootPath, dst))
+}
+
+// FileExists returns true if path exists
+func (storage *PublishedStorage) FileExists(path string) (bool, error) {
+	if _, err := os.Lstat(filepath.Join(storage.rootPath, path)); os.IsNotExist(err) {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// ReadLink returns the symbolic link pointed to by path
+func (storage *PublishedStorage) ReadLink(path string) (string, error) {
+	return os.Readlink(path)
+}

files/public_test.go

@@ -103,6 +103,48 @@ func (s *PublishedStorageSuite) TestRenameFile(c *C) {
 	c.Assert(err, IsNil)
 }
 
+func (s *PublishedStorageSuite) TestFileExists(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	exists, _ := s.storage.FileExists("ppa/dists/squeeze/Release")
+	c.Check(exists, Equals, false)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
+	c.Assert(err, IsNil)
+
+	exists, _ = s.storage.FileExists("ppa/dists/squeeze/Release")
+	c.Check(exists, Equals, true)
+}
+
+func (s *PublishedStorageSuite) TestSymLink(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
+	c.Assert(err, IsNil)
+
+	err = s.storage.SymLink("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease")
+	c.Assert(err, IsNil)
+
+	exists, _ := s.storage.FileExists("ppa/dists/squeeze/InRelease")
+	c.Check(exists, Equals, true)
+}
+
+func (s *PublishedStorageSuite) TestHardLink(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
+	c.Assert(err, IsNil)
+
+	err = s.storage.HardLink("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease")
+	c.Assert(err, IsNil)
+
+	exists, _ := s.storage.FileExists("ppa/dists/squeeze/InRelease")
+	c.Check(exists, Equals, true)
+}
+
 func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)