diff --git a/api/publish.go b/api/publish.go index 61da5b04e..c4a59ab9a 100644 --- a/api/publish.go +++ b/api/publish.go @@ -238,6 +238,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { Component string `binding:"required"` Name string `binding:"required"` } + AccessByHash *bool } if c.Bind(&b) != nil { @@ -317,6 +318,10 @@ func apiPublishUpdateSwitch(c *gin.Context) { published.SkipContents = *b.SkipContents } + if b.AccessByHash != nil { + published.AccessByHash = *b.AccessByHash + } + err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite) if err != nil { c.AbortWithError(500, fmt.Errorf("unable to update: %s", err)) diff --git a/cmd/publish_repo.go b/cmd/publish_repo.go index 129a20bbc..009ba9273 100644 --- a/cmd/publish_repo.go +++ b/cmd/publish_repo.go @@ -47,6 +47,7 @@ Example: cmd.Flag.String("butautomaticupgrades", "", "set value for ButAutomaticUpgrades field") cmd.Flag.String("label", "", "label to publish") cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") + cmd.Flag.Bool("access-by-hash", false, "provide index files by hash also") return cmd } diff --git a/cmd/publish_snapshot.go b/cmd/publish_snapshot.go index 98540955d..f76055da1 100644 --- a/cmd/publish_snapshot.go +++ b/cmd/publish_snapshot.go @@ -137,6 +137,10 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error { published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool) } + if context.Flags().IsSet("access-by-hash") { + published.AccessByHash = context.Flags().Lookup("access-by-hash").Value.Get().(bool) + } + duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published) if duplicate != nil { context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory()) diff --git a/deb/index_files.go b/deb/index_files.go index f8b7051fc..88a638f7f 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "os" + "path" "path/filepath" "strings" @@ -20,6 +21,7 @@ type indexFiles struct { tempDir string suffix string indexes map[string]*indexFile + accessByHash bool } type indexFile struct { @@ -28,6 +30,7 @@ type indexFile struct { compressable bool onlyGzip bool signable bool + accessByHash bool relativePath string tempFilename string tempFile *os.File @@ -91,11 +94,24 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { file.parent.generatedFiles[file.relativePath+ext] = checksumInfo } - err = file.parent.publishedStorage.MkDir(filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))) + filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath)) + + err = file.parent.publishedStorage.MkDir(filedir) if err != nil { return fmt.Errorf("unable to create dir: %s", err) } + hashs := []string{} + if file.accessByHash { + hashs = append(hashs, "MD5", "SHA1", "SHA256", "SHA512") + for _, hash := range hashs { + err = file.parent.publishedStorage.MkDir(filepath.Join(filedir, "by-hash", hash)) + if err != nil { + return fmt.Errorf("unable to create dir: %s", err) + } + } + } + for _, ext := range exts { err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext), file.tempFilename+ext) @@ -107,6 +123,29 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext)] = filepath.Join(file.parent.basePath, file.relativePath+ext) } + + if file.accessByHash { + sums := file.parent.generatedFiles[file.relativePath+ext] + storage := file.parent.publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath() + src := filepath.Join(storage, file.parent.basePath, file.relativePath) + + err = packageIndexByHash(src, file.parent.suffix, ext, storage, filedir, "SHA512", sums.SHA512) + if err != nil { + fmt.Printf("%s\n", err) + } + err = packageIndexByHash(src, file.parent.suffix, ext, storage, filedir, "SHA256", sums.SHA256) + if err != nil { + fmt.Printf("%s\n", err) + } + err = packageIndexByHash(src, file.parent.suffix, ext, storage, filedir, "SHA1", sums.SHA1) + if err != nil { + fmt.Printf("%s\n", err) + } + err = packageIndexByHash(src, file.parent.suffix, ext, storage, filedir, "MD5", sums.MD5) + if err != nil { + fmt.Printf("%s\n", err) + } + } } if file.signable && signer != nil { @@ -143,7 +182,35 @@ func (file *indexFile) Finalize(signer pgp.Signer) error { return nil } -func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string) *indexFiles { +func packageIndexByHash(src string, suffix string, ext string, storage string, filedir string, hash string, sum string) error { + indexfile := path.Base(src + ext) + src = src + suffix + ext + dst := filepath.Join(storage, filedir, "by-hash", hash) + + if _, err := os.Stat(filepath.Join(dst, sum)); err == nil { + return nil + } + err := os.Link(src, filepath.Join(dst, sum)) + if err != nil { + return fmt.Errorf("Access-By-Hash: error creating hardlink %s", filepath.Join(dst, sum)) + } + + if _, err := os.Stat(filepath.Join(dst, indexfile)); err == nil { + if _, err := os.Stat(filepath.Join(dst, indexfile+".old")); err == nil { + link, _ := os.Readlink(filepath.Join(dst, indexfile+".old")) + os.Remove(filepath.Join(dst, link)) + os.Remove(filepath.Join(dst, indexfile+".old")) + } + os.Rename(filepath.Join(dst, indexfile), filepath.Join(dst, indexfile+".old")) + } + err = os.Symlink(sum, filepath.Join(dst, indexfile)) + if err != nil { + return fmt.Errorf("Access-By-Hash: error creating symlink %s", filepath.Join(dst, indexfile)) + } + return nil +} + +func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, accessByHash bool) *indexFiles { return &indexFiles{ publishedStorage: publishedStorage, basePath: basePath, @@ -152,6 +219,7 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s tempDir: tempDir, suffix: suffix, indexes: make(map[string]*indexFile), + accessByHash: accessByHash, } } @@ -179,6 +247,7 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF discardable: false, compressable: true, signable: false, + accessByHash: files.accessByHash, relativePath: relativePath, } @@ -212,6 +281,7 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF discardable: udeb, compressable: false, signable: false, + accessByHash: files.accessByHash, relativePath: relativePath, } @@ -242,6 +312,7 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index compressable: true, onlyGzip: true, signable: false, + accessByHash: files.accessByHash, relativePath: relativePath, } diff --git a/deb/publish.go b/deb/publish.go index 4e5885400..3d62400d8 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -64,6 +64,9 @@ type PublishedRepo struct { // True if repo is being re-published rePublishing bool + + // Provide index files per hash also + AccessByHash bool } // ParsePrefix splits [storage:]prefix into components @@ -556,7 +559,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } defer os.RemoveAll(tempDir) - indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix) + indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AccessByHash) for component, list := range lists { hadUdebs := false @@ -683,6 +686,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP release["Component"] = component release["Origin"] = p.GetOrigin() release["Label"] = p.GetLabel() + if p.AccessByHash { + release["Acquire-By-Hash"] = "yes" + } var bufWriter *bufio.Writer bufWriter, err = indexes.ReleaseIndex(component, arch, udeb).BufWriter() @@ -720,6 +726,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP release["Codename"] = p.Distribution release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST") release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ") + if p.AccessByHash { + release["Acquire-By-Hash"] = "yes" + } release["Description"] = " Generated by aptly\n" release["MD5Sum"] = "" release["SHA1"] = ""