From 9e6ff801016ffc9bd7c7fb35ac62360e497174d7 Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Wed, 11 Jan 2017 18:20:50 -0700 Subject: [PATCH 01/13] add GetAddonDir to context --- context/context.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/context/context.go b/context/context.go index 0db75aa0c..2b9df9b63 100644 --- a/context/context.go +++ b/context/context.go @@ -3,6 +3,14 @@ package context import ( "fmt" + "os" + "path/filepath" + "runtime" + "runtime/pprof" + "strings" + "sync" + "time" + "github.com/smira/aptly/aptly" "github.com/smira/aptly/console" "github.com/smira/aptly/database" @@ -14,13 +22,6 @@ import ( "github.com/smira/aptly/utils" "github.com/smira/commander" "github.com/smira/flag" - "os" - "path/filepath" - "runtime" - "runtime/pprof" - "strings" - "sync" - "time" ) // AptlyContext is a common context shared by all commands @@ -355,6 +356,11 @@ func (context *AptlyContext) UploadPath() string { return filepath.Join(context.Config().RootDir, "upload") } +// AddonPath builds the local addon folder +func (context *AptlyContext) AddonPath() string { + return filepath.Join(context.config().RootDir, "addon") +} + // UpdateFlags sets internal copy of flags in the context func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) { context.Lock() From 604a4dc684e3e2001d098fa0998cac54a9926027 Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Wed, 11 Jan 2017 18:21:25 -0700 Subject: [PATCH 02/13] add AddonIndex to index_files --- deb/index_files.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/deb/index_files.go b/deb/index_files.go index f643adbcb..823e97390 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -3,11 +3,12 @@ package deb import ( "bufio" "fmt" - "github.com/smira/aptly/aptly" - "github.com/smira/aptly/utils" "os" "path/filepath" "strings" + + "github.com/smira/aptly/aptly" + "github.com/smira/aptly/utils" ) type indexFiles struct { @@ -249,6 +250,28 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index return file } +func (files *indexFiles) AddonIndex(component, path string) *indexFile { + key := fmt.Sprintf("ai-%s-%s", component, path) + file, ok := files.indexes[key] + + if !ok { + relativePath := filepath.Join(component, path) + + file = &indexFile{ + parent: files, + discardable: false, + compressable: false, + onlyGzip: false, + signable: false, + relativePath: relativePath, + } + + files.indexes[key] = file + } + + return file +} + func (files *indexFiles) ReleaseFile() *indexFile { return &indexFile{ parent: files, From d9222b71fba1275818fe89e5854bdda216d5be12 Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Wed, 11 Jan 2017 18:21:39 -0700 Subject: [PATCH 03/13] add GetAddonPaths to publish file --- deb/publish.go | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/deb/publish.go b/deb/publish.go index 98deb4c52..271d633c9 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -5,11 +5,6 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/smira/aptly/aptly" - "github.com/smira/aptly/database" - "github.com/smira/aptly/utils" - "github.com/smira/go-uuid/uuid" - "github.com/ugorji/go/codec" "io/ioutil" "log" "os" @@ -18,6 +13,12 @@ import ( "strings" "sync" "time" + + "github.com/smira/aptly/aptly" + "github.com/smira/aptly/database" + "github.com/smira/aptly/utils" + "github.com/smira/go-uuid/uuid" + "github.com/ugorji/go/codec" ) type repoSourceItem struct { @@ -452,6 +453,40 @@ func (p *PublishedRepo) GetLabel() string { return p.Label } +// GetAddonFiles returns a map of files to be added to a repo. Key being the relative +// path from component folder, and value being the full local FS path. +func (p *PublishedRepo) GetAddonFiles(addonDir string, component string) (map[string]string, error) { + files := make(map[string]string) + + if addonDir == "" { + return files, nil + } + + fsPath := filepath.Join(addonDir, p.Prefix, "dists", p.Distribution, component) + if err := filepath.Walk(fsPath, func(path string, info os.FileInfo, err error) error { + stat, err := os.Stat(path) + if err != nil { + return err + } + + if !stat.Mode().IsRegular() { + return nil + } + + relativePath, err := filepath.Rel(fsPath, path) + if err != nil { + return err + } + + files[relativePath] = path + return nil + }); err != nil && !os.IsNotExist(err) { + return files, err + } + + return files, nil +} + // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider, collectionFactory *CollectionFactory, signer utils.Signer, progress aptly.Progress, forceOverwrite bool) error { From f9935956f9417d76e714ffa42b3f6b4eb3fd5ea5 Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Wed, 11 Jan 2017 18:25:47 -0700 Subject: [PATCH 04/13] revert changes from auto linting on save --- context/context.go | 15 +++++++-------- deb/index_files.go | 5 ++--- deb/publish.go | 11 +++++------ 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/context/context.go b/context/context.go index 2b9df9b63..474e56cbb 100644 --- a/context/context.go +++ b/context/context.go @@ -3,14 +3,6 @@ package context import ( "fmt" - "os" - "path/filepath" - "runtime" - "runtime/pprof" - "strings" - "sync" - "time" - "github.com/smira/aptly/aptly" "github.com/smira/aptly/console" "github.com/smira/aptly/database" @@ -22,6 +14,13 @@ import ( "github.com/smira/aptly/utils" "github.com/smira/commander" "github.com/smira/flag" + "os" + "path/filepath" + "runtime" + "runtime/pprof" + "strings" + "sync" + "time" ) // AptlyContext is a common context shared by all commands diff --git a/deb/index_files.go b/deb/index_files.go index 823e97390..ad577eb90 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -3,12 +3,11 @@ package deb import ( "bufio" "fmt" + "github.com/smira/aptly/aptly" + "github.com/smira/aptly/utils" "os" "path/filepath" "strings" - - "github.com/smira/aptly/aptly" - "github.com/smira/aptly/utils" ) type indexFiles struct { diff --git a/deb/publish.go b/deb/publish.go index 271d633c9..2781170e9 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -5,6 +5,11 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/smira/aptly/aptly" + "github.com/smira/aptly/database" + "github.com/smira/aptly/utils" + "github.com/smira/go-uuid/uuid" + "github.com/ugorji/go/codec" "io/ioutil" "log" "os" @@ -13,12 +18,6 @@ import ( "strings" "sync" "time" - - "github.com/smira/aptly/aptly" - "github.com/smira/aptly/database" - "github.com/smira/aptly/utils" - "github.com/smira/go-uuid/uuid" - "github.com/ugorji/go/codec" ) type repoSourceItem struct { From 1305f8c037def8cd25dd23a02323f03e6b762457 Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Wed, 11 Jan 2017 20:39:31 -0700 Subject: [PATCH 05/13] pass down addonpath from api and cmd context --- api/publish.go | 4 ++-- cmd/publish_snapshot.go | 2 +- cmd/publish_switch.go | 2 +- cmd/publish_update.go | 2 +- deb/publish.go | 32 +++++++++++++++++++++++++++++++- deb/publish_test.go | 10 +++++----- man/aptly.1.ronn.tmpl | 4 ++-- 7 files changed, 43 insertions(+), 13 deletions(-) diff --git a/api/publish.go b/api/publish.go index a8459a116..1b2d95849 100644 --- a/api/publish.go +++ b/api/publish.go @@ -196,7 +196,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { return } - err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite) + err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.AddonPath(), nil, b.ForceOverwrite) if err != nil { c.Fail(500, fmt.Errorf("unable to publish: %s", err)) return @@ -304,7 +304,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { published.SkipContents = *b.SkipContents } - err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite) + err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.AddonPath(), nil, b.ForceOverwrite) if err != nil { c.Fail(500, fmt.Errorf("unable to update: %s", err)) return diff --git a/cmd/publish_snapshot.go b/cmd/publish_snapshot.go index 32fb1a8eb..24089add0 100644 --- a/cmd/publish_snapshot.go +++ b/cmd/publish_snapshot.go @@ -142,7 +142,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error { "the same package pool.\n") } - err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite) + err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.AddonPath(), context.Progress(), forceOverwrite) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/cmd/publish_switch.go b/cmd/publish_switch.go index 23b3b7b51..92e5a37ca 100644 --- a/cmd/publish_switch.go +++ b/cmd/publish_switch.go @@ -94,7 +94,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool) } - err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite) + err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.AddonPath(), context.Progress(), forceOverwrite) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/cmd/publish_update.go b/cmd/publish_update.go index befb2312d..62a12120f 100644 --- a/cmd/publish_update.go +++ b/cmd/publish_update.go @@ -58,7 +58,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error { published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool) } - err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite) + err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.AddonPath(), context.Progress(), forceOverwrite) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/deb/publish.go b/deb/publish.go index 2781170e9..12f785a51 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -488,7 +488,8 @@ func (p *PublishedRepo) GetAddonFiles(addonDir string, component string) (map[st // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider, - collectionFactory *CollectionFactory, signer utils.Signer, progress aptly.Progress, forceOverwrite bool) error { + collectionFactory *CollectionFactory, signer utils.Signer, addonDirectory string, progress aptly.Progress, + forceOverwrite bool) error { publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage) err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool")) @@ -651,8 +652,37 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP if progress != nil { progress.ShutdownBar() + progress.Printf("Finding addon files...\n") } + for component := range p.sourceItems { + addonFiles, err := p.GetAddonFiles(addonDirectory, component) + if err != nil { + return fmt.Errorf("unable to get addon files: %v", err) + } + + for relPath, absPath := range addonFiles { + bufWriter, err := indexes.AddonIndex(component, relPath).BufWriter() + if err != nil { + return fmt.Errorf("unable to generate addon index: %v", err) + } + + file, err := os.Open(absPath) + if err != nil { + return fmt.Errorf("unable to read addon file: %v", err) + } + + _, err = bufio.NewReader(file).WriteTo(bufWriter) + if err != nil { + return fmt.Errorf("unable to write addon file: %v", err) + } + } + } + + if progress != nil { + progress.Printf("Generating index files...\n") + } + udebs := []bool{false} if hadUdebs { udebs = append(udebs, true) diff --git a/deb/publish_test.go b/deb/publish_test.go index d2be9a03f..08a11039a 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -296,7 +296,7 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) { } func (s *PublishedRepoSuite) TestPublish(c *C) { - err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false) + err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, "", nil, false) c.Assert(err, IsNil) c.Check(s.repo.Architectures, DeepEquals, []string{"i386"}) @@ -343,7 +343,7 @@ func (s *PublishedRepoSuite) TestPublish(c *C) { } func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) { - err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, "", nil, false) c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"), PathExists) @@ -351,7 +351,7 @@ func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) { } func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) { - err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, "", nil, false) c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists) @@ -359,7 +359,7 @@ func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) { } func (s *PublishedRepoSuite) TestPublishLocalSourceRepo(c *C) { - err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, "", nil, false) c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists) @@ -367,7 +367,7 @@ func (s *PublishedRepoSuite) TestPublishLocalSourceRepo(c *C) { } func (s *PublishedRepoSuite) TestPublishOtherStorage(c *C) { - err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, "", nil, false) c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/maverick/Release"), PathExists) diff --git a/man/aptly.1.ronn.tmpl b/man/aptly.1.ronn.tmpl index f9abf4f17..8734d42da 100644 --- a/man/aptly.1.ronn.tmpl +++ b/man/aptly.1.ronn.tmpl @@ -74,8 +74,8 @@ Configuration file is stored in JSON format (default values shown below): Options: * `rootDir`: - is root of directory storage to store database (`rootDir`/db), downloaded packages (`rootDir`/pool) and - published repositories (`rootDir`/public) + is root of directory storage to store database (`rootDir`/db), downloaded packages (`rootDir`/pool), + published repositories (`rootDir`/public) and addon files (`rootDir`/addon) * `downloadConcurrency`: is a number of parallel download threads to use when downloading packages From 8ac7c604d2599f8e3ddfbe3f943e4f86da1f9343 Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Wed, 11 Jan 2017 20:43:31 -0700 Subject: [PATCH 06/13] add name to AUTHORS list --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index dacec23e0..7a46fe793 100644 --- a/AUTHORS +++ b/AUTHORS @@ -22,3 +22,4 @@ List of contributors, in chronological order: * Phil Frost (https://github.com/bitglue) * Benoit Foucher (https://github.com/bentoi) * Geoffrey Thomas (https://github.com/geofft) +* Blake Kostner (https://github.com/btkostner) From 4c8338b87ea8fd4b1c80bb5a5506d64f4275b43a Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Wed, 11 Jan 2017 22:21:10 -0700 Subject: [PATCH 07/13] remove added status messages due to component loop --- deb/publish.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/deb/publish.go b/deb/publish.go index 12f785a51..4fa1b7d5d 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -652,7 +652,6 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP if progress != nil { progress.ShutdownBar() - progress.Printf("Finding addon files...\n") } for component := range p.sourceItems { @@ -679,10 +678,6 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } } - if progress != nil { - progress.Printf("Generating index files...\n") - } - udebs := []bool{false} if hadUdebs { udebs = append(udebs, true) From feb3ceb974cccfc92b6adadc90213a921a52f05b Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Wed, 11 Jan 2017 22:23:37 -0700 Subject: [PATCH 08/13] add first test of addon files --- system/lib.py | 7 +++ system/t06_publish/PublishRepo29Test_gold | 14 ++++++ system/t06_publish/repo.py | 57 +++++++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 system/t06_publish/PublishRepo29Test_gold diff --git a/system/lib.py b/system/lib.py index d0240f6a7..81904ea67 100644 --- a/system/lib.py +++ b/system/lib.py @@ -222,6 +222,13 @@ def check_cmd_output(self, command, gold_name, match_prepare=None, expected_code else: raise + def write_file(self, path, content): + fullPath = os.path.join(os.environ["HOME"], ".aptly", path) + if not os.path.exists(os.path.dirname(fullPath)): + os.makedirs(os.path.dirname(fullPath), 0755) + with open(fullPath, "w") as f: + f.write(content) + def read_file(self, path): with open(os.path.join(os.environ["HOME"], ".aptly", path), "r") as f: return f.read() diff --git a/system/t06_publish/PublishRepo29Test_gold b/system/t06_publish/PublishRepo29Test_gold new file mode 100644 index 000000000..365295fa9 --- /dev/null +++ b/system/t06_publish/PublishRepo29Test_gold @@ -0,0 +1,14 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: + +Local repo local-repo has been successfully published. +Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing. +Now you can add following line to apt sources: + deb http://your-server/ maverick main + deb-src http://your-server/ maverick main +Don't forget to add your GPG key to apt with apt-key. + +You can also use `aptly serve` to publish your repositories over HTTP quickly. diff --git a/system/t06_publish/repo.py b/system/t06_publish/repo.py index 0a26ccfbb..99e41f4c1 100644 --- a/system/t06_publish/repo.py +++ b/system/t06_publish/repo.py @@ -694,3 +694,60 @@ def check(self): self.check_not_exists('public/dists/maverick/main/Contents-i386.gz') self.check_exists('public/dists/maverick/main/debian-installer/binary-i386/Release') self.check_not_exists('public/dists/maverick/main/Contents-udeb-i386.gz') + +class PublishRepo29Test(BaseTest): + """ + publish repo: addon files + """ + fixtureCmds = [ + "aptly repo create local-repo", + "aptly repo add local-repo ${files}" + ] + runCmd = "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -skip-contents local-repo" + gold_processor = BaseTest.expand_environ + + def prepare_fixture(self): + super(PublishRepo29Test, self).prepare_fixture() + + self.write_file(os.path.join('addon', 'dists', 'maverick', 'main', 'dep11', 'README'), 'README test file') + + def check(self): + super(PublishRepo29Test, self).check() + + self.check_exists('public/dists/maverick/main/dep11/README') + + self.check_exists('public/dists/maverick/Release') + + readme = self.read_file('public/dists/maverick/main/dep11/README') + if readme != 'README test file': + raise Exception("README file not copied on publish") + + release = self.read_file('public/dists/maverick/Release').split("\n") + release = [l for l in release if l.startswith(" ")] + pathsSeen = set() + for l in release: + fileHash, fileSize, path = l.split() + pathsSeen.add(path) + + fileSize = int(fileSize) + + st = os.stat(os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/', path)) + if fileSize != st.st_size: + raise Exception("file size doesn't match for %s: %d != %d" % (path, fileSize, st.st_size)) + + if len(fileHash) == 32: + h = hashlib.md5() + elif len(fileHash) == 40: + h = hashlib.sha1() + elif len(fileHash) == 64: + h = hashlib.sha256() + else: + h = hashlib.sha512() + + h.update(self.read_file(os.path.join('public/dists/maverick', path))) + + if h.hexdigest() != fileHash: + raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest())) + + if 'main/dep11/README' not in pathsSeen: + raise Exception("README file not included in release file") From addf3164ffcb0ac4f6a414e8b9d4236eef0d8d77 Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Wed, 3 May 2017 18:28:47 -0700 Subject: [PATCH 09/13] fix linting issue --- deb/publish.go | 48 +++++++++++------------ system/t06_publish/PublishRepo29Test_gold | 1 - system/t06_publish/repo.py | 10 ++--- 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/deb/publish.go b/deb/publish.go index ea6d237ae..e2a658840 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -482,7 +482,7 @@ func (p *PublishedRepo) GetAddonFiles(addonDir string, component string) (map[st // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider, collectionFactory *CollectionFactory, signer utils.Signer, addonDirectory string, progress aptly.Progress, - forceOverwrite bool) error { + forceOverwrite bool) error { publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage) err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool")) @@ -662,29 +662,29 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP progress.ShutdownBar() } - for component := range p.sourceItems { - addonFiles, err := p.GetAddonFiles(addonDirectory, component) - if err != nil { - return fmt.Errorf("unable to get addon files: %v", err) - } - - for relPath, absPath := range addonFiles { - bufWriter, err := indexes.AddonIndex(component, relPath).BufWriter() - if err != nil { - return fmt.Errorf("unable to generate addon index: %v", err) - } - - file, err := os.Open(absPath) - if err != nil { - return fmt.Errorf("unable to read addon file: %v", err) - } - - _, err = bufio.NewReader(file).WriteTo(bufWriter) - if err != nil { - return fmt.Errorf("unable to write addon file: %v", err) - } - } - } + for component := range p.sourceItems { + addonFiles, err := p.GetAddonFiles(addonDirectory, component) + if err != nil { + return fmt.Errorf("unable to get addon files: %v", err) + } + + for relPath, absPath := range addonFiles { + bufWriter, err := indexes.AddonIndex(component, relPath).BufWriter() + if err != nil { + return fmt.Errorf("unable to generate addon index: %v", err) + } + + file, err := os.Open(absPath) + if err != nil { + return fmt.Errorf("unable to read addon file: %v", err) + } + + _, err = bufio.NewReader(file).WriteTo(bufWriter) + if err != nil { + return fmt.Errorf("unable to write addon file: %v", err) + } + } + } udebs := []bool{false} if hadUdebs { diff --git a/system/t06_publish/PublishRepo29Test_gold b/system/t06_publish/PublishRepo29Test_gold index 53960b087..27dcf24ce 100644 --- a/system/t06_publish/PublishRepo29Test_gold +++ b/system/t06_publish/PublishRepo29Test_gold @@ -9,7 +9,6 @@ Local repo local-repo has been successfully published. Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing. Now you can add following line to apt sources: deb http://your-server/ maverick main - deb-src http://your-server/ maverick main Don't forget to add your GPG key to apt with apt-key. You can also use `aptly serve` to publish your repositories over HTTP quickly. diff --git a/system/t06_publish/repo.py b/system/t06_publish/repo.py index c2d1f2864..295be9075 100644 --- a/system/t06_publish/repo.py +++ b/system/t06_publish/repo.py @@ -699,6 +699,7 @@ def check(self): self.check_exists('public/dists/maverick/main/debian-installer/binary-i386/Release') self.check_not_exists('public/dists/maverick/main/Contents-udeb-i386.gz') + class PublishRepo29Test(BaseTest): """ publish repo: broken .deb file for contents @@ -710,6 +711,7 @@ class PublishRepo29Test(BaseTest): runCmd = "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo" gold_processor = BaseTest.expand_environ + class PublishRepo30Test(BaseTest): """ publish repo: addon files @@ -722,21 +724,17 @@ class PublishRepo30Test(BaseTest): gold_processor = BaseTest.expand_environ def prepare_fixture(self): - super(PublishRepo29Test, self).prepare_fixture() + super(PublishRepo30Test, self).prepare_fixture() self.write_file(os.path.join('addon', 'dists', 'maverick', 'main', 'dep11', 'README'), 'README test file') def check(self): - super(PublishRepo29Test, self).check() + super(PublishRepo30Test, self).check() self.check_exists('public/dists/maverick/main/dep11/README') self.check_exists('public/dists/maverick/Release') - readme = self.read_file('public/dists/maverick/main/dep11/README') - if readme != 'README test file': - raise Exception("README file not copied on publish") - release = self.read_file('public/dists/maverick/Release').split("\n") release = [l for l in release if l.startswith(" ")] pathsSeen = set() From fd0a980597b2f6289dbc7ca80da524f0ccf8772e Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Mon, 26 Feb 2018 19:17:04 -0700 Subject: [PATCH 10/13] error check filepath.Walk --- deb/publish.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deb/publish.go b/deb/publish.go index 3e8f581c5..d9b8f5d55 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -496,6 +496,10 @@ func (p *PublishedRepo) GetAddonFiles(addonDir string, component string) (map[st fsPath := filepath.Join(addonDir, p.Prefix, "dists", p.Distribution, component) if err := filepath.Walk(fsPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + stat, err := os.Stat(path) if err != nil { return err From b6f4f1d33219f7e1cdca10a1b57a56b37633e501 Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Mon, 26 Feb 2018 19:24:24 -0700 Subject: [PATCH 11/13] fix linting on python test files --- system/t06_publish/repo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/system/t06_publish/repo.py b/system/t06_publish/repo.py index 3f82696c0..9423db857 100644 --- a/system/t06_publish/repo.py +++ b/system/t06_publish/repo.py @@ -735,6 +735,7 @@ def check(self): "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release.gpg'), os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release')]) + class PublishRepo31Test(BaseTest): """ publish repo: sign with passphrase (internal PGP implementation) @@ -760,6 +761,7 @@ def check(self): "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release.gpg'), os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release')]) + class PublishRepo32Test(BaseTest): """ publish repo: addon files From 4f0952e2521a5f181b11fa6cf4898e6a65abfee6 Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Mon, 26 Feb 2018 23:03:21 -0700 Subject: [PATCH 12/13] update repo test for addon files --- system/t06_publish/repo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system/t06_publish/repo.py b/system/t06_publish/repo.py index 9423db857..955686918 100644 --- a/system/t06_publish/repo.py +++ b/system/t06_publish/repo.py @@ -774,12 +774,12 @@ class PublishRepo32Test(BaseTest): gold_processor = BaseTest.expand_environ def prepare_fixture(self): - super(PublishRepo30Test, self).prepare_fixture() + super(PublishRepo32Test, self).prepare_fixture() self.write_file(os.path.join('addon', 'dists', 'maverick', 'main', 'dep11', 'README'), 'README test file') def check(self): - super(PublishRepo30Test, self).check() + super(PublishRepo32Test, self).check() self.check_exists('public/dists/maverick/main/dep11/README') From 7910e530ccf8bdfa388044eda297360ed3036943 Mon Sep 17 00:00:00 2001 From: Blake Kostner Date: Mon, 30 Apr 2018 10:47:12 -0600 Subject: [PATCH 13/13] rename addon files to skel files --- api/publish.go | 4 ++-- cmd/publish_snapshot.go | 2 +- cmd/publish_switch.go | 2 +- cmd/publish_update.go | 2 +- context/context.go | 6 +++--- deb/index_files.go | 4 ++-- deb/publish.go | 24 ++++++++++++------------ man/aptly.1.ronn.tmpl | 2 +- system/t06_publish/repo.py | 4 ++-- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/api/publish.go b/api/publish.go index 0ef1dec7e..5d43fe37a 100644 --- a/api/publish.go +++ b/api/publish.go @@ -213,7 +213,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { return } - err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.AddonPath(), nil, b.ForceOverwrite) + err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.SkelPath(), nil, b.ForceOverwrite) if err != nil { c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err)) return @@ -327,7 +327,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { published.AcquireByHash = *b.AcquireByHash } - err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.AddonPath(), nil, b.ForceOverwrite) + err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.SkelPath(), nil, b.ForceOverwrite) if err != nil { c.AbortWithError(500, fmt.Errorf("unable to update: %s", err)) return diff --git a/cmd/publish_snapshot.go b/cmd/publish_snapshot.go index b8d5525f6..47fdcc8fa 100644 --- a/cmd/publish_snapshot.go +++ b/cmd/publish_snapshot.go @@ -158,7 +158,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error { "the same package pool.\n") } - err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.AddonPath(), context.Progress(), forceOverwrite) + err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.SkelPath(), context.Progress(), forceOverwrite) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/cmd/publish_switch.go b/cmd/publish_switch.go index a65517f32..82f4eb2e8 100644 --- a/cmd/publish_switch.go +++ b/cmd/publish_switch.go @@ -95,7 +95,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool) } - err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.AddonPath(), context.Progress(), forceOverwrite) + err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.SkelPath(), context.Progress(), forceOverwrite) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/cmd/publish_update.go b/cmd/publish_update.go index 78a41b1c3..309ce867b 100644 --- a/cmd/publish_update.go +++ b/cmd/publish_update.go @@ -59,7 +59,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error { published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool) } - err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.AddonPath(), context.Progress(), forceOverwrite) + err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.SkelPath(), context.Progress(), forceOverwrite) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/context/context.go b/context/context.go index 084e2ce21..155b10efe 100644 --- a/context/context.go +++ b/context/context.go @@ -418,9 +418,9 @@ func (context *AptlyContext) GetVerifier() pgp.Verifier { return &pgp.GoVerifier{} } -// AddonPath builds the local addon folder -func (context *AptlyContext) AddonPath() string { - return filepath.Join(context.config().RootDir, "addon") +// SkelPath builds the local skeleton folder +func (context *AptlyContext) SkelPath() string { + return filepath.Join(context.config().RootDir, "skel") } // UpdateFlags sets internal copy of flags in the context diff --git a/deb/index_files.go b/deb/index_files.go index 38a9f8059..d345059f8 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -329,8 +329,8 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index return file } -func (files *indexFiles) AddonIndex(component, path string) *indexFile { - key := fmt.Sprintf("ai-%s-%s", component, path) +func (files *indexFiles) SkelIndex(component, path string) *indexFile { + key := fmt.Sprintf("si-%s-%s", component, path) file, ok := files.indexes[key] if !ok { diff --git a/deb/publish.go b/deb/publish.go index 62b50878d..409ad9751 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -485,16 +485,16 @@ func (p *PublishedRepo) GetLabel() string { return p.Label } -// GetAddonFiles returns a map of files to be added to a repo. Key being the relative +// GetSkelFiles returns a map of files to be added to a repo. Key being the relative // path from component folder, and value being the full local FS path. -func (p *PublishedRepo) GetAddonFiles(addonDir string, component string) (map[string]string, error) { +func (p *PublishedRepo) GetSkelFiles(skelDir string, component string) (map[string]string, error) { files := make(map[string]string) - if addonDir == "" { + if skelDir == "" { return files, nil } - fsPath := filepath.Join(addonDir, p.Prefix, "dists", p.Distribution, component) + fsPath := filepath.Join(skelDir, p.Prefix, "dists", p.Distribution, component) if err := filepath.Walk(fsPath, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -525,7 +525,7 @@ func (p *PublishedRepo) GetAddonFiles(addonDir string, component string) (map[st // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider, - collectionFactory *CollectionFactory, signer pgp.Signer, addonDirectory string, progress aptly.Progress, + collectionFactory *CollectionFactory, signer pgp.Signer, skelDir string, progress aptly.Progress, forceOverwrite bool) error { publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage) @@ -722,25 +722,25 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } for component := range p.sourceItems { - addonFiles, err := p.GetAddonFiles(addonDirectory, component) + skelFiles, err := p.GetSkelFiles(skelDir, component) if err != nil { - return fmt.Errorf("unable to get addon files: %v", err) + return fmt.Errorf("unable to get skeleton files: %v", err) } - for relPath, absPath := range addonFiles { - bufWriter, err := indexes.AddonIndex(component, relPath).BufWriter() + for relPath, absPath := range skelFiles { + bufWriter, err := indexes.SkelIndex(component, relPath).BufWriter() if err != nil { - return fmt.Errorf("unable to generate addon index: %v", err) + return fmt.Errorf("unable to generate skeleton index: %v", err) } file, err := os.Open(absPath) if err != nil { - return fmt.Errorf("unable to read addon file: %v", err) + return fmt.Errorf("unable to read skeleton file: %v", err) } _, err = bufio.NewReader(file).WriteTo(bufWriter) if err != nil { - return fmt.Errorf("unable to write addon file: %v", err) + return fmt.Errorf("unable to write skeleton file: %v", err) } } } diff --git a/man/aptly.1.ronn.tmpl b/man/aptly.1.ronn.tmpl index abd51ad30..9f949ba6b 100644 --- a/man/aptly.1.ronn.tmpl +++ b/man/aptly.1.ronn.tmpl @@ -93,7 +93,7 @@ Options: * `rootDir`: is root of directory storage to store database (`rootDir`/db), downloaded packages (`rootDir`/pool), - published repositories (`rootDir`/public) and addon files (`rootDir`/addon) + published repositories (`rootDir`/public) and skeleton files (`rootDir`/skel) * `downloadConcurrency`: is a number of parallel download threads to use when downloading packages diff --git a/system/t06_publish/repo.py b/system/t06_publish/repo.py index 1464183e4..98d37080c 100644 --- a/system/t06_publish/repo.py +++ b/system/t06_publish/repo.py @@ -766,7 +766,7 @@ def check(self): class PublishRepo32Test(BaseTest): """ - publish repo: addon files + publish repo: skeleton files """ fixtureCmds = [ "aptly repo create local-repo", @@ -778,7 +778,7 @@ class PublishRepo32Test(BaseTest): def prepare_fixture(self): super(PublishRepo32Test, self).prepare_fixture() - self.write_file(os.path.join('addon', 'dists', 'maverick', 'main', 'dep11', 'README'), 'README test file') + self.write_file(os.path.join('skel', 'dists', 'maverick', 'main', 'dep11', 'README'), 'README test file') def check(self): super(PublishRepo32Test, self).check()