Addon files

AUTHORS

@@ -32,3 +32,4 @@ List of contributors, in chronological order:
 * Ludovico Cavedon (https://github.com/cavedon)
 * Petr Jediny (https://github.com/pjediny)
 * Maximilian Stein (https://github.com/steinymity)
+* Blake Kostner (https://github.com/btkostner)

api/publish.go

@@ -213,7 +213,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		return
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.SkelPath(), nil, b.ForceOverwrite)
 	if err != nil {
 		c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
 		return
@@ -327,7 +327,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		published.AcquireByHash = *b.AcquireByHash
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.SkelPath(), nil, b.ForceOverwrite)
 	if err != nil {
 		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
 		return

cmd/publish_snapshot.go

@@ -158,7 +158,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 			"the same package pool.\n")
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.SkelPath(), context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}

cmd/publish_switch.go

@@ -95,7 +95,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.SkelPath(), context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}

cmd/publish_update.go

@@ -59,7 +59,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.SkelPath(), context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}

context/context.go

@@ -418,6 +418,11 @@ func (context *AptlyContext) GetVerifier() pgp.Verifier {
 	return &pgp.GoVerifier{}
 }
 
+// SkelPath builds the local skeleton folder
+func (context *AptlyContext) SkelPath() string {
+	return filepath.Join(context.config().RootDir, "skel")
+}
+
 // UpdateFlags sets internal copy of flags in the context
 func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) {
 	context.Lock()

deb/index_files.go

@@ -329,6 +329,28 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index
 	return file
 }
 
+func (files *indexFiles) SkelIndex(component, path string) *indexFile {
+	key := fmt.Sprintf("si-%s-%s", component, path)
+	file, ok := files.indexes[key]
+
+	if !ok {
+		relativePath := filepath.Join(component, path)
+
+		file = &indexFile{
+			parent:       files,
+			discardable:  false,
+			compressable: false,
+			onlyGzip:     false,
+			signable:     false,
+			relativePath: relativePath,
+		}
+
+		files.indexes[key] = file
+	}
+
+	return file
+}
+
 func (files *indexFiles) LegacyContentsIndex(arch string, udeb bool) *indexFile {
 	if arch == ArchitectureSource {
 		udeb = false

deb/publish.go

@@ -485,9 +485,49 @@ func (p *PublishedRepo) GetLabel() string {
 	return p.Label
 }
 
+// GetSkelFiles returns a map of files to be added to a repo. Key being the relative
+// path from component folder, and value being the full local FS path.
+func (p *PublishedRepo) GetSkelFiles(skelDir string, component string) (map[string]string, error) {
+	files := make(map[string]string)
+
+	if skelDir == "" {
+		return files, nil
+	}
+
+	fsPath := filepath.Join(skelDir, p.Prefix, "dists", p.Distribution, component)
+	if err := filepath.Walk(fsPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		stat, err := os.Stat(path)
+		if err != nil {
+			return err
+		}
+
+		if !stat.Mode().IsRegular() {
+			return nil
+		}
+
+		relativePath, err := filepath.Rel(fsPath, path)
+		if err != nil {
+			return err
+		}
+
+		files[relativePath] = path
+		return nil
+	}); err != nil && !os.IsNotExist(err) {
+		return files, err
+	}
+
+	return files, nil
+}
+
 // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
 func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider,
-	collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool) error {
+	collectionFactory *CollectionFactory, signer pgp.Signer, skelDir string, progress aptly.Progress,
+	forceOverwrite bool) error {
+
 	publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage)
 
 	err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
@@ -681,6 +721,30 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			progress.ShutdownBar()
 		}
 
+		for component := range p.sourceItems {
+			skelFiles, err := p.GetSkelFiles(skelDir, component)
+			if err != nil {
+				return fmt.Errorf("unable to get skeleton files: %v", err)
+			}
+
+			for relPath, absPath := range skelFiles {
+				bufWriter, err := indexes.SkelIndex(component, relPath).BufWriter()
+				if err != nil {
+					return fmt.Errorf("unable to generate skeleton index: %v", err)
+				}
+
+				file, err := os.Open(absPath)
+				if err != nil {
+					return fmt.Errorf("unable to read skeleton file: %v", err)
+				}
+
+				_, err = bufio.NewReader(file).WriteTo(bufWriter)
+				if err != nil {
+					return fmt.Errorf("unable to write skeleton file: %v", err)
+				}
+			}
+		}
+
 		udebs := []bool{false}
 		if hadUdebs {
 			udebs = append(udebs, true)

deb/publish_test.go

@@ -306,7 +306,7 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
 }
 
 func (s *PublishedRepoSuite) TestPublish(c *C) {
-	err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false)
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, "", nil, false)
 	c.Assert(err, IsNil)
 
 	c.Check(s.repo.Architectures, DeepEquals, []string{"i386"})
@@ -353,31 +353,31 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 }
 
 func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
-	err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, "", nil, false)
 	c.Assert(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"), PathExists)
 }
 
 func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) {
-	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
+	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, "", nil, false)
 	c.Assert(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/main/binary-i386/Release"), PathExists)
 }
 
 func (s *PublishedRepoSuite) TestPublishLocalSourceRepo(c *C) {
-	err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
+	err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, "", nil, false)
 	c.Assert(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/main/source/Release"), PathExists)
 }
 
 func (s *PublishedRepoSuite) TestPublishOtherStorage(c *C) {
-	err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
+	err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, "", nil, false)
 	c.Assert(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/maverick/Release"), PathExists)

man/aptly.1.ronn.tmpl

@@ -92,8 +92,8 @@ Configuration file is stored in JSON format (default values shown below):
 Options:
 
   * `rootDir`:
-    is root of directory storage to store database (`rootDir`/db), downloaded packages (`rootDir`/pool) and
-    the default for published repositories (`rootDir`/public)
+    is root of directory storage to store database (`rootDir`/db), downloaded packages (`rootDir`/pool),
+    published repositories (`rootDir`/public) and skeleton files (`rootDir`/skel)
 
   * `downloadConcurrency`:
     is a number of parallel download threads to use when downloading packages

system/lib.py

@@ -221,6 +221,13 @@ def check_cmd_output(self, command, gold_name, match_prepare=None, expected_code
             else:
                 raise
 
+    def write_file(self, path, content):
+        fullPath = os.path.join(os.environ["HOME"], ".aptly", path)
+        if not os.path.exists(os.path.dirname(fullPath)):
+            os.makedirs(os.path.dirname(fullPath), 0755)
+        with open(fullPath, "w") as f:
+            f.write(content)
+
     def read_file(self, path):
         with open(os.path.join(os.environ["HOME"], ".aptly", path), "r") as f:
             return f.read()

system/t06_publish/PublishRepo32Test_gold

@@ -0,0 +1,14 @@
+Loading packages...
+Generating metadata files and linking package files...
+Finalizing metadata files...
+Signing file 'Release' with gpg, please enter your passphrase when prompted:
+Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
+
+Local repo local-repo has been successfully published.
+Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing.
+Now you can add following line to apt sources:
+  deb http://your-server/ maverick main
+  deb-src http://your-server/ maverick main
+Don't forget to add your GPG key to apt with apt-key.
+
+You can also use `aptly serve` to publish your repositories over HTTP quickly.

system/t06_publish/repo.py

@@ -762,3 +762,57 @@ def check(self):
         self.run_cmd(["gpg", "--no-auto-check-trustdb",  "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly_passphrase.pub"),
                       "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release.gpg'),
                       os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release')])
+
+
+class PublishRepo32Test(BaseTest):
+    """
+    publish repo: skeleton files
+    """
+    fixtureCmds = [
+        "aptly repo create local-repo",
+        "aptly repo add local-repo ${files}"
+    ]
+    runCmd = "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -skip-contents local-repo"
+    gold_processor = BaseTest.expand_environ
+
+    def prepare_fixture(self):
+        super(PublishRepo32Test, self).prepare_fixture()
+
+        self.write_file(os.path.join('skel', 'dists', 'maverick', 'main', 'dep11', 'README'), 'README test file')
+
+    def check(self):
+        super(PublishRepo32Test, self).check()
+
+        self.check_exists('public/dists/maverick/main/dep11/README')
+
+        self.check_exists('public/dists/maverick/Release')
+
+        release = self.read_file('public/dists/maverick/Release').split("\n")
+        release = [l for l in release if l.startswith(" ")]
+        pathsSeen = set()
+        for l in release:
+            fileHash, fileSize, path = l.split()
+            pathsSeen.add(path)
+
+            fileSize = int(fileSize)
+
+            st = os.stat(os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/', path))
+            if fileSize != st.st_size:
+                raise Exception("file size doesn't match for %s: %d != %d" % (path, fileSize, st.st_size))
+
+            if len(fileHash) == 32:
+                h = hashlib.md5()
+            elif len(fileHash) == 40:
+                h = hashlib.sha1()
+            elif len(fileHash) == 64:
+                h = hashlib.sha256()
+            else:
+                h = hashlib.sha512()
+
+            h.update(self.read_file(os.path.join('public/dists/maverick', path)))
+
+            if h.hexdigest() != fileHash:
+                raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest()))
+
+        if 'main/dep11/README' not in pathsSeen:
+            raise Exception("README file not included in release file")