Add option -skip-existing-packages to mirror update to speed up download queue setup

AUTHORS

@@ -26,3 +26,4 @@ List of contributors, in chronological order:
 * Harald Sitter (https://github.com/apachelogger)
 * Johannes Layher (https://github.com/jola5)
 * Charles Hsu (https://github.com/charz)
+* Clemens Rabe (https://github.com/seeraven)

cmd/mirror_update.go

@@ -81,8 +81,10 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		queue        []deb.PackageDownloadTask
 	)
 
+	skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
+
 	context.Progress().Printf("Building download queue...\n")
-	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool())
+	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), skipExistingPackages)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -188,6 +190,7 @@ Example:
 	cmd.Flag.Bool("force", false, "force update mirror even if it is locked by another process")
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
 	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")

deb/remote.go

@@ -506,27 +506,33 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu
 }
 
 // BuildDownloadQueue builds queue, discards current PackageList
-func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool) (queue []PackageDownloadTask, downloadSize int64, err error) {
+func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) {
 	queue = make([]PackageDownloadTask, 0, repo.packageList.Len())
 	seen := make(map[string]struct{}, repo.packageList.Len())
 
 	err = repo.packageList.ForEach(func(p *Package) error {
-		list, err2 := p.DownloadList(packagePool)
-		if err2 != nil {
-			return err2
+		download := true
+		if repo.packageRefs != nil && skipExistingPackages {
+			download = !repo.packageRefs.Has(p)
 		}
-		p.files = nil
-
-		for _, task := range list {
-			key := task.RepoURI + "-" + task.DestinationPath
-			_, found := seen[key]
-			if !found {
-				queue = append(queue, task)
-				downloadSize += task.Checksums.Size
-				seen[key] = struct{}{}
+
+		if download {
+			list, err2 := p.DownloadList(packagePool)
+			if err2 != nil {
+				return err2
+			}
+			p.files = nil
+
+			for _, task := range list {
+				key := task.RepoURI + "-" + task.DestinationPath
+				_, found := seen[key]
+				if !found {
+					queue = append(queue, task)
+					downloadSize += task.Checksums.Size
+					seen[key] = struct{}{}
+				}
 			}
 		}
-
 		return nil
 	})
 	if err != nil {

deb/remote_test.go

@@ -266,7 +266,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
-	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, false)
 	c.Check(size, Equals, int64(3))
 	c.Check(queue, HasLen, 1)
 	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
@@ -278,6 +278,47 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "amanda-client")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, true)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.repo.FinalizeDownload()
+	c.Assert(s.repo.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, false)
+	c.Check(size, Equals, int64(3))
+	c.Check(queue, HasLen, 1)
+	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.repo.FinalizeDownload()
+	c.Assert(s.repo.packageRefs, NotNil)
 }
 
 func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
@@ -298,7 +339,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
-	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, false)
 	c.Check(size, Equals, int64(15))
 	c.Check(queue, HasLen, 4)
 
@@ -323,6 +364,54 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[1])
 	c.Assert(err, IsNil)
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, true)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.repo.FinalizeDownload()
+	c.Assert(s.repo.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, false)
+	c.Check(size, Equals, int64(15))
+	c.Check(queue, HasLen, 4)
+
+	s.repo.FinalizeDownload()
+	c.Assert(s.repo.packageRefs, NotNil)
 }
 
 func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
@@ -340,7 +429,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
-	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, false)
 	c.Check(size, Equals, int64(3))
 	c.Check(queue, HasLen, 1)
 	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
@@ -352,6 +441,49 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "amanda-client")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, true)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.flat.FinalizeDownload()
+	c.Assert(s.flat.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, false)
+	c.Check(size, Equals, int64(3))
+	c.Check(queue, HasLen, 1)
+	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.flat.FinalizeDownload()
+	c.Assert(s.flat.packageRefs, NotNil)
 }
 
 func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
@@ -375,7 +507,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
-	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, false)
 	c.Check(size, Equals, int64(15))
 	c.Check(queue, HasLen, 4)
 
@@ -401,6 +533,56 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, true)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.flat.FinalizeDownload()
+	c.Assert(s.flat.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, false)
+	c.Check(size, Equals, int64(15))
+	c.Check(queue, HasLen, 4)
+
+	s.flat.FinalizeDownload()
+	c.Assert(s.flat.packageRefs, NotNil)
 }
 
 type RemoteRepoCollectionSuite struct {

man/aptly.1

@@ -510,6 +510,10 @@ gpg keyring to use when verifying Release file (could be specified multiple time
 \-\fBmax\-tries\fR=1
 max download tries till process fails with download error
 .
+.TP
+\-\fBskip\-existing\-packages\fR=false
+do not check file existence for packages listed in the internal database of the mirror
+.
 .SH "RENAMES MIRROR"
 \fBaptly\fR \fBmirror\fR \fBrename\fR \fIold\-name\fR \fInew\-name\fR
 .
@@ -1891,5 +1895,8 @@ Johannes Layher (https://github\.com/jola5)
 .IP "\[ci]" 4
 Charles Hsu (https://github\.com/charz)
 .
+.IP "\[ci]" 4
+Clemens Rabe (https://github\.com/seeraven)
+.
 .IP "" 0