diff --git a/aptly/interfaces.go b/aptly/interfaces.go index 3eb5102eb..1522b4b23 100644 --- a/aptly/interfaces.go +++ b/aptly/interfaces.go @@ -121,6 +121,8 @@ type Downloader interface { DownloadWithChecksum(url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error // GetProgress returns Progress object GetProgress() Progress + // GetLength of given url + GetLength(url string) (int64, error) } // ChecksumStorage is stores checksums in some (persistent) storage diff --git a/bash_completion.d/aptly b/bash_completion.d/aptly index cb9d9d5d3..025f008fa 100644 --- a/bash_completion.d/aptly +++ b/bash_completion.d/aptly @@ -156,7 +156,7 @@ _aptly() "create") if [[ $numargs -eq 0 ]]; then if [[ "$cur" == -* ]]; then - COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-sources -with-udebs" -- ${cur})) + COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur})) return 0 fi fi diff --git a/cmd/mirror_create.go b/cmd/mirror_create.go index 9db0970f7..72bcb7a2f 100644 --- a/cmd/mirror_create.go +++ b/cmd/mirror_create.go @@ -19,6 +19,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error { downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources") downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool) + downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool) var ( mirrorName, archiveURL, distribution string @@ -36,7 +37,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error { } repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(), - downloadSources, downloadUdebs) + downloadSources, downloadUdebs, downloadInstaller) if err != nil { return fmt.Errorf("unable to create mirror: %s", err) } @@ -94,6 +95,7 @@ Example: } cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures") + cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files") cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages") cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)") cmd.Flag.String("filter", "", "filter packages in mirror") diff --git a/cmd/mirror_update.go b/cmd/mirror_update.go index 48dcaa22d..3a4264b06 100644 --- a/cmd/mirror_update.go +++ b/cmd/mirror_update.go @@ -56,7 +56,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { } context.Progress().Printf("Downloading & parsing package files...\n") - err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch, maxTries) + err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, context.CollectionFactory(), ignoreMismatch, maxTries) if err != nil { return fmt.Errorf("unable to update: %s", err) } diff --git a/cmd/package_show.go b/cmd/package_show.go index bfac059e2..65d0e8efa 100644 --- a/cmd/package_show.go +++ b/cmd/package_show.go @@ -79,7 +79,7 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error { result := q.Query(context.CollectionFactory().PackageCollection()) err = result.ForEach(func(p *deb.Package) error { - p.Stanza().WriteTo(w, p.IsSource, false) + p.Stanza().WriteTo(w, p.IsSource, false, false) w.Flush() fmt.Printf("\n") diff --git a/deb/changes.go b/deb/changes.go index 6b852ea0e..729f35d1c 100644 --- a/deb/changes.go +++ b/deb/changes.go @@ -93,8 +93,8 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier text = input } - reader := NewControlFileReader(text) - c.Stanza, err = reader.ReadStanza(false) + reader := NewControlFileReader(text, false, false) + c.Stanza, err = reader.ReadStanza() if err != nil { return err } diff --git a/deb/deb.go b/deb/deb.go index 929407c4e..e14897205 100644 --- a/deb/deb.go +++ b/deb/deb.go @@ -87,8 +87,8 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) { } if tarHeader.Name == "./control" || tarHeader.Name == "control" { - reader := NewControlFileReader(untar) - stanza, err := reader.ReadStanza(false) + reader := NewControlFileReader(untar, false, false) + stanza, err := reader.ReadStanza() if err != nil { return nil, err } @@ -127,8 +127,8 @@ func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error text = file } - reader := NewControlFileReader(text) - stanza, err := reader.ReadStanza(false) + reader := NewControlFileReader(text, false, false) + stanza, err := reader.ReadStanza() if err != nil { return nil, err } diff --git a/deb/format.go b/deb/format.go index afd99a1e3..f6d0b5c2d 100644 --- a/deb/format.go +++ b/deb/format.go @@ -85,6 +85,9 @@ var ( "Directory", "Files", } + canonicalOrderInstaller = []string{ + "", + } ) // Copy returns copy of Stanza @@ -98,6 +101,9 @@ func (s Stanza) Copy() (result Stanza) { func isMultilineField(field string, isRelease bool) bool { switch field { + // file without a section + case "": + return true case "Description": return true case "Files": @@ -129,21 +135,26 @@ func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error if !isMultilineField(field, isRelease) { _, err = w.WriteString(field + ": " + value + "\n") } else { - if !strings.HasSuffix(value, "\n") { + if field != "" && !strings.HasSuffix(value, "\n") { value = value + "\n" } - if field != "Description" { + if field != "Description" && field != "" { value = "\n" + value } - _, err = w.WriteString(field + ":" + value) + + if field != "" { + _, err = w.WriteString(field + ":" + value) + } else { + _, err = w.WriteString(value) + } } return } // WriteTo saves stanza back to stream, modifying itself on the fly -func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error { +func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease, isInstaller bool) error { canonicalOrder := canonicalOrderBinary if isSource { canonicalOrder = canonicalOrderSource @@ -151,6 +162,9 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error { if isRelease { canonicalOrder = canonicalOrderRelease } + if isInstaller { + canonicalOrder = canonicalOrderInstaller + } for _, field := range canonicalOrder { value, ok := s[field] @@ -163,10 +177,13 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error { } } - for field, value := range s { - err := writeField(w, field, value, isRelease) - if err != nil { - return err + // no extra fields in installer + if !isInstaller { + for field, value := range s { + err := writeField(w, field, value, isRelease) + if err != nil { + return err + } } } @@ -209,19 +226,25 @@ func canonicalCase(field string) string { // ControlFileReader implements reading of control files stanza by stanza type ControlFileReader struct { - scanner *bufio.Scanner + scanner *bufio.Scanner + isRelease bool + isInstaller bool } // NewControlFileReader creates ControlFileReader, it wraps with buffering -func NewControlFileReader(r io.Reader) *ControlFileReader { - return &ControlFileReader{scanner: bufio.NewScanner(bufio.NewReaderSize(r, 32768))} +func NewControlFileReader(r io.Reader, isRelease, isInstaller bool) *ControlFileReader { + return &ControlFileReader{ + scanner: bufio.NewScanner(bufio.NewReaderSize(r, 32768)), + isRelease: isRelease, + isInstaller: isInstaller, + } } // ReadStanza reeads one stanza from control file -func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) { +func (c *ControlFileReader) ReadStanza() (Stanza, error) { stanza := make(Stanza, 32) lastField := "" - lastFieldMultiline := false + lastFieldMultiline := c.isInstaller for c.scanner.Scan() { line := c.scanner.Text() @@ -234,7 +257,7 @@ func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) { continue } - if line[0] == ' ' || line[0] == '\t' { + if line[0] == ' ' || line[0] == '\t' || c.isInstaller { if lastFieldMultiline { stanza[lastField] += line + "\n" } else { @@ -246,7 +269,7 @@ func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) { return nil, ErrMalformedStanza } lastField = canonicalCase(parts[0]) - lastFieldMultiline = isMultilineField(lastField, isRelease) + lastFieldMultiline = isMultilineField(lastField, c.isRelease) if lastFieldMultiline { stanza[lastField] = parts[1] if parts[1] != "" { diff --git a/deb/format_test.go b/deb/format_test.go index 92e02f475..a022132c2 100644 --- a/deb/format_test.go +++ b/deb/format_test.go @@ -14,6 +14,10 @@ type ControlFileSuite struct { var _ = Suite(&ControlFileSuite{}) +const installerFile = `dab96042d8e25e0f6bbb8d7c5bd78543afb5eb31a4a8b122ece68ab197228028 ./udeb.list +9d8bb14044dee520f4706ab197dfff10e9e39ecb3c1a402331712154e8284b2e ./MANIFEST.udebs +` + const controlFile = `Package: bti Binary: bti Version: 032-1 @@ -82,15 +86,15 @@ func (s *ControlFileSuite) SetUpTest(c *C) { } func (s *ControlFileSuite) TestReadStanza(c *C) { - r := NewControlFileReader(s.reader) + r := NewControlFileReader(s.reader, false, false) - stanza1, err := r.ReadStanza(false) + stanza1, err := r.ReadStanza() c.Assert(err, IsNil) - stanza2, err := r.ReadStanza(false) + stanza2, err := r.ReadStanza() c.Assert(err, IsNil) - stanza3, err := r.ReadStanza(false) + stanza3, err := r.ReadStanza() c.Assert(err, IsNil) c.Assert(stanza3, IsNil) @@ -102,27 +106,47 @@ func (s *ControlFileSuite) TestReadStanza(c *C) { } func (s *ControlFileSuite) TestReadWriteStanza(c *C) { - r := NewControlFileReader(s.reader) - stanza, err := r.ReadStanza(false) + r := NewControlFileReader(s.reader, false, false) + stanza, err := r.ReadStanza() c.Assert(err, IsNil) buf := &bytes.Buffer{} w := bufio.NewWriter(buf) - err = stanza.Copy().WriteTo(w, true, false) + err = stanza.Copy().WriteTo(w, true, false, false) c.Assert(err, IsNil) err = w.Flush() c.Assert(err, IsNil) str := buf.String() - r = NewControlFileReader(buf) - stanza2, err := r.ReadStanza(false) + r = NewControlFileReader(buf, false, false) + stanza2, err := r.ReadStanza() c.Assert(err, IsNil) c.Assert(stanza2, DeepEquals, stanza) c.Assert(strings.HasPrefix(str, "Package: "), Equals, true) } +func (s *ControlFileSuite) TestReadWriteInstallerStanza(c *C) { + s.reader = bytes.NewBufferString(installerFile) + r := NewControlFileReader(s.reader, false, true) + stanza, err := r.ReadStanza() + c.Assert(err, IsNil) + + buf := &bytes.Buffer{} + w := bufio.NewWriter(buf) + err = stanza.Copy().WriteTo(w, false, false, true) + c.Assert(err, IsNil) + err = w.Flush() + c.Assert(err, IsNil) + + r = NewControlFileReader(buf, false, true) + stanza2, err := r.ReadStanza() + c.Assert(err, IsNil) + + c.Assert(stanza2, DeepEquals, stanza) +} + func (s *ControlFileSuite) TestCanonicalCase(c *C) { c.Check(canonicalCase("Package"), Equals, "Package") c.Check(canonicalCase("package"), Equals, "Package") @@ -138,9 +162,9 @@ func (s *ControlFileSuite) TestCanonicalCase(c *C) { func (s *ControlFileSuite) BenchmarkReadStanza(c *C) { for i := 0; i < c.N; i++ { reader := bytes.NewBufferString(controlFile) - r := NewControlFileReader(reader) + r := NewControlFileReader(reader, false, false) for { - s, e := r.ReadStanza(false) + s, e := r.ReadStanza() if s == nil && e == nil { break } diff --git a/deb/index_files.go b/deb/index_files.go index f8b7051fc..6ff7f055e 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -155,11 +155,11 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s } } -func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile { +func (files *indexFiles) PackageIndex(component, arch string, udeb, installer bool) *indexFile { if arch == ArchitectureSource { udeb = false } - key := fmt.Sprintf("pi-%s-%s-%v", component, arch, udeb) + key := fmt.Sprintf("pi-%s-%s-%v-%v", component, arch, udeb, installer) file, ok := files.indexes[key] if !ok { var relativePath string @@ -169,6 +169,8 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF } else { if udeb { relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages") + } else if installer { + relativePath = filepath.Join(component, fmt.Sprintf("installer-%s", arch), "current", "images", "SHA256SUMS") } else { relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages") } @@ -177,7 +179,7 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF file = &indexFile{ parent: files, discardable: false, - compressable: true, + compressable: !installer, signable: false, relativePath: relativePath, } diff --git a/deb/package.go b/deb/package.go index 996aef457..12b337dde 100644 --- a/deb/package.go +++ b/deb/package.go @@ -26,6 +26,8 @@ type Package struct { Provides []string // Hash of files section FilesHash uint64 + // Is this package a dummy installer package + IsInstaller bool // Is this source package IsSource bool // Is this udeb package @@ -43,9 +45,10 @@ type Package struct { // Package types const ( - PackageTypeBinary = "deb" - PackageTypeUdeb = "udeb" - PackageTypeSource = "source" + PackageTypeBinary = "deb" + PackageTypeUdeb = "udeb" + PackageTypeSource = "source" + PackageTypeInstaller = "installer" ) // Special arhictectures @@ -168,6 +171,41 @@ func NewUdebPackageFromControlFile(input Stanza) *Package { return p } +// NewInstallerPackageFromControlFile creates a dummy installer Package from parsed hash sum file +func NewInstallerPackageFromControlFile(input Stanza, repo *RemoteRepo, component, architecture string, d aptly.Downloader) (*Package, error) { + p := &Package{ + Name: "installer", + Architecture: architecture, + IsInstaller: true, + V06Plus: true, + extra: &Stanza{}, + deps: &PackageDependencies{}, + } + + files := make(PackageFiles, 0) + files, err := files.ParseSumField(input[""], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data }, false, false) + if err != nil { + return nil, err + } + + relPath := filepath.Join("dists", repo.Distribution, component, fmt.Sprintf("%s-%s", p.Name, architecture), "current", "images") + for i := range files { + files[i].downloadPath = relPath + + url := repo.PackageURL(files[i].DownloadURL()).String() + var size int64 + size, err = d.GetLength(url) + if err != nil { + return nil, err + } + + files[i].Checksums.Size = size + } + + p.UpdateFiles(files) + return p, nil +} + // Key returns unique key identifying package func (p *Package) Key(prefix string) []byte { if p.V06Plus { @@ -509,6 +547,12 @@ func (p *Package) Stanza() (result Stanza) { if len(sha512) > 0 { result["Checksums-Sha512"] = strings.Join(sha512, "") } + } else if p.IsInstaller { + sha256 := []string{} + for _, f := range p.Files() { + sha256 = append(sha256, fmt.Sprintf("%s %s", f.Checksums.SHA256, f.Filename)) + } + result[""] = strings.Join(sha256, "\n") } else { f := p.Files()[0] result["Filename"] = f.DownloadURL() @@ -563,7 +607,7 @@ func (p *Package) Equals(p2 *Package) bool { // LinkFromPool links package file from pool to dist's pool location func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool, - prefix, component string, force bool) error { + prefix, distribution, component, architecture string, force bool) error { poolDir, err := p.PoolDirectory() if err != nil { return err @@ -576,6 +620,9 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP } relPath := filepath.Join("pool", component, poolDir) + if p.IsInstaller { + relPath = filepath.Join("dists", distribution, component, fmt.Sprintf("%s-%s", p.Name, architecture), "current", "images") + } publishedDirectory := filepath.Join(prefix, relPath) err = publishedStorage.LinkFromPool(publishedDirectory, f.Filename, packagePool, sourcePoolPath, f.Checksums, force) diff --git a/deb/package_files.go b/deb/package_files.go index 4716219c4..0904eed12 100644 --- a/deb/package_files.go +++ b/deb/package_files.go @@ -88,7 +88,8 @@ func (files PackageFiles) Less(i, j int) bool { return files[i].Filename < files[j].Filename } -func (files PackageFiles) parseSumField(input string, setter func(sum *utils.ChecksumInfo, data string)) (PackageFiles, error) { +// ParseSumField populates PackageFiles by parsing given input +func (files PackageFiles) ParseSumField(input string, setter func(sum *utils.ChecksumInfo, data string), withSize bool, onlyBasePath bool) (PackageFiles, error) { for _, line := range strings.Split(input, "\n") { line = strings.TrimSpace(line) if line == "" { @@ -96,16 +97,23 @@ func (files PackageFiles) parseSumField(input string, setter func(sum *utils.Che } parts := strings.Fields(line) - if len(parts) < 3 { + if withSize && len(parts) < 3 || !withSize && len(parts) < 2 { return nil, fmt.Errorf("unparseable hash sum line: %#v", line) } - size, err := strconv.ParseInt(parts[1], 10, 64) - if err != nil { - return nil, fmt.Errorf("unable to parse size: %s", err) + var size int64 + var err error + if withSize { + size, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("unable to parse size: %s", err) + } } - filename := filepath.Base(parts[len(parts)-1]) + filename := parts[len(parts)-1] + if onlyBasePath { + filename = filepath.Base(filename) + } found := false pos := 0 @@ -133,22 +141,22 @@ func (files PackageFiles) parseSumField(input string, setter func(sum *utils.Che func (files PackageFiles) ParseSumFields(stanza Stanza) (PackageFiles, error) { var err error - files, err = files.parseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data }) + files, err = files.ParseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data }, true, true) if err != nil { return nil, err } - files, err = files.parseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data }) + files, err = files.ParseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data }, true, true) if err != nil { return nil, err } - files, err = files.parseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data }) + files, err = files.ParseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data }, true, true) if err != nil { return nil, err } - files, err = files.parseSumField(stanza["Checksums-Sha512"], func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data }) + files, err = files.ParseSumField(stanza["Checksums-Sha512"], func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data }, true, true) if err != nil { return nil, err } diff --git a/deb/package_test.go b/deb/package_test.go index b3d34e762..783e24643 100644 --- a/deb/package_test.go +++ b/deb/package_test.go @@ -7,6 +7,7 @@ import ( "regexp" "github.com/smira/aptly/files" + "github.com/smira/aptly/http" . "gopkg.in/check.v1" ) @@ -22,7 +23,7 @@ func (s *PackageSuite) SetUpTest(c *C) { s.stanza = packageStanza.Copy() buf := bytes.NewBufferString(sourcePackageMeta) - s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza(false) + s.sourceStanza, _ = NewControlFileReader(buf, false, false).ReadStanza() } func (s *PackageSuite) TestNewFromPara(c *C) { @@ -43,7 +44,7 @@ func (s *PackageSuite) TestNewFromPara(c *C) { } func (s *PackageSuite) TestNewUdebFromPara(c *C) { - stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false) + stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta), false, false).ReadStanza() p := NewUdebPackageFromControlFile(stanza) c.Check(p.IsSource, Equals, false) @@ -57,6 +58,29 @@ func (s *PackageSuite) TestNewUdebFromPara(c *C) { c.Check(p.deps.Depends, DeepEquals, []string{"libc6-udeb (>= 2.13)"}) } +func (s *PackageSuite) TestNewInstallerFromPara(c *C) { + repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false) + downloader := http.NewFakeDownloader() + downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST.udebs", "MANIFEST.udebs") + downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/udeb.list", "udeb.list") + downloader.ExpectResponse("", "udeb.list") + stanza, _ := NewControlFileReader(bytes.NewBufferString(installerPackageMeta), false, true).ReadStanza() + p, err := NewInstallerPackageFromControlFile(stanza, repo, "main", "i386", downloader) + c.Check(err, IsNil) + + c.Check(p.IsSource, Equals, false) + c.Check(p.IsUdeb, Equals, false) + c.Check(p.IsInstaller, Equals, true) + c.Check(p.Name, Equals, "installer") + c.Check(p.Version, Equals, "") + c.Check(p.Architecture, Equals, "i386") + c.Check(p.Files(), HasLen, 2) + c.Check(p.Files()[0].Filename, Equals, "./MANIFEST.udebs") + c.Check(p.Files()[0].Checksums.Size, Equals, int64(14)) + c.Check(p.Files()[1].Filename, Equals, "./udeb.list") + c.Check(p.Files()[1].Checksums.Size, Equals, int64(9)) +} + func (s *PackageSuite) TestNewSourceFromPara(c *C) { p, err := NewSourcePackageFromControlFile(s.sourceStanza) @@ -156,7 +180,7 @@ func (s *PackageSuite) TestGetField(c *C) { p4, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy()) - stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false) + stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta), false, false).ReadStanza() p5 := NewUdebPackageFromControlFile(stanza5) c.Check(p.GetField("$Source"), Equals, "alien-arena") @@ -372,13 +396,13 @@ func (s *PackageSuite) TestLinkFromPool(c *C) { p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs) - err := p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false) + err := p.LinkFromPool(publishedStorage, packagePool, "", "jessie", "non-free", "i386", false) c.Check(err, IsNil) c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb") c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena") p.IsSource = true - err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false) + err = p.LinkFromPool(publishedStorage, packagePool, "", "jessie", "non-free", "i386", false) c.Check(err, IsNil) c.Check(p.Extra()["Directory"], Equals, "pool/non-free/a/alien-arena") } @@ -490,3 +514,6 @@ Size: 29188 MD5sum: ae70341c4d96dcded89fa670bcfea31e SHA1: 9532ae4226a85805189a671ee0283f719d48a5ba SHA256: bbb3a2cb07f741c3995b6d4bb08d772d83582b93a0236d4ea7736bc0370fc320` + +const installerPackageMeta = `9d8bb14044dee520f4706ab197dfff10e9e39ecb3c1a402331712154e8284b2e ./MANIFEST.udebs +dab96042d8e25e0f6bbb8d7c5bd78543afb5eb31a4a8b122ece68ab197228028 ./udeb.list` diff --git a/deb/publish.go b/deb/publish.go index 4e5885400..8d76db0f3 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -563,7 +563,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP // For all architectures, pregenerate packages/sources files for _, arch := range p.Architectures { - indexes.PackageIndex(component, arch, false) + indexes.PackageIndex(component, arch, false, false) } if progress != nil { @@ -579,27 +579,22 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP progress.AddBar(1) } - matches := false for _, arch := range p.Architectures { if pkg.MatchesArchitecture(arch) { - matches = true + hadUdebs = hadUdebs || pkg.IsUdeb + err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, p.Distribution, component, arch, forceOverwrite) + if err != nil { + return err + } break } } - if matches { - hadUdebs = hadUdebs || pkg.IsUdeb - err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, component, forceOverwrite) - if err != nil { - return err - } - } - for _, arch := range p.Architectures { if pkg.MatchesArchitecture(arch) { var bufWriter *bufio.Writer - if !p.SkipContents { + if !p.SkipContents && !pkg.IsInstaller { key := fmt.Sprintf("%s-%v", arch, pkg.IsUdeb) contentIndex := contentIndexes[key] @@ -612,12 +607,12 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP contentIndex.Push(pkg, packagePool, progress) } - bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter() + bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb, pkg.IsInstaller).BufWriter() if err != nil { return err } - err = pkg.Stanza().WriteTo(bufWriter, pkg.IsSource, false) + err = pkg.Stanza().WriteTo(bufWriter, pkg.IsSource, false, pkg.IsInstaller) if err != nil { return err } @@ -670,7 +665,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP // For all architectures, pregenerate .udeb indexes for _, arch := range p.Architectures { - indexes.PackageIndex(component, arch, true) + indexes.PackageIndex(component, arch, true, false) } } @@ -690,7 +685,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP return fmt.Errorf("unable to get ReleaseIndex writer: %s", err) } - err = release.WriteTo(bufWriter, false, true) + err = release.WriteTo(bufWriter, false, true, false) if err != nil { return fmt.Errorf("unable to create Release file: %s", err) } @@ -748,7 +743,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP return err } - err = release.WriteTo(bufWriter, false, true) + err = release.WriteTo(bufWriter, false, true, false) if err != nil { return fmt.Errorf("unable to create Release file: %s", err) } diff --git a/deb/publish_test.go b/deb/publish_test.go index c28d1f5fe..4ca7e844f 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -113,7 +113,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) { s.reflist = NewPackageRefListFromPackageList(s.list) - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) repo.packageRefs = s.reflist s.factory.RemoteRepoCollection().Add(repo) @@ -314,8 +314,8 @@ func (s *PublishedRepoSuite) TestPublish(c *C) { rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release")) c.Assert(err, IsNil) - cfr := NewControlFileReader(rf) - st, err := cfr.ReadStanza(true) + cfr := NewControlFileReader(rf, true, false) + st, err := cfr.ReadStanza() c.Assert(err, IsNil) c.Check(st["Origin"], Equals, "ppa squeeze") @@ -325,24 +325,24 @@ func (s *PublishedRepoSuite) TestPublish(c *C) { pf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages")) c.Assert(err, IsNil) - cfr = NewControlFileReader(pf) + cfr = NewControlFileReader(pf, false, false) for i := 0; i < 3; i++ { - st, err = cfr.ReadStanza(false) + st, err = cfr.ReadStanza() c.Assert(err, IsNil) c.Check(st["Filename"], Equals, "pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb") } - st, err = cfr.ReadStanza(false) + st, err = cfr.ReadStanza() c.Assert(err, IsNil) c.Assert(st, IsNil) drf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release")) c.Assert(err, IsNil) - cfr = NewControlFileReader(drf) - st, err = cfr.ReadStanza(true) + cfr = NewControlFileReader(drf, true, false) + st, err = cfr.ReadStanza() c.Assert(err, IsNil) c.Check(st["Archive"], Equals, "squeeze") diff --git a/deb/remote.go b/deb/remote.go index d8152ed84..8e3fe9772 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -3,6 +3,7 @@ package deb import ( "bytes" "fmt" + "io" "log" "net/url" "os" @@ -67,6 +68,8 @@ type RemoteRepo struct { DownloadSources bool // Should we download .udebs? DownloadUdebs bool + // Should we download installer files? + DownloadInstaller bool // "Snapshot" of current list of packages packageRefs *PackageRefList // Parsed archived root @@ -77,16 +80,17 @@ type RemoteRepo struct { // NewRemoteRepo creates new instance of Debian remote repository with specified params func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string, - architectures []string, downloadSources bool, downloadUdebs bool) (*RemoteRepo, error) { + architectures []string, downloadSources bool, downloadUdebs bool, downloadInstaller bool) (*RemoteRepo, error) { result := &RemoteRepo{ - UUID: uuid.New(), - Name: name, - ArchiveRoot: archiveRoot, - Distribution: distribution, - Components: components, - Architectures: architectures, - DownloadSources: downloadSources, - DownloadUdebs: downloadUdebs, + UUID: uuid.New(), + Name: name, + ArchiveRoot: archiveRoot, + Distribution: distribution, + Components: components, + Architectures: architectures, + DownloadSources: downloadSources, + DownloadUdebs: downloadUdebs, + DownloadInstaller: downloadInstaller, } err := result.prepare() @@ -133,6 +137,9 @@ func (repo *RemoteRepo) String() string { if repo.DownloadUdebs { srcFlag += " [udeb]" } + if repo.DownloadInstaller { + srcFlag += " [installer]" + } distribution := repo.Distribution if distribution == "" { distribution = "./" @@ -236,6 +243,12 @@ func (repo *RemoteRepo) UdebPath(component string, architecture string) string { return fmt.Sprintf("%s/debian-installer/binary-%s/Packages", component, architecture) } +// InstallerPath returns path of Packages files for given component and +// architecture +func (repo *RemoteRepo) InstallerPath(component string, architecture string) string { + return fmt.Sprintf("%s/installer-%s/current/images/SHA256SUMS", component, architecture) +} + // PackageURL returns URL of package file relative to repository root // architecture func (repo *RemoteRepo) PackageURL(filename string) *url.URL { @@ -304,8 +317,8 @@ ok: defer release.Close() - sreader := NewControlFileReader(release) - stanza, err := sreader.ReadStanza(true) + sreader := NewControlFileReader(release, true, false) + stanza, err := sreader.ReadStanza() if err != nil { return err } @@ -402,7 +415,7 @@ ok: } // DownloadPackageIndexes downloads & parses package index files -func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory, +func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, verifier pgp.Verifier, collectionFactory *CollectionFactory, ignoreMismatch bool, maxTries int) error { if repo.packageList != nil { panic("packageList != nil") @@ -413,39 +426,81 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly. packagesPaths := [][]string{} if repo.IsFlat() { - packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary}) + packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary, "", ""}) if repo.DownloadSources { - packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource}) + packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource, "", ""}) } } else { for _, component := range repo.Components { for _, architecture := range repo.Architectures { - packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary}) + packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary, component, architecture}) if repo.DownloadUdebs { - packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb}) + packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb, component, architecture}) + } + if repo.DownloadInstaller { + packagesPaths = append(packagesPaths, []string{repo.InstallerPath(component, architecture), PackageTypeInstaller, component, architecture}) } } if repo.DownloadSources { - packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource}) + packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource, component, "source"}) } } } for _, info := range packagesPaths { - path, kind := info[0], info[1] + path, kind, component, architecture := info[0], info[1], info[2], info[3] packagesReader, packagesFile, err := http.DownloadTryCompression(d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch, maxTries) + + isInstaller := kind == PackageTypeInstaller if err != nil { - return err + if _, ok := err.(*http.NoCandidateFoundError); isInstaller && ok { + // checking of gpg file is only needed when checksums matches are required. + // otherwise there actually has been no candidate found and we can continue + if ignoreMismatch { + continue + } + + // some repos do not have installer hashsum file listed in release file but provide a separate gpg file + packagesReader, packagesFile, err = http.DownloadTryCompression(d, repo.IndexesRootURL(), path, repo.ReleaseFiles, true, maxTries) + if err != nil { + if _, ok := err.(*http.NoCandidateFoundError); ok { + // installer files are not available in all components and architectures + // so ignore it if not found + continue + } + + return err + } + + if verifier != nil { + var filesigReader io.Reader + filesigReader, _, err = http.DownloadTryCompression(d, repo.IndexesRootURL(), path+".gpg", repo.ReleaseFiles, true, maxTries) + if err != nil { + return err + } + + err = verifier.VerifyDetachedSignature(filesigReader, packagesReader, false) + if err != nil { + return err + } + + _, err = packagesFile.Seek(0, 0) + } + } + + if err != nil { + return err + } } defer packagesFile.Close() stat, _ := packagesFile.Stat() progress.InitBar(stat.Size(), true) - sreader := NewControlFileReader(packagesReader) + sreader := NewControlFileReader(packagesReader, false, isInstaller) for { - stanza, err := sreader.ReadStanza(false) + stanza, err := sreader.ReadStanza() if err != nil { return err } @@ -467,6 +522,11 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly. if err != nil { return err } + } else if kind == PackageTypeInstaller { + p, err = NewInstallerPackageFromControlFile(stanza, repo, component, architecture, d) + if err != nil { + return err + } } err = repo.packageList.Add(p) if err != nil { diff --git a/deb/remote_test.go b/deb/remote_test.go index 5fe9a72a6..6d3ba10d5 100644 --- a/deb/remote_test.go +++ b/deb/remote_test.go @@ -88,8 +88,8 @@ type RemoteRepoSuite struct { var _ = Suite(&RemoteRepoSuite{}) func (s *RemoteRepoSuite) SetUpTest(c *C) { - s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false) - s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false) + s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false, false) + s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false) s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile) s.progress = console.NewProgress() s.db, _ = database.NewOpenDB(c.MkDir()) @@ -106,7 +106,7 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) { } func (s *RemoteRepoSuite) TestInvalidURL(c *C) { - _, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false) + _, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false, false) c.Assert(err, ErrorMatches, ".*(hexadecimal escape in host|percent-encoded characters in host|invalid URL escape).*") } @@ -116,11 +116,11 @@ func (s *RemoteRepoSuite) TestFlatCreation(c *C) { c.Check(s.flat.Architectures, IsNil) c.Check(s.flat.Components, IsNil) - flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false) + flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false, false) c.Check(flat2.IsFlat(), Equals, true) c.Check(flat2.Distribution, Equals, "./binary/") - _, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false) + _, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false, false) c.Check(err, ErrorMatches, "components aren't supported for flat repos") } @@ -130,8 +130,9 @@ func (s *RemoteRepoSuite) TestString(c *C) { s.repo.DownloadSources = true s.repo.DownloadUdebs = true + s.repo.DownloadInstaller = true s.flat.DownloadSources = true - c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb]") + c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb] [installer]") c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./ [src]") } @@ -176,6 +177,10 @@ func (s *RemoteRepoSuite) TestSourcesPath(c *C) { c.Assert(s.repo.SourcesPath("main"), Equals, "main/source/Sources") } +func (s *RemoteRepoSuite) TestInstallerPath(c *C) { + c.Assert(s.repo.InstallerPath("main", "amd64"), Equals, "main/installer-amd64/current/images/SHA256SUMS") +} + func (s *RemoteRepoSuite) TestFlatBinaryPath(c *C) { c.Assert(s.flat.FlatBinaryPath(), Equals, "Packages") } @@ -230,13 +235,13 @@ func (s *RemoteRepoSuite) TestFetchNullVerifier2(c *C) { } func (s *RemoteRepoSuite) TestFetchWrongArchitecture(c *C) { - s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false) + s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false, false) err := s.repo.Fetch(s.downloader, nil) c.Assert(err, ErrorMatches, "architecture xyz not available in repo.*") } func (s *RemoteRepoSuite) TestFetchWrongComponent(c *C) { - s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false) + s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false, false) err := s.repo.Fetch(s.downloader, nil) c.Assert(err, ErrorMatches, "component xyz not available in repo.*") } @@ -271,7 +276,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404}) s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile) - err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1) + err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1) c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) @@ -298,7 +303,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404}) s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile) - err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1) + err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1) c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) @@ -319,7 +324,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404}) s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile) - err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1) + err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1) c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) @@ -347,7 +352,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404}) s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile) - err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1) + err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1) c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) @@ -391,7 +396,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404}) s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile) - err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1) + err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1) c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) @@ -416,7 +421,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404}) s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile) - err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1) + err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1) c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) @@ -440,7 +445,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { err := s.flat.Fetch(downloader, nil) c.Assert(err, IsNil) - err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1) + err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1) c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) @@ -468,7 +473,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { err = s.flat.Fetch(downloader, nil) c.Assert(err, IsNil) - err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1) + err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1) c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) @@ -490,7 +495,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { err = s.flat.Fetch(downloader, nil) c.Assert(err, IsNil) - err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1) + err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1) c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) @@ -521,7 +526,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { err := s.flat.Fetch(downloader, nil) c.Assert(err, IsNil) - err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1) + err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1) c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) @@ -567,7 +572,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { err = s.flat.Fetch(downloader, nil) c.Assert(err, IsNil) - err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1) + err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1) c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) @@ -593,7 +598,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { err = s.flat.Fetch(downloader, nil) c.Assert(err, IsNil) - err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1) + err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1) c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) @@ -628,7 +633,7 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) { _, err := s.collection.ByName("yandex") c.Assert(err, ErrorMatches, "*.not found") - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) c.Assert(s.collection.Add(repo), IsNil) c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists") @@ -646,7 +651,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) { _, err := s.collection.ByUUID("some-uuid") c.Assert(err, ErrorMatches, "*.not found") - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) c.Assert(s.collection.Add(repo), IsNil) r, err := s.collection.ByUUID(repo.UUID) @@ -655,7 +660,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) { } func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) { - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) c.Assert(s.collection.Update(repo), IsNil) collection := NewRemoteRepoCollection(s.db) @@ -676,7 +681,7 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) { } func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) { - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) s.collection.Add(repo) count := 0 @@ -698,10 +703,10 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) { } func (s *RemoteRepoCollectionSuite) TestDrop(c *C) { - repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) s.collection.Add(repo1) - repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false) + repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false, false) s.collection.Add(repo2) r1, _ := s.collection.ByUUID(repo1.UUID) diff --git a/deb/snapshot_test.go b/deb/snapshot_test.go index ba1bb2479..a9d845dc4 100644 --- a/deb/snapshot_test.go +++ b/deb/snapshot_test.go @@ -17,7 +17,7 @@ var _ = Suite(&SnapshotSuite{}) func (s *SnapshotSuite) SetUpTest(c *C) { s.SetUpPackages() - s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) s.repo.packageRefs = s.reflist } @@ -116,11 +116,11 @@ func (s *SnapshotCollectionSuite) SetUpTest(c *C) { s.collection = NewSnapshotCollection(s.db) s.SetUpPackages() - s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) s.repo1.packageRefs = s.reflist s.snapshot1, _ = NewSnapshotFromRepository("snap1", s.repo1) - s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false) + s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false) s.repo2.packageRefs = s.reflist s.snapshot2, _ = NewSnapshotFromRepository("snap2", s.repo2) @@ -200,7 +200,7 @@ func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) { c.Check(s.collection.ByRemoteRepoSource(s.repo1), DeepEquals, []*Snapshot{s.snapshot1}) c.Check(s.collection.ByRemoteRepoSource(s.repo2), DeepEquals, []*Snapshot{s.snapshot2}) - repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false) + repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false) c.Check(s.collection.ByRemoteRepoSource(repo3), DeepEquals, []*Snapshot(nil)) } diff --git a/files/public.go b/files/public.go index 4147da1a4..f8bfe64ec 100644 --- a/files/public.go +++ b/files/public.go @@ -117,10 +117,11 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress // sourcePath is a relative path to package file in package pool // // LinkFromPool returns relative path for the published file to be included in package index -func (storage *PublishedStorage) LinkFromPool(publishedDirectory, baseName string, sourcePool aptly.PackagePool, +func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { - poolPath := filepath.Join(storage.rootPath, publishedDirectory) + baseName := filepath.Base(fileName) + poolPath := filepath.Join(storage.rootPath, publishedDirectory, filepath.Dir(fileName)) err := os.MkdirAll(poolPath, 0777) if err != nil { diff --git a/http/compression.go b/http/compression.go index b0eab5565..344d370f6 100644 --- a/http/compression.go +++ b/http/compression.go @@ -3,7 +3,6 @@ package http import ( "compress/bzip2" "compress/gzip" - "fmt" "io" "net/url" "os" @@ -89,7 +88,7 @@ func DownloadTryCompression(downloader aptly.Downloader, baseURL *url.URL, path } if err == nil { - err = fmt.Errorf("no candidates for %s found", baseURL.ResolveReference(&url.URL{Path: path})) + return nil, nil, &NoCandidateFoundError{URL: baseURL.ResolveReference(&url.URL{Path: path})} } return nil, nil, err } diff --git a/http/download.go b/http/download.go index 07a51009e..00e580999 100644 --- a/http/download.go +++ b/http/download.go @@ -61,20 +61,38 @@ func (downloader *downloaderImpl) GetProgress() aptly.Progress { return downloader.progress } +// GetLength of given url +func (downloader *downloaderImpl) GetLength(url string) (int64, error) { + req, err := downloader.newRequest("HEAD", url) + if err != nil { + return -1, err + } + + resp, err := downloader.client.Do(req) + if err != nil { + return -1, errors.Wrap(err, url) + } + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return -1, &Error{Code: resp.StatusCode, URL: url} + } + + if resp.ContentLength < 0 { + return -1, fmt.Errorf("Could not determine length of %s", url) + } + + return resp.ContentLength, nil +} + // Download starts new download task func (downloader *downloaderImpl) Download(url string, destination string) error { return downloader.DownloadWithChecksum(url, destination, nil, false, 1) } -// DownloadWithChecksum starts new download task with checksum verification -func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string, - expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error { - - downloader.progress.Printf("Downloading %s...\n", url) - - req, err := http.NewRequest("GET", url, nil) +func (downloader *downloaderImpl) newRequest(method, url string) (*http.Request, error) { + req, err := http.NewRequest(method, url, nil) if err != nil { - return errors.Wrap(err, url) + return nil, errors.Wrap(err, url) } req.Close = true @@ -84,6 +102,16 @@ func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination s req.URL.RawQuery = "" } + return req, nil +} + +// DownloadWithChecksum starts new download task with checksum verification +func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string, + expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error { + + downloader.progress.Printf("Downloading %s...\n", url) + req, err := downloader.newRequest("GET", url) + var temppath string for maxTries > 0 { temppath, err = downloader.download(req, url, destination, expected, ignoreMismatch) diff --git a/http/fake.go b/http/fake.go index 19bf0af49..2f9731f0e 100644 --- a/http/fake.go +++ b/http/fake.go @@ -59,8 +59,17 @@ func (f *FakeDownloader) Empty() bool { return len(f.expected) == 0 } -// DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification -func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error { +// GetLength returns content length of given url +func (f *FakeDownloader) GetLength(url string) (int64, error) { + expectation, err := f.getExpectedRequest(url) + if err != nil { + return -1, err + } + + return int64(len(expectation.Response)), nil +} + +func (f *FakeDownloader) getExpectedRequest(url string) (*expectedRequest, error) { var expectation expectedRequest if len(f.expected) > 0 && f.expected[0].URL == url { expectation, f.expected = f.expected[0], f.expected[1:] @@ -68,14 +77,24 @@ func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, expec expectation = f.anyExpected[url] delete(f.anyExpected, url) } else { - return fmt.Errorf("unexpected request for %s", url) + return nil, fmt.Errorf("unexpected request for %s", url) } if expectation.Err != nil { - return expectation.Err + return nil, expectation.Err + } + + return &expectation, nil +} + +// DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification +func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error { + expectation, err := f.getExpectedRequest(url) + if err != nil { + return err } - err := os.MkdirAll(filepath.Dir(filename), 0755) + err = os.MkdirAll(filepath.Dir(filename), 0755) if err != nil { return err } diff --git a/http/http.go b/http/http.go index df6ccc50e..480a7830f 100644 --- a/http/http.go +++ b/http/http.go @@ -3,6 +3,7 @@ package http import ( "fmt" + "net/url" ) // Error is download error connected to HTTP code @@ -15,3 +16,13 @@ type Error struct { func (e *Error) Error() string { return fmt.Sprintf("HTTP code %d while fetching %s", e.Code, e.URL) } + +// NoCandidateFoundError indicates that now candidate of given url could be found +type NoCandidateFoundError struct { + URL *url.URL +} + +// Error message +func (e *NoCandidateFoundError) Error() string { + return fmt.Sprintf("no candidates for %s found", e.URL) +} diff --git a/man/aptly.1 b/man/aptly.1 index 179aa667e..4f4953a53 100644 --- a/man/aptly.1 +++ b/man/aptly.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "APTLY" "1" "September 2017" "" "" +.TH "APTLY" "1" "November 2017" "" "" . .SH "NAME" \fBaptly\fR \- Debian repository management tool @@ -482,6 +482,10 @@ disable verification of Release file signatures gpg keyring to use when verifying Release file (could be specified multiple times) . .TP +\-\fBwith\-installer\fR +download additional not packaged installer files +. +.TP \-\fBwith\-sources\fR download source packages in addition to binary packages . diff --git a/system/files/ubuntu-archive-keyring.gpg b/system/files/ubuntu-archive-keyring.gpg new file mode 100644 index 000000000..b18548d05 Binary files /dev/null and b/system/files/ubuntu-archive-keyring.gpg differ diff --git a/system/lib.py b/system/lib.py index 619170d58..bc0c392fc 100644 --- a/system/lib.py +++ b/system/lib.py @@ -83,6 +83,7 @@ class BaseTest(object): fixtureDBDir = os.path.join(os.environ["HOME"], "aptly-fixture-db") fixturePoolDir = os.path.join(os.environ["HOME"], "aptly-fixture-pool") fixtureGpgKeys = ["debian-archive-keyring.gpg", + "ubuntu-archive-keyring.gpg", "launchpad.key", "flat.key", "pagerduty.key", diff --git a/system/t03_help/MirrorCreateHelpTest_gold b/system/t03_help/MirrorCreateHelpTest_gold index 8830ee1e2..10e191c97 100644 --- a/system/t03_help/MirrorCreateHelpTest_gold +++ b/system/t03_help/MirrorCreateHelpTest_gold @@ -28,6 +28,7 @@ Options: -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation) -ignore-signatures: disable verification of Release file signatures -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times) + -with-installer: download additional not packaged installer files -with-sources: download source packages in addition to binary packages -with-udebs: download .udeb packages (Debian installer support) diff --git a/system/t03_help/MirrorCreateTest_gold b/system/t03_help/MirrorCreateTest_gold index ced808083..7b138a518 100644 --- a/system/t03_help/MirrorCreateTest_gold +++ b/system/t03_help/MirrorCreateTest_gold @@ -19,6 +19,7 @@ Options: -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation) -ignore-signatures: disable verification of Release file signatures -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times) + -with-installer: download additional not packaged installer files -with-sources: download source packages in addition to binary packages -with-udebs: download .udeb packages (Debian installer support) ERROR: unable to parse command diff --git a/system/t03_help/WrongFlagTest_gold b/system/t03_help/WrongFlagTest_gold index 8824d9ae2..811e4d646 100644 --- a/system/t03_help/WrongFlagTest_gold +++ b/system/t03_help/WrongFlagTest_gold @@ -20,6 +20,7 @@ Options: -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation) -ignore-signatures: disable verification of Release file signatures -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times) + -with-installer: download additional not packaged installer files -with-sources: download source packages in addition to binary packages -with-udebs: download .udeb packages (Debian installer support) ERROR: unable to parse flags diff --git a/system/t04_mirror/UpdateMirror23Test_gold b/system/t04_mirror/UpdateMirror23Test_gold new file mode 100644 index 000000000..ace39f3ea --- /dev/null +++ b/system/t04_mirror/UpdateMirror23Test_gold @@ -0,0 +1,32 @@ + + +Applying filter... +Building download queue... +Download queue: 12 items (25.14 MiB) +Downloading & parsing package files... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/InRelease... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/Release... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/Release.gpg... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/binary-s390x/Packages.bz2... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/MANIFEST... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/MANIFEST.udebs... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/MD5SUMS... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/SHA256SUMS... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/generic/debian.exec... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/generic/initrd.debian... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/generic/kernel.debian... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/generic/parmfile.debian... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/tape/initrd.debian... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/tape/kernel.debian-nolabel... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/tape/kernel.debian... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/tape/parmfile.debian... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/udeb.list... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/non-free/binary-s390x/Packages.bz2... +Mirror `wheezy` has been successfully updated. +Packages filtered: 35003 -> 1. +gpgv: Good signature from "Debian Archive Automatic Signing Key (7.0/wheezy) " +gpgv: Good signature from "Debian Archive Automatic Signing Key (8/jessie) " +gpgv: Good signature from "Wheezy Stable Release Key " +gpgv: RSA key ID 2B90D010 +gpgv: RSA key ID 46925553 +gpgv: RSA key ID 65FFB764 \ No newline at end of file diff --git a/system/t04_mirror/UpdateMirror24Test_gold b/system/t04_mirror/UpdateMirror24Test_gold new file mode 100644 index 000000000..f8556e515 --- /dev/null +++ b/system/t04_mirror/UpdateMirror24Test_gold @@ -0,0 +1,66 @@ + + +Applying filter... +Building download queue... +Download queue: 40 items (178.88 MiB) +Downloading & parsing package files... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/InRelease... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/binary-amd64/Packages.bz2... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/MANIFEST... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/MANIFEST.udebs... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/SHA256SUMS... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/SHA256SUMS.bz2... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/SHA256SUMS.gpg... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/SHA256SUMS.gpg.bz2... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/SHA256SUMS.gpg.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/SHA256SUMS.gpg.xz... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/SHA256SUMS.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/SHA256SUMS.xz... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/cdrom/debian-cd_info.tar.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/cdrom/initrd.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/cdrom/vmlinuz... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/cdrom/xen/xm-debian.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/hd-media/boot.img.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/hd-media/initrd.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/hd-media/vmlinuz... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/boot.img.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/mini.iso... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/netboot.tar.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/adtxt.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/exithelp.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f1.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f10.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f2.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f3.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f4.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f5.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f6.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f7.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f8.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f9.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/ldlinux.c32... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/libcom32.c32... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/libutil.c32... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/menu.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/prompt.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/rqtxt.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/splash.png... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/stdmenu.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/syslinux.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/txt.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/vesamenu.c32... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/initrd.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/linux... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/pxelinux.0... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/netboot/xen/xm-debian.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/vivid/main/installer-amd64/current/images/udeb.list... +Mirror `vivid` has been successfully updated. +Packages filtered: 8454 -> 1. +gpgv: Good signature from "Ubuntu Archive Automatic Signing Key (2012) " +gpgv: Good signature from "Ubuntu Archive Automatic Signing Key (2012) " +gpgv: Good signature from "Ubuntu Archive Automatic Signing Key " +gpgv: Good signature from "Ubuntu Archive Automatic Signing Key " +gpgv: DSA key ID 437D05B5 +gpgv: RSA key ID C0B21F32 +gpgv: DSA key ID 437D05B5 +gpgv: RSA key ID C0B21F32 \ No newline at end of file diff --git a/system/t04_mirror/update.py b/system/t04_mirror/update.py index 865eaf960..f7a3d2cbc 100644 --- a/system/t04_mirror/update.py +++ b/system/t04_mirror/update.py @@ -373,3 +373,35 @@ class UpdateMirror22Test(BaseTest): ] runCmd = "aptly mirror update --keyring=aptlytest.gpg libnvidia-container" outputMatchPrepare = filterOutSignature + + +class UpdateMirror23Test(BaseTest): + """ + update mirrors: update with installer + """ + longTest = False + fixtureGpg = True + fixtureCmds = [ + "aptly -architectures=s390x mirror create -keyring=aptlytest.gpg -filter='installer' -with-installer wheezy http://mirror.yandex.ru/debian/ wheezy main non-free", + ] + runCmd = "aptly mirror update -keyring=aptlytest.gpg wheezy" + outputMatchPrepare = filterOutSignature + + def output_processor(self, output): + return "\n".join(sorted(output.split("\n"))) + + +class UpdateMirror24Test(BaseTest): + """ + update mirrors: update with installer with separate gpg file + """ + longTest = False + fixtureGpg = True + fixtureCmds = [ + "aptly -architectures=amd64 mirror create -keyring=aptlytest.gpg -filter='installer' -with-installer vivid http://mirror.yandex.ru/ubuntu/ vivid main", + ] + runCmd = "aptly mirror update -keyring=aptlytest.gpg vivid" + outputMatchPrepare = filterOutSignature + + def output_processor(self, output): + return "\n".join(sorted(output.split("\n"))) diff --git a/system/t06_publish/PublishSnapshot38Test_gold b/system/t06_publish/PublishSnapshot38Test_gold new file mode 100644 index 000000000..0365afcb4 --- /dev/null +++ b/system/t06_publish/PublishSnapshot38Test_gold @@ -0,0 +1,13 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: + +Snapshot wheezy has been successfully published. +Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing. +Now you can add following line to apt sources: + deb http://your-server/ wheezy main +Don't forget to add your GPG key to apt with apt-key. + +You can also use `aptly serve` to publish your repositories over HTTP quickly. diff --git a/system/t06_publish/PublishSnapshot38Test_installer_s390x b/system/t06_publish/PublishSnapshot38Test_installer_s390x new file mode 100644 index 000000000..dfbcaf299 --- /dev/null +++ b/system/t06_publish/PublishSnapshot38Test_installer_s390x @@ -0,0 +1,12 @@ +a29b0c17f98afb5ebc5a65bd03411e430dd372223d931b2c2441604fd6942472 ./MD5SUMS +a5a8e9029fc31df4a892644524a28c7d31f12a012d72cf1ac5672178b982ecac ./udeb.list +d592bca155b709066bcd48070358e6a2dd4f401c09f594b37b15caaf9fc45d80 ./tape/parmfile.debian +5e4d45f22a87e36d21685043f85106a6380b3ae7fbdccff70097ccbf6757a287 ./tape/kernel.debian-nolabel +57afafa42e61248d96e9d26efb2db075a17f484c51445c6420a32220b05c1fb2 ./tape/initrd.debian +0ca15998b2156af0a2ecffc771f9a418b4918956049fac4250c01d163b143006 ./tape/kernel.debian +1a907f670fc825f70827608ec32bb8a194971d0e1126d1caf33de87fc27d0d08 ./MANIFEST.udebs +62ba2aaafac85c163c3cbf25ee52724a59b12d805382a0122080b6a86e229b0a ./MANIFEST +d592bca155b709066bcd48070358e6a2dd4f401c09f594b37b15caaf9fc45d80 ./generic/parmfile.debian +48d2cbebbc8582f546232c1acf68b1b73125731441fda528f8d71795a60e14a4 ./generic/debian.exec +5194c4cfc1d527ce6aa45ced5a7f5102a3c85851ac8409c32b53c1004d0717da ./generic/kernel.debian +57afafa42e61248d96e9d26efb2db075a17f484c51445c6420a32220b05c1fb2 ./generic/initrd.debian diff --git a/system/t06_publish/snapshot.py b/system/t06_publish/snapshot.py index 3a2af0f6e..5335ba007 100644 --- a/system/t06_publish/snapshot.py +++ b/system/t06_publish/snapshot.py @@ -1016,3 +1016,25 @@ class PublishSnapshot37Test(BaseTest): ] runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec wheezy" gold_processor = BaseTest.expand_environ + + +class PublishSnapshot38Test(BaseTest): + """ + publish snapshot: mirror with installer + """ + fixtureGpg = True + fixtureCmds = [ + "aptly -architectures=s390x mirror create -keyring=aptlytest.gpg -filter='installer' -with-installer wheezy http://mirror.yandex.ru/debian/ wheezy main", + "aptly mirror update -keyring=aptlytest.gpg wheezy", + "aptly snapshot create wheezy from mirror wheezy", + ] + runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec wheezy" + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishSnapshot38Test, self).check() + self.check_exists('public/dists/wheezy/main/installer-s390x/current/images/SHA256SUMS') + self.check_exists('public/dists/wheezy/main/installer-s390x/current/images/generic/debian.exec') + self.check_exists('public/dists/wheezy/main/installer-s390x/current/images/MANIFEST') + + self.check_file_contents('public/dists/wheezy/main/installer-s390x/current/images/SHA256SUMS', "installer_s390x", match_prepare=sorted_processor)