Skip to content

Commit

Permalink
Add support to mirror non package installer files
Browse files Browse the repository at this point in the history
  • Loading branch information
Oliver Sauder committed Nov 27, 2017
1 parent 10e0966 commit 9ab2be3
Show file tree
Hide file tree
Showing 35 changed files with 590 additions and 152 deletions.
2 changes: 2 additions & 0 deletions aptly/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,8 @@ type Downloader interface {
DownloadWithChecksum(url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error
// GetProgress returns Progress object
GetProgress() Progress
// GetLength of given url
GetLength(url string) (int64, error)
}

// ChecksumStorage is stores checksums in some (persistent) storage
Expand Down
2 changes: 1 addition & 1 deletion bash_completion.d/aptly
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ _aptly()
"create")
if [[ $numargs -eq 0 ]]; then
if [[ "$cur" == -* ]]; then
COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-sources -with-udebs" -- ${cur}))
COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
return 0
fi
fi
Expand Down
4 changes: 3 additions & 1 deletion cmd/mirror_create.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {

downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)

var (
mirrorName, archiveURL, distribution string
Expand All @@ -36,7 +37,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
}

repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
downloadSources, downloadUdebs)
downloadSources, downloadUdebs, downloadInstaller)
if err != nil {
return fmt.Errorf("unable to create mirror: %s", err)
}
Expand Down Expand Up @@ -94,6 +95,7 @@ Example:
}

cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
cmd.Flag.String("filter", "", "filter packages in mirror")
Expand Down
2 changes: 1 addition & 1 deletion cmd/mirror_update.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
}

context.Progress().Printf("Downloading & parsing package files...\n")
err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch, maxTries)
err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, context.CollectionFactory(), ignoreMismatch, maxTries)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/package_show.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
result := q.Query(context.CollectionFactory().PackageCollection())

err = result.ForEach(func(p *deb.Package) error {
p.Stanza().WriteTo(w, p.IsSource, false)
p.Stanza().WriteTo(w, p.IsSource, false, false)
w.Flush()
fmt.Printf("\n")

Expand Down
4 changes: 2 additions & 2 deletions deb/changes.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,8 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
text = input
}

reader := NewControlFileReader(text)
c.Stanza, err = reader.ReadStanza(false)
reader := NewControlFileReader(text, false, false)
c.Stanza, err = reader.ReadStanza()
if err != nil {
return err
}
Expand Down
8 changes: 4 additions & 4 deletions deb/deb.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,8 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
}

if tarHeader.Name == "./control" || tarHeader.Name == "control" {
reader := NewControlFileReader(untar)
stanza, err := reader.ReadStanza(false)
reader := NewControlFileReader(untar, false, false)
stanza, err := reader.ReadStanza()
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -127,8 +127,8 @@ func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error
text = file
}

reader := NewControlFileReader(text)
stanza, err := reader.ReadStanza(false)
reader := NewControlFileReader(text, false, false)
stanza, err := reader.ReadStanza()
if err != nil {
return nil, err
}
Expand Down
53 changes: 38 additions & 15 deletions deb/format.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,9 @@ var (
"Directory",
"Files",
}
canonicalOrderInstaller = []string{
"",
}
)

// Copy returns copy of Stanza
Expand All @@ -98,6 +101,9 @@ func (s Stanza) Copy() (result Stanza) {

func isMultilineField(field string, isRelease bool) bool {
switch field {
// file without a section
case "":
return true
case "Description":
return true
case "Files":
Expand Down Expand Up @@ -129,28 +135,36 @@ func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error
if !isMultilineField(field, isRelease) {
_, err = w.WriteString(field + ": " + value + "\n")
} else {
if !strings.HasSuffix(value, "\n") {
if field != "" && !strings.HasSuffix(value, "\n") {
value = value + "\n"
}

if field != "Description" {
if field != "Description" && field != "" {
value = "\n" + value
}
_, err = w.WriteString(field + ":" + value)

if field != "" {
_, err = w.WriteString(field + ":" + value)
} else {
_, err = w.WriteString(value)
}
}

return
}

// WriteTo saves stanza back to stream, modifying itself on the fly
func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease, isInstaller bool) error {
canonicalOrder := canonicalOrderBinary
if isSource {
canonicalOrder = canonicalOrderSource
}
if isRelease {
canonicalOrder = canonicalOrderRelease
}
if isInstaller {
canonicalOrder = canonicalOrderInstaller
}

for _, field := range canonicalOrder {
value, ok := s[field]
Expand All @@ -163,10 +177,13 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
}
}

for field, value := range s {
err := writeField(w, field, value, isRelease)
if err != nil {
return err
// no extra fields in installer
if !isInstaller {
for field, value := range s {
err := writeField(w, field, value, isRelease)
if err != nil {
return err
}
}
}

Expand Down Expand Up @@ -209,19 +226,25 @@ func canonicalCase(field string) string {

// ControlFileReader implements reading of control files stanza by stanza
type ControlFileReader struct {
scanner *bufio.Scanner
scanner *bufio.Scanner
isRelease bool
isInstaller bool
}

// NewControlFileReader creates ControlFileReader, it wraps with buffering
func NewControlFileReader(r io.Reader) *ControlFileReader {
return &ControlFileReader{scanner: bufio.NewScanner(bufio.NewReaderSize(r, 32768))}
func NewControlFileReader(r io.Reader, isRelease, isInstaller bool) *ControlFileReader {
return &ControlFileReader{
scanner: bufio.NewScanner(bufio.NewReaderSize(r, 32768)),
isRelease: isRelease,
isInstaller: isInstaller,
}
}

// ReadStanza reeads one stanza from control file
func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
func (c *ControlFileReader) ReadStanza() (Stanza, error) {
stanza := make(Stanza, 32)
lastField := ""
lastFieldMultiline := false
lastFieldMultiline := c.isInstaller

for c.scanner.Scan() {
line := c.scanner.Text()
Expand All @@ -234,7 +257,7 @@ func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
continue
}

if line[0] == ' ' || line[0] == '\t' {
if line[0] == ' ' || line[0] == '\t' || c.isInstaller {
if lastFieldMultiline {
stanza[lastField] += line + "\n"
} else {
Expand All @@ -246,7 +269,7 @@ func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
return nil, ErrMalformedStanza
}
lastField = canonicalCase(parts[0])
lastFieldMultiline = isMultilineField(lastField, isRelease)
lastFieldMultiline = isMultilineField(lastField, c.isRelease)
if lastFieldMultiline {
stanza[lastField] = parts[1]
if parts[1] != "" {
Expand Down
46 changes: 35 additions & 11 deletions deb/format_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ type ControlFileSuite struct {

var _ = Suite(&ControlFileSuite{})

const installerFile = `dab96042d8e25e0f6bbb8d7c5bd78543afb5eb31a4a8b122ece68ab197228028 ./udeb.list
9d8bb14044dee520f4706ab197dfff10e9e39ecb3c1a402331712154e8284b2e ./MANIFEST.udebs
`

const controlFile = `Package: bti
Binary: bti
Version: 032-1
Expand Down Expand Up @@ -82,15 +86,15 @@ func (s *ControlFileSuite) SetUpTest(c *C) {
}

func (s *ControlFileSuite) TestReadStanza(c *C) {
r := NewControlFileReader(s.reader)
r := NewControlFileReader(s.reader, false, false)

stanza1, err := r.ReadStanza(false)
stanza1, err := r.ReadStanza()
c.Assert(err, IsNil)

stanza2, err := r.ReadStanza(false)
stanza2, err := r.ReadStanza()
c.Assert(err, IsNil)

stanza3, err := r.ReadStanza(false)
stanza3, err := r.ReadStanza()
c.Assert(err, IsNil)
c.Assert(stanza3, IsNil)

Expand All @@ -102,27 +106,47 @@ func (s *ControlFileSuite) TestReadStanza(c *C) {
}

func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
r := NewControlFileReader(s.reader)
stanza, err := r.ReadStanza(false)
r := NewControlFileReader(s.reader, false, false)
stanza, err := r.ReadStanza()
c.Assert(err, IsNil)

buf := &bytes.Buffer{}
w := bufio.NewWriter(buf)
err = stanza.Copy().WriteTo(w, true, false)
err = stanza.Copy().WriteTo(w, true, false, false)
c.Assert(err, IsNil)
err = w.Flush()
c.Assert(err, IsNil)

str := buf.String()

r = NewControlFileReader(buf)
stanza2, err := r.ReadStanza(false)
r = NewControlFileReader(buf, false, false)
stanza2, err := r.ReadStanza()
c.Assert(err, IsNil)

c.Assert(stanza2, DeepEquals, stanza)
c.Assert(strings.HasPrefix(str, "Package: "), Equals, true)
}

func (s *ControlFileSuite) TestReadWriteInstallerStanza(c *C) {
s.reader = bytes.NewBufferString(installerFile)
r := NewControlFileReader(s.reader, false, true)
stanza, err := r.ReadStanza()
c.Assert(err, IsNil)

buf := &bytes.Buffer{}
w := bufio.NewWriter(buf)
err = stanza.Copy().WriteTo(w, false, false, true)
c.Assert(err, IsNil)
err = w.Flush()
c.Assert(err, IsNil)

r = NewControlFileReader(buf, false, true)
stanza2, err := r.ReadStanza()
c.Assert(err, IsNil)

c.Assert(stanza2, DeepEquals, stanza)
}

func (s *ControlFileSuite) TestCanonicalCase(c *C) {
c.Check(canonicalCase("Package"), Equals, "Package")
c.Check(canonicalCase("package"), Equals, "Package")
Expand All @@ -138,9 +162,9 @@ func (s *ControlFileSuite) TestCanonicalCase(c *C) {
func (s *ControlFileSuite) BenchmarkReadStanza(c *C) {
for i := 0; i < c.N; i++ {
reader := bytes.NewBufferString(controlFile)
r := NewControlFileReader(reader)
r := NewControlFileReader(reader, false, false)
for {
s, e := r.ReadStanza(false)
s, e := r.ReadStanza()
if s == nil && e == nil {
break
}
Expand Down
8 changes: 5 additions & 3 deletions deb/index_files.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,11 +155,11 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
}
}

func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
func (files *indexFiles) PackageIndex(component, arch string, udeb, installer bool) *indexFile {
if arch == ArchitectureSource {
udeb = false
}
key := fmt.Sprintf("pi-%s-%s-%v", component, arch, udeb)
key := fmt.Sprintf("pi-%s-%s-%v-%v", component, arch, udeb, installer)
file, ok := files.indexes[key]
if !ok {
var relativePath string
Expand All @@ -169,6 +169,8 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
} else {
if udeb {
relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages")
} else if installer {
relativePath = filepath.Join(component, fmt.Sprintf("installer-%s", arch), "current", "images", "SHA256SUMS")
} else {
relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
}
Expand All @@ -177,7 +179,7 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
file = &indexFile{
parent: files,
discardable: false,
compressable: true,
compressable: !installer,
signable: false,
relativePath: relativePath,
}
Expand Down
Loading

0 comments on commit 9ab2be3

Please sign in to comment.