Commit 9141f1b3 authored by Sébastien Delafond's avatar Sébastien Delafond

Imported Upstream version 0.9.5

parent fbac9260
......@@ -572,5 +572,17 @@ _aptly()
;;
esac
;;
"db")
case "$subcmd" in
"cleanup")
if [[ $numargs -eq 0 ]]; then
if [[ "$cur" == -* ]]; then
COMPREPLY=($(compgen -W "-dry-run -verbose" -- ${cur}))
fi
return 0
fi
;;
esac
;;
esac
} && complete -F _aptly aptly
......@@ -312,6 +312,8 @@ func apiPublishUpdateSwitch(c *gin.Context) {
// DELETE /publish/:prefix/:distribution
func apiPublishDrop(c *gin.Context) {
force := c.Request.URL.Query().Get("force") == "1"
param := parseEscapedPath(c.Params.ByName("prefix"))
storage, prefix := deb.ParsePrefix(param)
distribution := c.Params.ByName("distribution")
......@@ -326,7 +328,7 @@ func apiPublishDrop(c *gin.Context) {
defer collection.Unlock()
err := collection.Remove(context, storage, prefix, distribution,
context.CollectionFactory(), context.Progress())
context.CollectionFactory(), context.Progress(), force)
if err != nil {
c.Fail(500, fmt.Errorf("unable to drop: %s", err))
return
......
package aptly
// Version of aptly
const Version = "0.9.1"
const Version = "0.9.5"
// Enable debugging features?
const EnableDebug = false
......@@ -6,6 +6,7 @@ import (
"github.com/smira/aptly/utils"
"github.com/smira/commander"
"sort"
"strings"
)
// aptly db cleanup
......@@ -17,51 +18,112 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
return commander.ErrCommandError
}
verbose := context.Flags().Lookup("verbose").Value.Get().(bool)
dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool)
// collect information about references packages...
existingPackageRefs := deb.NewPackageRefList()
context.Progress().Printf("Loading mirrors, local repos, snapshots and published repos...\n")
// used only in verbose mode to report package use source
packageRefSources := map[string][]string{}
context.Progress().ColoredPrintf("@{w!}Loading mirrors, local repos, snapshots and published repos...@|")
if verbose {
context.Progress().ColoredPrintf("@{y}Loading mirrors:@|")
}
err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
if verbose {
context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
}
err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
if err != nil {
return err
}
if repo.RefList() != nil {
existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
if verbose {
description := fmt.Sprintf("mirror %s", repo.Name)
repo.RefList().ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
}
}
return nil
})
if err != nil {
return err
}
if verbose {
context.Progress().ColoredPrintf("@{y}Loading local repos:@|")
}
err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
if verbose {
context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
}
err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
if err != nil {
return err
}
if repo.RefList() != nil {
existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
if verbose {
description := fmt.Sprintf("local repo %s", repo.Name)
repo.RefList().ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
}
}
return nil
})
if err != nil {
return err
}
if verbose {
context.Progress().ColoredPrintf("@{y}Loading snapshots:@|")
}
err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
if verbose {
context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
}
err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
if err != nil {
return err
}
existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
if verbose {
description := fmt.Sprintf("snapshot %s", snapshot.Name)
snapshot.RefList().ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
}
return nil
})
if err != nil {
return err
}
if verbose {
context.Progress().ColoredPrintf("@{y}Loading published repositories:@|")
}
err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
if verbose {
context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution)
}
if published.SourceKind != "local" {
return nil
}
......@@ -72,6 +134,14 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
for _, component := range published.Components() {
existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
if verbose {
description := fmt.Sprintf("published repository %s:%s/%s component %s",
published.Storage, published.Prefix, published.Distribution, component)
published.RefList(component).ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
}
}
return nil
})
......@@ -80,38 +150,65 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
}
// ... and compare it to the list of all packages
context.Progress().Printf("Loading list of all packages...\n")
context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
toDelete := allPackageRefs.Substract(existingPackageRefs)
// delete packages that are no longer referenced
context.Progress().Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len())
context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len())
// database can't err as collection factory already constructed
db, _ := context.Database()
db.StartBatch()
err = toDelete.ForEach(func(ref []byte) error {
return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
})
if err != nil {
return err
}
err = db.FinishBatch()
if err != nil {
return fmt.Errorf("unable to write to DB: %s", err)
if toDelete.Len() > 0 {
if verbose {
context.Progress().ColoredPrintf("@{r}List of package keys to delete:@|")
err = toDelete.ForEach(func(ref []byte) error {
context.Progress().ColoredPrintf(" - @{r}%s@|", string(ref))
return nil
})
if err != nil {
return err
}
}
if !dryRun {
db.StartBatch()
err = toDelete.ForEach(func(ref []byte) error {
return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
})
if err != nil {
return err
}
err = db.FinishBatch()
if err != nil {
return fmt.Errorf("unable to write to DB: %s", err)
}
} else {
context.Progress().ColoredPrintf("@{y!}Skipped deletion, as -dry-run has been requested.@|")
}
}
// now, build a list of files that should be present in Repository (package pool)
context.Progress().Printf("Building list of files referenced by packages...\n")
context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
referencedFiles := make([]string, 0, existingPackageRefs.Len())
context.Progress().InitBar(int64(existingPackageRefs.Len()), false)
err = existingPackageRefs.ForEach(func(key []byte) error {
pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
if err2 != nil {
return err2
tail := ""
if verbose {
tail = fmt.Sprintf(" (sources: %s)", strings.Join(packageRefSources[string(key)], ", "))
}
if dryRun {
context.Progress().ColoredPrintf("@{r!}Unresolvable package reference, skipping (-dry-run): %s: %s%s",
string(key), err2, tail)
return nil
}
return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
}
paths, err2 := pkg.FilepathList(context.PackagePool())
if err2 != nil {
......@@ -130,7 +227,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
context.Progress().ShutdownBar()
// build a list of files in the package pool
context.Progress().Printf("Building list of files in package pool...\n")
context.Progress().ColoredPrintf("@{w!}Building list of files in package pool...@|")
existingFiles, err := context.PackagePool().FilepathList(context.Progress())
if err != nil {
return fmt.Errorf("unable to collect file paths: %s", err)
......@@ -140,28 +237,43 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
// delete files that are no longer referenced
context.Progress().Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete))
context.Progress().ColoredPrintf("@{r!}Deleting unreferenced files (%d)...@|", len(filesToDelete))
if len(filesToDelete) > 0 {
context.Progress().InitBar(int64(len(filesToDelete)), false)
if verbose {
context.Progress().ColoredPrintf("@{r}List of files to be deleted:@|")
for _, file := range filesToDelete {
context.Progress().ColoredPrintf(" - @{r}%s@|", file)
}
}
var size, totalSize int64
for _, file := range filesToDelete {
size, err = context.PackagePool().Remove(file)
if err != nil {
return err
if !dryRun {
context.Progress().InitBar(int64(len(filesToDelete)), false)
var size, totalSize int64
for _, file := range filesToDelete {
size, err = context.PackagePool().Remove(file)
if err != nil {
return err
}
context.Progress().AddBar(1)
totalSize += size
}
context.Progress().ShutdownBar()
context.Progress().AddBar(1)
totalSize += size
context.Progress().ColoredPrintf("@{w!}Disk space freed: %s...@|", utils.HumanBytes(totalSize))
} else {
context.Progress().ColoredPrintf("@{y!}Skipped file deletion, as -dry-run has been requested.@|")
}
context.Progress().ShutdownBar()
context.Progress().Printf("Disk space freed: %s...\n", utils.HumanBytes(totalSize))
}
context.Progress().Printf("Compacting database...\n")
err = db.CompactDB()
if !dryRun {
context.Progress().ColoredPrintf("@{w!}Compacting database...@|")
err = db.CompactDB()
} else {
context.Progress().ColoredPrintf("@{y!}Skipped DB compaction, as -dry-run has been requested.@|")
}
return err
}
......@@ -181,5 +293,8 @@ Example:
`,
}
cmd.Flag.Bool("verbose", false, "be verbose when loading objects/removing them")
cmd.Flag.Bool("dry-run", false, "don't delete anything")
return cmd
}
......@@ -23,7 +23,7 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
storage, prefix := deb.ParsePrefix(param)
err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
context.CollectionFactory(), context.Progress())
context.CollectionFactory(), context.Progress(), context.Flags().Lookup("force-drop").Value.Get().(bool))
if err != nil {
return fmt.Errorf("unable to remove: %s", err)
}
......@@ -48,5 +48,7 @@ Example:
`,
}
cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up")
return cmd
}
......@@ -96,6 +96,10 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
return fmt.Errorf("unable to search: %s", err)
}
if result.Len() == 0 {
return fmt.Errorf("no results")
}
result.ForEach(func(p *deb.Package) error {
context.Progress().Printf("%s\n", p)
return nil
......
......@@ -133,19 +133,30 @@ func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
p.queue <- printTask{code: codePrint, message: color.Sprintf(msg, a...) + "\n"}
} else {
// stip color marks
var prev rune
var inColorMark, inCurly bool
msg = strings.Map(func(r rune) rune {
if prev == '@' {
prev = 0
if r == '@' {
return r
if inColorMark {
if inCurly {
if r == '}' {
inCurly = false
inColorMark = false
return -1
}
} else {
if r == '{' {
inCurly = true
} else if r == '@' {
return '@'
} else {
inColorMark = false
}
}
return -1
}
prev = r
if r == '@' {
inColorMark = true
return -1
}
return r
......
......@@ -5,6 +5,7 @@ import (
"errors"
"io"
"strings"
"unicode"
)
// Stanza or paragraph of Debian control file
......@@ -157,6 +158,35 @@ func init() {
multilineFields["MD5Sum"] = true
}
func canonicalCase(field string) string {
upper := strings.ToUpper(field)
switch upper {
case "SHA1", "SHA256", "SHA512":
return upper
case "MD5SUM":
return "MD5Sum"
case "NOTAUTOMATIC":
return "NotAutomatic"
case "BUTAUTOMATICUPGRADES":
return "ButAutomaticUpgrades"
}
startOfWord := true
return strings.Map(func(r rune) rune {
if startOfWord {
startOfWord = false
return unicode.ToUpper(r)
}
if r == '-' {
startOfWord = true
}
return unicode.ToLower(r)
}, field)
}
// ControlFileReader implements reading of control files stanza by stanza
type ControlFileReader struct {
scanner *bufio.Scanner
......@@ -195,7 +225,7 @@ func (c *ControlFileReader) ReadStanza() (Stanza, error) {
if len(parts) != 2 {
return nil, ErrMalformedStanza
}
lastField = parts[0]
lastField = canonicalCase(parts[0])
_, lastFieldMultiline = multilineFields[lastField]
if lastFieldMultiline {
stanza[lastField] = parts[1]
......
......@@ -123,6 +123,18 @@ func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
c.Assert(strings.HasPrefix(str, "Package: "), Equals, true)
}
func (s *ControlFileSuite) TestCanonicalCase(c *C) {
c.Check(canonicalCase("Package"), Equals, "Package")
c.Check(canonicalCase("package"), Equals, "Package")
c.Check(canonicalCase("pAckaGe"), Equals, "Package")
c.Check(canonicalCase("MD5Sum"), Equals, "MD5Sum")
c.Check(canonicalCase("SHA1"), Equals, "SHA1")
c.Check(canonicalCase("SHA256"), Equals, "SHA256")
c.Check(canonicalCase("Package-List"), Equals, "Package-List")
c.Check(canonicalCase("package-list"), Equals, "Package-List")
c.Check(canonicalCase("packaGe-lIst"), Equals, "Package-List")
}
func (s *ControlFileSuite) BenchmarkReadStanza(c *C) {
for i := 0; i < c.N; i++ {
reader := bytes.NewBufferString(controlFile)
......
......@@ -91,6 +91,24 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
continue
}
if p.Name == "" {
reporter.Warning("Empty package name on %s", file)
failedFiles = append(failedFiles, file)
continue
}
if p.Version == "" {
reporter.Warning("Empty version on %s", file)
failedFiles = append(failedFiles, file)
continue
}
if p.Architecture == "" {
reporter.Warning("Empty architecture on %s", file)
failedFiles = append(failedFiles, file)
continue
}
var checksums utils.ChecksumInfo
checksums, err = utils.ChecksumsForFile(file)
if err != nil {
......
......@@ -1000,7 +1000,8 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st
// Remove removes published repository, cleaning up directories, files
func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly.PublishedStorageProvider,
storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress) error {
storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress,
force bool) error {
repo, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
return err
......@@ -1041,7 +1042,9 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
err = collection.CleanupPrefixComponentFiles(repo.Prefix, cleanComponents,
publishedStorageProvider.GetPublishedStorage(storage), collectionFactory, progress)
if err != nil {
return err
if !force {
return fmt.Errorf("cleanup failed, use -force-drop to override: %s", err)
}
}
}
......
......@@ -740,7 +740,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
}
func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
c.Check(err, IsNil)
_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
......@@ -760,10 +760,10 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
c.Check(err, ErrorMatches, ".*not found")
err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil)
err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false)
c.Check(err, IsNil)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
......@@ -778,7 +778,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
}
func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil)
err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil, false)
c.Check(err, IsNil)
_, err = s.collection.ByStoragePrefixDistribution("", ".", "anaconda")
......@@ -800,7 +800,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
}
func (s *PublishedRepoRemoveSuite) TestRemoveRepo5(c *C) {
err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil)
err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil, false)
c.Check(err, IsNil)
_, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")
......
.\" generated with Ronn/v0.7.3
.\" http://github.com/rtomayko/ronn/tree/0.7.3
.
.TH "APTLY" "1" "February 2015" "" ""
.TH "APTLY" "1" "March 2015" "" ""
.
.SH "NAME"
\fBaptly\fR \- Debian repository management tool
......@@ -133,6 +133,10 @@ specifies paramaters for short PPA url expansion, if left blank they default to
\fBS3PublishEndpoints\fR
configuration of Amazon S3 publishing endpoints (see below)
.
.TP
\fBSwiftPublishEndpoints\fR
configuration of OpenStack Swift publishing endpoints (see below)
.
.SH "S3 PUBLISHING ENDPOINTS"
aptly could be configured to publish repository directly to Amazon S3\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings:
.
......@@ -1082,6 +1086,13 @@ $ aptly publish drop wheezy
.
.IP "" 0
.
.P
Options:
.
.TP
\-\fBforce\-drop\fR=false
remove published repository even if some files could not be cleaned up
.
.SH "LIST OF PUBLISHED REPOSITORIES"
\fBaptly\fR \fBpublish\fR \fBlist\fR
.
......@@ -1469,6 +1480,17 @@ Example:
.P
$ aptly db cleanup
.
.P
Options:
.
.TP
\-\fBdry\-run\fR=false
don\(cqt delete anything
.
.TP
\-\fBverbose\fR=false
be verbose when loading objects/removing them
.
.SH "RECOVER DB AFTER CRASH"
\fBaptly\fR \fBdb\fR \fBrecover\fR
.
......
......@@ -37,3 +37,13 @@ class SearchSnapshot4Test(BaseTest):
fixtureCmds = ["aptly snapshot create wheezy-main from mirror wheezy-main"]
outputMatchPrepare = lambda _, s: "\n".join(sorted(s.split("\n")))
runCmd = "aptly snapshot search -with-deps wheezy-main 'Name (nginx)'"
class SearchSnapshot5Test(BaseTest):
"""
search snapshot: no results
"""
fixtureDB = True
fixtureCmds = ["aptly snapshot create wheezy-main from mirror wheezy-main"]
runCmd = "aptly snapshot search -with-deps wheezy-main 'Name (no-such-package)'"
expectedCode = 1
Loading mirrors, local repos, snapshots and published repos...
Loading mirrors:
- wheezy-backports
- wheezy-updates-src
- wheezy-main
- wheezy-contrib
- sensu
- wheezy-main-src
- wheezy-backports-src
- wheezy-contrib-src
- wheezy-non-free
- wheezy-non-free-src
- wheezy-updates
Loading local repos:
Loading snapshots:
Loading published repositories:
Loading list of all packages...
Deleting unreferenced packages (7)...
List of package keys to delete:
- Pall gnuplot 4.6.1-1~maverick2 36650cfe603d11a1
- Pall gnuplot-doc 4.6.1-1~maverick2 10c388f966074d29
- Pamd64 gnuplot-nox 4.6.1-1~maverick2 336b8733c4444003
- Pamd64 gnuplot-x11 4.6.1-1~maverick2 7300d8122b81b641
- Pi386 gnuplot-nox 4.6.1-1~maverick2 17785995cf0f815
- Pi386 gnuplot-x11 4.6.1-1~maverick2 d42e1d0d2f23740
- Psource gnuplot 4.6.1-1~maverick2 b8cd36358f5db41f
Building list of files referenced by packages...
Building list of files in package pool...
Deleting unreferenced files (9)...
List of files to be deleted:
- 02/1d/gnuplot_4.6.1-1~maverick2.dsc
- 10/32/gnuplot_4.6.1-1~maverick2.debian.tar.gz
- 17/ab/gnuplot-x11_4.6.1-1~maverick2_amd64.deb
- 25/a5/gnuplot-doc_4.6.1-1~maverick2_all.deb
- 49/12/gnuplot_4.6.1-1~maverick2_all.deb
- 4c/9a/gnuplot_4.6.1.orig.tar.gz
- a7/ef/gnuplot-nox_4.6.1-1~maverick2_i386.deb
- db/55/gnuplot-nox_4.6.1-1~maverick2_amd64.deb
- fc/ad/gnuplot-x11_4.6.1-1~maverick2_i386.deb
Disk space freed: 10.85 MiB...
Compacting database...
Loading mirrors, local repos, snapshots and published repos...
Loading mirrors:
- wheezy-backports
- wheezy-updates-src
- wheezy-main
- wheezy-contrib
- sensu
- wheezy-main-src
- wheezy-backports-src
- wheezy-contrib-src
- wheezy-non-free
- wheezy-non-free-src
- wheezy-updates
Loading local repos:
Loading snapshots:
Loading published repositories:
Loading list of all packages...
Deleting unreferenced packages (7)...
List of package keys to delete:
- Pall gnuplot 4.6.1-1~maverick2 36650cfe603d11a1
- Pall gnuplot-doc 4.6.1-1~maverick2 10c388f966074d29
- Pamd64 gnuplot-nox 4.6.1-1~maverick2 336b8733c4444003
- Pamd64 gnuplot-x11 4.6.1-1~maverick2 7300d8122b81b641
- Pi386 gnuplot-nox 4.6.1-1~maverick2 17785995cf0f815
- Pi386 gnuplot-x11 4.6.1-1~maverick2 d42e1d0d2f23740
- Psource gnuplot 4.6.1-1~maverick2 b8cd36358f5db41f
Skipped deletion, as -dry-run has been requested.
Building list of files referenced by packages...