reposync.py 6.81 KB
Newer Older
1 2
#!/usr/bin/python -tt

Seth Vidal's avatar
Seth Vidal committed
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# copyright 2006 Duke University
# author seth vidal

# sync all or the newest packages from a repo to the local path
# TODO:
#     have it print out list of changes
#     make it work with mirrorlists (silly, really)
Seth Vidal's avatar
Seth Vidal committed
23
#     man page/more useful docs
Seth Vidal's avatar
Seth Vidal committed
24
#     deal nicely with a package changing but not changing names (ie: replacement)
25 26
#     maybe have it iterate the dir, if it exists, and delete files not listed
#     in a repo
Seth Vidal's avatar
Seth Vidal committed
27

Seth Vidal's avatar
Seth Vidal committed
28 29 30 31 32 33 34
# criteria
# if a package is not the same and smaller then reget it
# if a package is not the same and larger, delete it and get it again
# always replace metadata files if they're not the same.



Seth Vidal's avatar
Seth Vidal committed
35 36
import os
import sys
37
import shutil
38

Seth Vidal's avatar
Seth Vidal committed
39 40 41 42 43 44 45 46
from optparse import OptionParser
from urlparse import urljoin

import yum
import yum.Errors
from yum.misc import getCacheDir
from yum.constants import *
from yum.packages import parsePackages
47
from yum.packageSack import ListPackageSack
Seth Vidal's avatar
Seth Vidal committed
48
import rpmUtils.arch
49
import logging
Seth Vidal's avatar
Seth Vidal committed
50 51 52 53 54 55 56 57 58 59 60 61 62 63

# for yum 2.4.X compat
def sortPkgObj(pkg1 ,pkg2):
    """sorts a list of yum package objects by name"""
    if pkg1.name > pkg2.name:
        return 1
    elif pkg1.name == pkg2.name:
        return 0
    else:
        return -1
        
class RepoSync(yum.YumBase):
    def __init__(self, opts):
        yum.YumBase.__init__(self)
64
        self.logger = logging.getLogger('yum.verbose.reposync')
Seth Vidal's avatar
Seth Vidal committed
65 66 67
        self.opts = opts

def parseArgs():
68 69 70 71 72 73 74
    usage = """
    Reposync is used to synchronize a remote yum repository to a local 
    directory using yum to retrieve the packages.
    
    %s [options]
    """ % sys.argv[0]

Seth Vidal's avatar
Seth Vidal committed
75 76 77
    parser = OptionParser(usage=usage)
    parser.add_option("-c", "--config", default='/etc/yum.conf',
        help='config file to use (defaults to /etc/yum.conf)')
Seth Vidal's avatar
Seth Vidal committed
78
    parser.add_option("-a", "--arch", default=None,
79
        help='act as if running the specified arch (default: current arch, note: does not override $releasever)')
Seth Vidal's avatar
Seth Vidal committed
80 81 82 83 84
    parser.add_option("-r", "--repoid", default=[], action='append',
        help="specify repo ids to query, can be specified multiple times (default is all enabled)")
    parser.add_option("-t", "--tempcache", default=False, action="store_true", 
        help="Use a temp dir for storing/accessing yum-cache")
    parser.add_option("-p", "--download_path", dest='destdir', 
85
        default=os.getcwd(), help="Path to download packages to: defaults to current dir")
Seth Vidal's avatar
Seth Vidal committed
86 87
    parser.add_option("-u", "--urls", default=False, action="store_true", 
        help="Just list urls of what would be downloaded, don't download")
88 89
    parser.add_option("-n", "--newest-only", dest='newest', default=False, action="store_true", 
        help="Download only newest packages per-repo")
Seth Vidal's avatar
Seth Vidal committed
90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
    parser.add_option("-q", "--quiet", default=False, action="store_true", 
        help="Output as little as possible")
        
    (opts, args) = parser.parse_args()
    return (opts, args)


def main():
# TODO/FIXME
# gpg/sha checksum them
    (opts, junk) = parseArgs()
    
    if not os.path.exists(opts.destdir) and not opts.urls:
        try:
            os.makedirs(opts.destdir)
        except OSError, e:
            print >> sys.stderr, "Error: Cannot create destination dir %s" % opts.destdir
            sys.exit(1)
    
    if not os.access(opts.destdir, os.W_OK) and not opts.urls:
        print >> sys.stderr, "Error: Cannot write to  destination dir %s" % opts.destdir
        sys.exit(1)
        
    my = RepoSync(opts=opts)
114
    my.doConfigSetup(fn=opts.config, init_plugins=False)
Seth Vidal's avatar
Seth Vidal committed
115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
    
    # do the happy tmpdir thing if we're not root
    if os.geteuid() != 0 or opts.tempcache:
        cachedir = getCacheDir()
        if cachedir is None:
            print >> sys.stderr, "Error: Could not make cachedir, exiting"
            sys.exit(50)
            
        my.repos.setCacheDir(cachedir)

    if len(opts.repoid) > 0:
        myrepos = []
        
        # find the ones we want
        for glob in opts.repoid:
            myrepos.extend(my.repos.findRepos(glob))
        
        # disable them all
        for repo in my.repos.repos.values():
            repo.disable()
        
        # enable the ones we like
        for repo in myrepos:
            repo.enable()

140
    my.doRepoSetup()
Seth Vidal's avatar
Seth Vidal committed
141
    my.doSackSetup(rpmUtils.arch.getArchList(opts.arch))
Seth Vidal's avatar
Seth Vidal committed
142 143 144
    
    download_list = []
    
145 146 147 148 149 150

    for repo in my.repos.listEnabled():
        local_repo_path = opts.destdir + '/' + repo.id
            
        reposack = ListPackageSack(my.pkgSack.returnPackages(repoid=repo.id))
            
151
        if opts.newest:
152
            download_list = reposack.returnNewestByNameArch()
153 154
        else:
            download_list = list(reposack)
Seth Vidal's avatar
Seth Vidal committed
155
        
156 157 158 159
        download_list.sort(sortPkgObj)
        for pkg in download_list:
            repo = my.repos.getRepo(pkg.repoid)
            remote = pkg.returnSimple('relativepath')
160 161 162 163 164
            local = local_repo_path + '/' + remote
            localdir = os.path.dirname(local)
            if not os.path.exists(localdir):
                os.makedirs(localdir)

165 166 167 168
            if (os.path.exists(local) and 
                str(os.path.getsize(local)) == pkg.returnSimple('packagesize')):
                
                if not opts.quiet:
169
                    my.logger.error("%s already exists and appears to be complete" % local)
170 171 172 173 174 175 176 177 178 179 180 181
                continue
    
            if opts.urls:
                url = urljoin(repo.urls[0],remote)
                print '%s' % url
                continue
    
            # make sure the repo subdir is here before we go on.
            if not os.path.exists(local_repo_path):
                try:
                    os.makedirs(local_repo_path)
                except IOError, e:
182
                    my.logger.error("Could not make repo subdir: %s" % e)
183
                    sys.exit(1)
Seth Vidal's avatar
Seth Vidal committed
184
            
185 186
            # Disable cache otherwise things won't download            
            repo.cache = 0
187 188 189
            if not opts.quiet:
                my.logger.info( 'Downloading %s' % os.path.basename(remote))
            pkg.localpath = local # Hack: to set the localpath we want.
190
            path = repo.getPackage(pkg)
Seth Vidal's avatar
Seth Vidal committed
191

192 193
            if not os.path.exists(local) or not os.path.samefile(path, local):
                shutil.copy2(path, local)
Seth Vidal's avatar
Seth Vidal committed
194 195 196 197

if __name__ == "__main__":
    main()