| Server IP : 182.53.201.61 / Your IP : 216.73.217.175 Web Server : Apache/2.2.15 (Fedora) System : Linux km10.dyndns.org 2.6.31.5-127.fc12.i686.PAE #1 SMP Sat Nov 7 21:25:57 EST 2009 i686 User : apache ( 48) PHP Version : 5.3.3 Disable Function : NONE MySQL : ON | cURL : ON | WGET : ON | Perl : ON | Python : ON | Sudo : ON | Pkexec : ON Directory : /usr/lib/python2.6/site-packages/createrepo/ |
Upload File : |
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# Copyright 2009 Red Hat, Inc -
# written by seth vidal skvidal at fedoraproject.org
import os
import sys
import libxml2
import fnmatch
import time
import yumbased
import shutil
from bz2 import BZ2File
from urlgrabber import grabber
import tempfile
import stat
from yum import misc, Errors, to_unicode
from yum.sqlutils import executeSQL
from yum.packageSack import MetaSack
from yum.packages import YumAvailablePackage
import rpmUtils.transaction
from utils import _, errorprint, MDError
import readMetadata
try:
import sqlite3 as sqlite
except ImportError:
import sqlite
try:
import sqlitecachec
except ImportError:
pass
from utils import _gzipOpen, bzipFile, checkAndMakeDir, GzipFile, \
checksum_and_rename
import deltarpms
__version__ = '0.9.8'
class MetaDataConfig(object):
def __init__(self):
self.quiet = False
self.verbose = False
self.profile = False
self.excludes = []
self.baseurl = None
self.groupfile = None
self.sumtype = 'sha256'
self.pretty = False
self.cachedir = None
self.use_cache = False
self.basedir = os.getcwd()
self.checkts = False
self.split = False
self.update = False
self.deltas = False # do the deltarpm thing
# where to put the .drpms - defaults to 'drpms' inside 'repodata'
self.deltadir = None
self.delta_relative = 'drpms/'
self.oldpackage_paths = [] # where to look for the old packages -
self.deltafile = 'prestodelta.xml.gz'
self.num_deltas = 1 # number of older versions to delta (max)
self.max_delta_rpm_size = 100000000
self.update_md_path = None
self.skip_stat = False
self.database = False
self.outputdir = None
self.file_patterns = ['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$']
self.dir_patterns = ['.*bin\/.*', '^\/etc\/.*']
self.skip_symlinks = False
self.pkglist = []
self.database_only = False
self.primaryfile = 'primary.xml.gz'
self.filelistsfile = 'filelists.xml.gz'
self.otherfile = 'other.xml.gz'
self.repomdfile = 'repomd.xml'
self.tempdir = '.repodata'
self.finaldir = 'repodata'
self.olddir = '.olddata'
self.mdtimestamp = 0
self.directory = None
self.directories = []
self.changelog_limit = 10 # needs to be an int or None
self.unique_md_filenames = False
self.additional_metadata = {} # dict of 'type':'filename'
self.revision = str(int(time.time()))
self.content_tags = [] # flat list of strings (like web 2.0 tags)
self.distro_tags = []# [(cpeid(None allowed), human-readable-string)]
class SimpleMDCallBack(object):
def errorlog(self, thing):
print >> sys.stderr, thing
def log(self, thing):
print thing
def progress(self, item, current, total):
sys.stdout.write('\r' + ' ' * 80)
sys.stdout.write("\r%d/%d - %s" % (current, total, item))
sys.stdout.flush()
class MetaDataGenerator:
def __init__(self, config_obj=None, callback=None):
self.conf = config_obj
if config_obj == None:
self.conf = MetaDataConfig()
if not callback:
self.callback = SimpleMDCallBack()
else:
self.callback = callback
self.ts = rpmUtils.transaction.initReadOnlyTransaction()
self.pkgcount = 0
self.current_pkg = 0
self.files = []
self.rpmlib_reqs = {}
if not self.conf.directory and not self.conf.directories:
raise MDError, "No directory given on which to run."
if not self.conf.directories: # just makes things easier later
self.conf.directories = [self.conf.directory]
if not self.conf.directory: # ensure we have both in the config object
self.conf.directory = self.conf.directories[0]
# the cachedir thing:
if self.conf.cachedir:
self.conf.use_cache = True
# this does the dir setup we need done
self._parse_directory()
self._test_setup_dirs()
def _parse_directory(self):
"""pick up the first directory given to us and make sure we know
where things should go"""
if os.path.isabs(self.conf.directory):
self.conf.basedir = os.path.dirname(self.conf.directory)
self.conf.relative_dir = os.path.basename(self.conf.directory)
else:
self.conf.basedir = os.path.realpath(self.conf.basedir)
self.conf.relative_dir = self.conf.directory
self.package_dir = os.path.join(self.conf.basedir,
self.conf.relative_dir)
if not self.conf.outputdir:
self.conf.outputdir = os.path.join(self.conf.basedir,
self.conf.relative_dir)
def _test_setup_dirs(self):
# start the sanity/stupidity checks
for mydir in self.conf.directories:
if os.path.isabs(mydir):
testdir = mydir
else:
if mydir.startswith('../'):
testdir = os.path.realpath(mydir)
else:
testdir = os.path.join(self.conf.basedir, mydir)
if not os.path.exists(testdir):
raise MDError, _('Directory %s must exist') % mydir
if not os.path.isdir(testdir):
raise MDError, _('%s must be a directory') % mydir
if not os.access(self.conf.outputdir, os.W_OK):
raise MDError, _('Directory %s must be writable.') % self.conf.outputdir
temp_output = os.path.join(self.conf.outputdir, self.conf.tempdir)
if not checkAndMakeDir(temp_output):
raise MDError, _('Cannot create/verify %s') % temp_output
temp_final = os.path.join(self.conf.outputdir, self.conf.finaldir)
if not checkAndMakeDir(temp_final):
raise MDError, _('Cannot create/verify %s') % temp_final
if self.conf.deltas:
temp_delta = os.path.join(self.conf.outputdir,
self.conf.delta_relative)
if not checkAndMakeDir(temp_delta):
raise MDError, _('Cannot create/verify %s') % temp_delta
self.conf.deltadir = temp_delta
if os.path.exists(os.path.join(self.conf.outputdir, self.conf.olddir)):
raise MDError, _('Old data directory exists, please remove: %s') % self.conf.olddir
# make sure we can write to where we want to write to:
# and pickup the mdtimestamps while we're at it
direcs = ['tempdir' , 'finaldir']
if self.conf.deltas:
direcs.append('deltadir')
for direc in direcs:
filepath = os.path.join(self.conf.outputdir, getattr(self.conf,
direc))
if os.path.exists(filepath):
if not os.access(filepath, os.W_OK):
raise MDError, _('error in must be able to write to metadata dir:\n -> %s') % filepath
if self.conf.checkts:
# checking for repodata/repomd.xml - not just the data dir
rxml = filepath + '/repomd.xml'
if os.path.exists(rxml):
timestamp = os.path.getctime(rxml)
if timestamp > self.conf.mdtimestamp:
self.conf.mdtimestamp = timestamp
if self.conf.groupfile:
a = self.conf.groupfile
if self.conf.split:
a = os.path.join(self.package_dir, self.conf.groupfile)
elif not os.path.isabs(a):
a = os.path.join(self.package_dir, self.conf.groupfile)
if not os.path.exists(a):
raise MDError, _('Error: groupfile %s cannot be found.' % a)
self.conf.groupfile = a
if self.conf.cachedir:
a = self.conf.cachedir
if not os.path.isabs(a):
a = os.path.join(self.conf.outputdir, a)
if not checkAndMakeDir(a):
raise MDError, _('Error: cannot open/write to cache dir %s' % a)
self.conf.cachedir = a
def _os_path_walk(self, top, func, arg):
"""Directory tree walk with callback function.
copy of os.path.walk, fixes the link/stating problem
"""
try:
names = os.listdir(top)
except os.error:
return
func(arg, top, names)
for name in names:
name = os.path.join(top, name)
if os.path.isdir(name):
self._os_path_walk(name, func, arg)
def getFileList(self, directory, ext):
"""Return all files in path matching ext, store them in filelist,
recurse dirs. Returns a list object"""
extlen = len(ext)
def extension_visitor(filelist, dirname, names):
for fn in names:
if os.path.isdir(fn):
continue
if self.conf.skip_symlinks and os.path.islink(fn):
continue
elif fn[-extlen:].lower() == '%s' % (ext):
relativepath = dirname.replace(startdir, "", 1)
relativepath = relativepath.lstrip("/")
filelist.append(os.path.join(relativepath, fn))
filelist = []
startdir = directory + '/'
self._os_path_walk(startdir, extension_visitor, filelist)
return filelist
def errorlog(self, thing):
"""subclass this if you want something different...."""
errorprint(thing)
def checkTimeStamps(self):
"""check the timestamp of our target dir. If it is not newer than
the repodata return False, else True"""
if self.conf.checkts:
dn = os.path.join(self.conf.basedir, self.conf.directory)
files = self.getFileList(dn, '.rpm')
files = self.trimRpms(files)
for f in files:
fn = os.path.join(self.conf.basedir, self.conf.directory, f)
if not os.path.exists(fn):
self.callback.errorlog(_('cannot get to file: %s') % fn)
if os.path.getctime(fn) > self.conf.mdtimestamp:
return False
else:
return True
return False
def trimRpms(self, files):
badrpms = []
for rpm_file in files:
for glob in self.conf.excludes:
if fnmatch.fnmatch(rpm_file, glob):
if rpm_file not in badrpms:
badrpms.append(rpm_file)
for rpm_file in badrpms:
if rpm_file in files:
files.remove(rpm_file)
return files
def _setup_old_metadata_lookup(self):
"""sets up the .oldData object for handling the --update call. Speeds
up generating updates for new metadata"""
#FIXME - this only actually works for single dirs. It will only
# function for the first dir passed to --split, not all of them
# this needs to be fixed by some magic in readMetadata.py
# using opts.pkgdirs as a list, I think.
if self.conf.update:
#build the paths
opts = {
'verbose' : self.conf.verbose,
'pkgdir' : os.path.normpath(self.package_dir)
}
if self.conf.skip_stat:
opts['do_stat'] = False
if self.conf.update_md_path:
old_repo_path = os.path.normpath(self.conf.update_md_path)
else:
old_repo_path = self.conf.outputdir
#and scan the old repo
self.oldData = readMetadata.MetadataIndex(old_repo_path, opts)
def _setup_grabber(self):
if not hasattr(self, '_grabber'):
self._grabber = grabber.URLGrabber()
return self._grabber
grabber = property(fget = lambda self: self._setup_grabber())
def doPkgMetadata(self):
"""all the heavy lifting for the package metadata"""
if self.conf.update:
self._setup_old_metadata_lookup()
# rpms we're going to be dealing with
if self.conf.pkglist:
packages = self.conf.pkglist
else:
packages = self.getFileList(self.package_dir, '.rpm')
if not isinstance(packages, MetaSack):
packages = self.trimRpms(packages)
self.pkgcount = len(packages)
try:
self.openMetadataDocs()
self.writeMetadataDocs(packages)
self.closeMetadataDocs()
except (IOError, OSError), e:
raise MDError, _('Cannot access/write repodata files: %s') % e
def openMetadataDocs(self):
if self.conf.database_only:
self.setup_sqlite_dbs()
else:
self.primaryfile = self._setupPrimary()
self.flfile = self._setupFilelists()
self.otherfile = self._setupOther()
if self.conf.deltas:
self.deltafile = self._setupDelta()
def _setupPrimary(self):
# setup the primary metadata file
primaryfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir,
self.conf.primaryfile)
fo = _gzipOpen(primaryfilepath, 'w')
fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
fo.write('<metadata xmlns="http://linux.duke.edu/metadata/common"' \
' xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s">' %
self.pkgcount)
return fo
def _setupFilelists(self):
# setup the filelist file
filelistpath = os.path.join(self.conf.outputdir, self.conf.tempdir,
self.conf.filelistsfile)
fo = _gzipOpen(filelistpath, 'w')
fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
fo.write('<filelists xmlns="http://linux.duke.edu/metadata/filelists"' \
' packages="%s">' % self.pkgcount)
return fo
def _setupOther(self):
# setup the other file
otherfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir,
self.conf.otherfile)
fo = _gzipOpen(otherfilepath, 'w')
fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
fo.write('<otherdata xmlns="http://linux.duke.edu/metadata/other"' \
' packages="%s">' %
self.pkgcount)
return fo
def _setupDelta(self):
# setup the other file
deltafilepath = os.path.join(self.conf.outputdir, self.conf.tempdir,
self.conf.deltafile)
fo = _gzipOpen(deltafilepath, 'w')
fo.write('<?xml version="1.0" encoding="UTF-8"?>\n')
fo.write('<prestodelta>\n')
return fo
def read_in_package(self, rpmfile, pkgpath=None, reldir=None):
"""rpmfile == relative path to file from self.packge_dir"""
baseurl = self.conf.baseurl
if not pkgpath:
pkgpath = self.package_dir
if not rpmfile.strip():
raise MDError, "Blank filename passed in, skipping"
if rpmfile.find("://") != -1:
if not hasattr(self, 'tempdir'):
self.tempdir = tempfile.mkdtemp()
pkgname = os.path.basename(rpmfile)
baseurl = os.path.dirname(rpmfile)
reldir = self.tempdir
dest = os.path.join(self.tempdir, pkgname)
if not self.conf.quiet:
self.callback.log('\nDownloading %s' % rpmfile)
try:
rpmfile = self.grabber.urlgrab(rpmfile, dest)
except grabber.URLGrabError, e:
raise MDError, "Unable to retrieve remote package %s: %s" % (
rpmfile, e)
else:
rpmfile = '%s/%s' % (pkgpath, rpmfile)
try:
po = yumbased.CreateRepoPackage(self.ts, rpmfile,
sumtype=self.conf.sumtype)
except Errors.MiscError, e:
raise MDError, "Unable to open package: %s" % e
# external info we need
po._cachedir = self.conf.cachedir
po._baseurl = baseurl
po._reldir = reldir
po._packagenumber = self.current_pkg
for r in po.requires_print:
if r.startswith('rpmlib('):
self.rpmlib_reqs[r] = 1
if po.checksum in (None, ""):
raise MDError, "No Package ID found for package %s, not going to" \
" add it" % po
return po
def writeMetadataDocs(self, pkglist=[], pkgpath=None):
if not pkglist:
pkglist = self.conf.pkglist
if not pkgpath:
directory = self.conf.directory
else:
directory = pkgpath
for pkg in pkglist:
self.current_pkg += 1
recycled = False
# look to see if we can get the data from the old repodata
# if so write this one out that way
if self.conf.update:
#see if we can pull the nodes from the old repo
#print self.oldData.basenodes.keys()
old_pkg = pkg
if pkg.find("://") != -1:
old_pkg = os.path.basename(pkg)
nodes = self.oldData.getNodes(old_pkg)
if nodes is not None:
recycled = True
# FIXME also open up the delta file
# otherwise do it individually
if not recycled:
#scan rpm files
if not pkgpath:
reldir = os.path.join(self.conf.basedir, directory)
else:
reldir = pkgpath
if not isinstance(pkg, YumAvailablePackage):
try:
po = self.read_in_package(pkg, pkgpath=pkgpath,
reldir=reldir)
except MDError, e:
# need to say something here
self.callback.errorlog("\nError %s: %s\n" % (pkg, e))
continue
# we can use deltas:
if self.conf.deltas:
self._do_delta_rpm_package(po)
else:
po = pkg
if self.conf.database_only:
pass # disabled right now for sanity reasons (mine)
#po.do_sqlite_dump(self.md_sqlite)
else:
self.primaryfile.write(po.xml_dump_primary_metadata())
self.flfile.write(po.xml_dump_filelists_metadata())
self.otherfile.write(po.xml_dump_other_metadata(
clog_limit=self.conf.changelog_limit))
else:
if self.conf.verbose:
self.callback.log(_("Using data from old metadata for %s")
% pkg)
(primarynode, filenode, othernode) = nodes
for node, outfile in ((primarynode, self.primaryfile),
(filenode, self.flfile),
(othernode, self.otherfile)):
if node is None:
break
output = node.serialize('UTF-8', self.conf.pretty)
if output:
outfile.write(output)
else:
if self.conf.verbose:
self.callback.log(_("empty serialize on write to" \
"%s in %s") % (outfile, pkg))
outfile.write('\n')
self.oldData.freeNodes(pkg)
#FIXME - if we're in update and we have deltas enabled
# check the presto data for this pkg and write its info back out
# to our deltafile
if not self.conf.quiet:
if self.conf.verbose:
self.callback.log('%d/%d - %s' % (self.current_pkg,
self.pkgcount, pkg))
else:
self.callback.progress(pkg, self.current_pkg, self.pkgcount)
return self.current_pkg
def closeMetadataDocs(self):
if not self.conf.quiet:
self.callback.log('')
# save them up to the tmp locations:
if not self.conf.quiet:
self.callback.log(_('Saving Primary metadata'))
if self.conf.database_only:
self.md_sqlite.pri_cx.close()
else:
self.primaryfile.write('\n</metadata>')
self.primaryfile.close()
if not self.conf.quiet:
self.callback.log(_('Saving file lists metadata'))
if self.conf.database_only:
self.md_sqlite.file_cx.close()
else:
self.flfile.write('\n</filelists>')
self.flfile.close()
if not self.conf.quiet:
self.callback.log(_('Saving other metadata'))
if self.conf.database_only:
self.md_sqlite.other_cx.close()
else:
self.otherfile.write('\n</otherdata>')
self.otherfile.close()
if self.conf.deltas:
deltam_st = time.time()
if not self.conf.quiet:
self.callback.log(_('Saving delta metadata'))
self.deltafile.write(self.generate_delta_xml())
self.deltafile.write('\n</prestodelta>')
self.deltafile.close()
if self.conf.profile:
self.callback.log('deltam time: %0.3f' % (time.time() - deltam_st))
def _do_delta_rpm_package(self, pkg):
"""makes the drpms, if possible, for this package object.
returns the presto/delta xml metadata as a string
"""
drpm_pkg_time = time.time()
# duck and cover if the pkg.size is > whatever
if int(pkg.size) > self.conf.max_delta_rpm_size:
if not self.conf.quiet:
self.callback.log("Skipping %s package " \
"that is > max_delta_rpm_size" % pkg)
return
# generate a list of all the potential 'old rpms'
opd = self._get_old_package_dict()
# for each of our old_package_paths -
# make a drpm from the newest of that pkg
# get list of potential candidates which are likely to match
for d in self.conf.oldpackage_paths:
pot_cand = []
if d not in opd:
continue
for fn in opd[d]:
if os.path.basename(fn).startswith(pkg.name):
pot_cand.append(fn)
candidates = []
for fn in pot_cand:
try:
thispo = yumbased.CreateRepoPackage(self.ts, fn,
sumtype=self.conf.sumtype)
except Errors.MiscError, e:
continue
if (thispo.name, thispo.arch) != (pkg.name, pkg.arch):
# not the same, doesn't matter
continue
if thispo == pkg: #exactly the same, doesn't matter
continue
if thispo.EVR >= pkg.EVR: # greater or equal, doesn't matter
continue
candidates.append(thispo)
candidates.sort()
candidates.reverse()
for delta_p in candidates[0:self.conf.num_deltas]:
#make drpm of pkg and delta_p
dt_st = time.time()
drpmfn = deltarpms.create_drpm(delta_p, pkg, self.conf.deltadir)
if not self.conf.quiet or self.conf.profile:
self.callback.log('created drpm from %s to %s: %s in %0.3f' % (
delta_p, pkg, drpmfn, (time.time() - dt_st)))
if self.conf.profile:
self.callback.log('total drpm time for %s: %0.3f' % (pkg,
(time.time() - drpm_pkg_time)))
def _get_old_package_dict(self):
if hasattr(self, '_old_package_dict'):
return self._old_package_dict
self._old_package_dict = {}
opl = []
for d in self.conf.oldpackage_paths:
for f in self.getFileList(d, 'rpm'):
fp = d + '/' + f
fpstat = os.stat(fp)
if int(fpstat[stat.ST_SIZE]) > self.conf.max_delta_rpm_size:
self.callback.log("Skipping %s package " \
"that is > max_delta_rpm_size" % f)
continue
if not self._old_package_dict.has_key(d):
self._old_package_dict[d] = []
self._old_package_dict[d].append(d + '/' + f)
return self._old_package_dict
def generate_delta_xml(self):
"""take the delta rpm output dir, process all the drpm files
produce the text output for the presto/delta xml metadata"""
# go through the drpm dir
# for each file -store the drpm info in a dict based on its target. Just
# appending the output. for each of the keys in the dict, return
# the tag for the target + each of the drpm infos + closure for the target
# tag
targets = {}
results = []
for drpm_fn in self.getFileList(self.conf.deltadir, 'drpm'):
drpm_rel_fn = os.path.normpath(self.conf.delta_relative +
'/' + drpm_fn) # this is annoying
drpm_po = yumbased.CreateRepoPackage(self.ts,
self.conf.deltadir + '/' + drpm_fn, sumtype=self.conf.sumtype)
drpm = deltarpms.DeltaRPMPackage(drpm_po, self.conf.outputdir,
drpm_rel_fn)
if not targets.has_key(drpm_po.pkgtup):
targets[drpm_po.pkgtup] = []
targets[drpm_po.pkgtup].append(drpm.xml_dump_metadata())
for (n, a, e, v, r) in targets.keys():
results.append(""" <newpackage name="%s" epoch="%s" version="%s" release="%s" arch="%s">\n""" % (
n, e, v, r, a))
results.extend(targets[(n,a,e,v,r)])
# for src in targets[(n, a, e, v, r)]:
# results.append(src)
results.append(" </newpackage>\n")
return ' '.join(results)
def addArbitraryMetadata(self, mdfile, mdtype, xml_node, compress=True,
compress_type='gzip', attribs={}):
"""add random metadata to the repodata dir and repomd.xml
mdfile = complete path to file
mdtype = the metadata type to use
xml_node = the node of the repomd xml object to append this
data onto
compress = compress the file before including it
"""
# copy the file over here
sfile = os.path.basename(mdfile)
fo = open(mdfile, 'r')
outdir = os.path.join(self.conf.outputdir, self.conf.tempdir)
if compress:
if compress_type == 'gzip':
sfile = '%s.gz' % sfile
outfn = os.path.join(outdir, sfile)
output = GzipFile(filename = outfn, mode='wb')
elif compress_type == 'bzip2':
sfile = '%s.bz2' % sfile
outfn = os.path.join(outdir, sfile)
output = BZ2File(filename = outfn, mode='wb')
else:
outfn = os.path.join(outdir, sfile)
output = open(outfn, 'w')
output.write(fo.read())
output.close()
fo.seek(0)
open_csum = misc.checksum(self.conf.sumtype, fo)
fo.close()
if self.conf.unique_md_filenames:
(csum, outfn) = checksum_and_rename(outfn, self.conf.sumtype)
sfile = os.path.basename(outfn)
else:
if compress:
csum = misc.checksum(self.conf.sumtype, outfn)
else:
csum = open_csum
timest = os.stat(outfn)[8]
# add all this garbage into the xml node like:
data = xml_node.newChild(None, 'data', None)
data.newProp('type', mdtype)
location = data.newChild(None, 'location', None)
if self.conf.baseurl is not None:
location.newProp('xml:base', self.conf.baseurl)
location.newProp('href', os.path.join(self.conf.finaldir, sfile))
checksum = data.newChild(None, 'checksum', csum)
checksum.newProp('type', self.conf.sumtype)
if compress:
opencsum = data.newChild(None, 'open-checksum', open_csum)
opencsum.newProp('type', self.conf.sumtype)
timestamp = data.newChild(None, 'timestamp', str(timest))
# add the random stuff
for (k, v) in attribs.items():
data.newChild(None, k, str(v))
def doRepoMetadata(self):
"""wrapper to generate the repomd.xml file that stores the info
on the other files"""
repodoc = libxml2.newDoc("1.0")
reporoot = repodoc.newChild(None, "repomd", None)
repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None)
reporoot.setNs(repons)
rpmns = reporoot.newNs("http://linux.duke.edu/metadata/rpm", 'rpm')
repopath = os.path.join(self.conf.outputdir, self.conf.tempdir)
repofilepath = os.path.join(repopath, self.conf.repomdfile)
revision = reporoot.newChild(None, 'revision', self.conf.revision)
if self.conf.content_tags or self.conf.distro_tags:
tags = reporoot.newChild(None, 'tags', None)
for item in self.conf.content_tags:
c_tags = tags.newChild(None, 'content', item)
for (cpeid, item) in self.conf.distro_tags:
d_tags = tags.newChild(None, 'distro', item)
if cpeid:
d_tags.newProp('cpeid', cpeid)
sumtype = self.conf.sumtype
if self.conf.database_only:
workfiles = []
db_workfiles = [(self.md_sqlite.pri_sqlite_file, 'primary_db'),
(self.md_sqlite.file_sqlite_file, 'filelists_db'),
(self.md_sqlite.other_sqlite_file, 'other_db')]
dbversion = '10'
else:
workfiles = [(self.conf.otherfile, 'other',),
(self.conf.filelistsfile, 'filelists'),
(self.conf.primaryfile, 'primary')]
db_workfiles = []
repoid = 'garbageid'
if self.conf.deltas:
workfiles.append((self.conf.deltafile, 'prestodelta'))
if self.conf.database:
if not self.conf.quiet: self.callback.log('Generating sqlite DBs')
try:
dbversion = str(sqlitecachec.DBVERSION)
except AttributeError:
dbversion = '9'
rp = sqlitecachec.RepodataParserSqlite(repopath, repoid, None)
for (rpm_file, ftype) in workfiles:
complete_path = os.path.join(repopath, rpm_file)
zfo = _gzipOpen(complete_path)
# This is misc.checksum() done locally so we can get the size too.
data = misc.Checksums([sumtype])
while data.read(zfo, 2**16):
pass
uncsum = data.hexdigest(sumtype)
unsize = len(data)
zfo.close()
csum = misc.checksum(sumtype, complete_path)
timestamp = os.stat(complete_path)[8]
db_csums = {}
db_compressed_sums = {}
if self.conf.database:
if ftype in ['primary', 'filelists', 'other']:
if self.conf.verbose:
self.callback.log("Starting %s db creation: %s" % (ftype,
time.ctime()))
if ftype == 'primary':
rp.getPrimary(complete_path, csum)
elif ftype == 'filelists':
rp.getFilelists(complete_path, csum)
elif ftype == 'other':
rp.getOtherdata(complete_path, csum)
if ftype in ['primary', 'filelists', 'other']:
tmp_result_name = '%s.xml.gz.sqlite' % ftype
tmp_result_path = os.path.join(repopath, tmp_result_name)
good_name = '%s.sqlite' % ftype
resultpath = os.path.join(repopath, good_name)
# rename from silly name to not silly name
os.rename(tmp_result_path, resultpath)
compressed_name = '%s.bz2' % good_name
result_compressed = os.path.join(repopath, compressed_name)
db_csums[ftype] = misc.checksum(sumtype, resultpath)
# compress the files
bzipFile(resultpath, result_compressed)
# csum the compressed file
db_compressed_sums[ftype] = misc.checksum(sumtype,
result_compressed)
# timestamp+size the uncompressed file
un_stat = os.stat(resultpath)
# remove the uncompressed file
os.unlink(resultpath)
if self.conf.unique_md_filenames:
csum_compressed_name = '%s-%s.bz2' % (
db_compressed_sums[ftype], good_name)
csum_result_compressed = os.path.join(repopath,
csum_compressed_name)
os.rename(result_compressed, csum_result_compressed)
result_compressed = csum_result_compressed
compressed_name = csum_compressed_name
# timestamp+size the compressed file
db_stat = os.stat(result_compressed)
# add this data as a section to the repomdxml
db_data_type = '%s_db' % ftype
data = reporoot.newChild(None, 'data', None)
data.newProp('type', db_data_type)
location = data.newChild(None, 'location', None)
if self.conf.baseurl is not None:
location.newProp('xml:base', self.conf.baseurl)
location.newProp('href', os.path.join(self.conf.finaldir,
compressed_name))
checksum = data.newChild(None, 'checksum',
db_compressed_sums[ftype])
checksum.newProp('type', sumtype)
db_tstamp = data.newChild(None, 'timestamp',
str(db_stat.st_mtime))
data.newChild(None, 'size', str(db_stat.st_size))
data.newChild(None, 'open-size', str(un_stat.st_size))
unchecksum = data.newChild(None, 'open-checksum',
db_csums[ftype])
unchecksum.newProp('type', sumtype)
database_version = data.newChild(None, 'database_version',
dbversion)
if self.conf.verbose:
self.callback.log("Ending %s db creation: %s" % (ftype,
time.ctime()))
data = reporoot.newChild(None, 'data', None)
data.newProp('type', ftype)
checksum = data.newChild(None, 'checksum', csum)
checksum.newProp('type', sumtype)
timestamp = data.newChild(None, 'timestamp', str(timestamp))
size = os.stat(os.path.join(repopath, rpm_file))
data.newChild(None, 'size', str(size.st_size))
data.newChild(None, 'open-size', str(unsize))
unchecksum = data.newChild(None, 'open-checksum', uncsum)
unchecksum.newProp('type', sumtype)
location = data.newChild(None, 'location', None)
if self.conf.baseurl is not None:
location.newProp('xml:base', self.conf.baseurl)
if self.conf.unique_md_filenames:
res_file = '%s-%s.xml.gz' % (csum, ftype)
orig_file = os.path.join(repopath, rpm_file)
dest_file = os.path.join(repopath, res_file)
os.rename(orig_file, dest_file)
else:
res_file = rpm_file
rpm_file = res_file
location.newProp('href', os.path.join(self.conf.finaldir, rpm_file))
if not self.conf.quiet and self.conf.database:
self.callback.log('Sqlite DBs complete')
for (fn, ftype) in db_workfiles:
attribs = {'database_version':dbversion}
self.addArbitraryMetadata(fn, ftype, reporoot, compress=True,
compress_type='bzip2', attribs=attribs)
try:
os.unlink(fn)
except (IOError, OSError), e:
pass
if self.conf.groupfile is not None:
self.addArbitraryMetadata(self.conf.groupfile, 'group_gz', reporoot)
self.addArbitraryMetadata(self.conf.groupfile, 'group', reporoot,
compress=False)
if self.conf.additional_metadata:
for md_type, mdfile in self.conf.additional_metadata.items():
self.addArbitraryMetadata(mdfile, md_type, reporoot)
# FIXME - disabled until we decide how best to use this
#if self.rpmlib_reqs:
# rpmlib = reporoot.newChild(rpmns, 'lib', None)
# for r in self.rpmlib_reqs.keys():
# req = rpmlib.newChild(rpmns, 'requires', r)
# save it down
try:
repodoc.saveFormatFileEnc(repofilepath, 'UTF-8', 1)
except:
self.callback.errorlog(
_('Error saving temp file for repomd.xml: %s') % repofilepath)
raise MDError, 'Could not save temp file: %s' % repofilepath
del repodoc
def doFinalMove(self):
"""move the just-created repodata from .repodata to repodata
also make sure to preserve any files we didn't mess with in the
metadata dir"""
output_final_dir = os.path.join(self.conf.outputdir, self.conf.finaldir)
output_old_dir = os.path.join(self.conf.outputdir, self.conf.olddir)
if os.path.exists(output_final_dir):
try:
os.rename(output_final_dir, output_old_dir)
except:
raise MDError, _('Error moving final %s to old dir %s' % (
output_final_dir, output_old_dir))
output_temp_dir = os.path.join(self.conf.outputdir, self.conf.tempdir)
try:
os.rename(output_temp_dir, output_final_dir)
except:
# put the old stuff back
os.rename(output_old_dir, output_final_dir)
raise MDError, _('Error moving final metadata into place')
for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile',
'groupfile']:
if getattr(self.conf, f):
fn = os.path.basename(getattr(self.conf, f))
else:
continue
oldfile = os.path.join(output_old_dir, fn)
if os.path.exists(oldfile):
try:
os.remove(oldfile)
except OSError, e:
raise MDError, _(
'Could not remove old metadata file: %s: %s') % (oldfile, e)
# Move everything else back from olddir (eg. repoview files)
for f in os.listdir(output_old_dir):
oldfile = os.path.join(output_old_dir, f)
finalfile = os.path.join(output_final_dir, f)
if f.find('-') != -1 and f.split('-')[1] in ('primary.sqlite.bz2',
'filelists.sqlite.bz2', 'primary.xml.gz','other.sqlite.bz2',
'other.xml.gz','filelists.xml.gz'):
os.remove(oldfile) # kill off the old ones
continue
if f in ('filelists.sqlite.bz2', 'other.sqlite.bz2',
'primary.sqlite.bz2'):
os.remove(oldfile)
continue
if os.path.exists(finalfile):
# Hmph? Just leave it alone, then.
try:
if os.path.isdir(oldfile):
shutil.rmtree(oldfile)
else:
os.remove(oldfile)
except OSError, e:
raise MDError, _(
'Could not remove old metadata file: %s: %s') % (oldfile, e)
else:
try:
os.rename(oldfile, finalfile)
except OSError, e:
msg = _('Could not restore old non-metadata file: %s -> %s') % (oldfile, finalfile)
msg += _('Error was %s') % e
raise MDError, msg
try:
os.rmdir(output_old_dir)
except OSError, e:
self.errorlog(_('Could not remove old metadata dir: %s')
% self.conf.olddir)
self.errorlog(_('Error was %s') % e)
self.errorlog(_('Please clean up this directory manually.'))
def setup_sqlite_dbs(self, initdb=True):
"""sets up the sqlite dbs w/table schemas and db_infos"""
destdir = os.path.join(self.conf.outputdir, self.conf.tempdir)
try:
self.md_sqlite = MetaDataSqlite(destdir)
except sqlite.OperationalError, e:
raise MDError, _('Cannot create sqlite databases: %s.\n'\
'Maybe you need to clean up a .repodata dir?') % e
class SplitMetaDataGenerator(MetaDataGenerator):
"""takes a series of dirs and creates repodata for all of them
most commonly used with -u media:// - if no outputdir is specified
it will create the repodata in the first dir in the list of dirs
"""
def __init__(self, config_obj=None, callback=None):
MetaDataGenerator.__init__(self, config_obj=config_obj, callback=None)
def _getFragmentUrl(self, url, fragment):
import urlparse
urlparse.uses_fragment.append('media')
if not url:
return url
(scheme, netloc, path, query, fragid) = urlparse.urlsplit(url)
return urlparse.urlunsplit((scheme, netloc, path, query, str(fragment)))
def getFileList(self, directory, ext):
extlen = len(ext)
def extension_visitor(arg, dirname, names):
for fn in names:
if os.path.isdir(fn):
continue
elif fn[-extlen:].lower() == '%s' % (ext):
reldir = os.path.basename(dirname)
if reldir == os.path.basename(directory):
reldir = ""
arg.append(os.path.join(reldir, fn))
rpmlist = []
os.path.walk(directory, extension_visitor, rpmlist)
return rpmlist
def doPkgMetadata(self):
"""all the heavy lifting for the package metadata"""
if len(self.conf.directories) == 1:
MetaDataGenerator.doPkgMetadata(self)
return
if self.conf.update:
self._setup_old_metadata_lookup()
filematrix = {}
for mydir in self.conf.directories:
if os.path.isabs(mydir):
thisdir = mydir
else:
if mydir.startswith('../'):
thisdir = os.path.realpath(mydir)
else:
thisdir = os.path.join(self.conf.basedir, mydir)
filematrix[mydir] = self.getFileList(thisdir, '.rpm')
self.trimRpms(filematrix[mydir])
self.pkgcount += len(filematrix[mydir])
mediano = 1
self.current_pkg = 0
self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, mediano)
try:
self.openMetadataDocs()
original_basedir = self.conf.basedir
for mydir in self.conf.directories:
self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, mediano)
self.writeMetadataDocs(filematrix[mydir], mydir)
mediano += 1
self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, 1)
self.closeMetadataDocs()
except (IOError, OSError), e:
raise MDError, _('Cannot access/write repodata files: %s') % e
class MetaDataSqlite(object):
def __init__(self, destdir):
self.pri_sqlite_file = os.path.join(destdir, 'primary.sqlite')
self.pri_cx = sqlite.Connection(self.pri_sqlite_file)
self.file_sqlite_file = os.path.join(destdir, 'filelists.sqlite')
self.file_cx = sqlite.Connection(self.file_sqlite_file)
self.other_sqlite_file = os.path.join(destdir, 'other.sqlite')
self.other_cx = sqlite.Connection(self.other_sqlite_file)
self.primary_cursor = self.pri_cx.cursor()
self.filelists_cursor = self.file_cx.cursor()
self.other_cursor = self.other_cx.cursor()
self.create_primary_db()
self.create_filelists_db()
self.create_other_db()
def create_primary_db(self):
# make the tables
schema = [
"""PRAGMA synchronous="OFF";""",
"""pragma locking_mode="EXCLUSIVE";""",
"""CREATE TABLE conflicts ( name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT, pkgKey INTEGER );""",
"""CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);""",
"""CREATE TABLE files ( name TEXT, type TEXT, pkgKey INTEGER);""",
"""CREATE TABLE obsoletes ( name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT, pkgKey INTEGER );""",
"""CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT, name TEXT, arch TEXT, version TEXT, epoch TEXT, release TEXT, summary TEXT, description TEXT, url TEXT, time_file INTEGER, time_build INTEGER, rpm_license TEXT, rpm_vendor TEXT, rpm_group TEXT, rpm_buildhost TEXT, rpm_sourcerpm TEXT, rpm_header_start INTEGER, rpm_header_end INTEGER, rpm_packager TEXT, size_package INTEGER, size_installed INTEGER, size_archive INTEGER, location_href TEXT, location_base TEXT, checksum_type TEXT);""",
"""CREATE TABLE provides ( name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT, pkgKey INTEGER );""",
"""CREATE TABLE requires ( name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT, pkgKey INTEGER , pre BOOL DEFAULT FALSE);""",
"""CREATE INDEX filenames ON files (name);""",
"""CREATE INDEX packageId ON packages (pkgId);""",
"""CREATE INDEX packagename ON packages (name);""",
"""CREATE INDEX pkgconflicts on conflicts (pkgKey);""",
"""CREATE INDEX pkgobsoletes on obsoletes (pkgKey);""",
"""CREATE INDEX pkgprovides on provides (pkgKey);""",
"""CREATE INDEX pkgrequires on requires (pkgKey);""",
"""CREATE INDEX providesname ON provides (name);""",
"""CREATE INDEX requiresname ON requires (name);""",
"""CREATE TRIGGER removals AFTER DELETE ON packages
BEGIN
DELETE FROM files WHERE pkgKey = old.pkgKey;
DELETE FROM requires WHERE pkgKey = old.pkgKey;
DELETE FROM provides WHERE pkgKey = old.pkgKey;
DELETE FROM conflicts WHERE pkgKey = old.pkgKey;
DELETE FROM obsoletes WHERE pkgKey = old.pkgKey;
END;""",
"""INSERT into db_info values (%s, 'direct_create');""" % sqlitecachec.DBVERSION,
]
for cmd in schema:
executeSQL(self.primary_cursor, cmd)
def create_filelists_db(self):
schema = [
"""PRAGMA synchronous="OFF";""",
"""pragma locking_mode="EXCLUSIVE";""",
"""CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);""",
"""CREATE TABLE filelist ( pkgKey INTEGER, dirname TEXT, filenames TEXT, filetypes TEXT);""",
"""CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT);""",
"""CREATE INDEX dirnames ON filelist (dirname);""",
"""CREATE INDEX keyfile ON filelist (pkgKey);""",
"""CREATE INDEX pkgId ON packages (pkgId);""",
"""CREATE TRIGGER remove_filelist AFTER DELETE ON packages
BEGIN
DELETE FROM filelist WHERE pkgKey = old.pkgKey;
END;""",
"""INSERT into db_info values (%s, 'direct_create');""" % sqlitecachec.DBVERSION,
]
for cmd in schema:
executeSQL(self.filelists_cursor, cmd)
def create_other_db(self):
schema = [
"""PRAGMA synchronous="OFF";""",
"""pragma locking_mode="EXCLUSIVE";""",
"""CREATE TABLE changelog ( pkgKey INTEGER, author TEXT, date INTEGER, changelog TEXT);""",
"""CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);""",
"""CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT);""",
"""CREATE INDEX keychange ON changelog (pkgKey);""",
"""CREATE INDEX pkgId ON packages (pkgId);""",
"""CREATE TRIGGER remove_changelogs AFTER DELETE ON packages
BEGIN
DELETE FROM changelog WHERE pkgKey = old.pkgKey;
END;""",
"""INSERT into db_info values (%s, 'direct_create');""" % sqlitecachec.DBVERSION,
]
for cmd in schema:
executeSQL(self.other_cursor, cmd)