From 20a13e7b3b11894cdd38377b0753f142df61b21c Mon Sep 17 00:00:00 2001 From: "Dr. Peter Poeml" Date: Fri, 14 Jul 2006 17:39:46 +0000 Subject: [PATCH] - bump version (0.7) - initial support for local builds (subcommand 'build') --- NEWS | 8 +- README | 6 + osc/build.py | 330 +++++++++++++++++++++++++++++++++++++++++++++++++++++ osc/commandline.py | 26 ++++- osc/core.py | 4 +- osc/fetch.py | 168 +++++++++++++++++++++++++++ osc/meter.py | 93 +++++++++++++++ setup.py | 2 +- 8 files changed, 632 insertions(+), 5 deletions(-) create mode 100644 osc/build.py create mode 100644 osc/fetch.py create mode 100644 osc/meter.py diff --git a/NEWS b/NEWS index a5f2d56..d3cf6ea 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,11 @@ -since 0.6: +since 0.7: +... + +0.7: +- initial support for local builds (subcommand 'build') +- better error handling +- new subcommands buildconfig, buildinfo, repos - remove requirement on pyxml package - editmeta: add examples for package/project templates - add support for streaming the build log (thanks to Christoph Thiel) diff --git a/README b/README index ba24456..6a4da09 100644 --- a/README +++ b/README @@ -113,6 +113,12 @@ Update package meta data with metadata taken from spec file osc updatepacmetafromspec +There are other commands, which you may not need (they may be useful in scripts): + osc repos + osc buildconfig + osc buildinfo + + HINT FOR W3M USERS diff --git a/osc/build.py b/osc/build.py new file mode 100644 index 0000000..fa67746 --- /dev/null +++ b/osc/build.py @@ -0,0 +1,330 @@ +#!/usr/bin/python + +# Copyright (C) 2006 Peter Poeml. All rights reserved. +# This program is free software; it may be used, copied, modified +# and distributed under the terms of the GNU General Public Licence, +# either version 2, or (at your option) any later version. + + + +import os +import sys +import ConfigParser +import cElementTree as ET +from tempfile import NamedTemporaryFile +from osc.fetch import * + + +DEFAULTS = { 'packagecachedir': '/var/tmp/osbuild-packagecache', + 'su-wrapper': 'su -c', + 'build-cmd': '/usr/bin/build', + 'build-root': '/var/tmp/build-root', + + # default list of download URLs, which will be tried in order + 'urllist': [ + # the normal repo server, redirecting to mirrors + 'http://software.opensuse.org/download/%(project)s/%(repository)s/%(arch)s/%(filename)s', + # direct access to "full" tree + 'http://api.opensuse.org/rpm/%(project)s/%(repository)s/_repository/%(buildarch)s/%(name)s', + ], +} + + + +text_config_incomplete = """ + +Your configuration is not complete. +Make sure that you have a [general] section in %%s: +(You can copy&paste it. Some commented defaults are shown.) + +[general] + +# Downloaded packages are cached here. Must be writable by you. +#packagecachedir: %(packagecachedir)s + +# Wrapper to call build as root (sudo, su -, ...) +#su-wrapper: %(su-wrapper)s + +# rootdir to setup the chroot environment +#build-root: %(build-root)s + + +Note: +Configuration can be overridden by envvars, e.g. +OSC_SU_WRAPPER overrides the setting of su-wrapper. +""" % DEFAULTS + +change_personality = { + 'i686': 'linux32', + 'i586': 'linux32', + 'ppc': 'powerpc32', + 's390': 's390', + } + +can_also_build = { + 'x86_64': ['i686', 'i586'], + 'i686': ['i586'], + 'ppc64': ['ppc'], + 's390x': ['s390'], + } + +hostarch = os.uname()[4] +if hostarch == 'i686': # FIXME + hostarch = 'i586' + + +class Buildinfo: + """represent the contents of a buildinfo file""" + + def __init__(self, filename): + + tree = ET.parse(filename) + root = tree.getroot() + + if root.find('error') != None: + sys.stderr.write('buildinfo is borken... it says:\n') + error = root.find('error').text + sys.stderr.write(error + '\n') + sys.exit(1) + + # are we building .rpm or .deb? + # need the right suffix for downloading + # if a package named debhelper is in the dependencies, it must be .deb + self.pacsuffix = 'rpm' + for node in root.findall('dep'): + if node.text == 'debhelper': + self.pacsuffix = 'deb' + break + + self.buildarch = root.find('arch').text + + self.deps = [] + for node in root.findall('bdep'): + p = Pac(node.get('name'), + node.get('version'), + node.get('release'), + node.get('project'), + node.get('repository'), + node.get('arch'), + self.buildarch, # buildarch is used only for the URL to access the full tree... + self.pacsuffix) + self.deps.append(p) + + self.pdeps = [] + for node in root.findall('pdep'): + self.pdeps.append(node.text) + + + +class Pac: + """represent a package to be downloaded""" + def __init__(self, name, version, release, project, repository, arch, buildarch, pacsuffix): + + self.name = name + self.version = version + self.release = release + self.arch = arch + self.project = project + self.repository = repository + self.buildarch = buildarch + self.pacsuffix = pacsuffix + + # build a map to fill our the URL templates + self.mp = {} + self.mp['name'] = self.name + self.mp['version'] = self.version + self.mp['release'] = self.release + self.mp['arch'] = self.arch + self.mp['project'] = self.project + self.mp['repository'] = self.repository + self.mp['buildarch'] = self.buildarch + self.mp['pacsuffix'] = self.pacsuffix + + self.filename = '%(name)s-%(version)s-%(release)s.%(arch)s.%(pacsuffix)s' % self.mp + + self.mp['filename'] = self.filename + + + def makeurls(self, cachedir, urllist): + + self.urllist = [] + + # build up local URL + # by using the urlgrabber with local urls, we basically build up a cache. + # the cache has no validation, since the package servers don't support etags, + # or if-modified-since, so the caching is simply name-based (on the assumption + # that the filename is suitable as identifier) + self.localdir = '%s/%s/%s/%s' % (cachedir, self.project, self.repository, self.arch) + self.fullfilename=os.path.join(self.localdir, self.filename) + self.url_local = 'file://%s/' % self.fullfilename + + # first, add the local URL + self.urllist.append(self.url_local) + + # remote URLs + for url in urllist: + self.urllist.append(url % self.mp) + + + def __str__(self): + return self.name + + + + +def get_build_conf(): + auth_dict = { } # to hold multiple usernames and passwords + + conffile = os.path.expanduser('~/.oscrc') + if not os.path.exists(conffile): + print >>sys.stderr, 'Error:' + print >>sys.stderr, 'You need to create ~/.oscrc.' + print >>sys.stderr, 'Running the osc command will do this for you.' + sys.exit(1) + + config = ConfigParser.ConfigParser(DEFAULTS) + config.read(conffile) + + + if not config.has_section('general'): + # FIXME: it might be sufficient to just assume defaults? + print >>sys.stderr, text_config_incomplete % conffile + sys.exit(1) + + + for host in [ x for x in config.sections() if x != 'general' ]: + auth_dict[host] = dict(config.items(host)) + + + config = dict(config.items('general')) + + # make it possible to override configuration of the rc file + for var in ['OSC_PACKAGECACHEDIR', 'BUILD_ROOT']: + val = os.getenv(var) + if val: + if var.startswith('OSC_'): var = var[4:] + var = var.lower().replace('_', '-') + config[var] = val + + return config, auth_dict + + +def get_built_files(pacdir, pactype): + if pactype == 'rpm': + b_built = os.popen('find %s -name *.rpm' \ + % os.path.join(pacdir, 'RPMS')).read().strip() + s_built = os.popen('find %s -name *.rpm' \ + % os.path.join(pacdir, 'SRPMS')).read().strip() + else: + b_built = os.popen('find %s -name *.deb' \ + % os.path.join(pacdir, 'DEBS')).read().strip() + s_built = None + return s_built, b_built + + +def main(argv): + + global config + config, auth = get_build_conf() + + if len(argv) < 2: + print 'you have to choose a repo to build on' + print 'possible repositories are:' + + (i, o) = os.popen4(['osc', 'repos']) + i.close() + + for line in o.readlines(): + if line.split()[1] == hostarch or line.split()[1] in can_also_build[hostarch]: + print line.strip() + sys.exit(1) + + repo = argv[1] + arch = argv[2] + spec = argv[3] + buildargs = [] + buildargs += argv[4:] + + if not os.path.exists(spec): + print >>sys.stderr, 'Error: specfile \'%s\' does not exist.' % spec + sys.exit(1) + + + print 'Getting buildinfo from server' + bi_file = NamedTemporaryFile(suffix='.xml', prefix='buildinfo.', dir = '/tmp') + os.system('osc buildinfo %s %s > %s' % (repo, arch, bi_file.name)) + bi = Buildinfo(bi_file.name) + + + print 'Updating cache of required packages' + fetcher = Fetcher(cachedir = config['packagecachedir'], + urllist = config['urllist'], + auth_dict = auth) + # now update the package cache + fetcher.run(bi) + + + if bi.pacsuffix == 'rpm': + """don't know how to verify .deb packages. They are verified on install + anyway, I assume... verifying package now saves time though, since we don't + even try to set up the buildroot if it wouldn't work.""" + + print 'Verifying integrity of cached packages' + verify_pacs([ i.fullfilename for i in bi.deps ]) + + + print 'Writing build configuration' + + buildconf = [ '%s %s\n' % (i.name, i.fullfilename) for i in bi.deps ] + + buildconf.append('preinstall: ' + ' '.join(bi.pdeps) + '\n') + + rpmlist = NamedTemporaryFile(prefix='rpmlist.', dir = '/tmp') + rpmlist.writelines(buildconf) + rpmlist.flush() + os.fsync(rpmlist) + + + + print 'Getting buildconfig from server' + bc_file = NamedTemporaryFile(prefix='buildconfig.', dir = '/tmp') + os.system('osc buildconfig %s %s > %s' % (repo, arch, bc_file.name)) + + + print 'Running build' + + buildargs = ' '.join(buildargs) + + cmd = '%s --root=%s --rpmlist=%s --dist=%s %s %s' \ + % (config['build-cmd'], + config['build-root'], + rpmlist.name, + bc_file.name, + spec, + buildargs) + + if config['su-wrapper'].startswith('su '): + tmpl = '%s \'%s\'' + else: + tmpl = '%s %s' + cmd = tmpl % (config['su-wrapper'], cmd) + + if hostarch != bi.buildarch: + cmd = change_personality[bi.buildarch] + ' ' + cmd + + print cmd + os.system(cmd) + + pacdirlink = os.readlink(os.path.join(config['build-root'], '.build.packages')) + pacdir = os.path.join(config['build-root'] + pacdirlink) + + if os.path.exists(pacdir): + (s_built, b_built) = get_built_files(pacdir, bi.pacsuffix) + + print + #print 'built source packages:' + if s_built: print s_built + #print 'built binary packages:' + print b_built + + diff --git a/osc/commandline.py b/osc/commandline.py index b4f3c50..16edf5b 100755 --- a/osc/commandline.py +++ b/osc/commandline.py @@ -607,6 +607,28 @@ usage: 1. osc repos # package = current dir print platform +def build(args): + """build: build a package _locally_ +You need to call the command inside a package directory. + +usage: osc build + """ + + if args is None or len(args) < 3: + print 'missing argument' + print build.func_doc + print 'Valid arguments are:' + print + repos(None) + print + sys.exit(1) + + import osc.build + osc.build.main(sys.argv[1:]) + + + + def history(args): """history: Shows the build history of a package (NOT IMPLEMENTED YET) @@ -662,10 +684,12 @@ usage: osc help [SUBCOMMAND...] print '\n'.join(lines) - +# all commands and aliases are defined here +# a function with the respective name is assumed to exist cmd_dict = { add: ['add'], addremove: ['addremove'], + build: ['build'], buildconfig: ['buildconfig'], buildinfo: ['buildinfo'], commit: ['commit', 'ci', 'checkin'], diff --git a/osc/core.py b/osc/core.py index 7a49cb0..a8396e5 100755 --- a/osc/core.py +++ b/osc/core.py @@ -5,7 +5,7 @@ # and distributed under the terms of the GNU General Public Licence, # either version 2, or (at your option) any later version. -__version__ = '0.6' +__version__ = '0.7' import os import sys @@ -700,7 +700,7 @@ def check_store_version(dir): sys.exit(1) if v != __version__: - if v in ['0.2', '0.3', '0.4', '0.5']: + if v in ['0.2', '0.3', '0.4', '0.5', '0.6']: # version is fine, no migration needed f = open(versionfile, 'w') f.write(__version__ + '\n') diff --git a/osc/fetch.py b/osc/fetch.py new file mode 100644 index 0000000..e74ba78 --- /dev/null +++ b/osc/fetch.py @@ -0,0 +1,168 @@ +#!/usr/bin/python + +# Copyright (C) 2006 Peter Poeml. All rights reserved. +# This program is free software; it may be used, copied, modified +# and distributed under the terms of the GNU General Public Licence, +# either version 2, or (at your option) any later version. + +import sys, os +import urllib2 +from urlgrabber.grabber import URLGrabber, URLGrabError +from urlgrabber.mirror import MirrorGroup +try: + from meter import TextMeter +except: + TextMeter = None + + +def join_url(self, base_url, rel_url): + """to override _join_url of MirrorGroup, because we want to + pass full URLs instead of base URL where relative_url is added later... + IOW, we make MirrorGroup ignore relative_url""" + return base_url + + +class Fetcher: + def __init__(self, cachedir = '/tmp', auth_dict = {}, urllist = []): + + __version__ = '0.1' + __user_agent__ = 'osbuild/%s' % __version__ + + # set up progress bar callback + if sys.stdout.isatty() and TextMeter: + self.progress_obj = TextMeter(fo=sys.stdout) + else: + self.progress_obj = None + + + self.cachedir = cachedir + self.urllist = urllist + + passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm() + for host in auth_dict.keys(): + passmgr.add_password(None, host, auth_dict[host]['user'], auth_dict[host]['pass']) + authhandler = urllib2.HTTPBasicAuthHandler(passmgr) + self.gr = URLGrabber(user_agent=__user_agent__, + keepalive=1, + opener = urllib2.build_opener(authhandler), + progress_obj=self.progress_obj, + failure_callback=(self.failureReport,(),{}), + ) + + + def failureReport(self, errobj): + """failure output for failovers from urlgrabber""" + + #log(0, '%s: %s' % (errobj.url, str(errobj.exception))) + #log(0, 'Trying other mirror.') + print 'Trying upstream server for %s (%s), since it is not on %s.' \ + % (self.curpac, self.curpac.project, errobj.url.split('/')[2]) + raise errobj.exception + + + def fetch(self, pac): + # for use by the failure callback + self.curpac = pac + + MirrorGroup._join_url = join_url + mg = MirrorGroup(self.gr, pac.urllist) + + try: + # it returns the filename + ret = mg.urlgrab(pac.filename, + filename=pac.fullfilename, + text = '(%s) %s' %(pac.project, pac.filename)) + + except URLGrabError, e: + print + print >>sys.stderr, 'Error:', e.strerror + print >>sys.stderr, 'Failed to retrieve %s from the following locations (in order):' % pac.filename + print >>sys.stderr, '\n'.join(pac.urllist) + + sys.exit(1) + + + def dirSetup(self, pac): + dir = os.path.join(self.cachedir, pac.localdir) + if not os.path.exists(dir): + os.makedirs(dir, mode=0755) + + + def run(self, buildinfo): + for i in buildinfo.deps: + i.makeurls(self.cachedir, self.urllist) + + if os.path.exists(os.path.join(i.localdir, i.fullfilename)): + #print 'cached:', i.fullfilename + pass + else: + self.dirSetup(i) + + try: + # if there isn't a progress bar, there is no output at all + if not self.progress_obj: + print '(%s) %s' % (i.project, i.filename) + self.fetch(i) + + except KeyboardInterrupt: + print 'Cancelled by user (ctrl-c)' + print 'Exiting.' + if os.path.exists(i.fullfilename): + print 'Cleaning up incomplete file', i.fullfilename + os.unlink(i.fullfilename) + sys.exit(0) + + + +def verify_pacs(pac_list): + """Take a list of rpm filenames and run rpm -K on them. + + In case of failure, exit. + + Check all packages in one go, since this takes only 6 seconds on my Athlon 700 + instead of 20 when calling 'rpm -K' for each of them. + """ + + + + # we can use os.popen4 because we don't care about the return value. + # we check the output anyway, and rpm always writes to stdout. + (i, o) = os.popen4(['/bin/rpm', '-K'] + pac_list) + + i.close() + + for line in o.readlines(): + + if not 'OK' in line: + print + print >>sys.stderr, 'The following package could not be verified:' + print >>sys.stderr, line + sys.exit(1) + + if 'NOT OK' in line: + print + print >>sys.stderr, 'The following package could not be verified:' + print >>sys.stderr, line + + if 'MISSING KEYS' in line: + missing_key = line.split('#')[-1].split(')')[0] + + print >>sys.stderr, """ +- If the key is missing, install it first. + For example, do the following as root: + gpg --keyserver pgp.mit.edu --recv-keys %s + gpg --armor --export %s > keyfile + rpm --import keyfile + + Then, just start the build again. +""" %(missing_key, missing_key) + + else: + print >>sys.stderr, """ +- If the signature is wrong, you may try deleting the package manually + and re-run this program, so it is fetched again. +""" + + sys.exit(1) + + diff --git a/osc/meter.py b/osc/meter.py new file mode 100644 index 0000000..a192400 --- /dev/null +++ b/osc/meter.py @@ -0,0 +1,93 @@ +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, +# Boston, MA 02111-1307 USA + +# this is basically a copy of python-urlgrabber's TextMeter class, +# with support added for dynamical sizing according to screen size. +# it uses getScreenWidth() scrapped from smart. +# Peter Poeml + + +from urlgrabber.progress import BaseMeter, format_time, format_number +import sys, os + +def getScreenWidth(): + import termios, struct, fcntl + s = struct.pack('HHHH', 0, 0, 0, 0) + try: + x = fcntl.ioctl(1, termios.TIOCGWINSZ, s) + except IOError: + return 80 + return struct.unpack('HHHH', x)[1] + + +class TextMeter(BaseMeter): + def __init__(self, fo=sys.stderr): + BaseMeter.__init__(self) + self.fo = fo + try: + width = int(os.environ['COLUMNS']) + except (KeyError, ValueError): + width = getScreenWidth() + + + #self.unsized_templ = '\r%-60.60s %5sB %s ' + self.unsized_templ = '\r%%-%s.%ss %%5sB %%s ' % (width *4/3, width*4/3) + #self.sized_templ = '\r%-45.45s %3i%% |%-15.15s| %5sB %8s ' + self.sized_templ = '\r%%-%s.%ss %%3i%%%% |%%-%s.%ss| %%5sB %%8s ' %(width*4/10, width*4/10, width/3, width/3) + self.bar_length = width/3 + + + + def _do_update(self, amount_read, now=None): + etime = self.re.elapsed_time() + fetime = format_time(etime) + fread = format_number(amount_read) + #self.size = None + if self.text is not None: + text = self.text + else: + text = self.basename + if self.size is None: + out = self.unsized_templ % \ + (text, fread, fetime) + else: + rtime = self.re.remaining_time() + frtime = format_time(rtime) + frac = self.re.fraction_read() + bar = '='*int(self.bar_length * frac) + + out = self.sized_templ % \ + (text, frac*100, bar, fread, frtime) + 'ETA ' + + self.fo.write(out) + self.fo.flush() + + def _do_end(self, amount_read, now=None): + total_time = format_time(self.re.elapsed_time()) + total_size = format_number(amount_read) + if self.text is not None: + text = self.text + else: + text = self.basename + if self.size is None: + out = self.unsized_templ % \ + (text, total_size, total_time) + else: + bar = '=' * self.bar_length + out = self.sized_templ % \ + (text, 100, bar, total_size, total_time) + ' ' + self.fo.write(out + '\n') + self.fo.flush() diff --git a/setup.py b/setup.py index f6a94f9..39d47b7 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from distutils.core import setup setup(name='osc', - version='0.6', + version='0.7', description='opensuse commander', author='Peter Poeml', author_email='poeml@suse.de', -- 2.1.4