Putting names at the top of files is is not recommended. Collective wisdom for
[opensuse:osc.git] / osc / build.py
1 #!/usr/bin/python
2
3 # Copyright (C) 2006 Novell Inc.  All rights reserved.
4 # This program is free software; it may be used, copied, modified
5 # and distributed under the terms of the GNU General Public Licence,
6 # either version 2, or (at your option) any later version.
7
8
9
10 import os
11 import re
12 import sys
13 from tempfile import NamedTemporaryFile
14 from osc.fetch import *
15 from osc.core import get_buildinfo, store_read_apiurl, store_read_project, store_read_package, meta_exists, quote_plus, get_buildconfig
16 import osc.conf
17 import oscerr
18 import subprocess
19 try:
20     from xml.etree import cElementTree as ET
21 except ImportError:
22     import cElementTree as ET
23
24 from conf import config, cookiejar
25
26 change_personality = {
27             'i686':  'linux32',
28             'i586':  'linux32',
29             'i386':  'linux32',
30             'ppc':   'powerpc32',
31             's390':  's390',
32         }
33
34 can_also_build = { 
35              'armv4l': [                                         'armv4l'                                 ],
36              'armv5el':[                                         'armv4l', 'armv5el'                      ],
37              'armv7el':[                                         'armv4l', 'armv5el', 'armv7el'           ],
38              's390x':  ['s390'                                                                            ],
39              'ppc64':  [                        'ppc', 'ppc64',                                           ],
40              'i386':   [        'i586',                          'armv4l', 'armv5el', 'armv7el',    'sh4' ],
41              'i586':   [                'i386',                  'armv4l', 'armv5el', 'armv7el',    'sh4' ],
42              'i686':   [        'i586',                          'armv4l', 'armv5el', 'armv7el',    'sh4' ],
43              'x86_64': ['i686', 'i586', 'i386',                  'armv4l', 'armv5el', 'armv7el',    'sh4' ],
44              }
45
46 # real arch of this machine
47 hostarch = os.uname()[4]
48 if hostarch == 'i686': # FIXME
49     hostarch = 'i586'
50
51
52 class Buildinfo:
53     """represent the contents of a buildinfo file"""
54
55     def __init__(self, filename, apiurl):
56
57         try:
58             tree = ET.parse(filename)
59         except:
60             print >>sys.stderr, 'could not parse the buildinfo:'
61             print >>sys.stderr, open(filename).read()
62             sys.exit(1)
63
64         root = tree.getroot()
65
66         if root.find('error') != None:
67             sys.stderr.write('buildinfo is broken... it says:\n')
68             error = root.find('error').text
69             sys.stderr.write(error + '\n')
70             sys.exit(1)
71
72         if not (apiurl.startswith('https://') or apiurl.startswith('http://')):
73             raise urllib2.URLError('invalid protocol for the apiurl: \'%s\'' % apiurl)
74
75         # are we building  .rpm or .deb?
76         # need the right suffix for downloading
77         # if a package named debhelper is in the dependencies, it must be .deb
78         # XXX: shouldn't we deliver the type via the buildinfo?
79         self.pacsuffix = 'rpm'
80         for node in root.findall('bdep'):
81             if node.get('name') == 'debhelper':
82                 self.pacsuffix = 'deb'
83                 break
84
85         self.buildarch = root.find('arch').text
86         self.downloadurl = root.get('downloadurl')
87         self.debuginfo = 0
88         if root.find('debuginfo') != None:
89             try:
90                 self.debuginfo = int(root.find('debuginfo').text)
91             except ValueError:
92                 pass
93
94         self.deps = []
95         for node in root.findall('bdep'):
96             p = Pac(node,
97                     self.buildarch,       # buildarch is used only for the URL to access the full tree...
98                     self.pacsuffix,
99                     apiurl)
100                     
101             self.deps.append(p)
102
103         self.vminstall_list = [ dep.name for dep in self.deps if dep.vminstall ]
104         self.preinstall_list = [ dep.name for dep in self.deps if dep.preinstall ]
105         self.runscripts_list = [ dep.name for dep in self.deps if dep.runscripts ]
106
107
108     def has_dep(self, name):
109         for i in self.deps:
110             if i.name == name:
111                 return True
112         return False
113
114     def remove_dep(self, name):
115         for i in self.deps:
116             if i.name == name:
117                 self.deps.remove(i)
118                 return True
119         return False
120
121
122 class Pac:
123     """represent a package to be downloaded
124
125     We build a map that's later used to fill our URL templates
126     """
127     def __init__(self, node, buildarch, pacsuffix, apiurl):
128
129         self.mp = {}
130         for i in ['name', 'package', 
131                   'version', 'release', 
132                   'project', 'repository', 
133                   'preinstall', 'vminstall', 'noinstall', 'runscripts',
134                  ]:
135             self.mp[i] = node.get(i)
136
137         self.mp['buildarch']  = buildarch
138         self.mp['pacsuffix']  = pacsuffix
139
140
141         self.mp['repopackage'] = node.get('package') or '_repository'
142         self.mp['arch'] = node.get('arch') or self.mp['buildarch']
143         self.mp['repoarch'] = node.get('repoarch') or self.mp['arch']
144
145         if not (self.mp['name'] and self.mp['arch'] and self.mp['version']):
146             raise oscerr.APIError(
147                 "buildinfo for package %s/%s/%s is incomplete" 
148                     % (self.mp['name'], self.mp['arch'], self.mp['version']))
149
150         self.mp['apiurl'] = apiurl
151
152         self.filename = '%(name)s-%(version)s-%(release)s.%(arch)s.%(pacsuffix)s' % self.mp
153
154         self.mp['filename'] = self.filename
155         if self.mp['repopackage'] == '_repository':
156             self.mp['repofilename'] = self.mp['name']
157         else:
158             self.mp['repofilename'] = self.mp['filename']
159
160         # make the content of the dictionary accessible as class attributes
161         self.__dict__.update(self.mp)
162
163
164     def makeurls(self, cachedir, urllist):
165
166         self.urllist = []
167
168         # build up local URL
169         # by using the urlgrabber with local urls, we basically build up a cache.
170         # the cache has no validation, since the package servers don't support etags,
171         # or if-modified-since, so the caching is simply name-based (on the assumption
172         # that the filename is suitable as identifier)
173         self.localdir = '%s/%s/%s/%s' % (cachedir, self.project, self.repository, self.arch)
174         self.fullfilename=os.path.join(self.localdir, self.filename)
175         self.url_local = 'file://%s/' % self.fullfilename
176
177         # first, add the local URL 
178         self.urllist.append(self.url_local)
179
180         # remote URLs
181         for url in urllist:
182             self.urllist.append(url % self.mp)
183
184     def __str__(self):
185         return self.name
186
187     def __repr__(self):
188         return "%s" % self.name
189
190
191
192 def get_built_files(pacdir, pactype):
193     if pactype == 'rpm':
194         b_built = subprocess.Popen(['find', os.path.join(pacdir, 'RPMS'), 
195                                     '-name', '*.rpm'],
196                                    stdout=subprocess.PIPE).stdout.read().strip()
197         s_built = subprocess.Popen(['find', os.path.join(pacdir, 'SRPMS'), 
198                                     '-name', '*.rpm'],
199                                    stdout=subprocess.PIPE).stdout.read().strip()
200     else:
201         b_built = subprocess.Popen(['find', os.path.join(pacdir, 'DEBS'),
202                                     '-name', '*.deb'],
203                                    stdout=subprocess.PIPE).stdout.read().strip()
204         s_built = ''
205     return s_built, b_built
206
207
208 def get_prefer_pkgs(dirs, wanted_arch):
209     # XXX learn how to do the same for Debian packages
210     import glob
211     paths = []
212     for dir in dirs:
213         paths += glob.glob(os.path.join(os.path.abspath(dir), '*.rpm'))
214     prefer_pkgs = []
215
216     for path in paths:
217         if path.endswith('src.rpm'):
218             continue
219         if path.find('-debuginfo-') > 0:
220             continue
221         arch, name = subprocess.Popen(['rpm', '-qp', 
222                                       '--nosignature', '--nodigest', 
223                                       '--qf', '%{arch} %{name}\n', path], 
224                                       stdout=subprocess.PIPE).stdout.read().split()
225         # instead of thip assumption, we should probably rather take the
226         # requested arch for this package from buildinfo
227         # also, it will ignore i686 packages, how to handle those?
228         if arch == wanted_arch or arch == 'noarch':
229             prefer_pkgs.append((name, path))
230
231     return dict(prefer_pkgs)
232
233
234 def main(opts, argv):
235
236     repo = argv[0]
237     arch = argv[1]
238     build_descr = argv[2]
239     xp = []
240
241     build_type = os.path.splitext(build_descr)[1][1:]
242     if build_type not in ['spec', 'dsc', 'kiwi']:
243         raise oscerr.WrongArgs(
244                 "Unknown build type: '%s'. Build description should end in .spec, .dsc or .kiwi." \
245                         % build_type)
246
247     buildargs = []
248     if not opts.userootforbuild:
249         buildargs.append('--norootforbuild')
250     if opts.clean:
251         buildargs.append('--clean')
252     if opts.noinit:
253         buildargs.append('--noinit')
254     if opts.nochecks:
255         buildargs.append('--no-checks')
256     if not opts.no_changelog:
257         buildargs.append('--changelog')
258     if opts.jobs:
259         buildargs.append('--jobs %s' % opts.jobs)
260     if opts.icecream:
261         buildargs.append('--icecream %s' % opts.icecream)
262         xp.append('icecream')
263     if opts.ccache:
264         buildargs.append('--ccache')
265         xp.append('ccache')
266     if opts.baselibs:
267         buildargs.append('--baselibs')
268     if opts.debuginfo:
269         buildargs.append('--debug')
270     if opts._with:
271         buildargs.append('--with %s' % opts._with)
272     if opts.without:
273         buildargs.append('--without %s' % opts.without)
274 # FIXME: quoting
275 #    if opts.define:
276 #        buildargs.append('--define "%s"' % opts.define)
277
278     if opts.local_package:
279         pac = '_repository'
280     if opts.alternative_project:
281         prj = opts.alternative_project
282         pac = '_repository'
283         apiurl = config['apiurl']
284     else:
285         prj = store_read_project(os.curdir)
286         pac = store_read_package(os.curdir)
287         apiurl = store_read_apiurl(os.curdir)
288
289     if not os.path.exists(build_descr):
290         print >>sys.stderr, 'Error: build description named \'%s\' does not exist.' % build_descr
291         return 1
292
293     # make it possible to override configuration of the rc file
294     for var in ['OSC_PACKAGECACHEDIR', 'OSC_SU_WRAPPER', 'OSC_BUILD_ROOT']: 
295         val = os.getenv(var)
296         if val:
297             if var.startswith('OSC_'): var = var[4:]
298             var = var.lower().replace('_', '-')
299             if config.has_key(var):
300                 print 'Overriding config value for %s=\'%s\' with \'%s\'' % (var, config[var], val)
301             config[var] = val
302
303     config['build-root'] = config['build-root'] % { 'repo': repo, 'arch': arch,
304                                                     'project' : prj, 'package' : pac
305                                                   }
306
307     if not opts.extra_pkgs:
308         extra_pkgs = config['extra-pkgs']
309     elif opts.extra_pkgs == ['']:
310         extra_pkgs = None
311     else:
312         extra_pkgs = opts.extra_pkgs
313
314     if xp:
315         extra_pkgs += xp
316
317
318     print 'Getting buildinfo from server'
319     bi_file = NamedTemporaryFile(suffix='.xml', prefix='buildinfo.', dir = '/tmp')
320     try:
321         bi_text = ''.join(get_buildinfo(apiurl, 
322                                         prj,
323                                         pac,
324                                         repo, 
325                                         arch, 
326                                         specfile=open(build_descr).read(), 
327                                         addlist=extra_pkgs))
328     except urllib2.HTTPError, e:
329         if e.code == 404:
330         # check what caused the 404
331             if meta_exists(metatype='prj', path_args=(quote_plus(prj), ),
332                            template_args=None, create_new=False):
333                 if pac == '_repository' or meta_exists(metatype='pkg', path_args=(quote_plus(prj), quote_plus(pac)),
334                                                        template_args=None, create_new=False):
335                     print >>sys.stderr, 'wrong repo/arch?'
336                     sys.exit(1)
337                 else:
338                     print >>sys.stderr, 'The package \'%s\' does not exists - please ' \
339                                         'rerun with \'--local-package\'' % pac
340                     sys.exit(1)
341             else:
342                 print >>sys.stderr, 'The project \'%s\' does not exists - please ' \
343                                     'rerun with \'--alternative-project <alternative_project>\'' % prj
344                 sys.exit(1)
345         else:
346             raise
347     bi_file.write(bi_text)
348     bi_file.flush()
349
350     bi = Buildinfo(bi_file.name, apiurl)
351     if bi.debuginfo:
352         buildargs.append('--debug')
353     buildargs = ' '.join(set(buildargs))
354
355     # real arch of this machine 
356     # vs.
357     # arch we are supposed to build for
358     if hostarch != bi.buildarch:
359         if not bi.buildarch in can_also_build.get(hostarch, []):
360             print >>sys.stderr, 'Error: hostarch \'%s\' cannot build \'%s\'.' % (hostarch, bi.buildarch)
361             return 1
362
363     rpmlist_prefers = []
364     if opts.prefer_pkgs:
365         print 'Evaluating preferred packages'
366         # the resulting dict will also contain packages which are not on the install list
367         # but they won't be installed
368         prefer_pkgs = get_prefer_pkgs(opts.prefer_pkgs, bi.buildarch)
369
370         for name, path in prefer_pkgs.iteritems():
371             if bi.has_dep(name):
372                 # We remove a preferred package from the buildinfo, so that the
373                 # fetcher doesn't take care about them.
374                 # Instead, we put it in a list which is appended to the rpmlist later.
375                 # At the same time, this will make sure that these packages are
376                 # not verified.
377                 bi.remove_dep(name)
378                 rpmlist_prefers.append((name, path))
379                 print ' - %s (%s)' % (name, path)
380                 continue
381
382     print 'Updating cache of required packages'
383
384     urllist = []
385     # OBS 1.5 and before has no downloadurl defined in buildinfo
386     if bi.downloadurl:
387         urllist.append( bi.downloadurl + '/%(project)s/%(repository)s/%(arch)s/%(filename)s' )
388     urllist.append( '%(apiurl)s/build/%(project)s/%(repository)s/%(buildarch)s/%(repopackage)s/%(repofilename)s' )
389
390     fetcher = Fetcher(cachedir = config['packagecachedir'], 
391                       urllist = urllist,
392                       api_host_options = config['api_host_options'],
393                       http_debug = config['http_debug'],
394                       cookiejar=cookiejar)
395
396     # now update the package cache
397     fetcher.run(bi)
398
399     if build_type == 'kiwi' and not os.path.exists('repos'):
400         os.symlink(config['packagecachedir'], 'repos')
401
402     if bi.pacsuffix == 'rpm':
403         """don't know how to verify .deb packages. They are verified on install
404         anyway, I assume... verifying package now saves time though, since we don't
405         even try to set up the buildroot if it wouldn't work."""
406
407         if config['build-type'] == "xen" or config['build-type'] == "kvm":
408             print 'Skipping verification of package signatures due to secure VM build'
409         elif opts.no_verify:
410             print 'Skipping verification of package signatures'
411         else:
412             print 'Verifying integrity of cached packages'
413             verify_pacs([ i.fullfilename for i in bi.deps ])
414
415     print 'Writing build configuration'
416
417     rpmlist = [ '%s %s\n' % (i.name, i.fullfilename) for i in bi.deps if not i.noinstall ]
418     rpmlist += [ '%s %s\n' % (i[0], i[1]) for i in rpmlist_prefers ]
419
420     rpmlist.append('preinstall: ' + ' '.join(bi.preinstall_list) + '\n')
421     rpmlist.append('vminstall: ' + ' '.join(bi.vminstall_list) + '\n')
422     rpmlist.append('runscripts: ' + ' '.join(bi.runscripts_list) + '\n')
423
424     rpmlist_file = NamedTemporaryFile(prefix='rpmlist.', dir = '/tmp')
425     rpmlist_file.writelines(rpmlist)
426     rpmlist_file.flush()
427     os.fsync(rpmlist_file)
428
429
430
431     print 'Getting buildconfig from server'
432     bc_file = NamedTemporaryFile(prefix='buildconfig.', dir = '/tmp')
433     bc_file.write(get_buildconfig(apiurl, prj, pac, repo, arch))
434     bc_file.flush()
435
436     vm_options=""
437     if config['build-device'] and config['build-memory'] and config['build-type']:
438        if config['build-type'] == "kvm":
439           vm_options="--kvm " + config['build-device']
440        elif config['build-type'] == "xen":
441           vm_options="--xen " + config['build-device']
442        else:
443           print "ERROR: unknown VM is set ! (" + config['build-type'] + ")"
444           sys.exit(1)
445        if config['build-swap']:
446           vm_options+=" --swap " + config['build-swap']
447        if config['build-memory']:
448           vm_options+=" --memory " + config['build-memory']
449     
450     print 'Running build'
451     cmd = '%s --root=%s --rpmlist=%s --dist=%s --arch=%s %s %s %s' \
452                  % (config['build-cmd'],
453                     config['build-root'],
454                     rpmlist_file.name, 
455                     bc_file.name, 
456                     bi.buildarch,
457                     vm_options,
458                     build_descr, 
459                     buildargs)
460
461     if config['su-wrapper'].startswith('su '):
462         tmpl = '%s \'%s\''
463     else:
464         tmpl = '%s %s'
465
466     # change personality, if needed
467     cmd = tmpl % (config['su-wrapper'], cmd)
468     if hostarch != bi.buildarch:
469         cmd = (change_personality.get(bi.buildarch, '') + ' ' + cmd).strip()
470
471     print cmd
472     rc = subprocess.call(cmd, shell=True)
473     if rc: 
474         print
475         print 'The buildroot was:', config['build-root']
476         sys.exit(rc)
477
478     pacdirlink = os.path.join(config['build-root'], '.build.packages')
479     if os.path.exists(pacdirlink):
480         pacdirlink = os.readlink(pacdirlink)
481         pacdir = os.path.join(config['build-root'], pacdirlink)
482
483         if os.path.exists(pacdir):
484             (s_built, b_built) = get_built_files(pacdir, bi.pacsuffix)
485
486             print
487             if s_built: print s_built
488             print
489             print b_built
490
491             if opts.keep_pkgs:
492                 for i in b_built.splitlines() + s_built.splitlines():
493                     import shutil
494                     shutil.copy2(i, os.path.join(opts.keep_pkgs, os.path.basename(i)))
495
496