Make 'trust always' more verbose, so that a user learns how to undo its effect.
[opensuse:osc.git] / osc / build.py
1 # Copyright (C) 2006 Novell Inc.  All rights reserved.
2 # This program is free software; it may be used, copied, modified
3 # and distributed under the terms of the GNU General Public Licence,
4 # either version 2, or (at your option) any later version.
5
6
7
8 import os
9 import re
10 import sys
11 from tempfile import NamedTemporaryFile, mkdtemp
12 from shutil import rmtree
13 from osc.fetch import *
14 from osc.core import get_buildinfo, store_read_apiurl, store_read_project, store_read_package, meta_exists, quote_plus, get_buildconfig, is_package_dir
15 from osc.core import get_binarylist, get_binary_file
16 from osc.util import rpmquery, debquery
17 import osc.conf
18 import oscerr
19 import subprocess
20 try:
21     from xml.etree import cElementTree as ET
22 except ImportError:
23     import cElementTree as ET
24
25 from conf import config, cookiejar
26
27 change_personality = {
28             'i686':  'linux32',
29             'i586':  'linux32',
30             'i386':  'linux32',
31             'ppc':   'powerpc32',
32             's390':  's390',
33             'sparc': 'linux32',
34             'sparcv8': 'linux32',
35         }
36
37 can_also_build = {
38              'armv4l': [                                         'armv4l'                                             ],
39              'armv5el':[                                         'armv4l', 'armv5el'                                  ],
40              'armv6el':[                                         'armv4l', 'armv5el', 'armv6el'                       ],
41              'armv6l' :[                                         'armv4l', 'armv5el', 'armv6el'                       ],
42              'armv7el':[                                         'armv4l', 'armv5el', 'armv6el', 'armv7el'            ],
43              'armv7l' :[                                         'armv4l', 'armv5el', 'armv6el', 'armv7el'            ],
44              'armv8el':[                                         'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el' ],
45              'armv8l' :[                                         'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el' ],
46              's390x':  ['s390' ],
47              'ppc64':  [                        'ppc', 'ppc64' ],
48              'sh4':    [                                                                                               'sh4' ],
49              'i386':   [        'i586',         'ppc', 'ppc64',  'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el', 'sh4', 'mips', 'mips64' ],
50              'i586':   [                'i386', 'ppc', 'ppc64',  'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el', 'sh4', 'mips', 'mips64' ],
51              'i686':   [        'i586',         'ppc', 'ppc64',  'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el', 'sh4', 'mips', 'mips64' ],
52              'x86_64': ['i686', 'i586', 'i386', 'ppc', 'ppc64',  'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el', 'sh4', 'mips', 'mips64' ],
53              'sparc64': ['sparc64v', 'sparcv9v', 'sparcv9', 'sparcv8', 'sparc'],
54              }
55
56 # real arch of this machine
57 hostarch = os.uname()[4]
58 if hostarch == 'i686': # FIXME
59     hostarch = 'i586'
60
61 class Buildinfo:
62     """represent the contents of a buildinfo file"""
63
64     def __init__(self, filename, apiurl, buildtype = 'spec', localpkgs = []):
65         try:
66             tree = ET.parse(filename)
67         except:
68             print >>sys.stderr, 'could not parse the buildinfo:'
69             print >>sys.stderr, open(filename).read()
70             sys.exit(1)
71
72         root = tree.getroot()
73
74         self.apiurl = apiurl
75
76         if root.find('error') != None:
77             sys.stderr.write('buildinfo is broken... it says:\n')
78             error = root.find('error').text
79             sys.stderr.write(error + '\n')
80             sys.exit(1)
81
82         if not (apiurl.startswith('https://') or apiurl.startswith('http://')):
83             raise urllib2.URLError('invalid protocol for the apiurl: \'%s\'' % apiurl)
84
85         self.buildtype = buildtype
86         self.apiurl = apiurl
87
88         # are we building .rpm or .deb?
89         # XXX: shouldn't we deliver the type via the buildinfo?
90         self.pacsuffix = 'rpm'
91         if self.buildtype == 'dsc':
92             self.pacsuffix = 'deb'
93
94         self.buildarch = root.find('arch').text
95         if root.find('release') != None:
96             self.release = root.find('release').text
97         else:
98             self.release = None
99         self.downloadurl = root.get('downloadurl')
100         self.debuginfo = 0
101         if root.find('debuginfo') != None:
102             try:
103                 self.debuginfo = int(root.find('debuginfo').text)
104             except ValueError:
105                 pass
106
107         self.deps = []
108         self.projects = {}
109         self.keys = []
110         self.prjkeys = []
111         for node in root.findall('bdep'):
112             p = Pac(node, self.buildarch, self.pacsuffix,
113                     apiurl, localpkgs)
114             if p.project:
115                 self.projects[p.project] = 1
116             self.deps.append(p)
117
118         self.vminstall_list = [ dep.name for dep in self.deps if dep.vminstall ]
119         self.cbinstall_list = [ dep.name for dep in self.deps if dep.cbinstall ]
120         self.cbpreinstall_list = [ dep.name for dep in self.deps if dep.cbpreinstall ]
121         self.preinstall_list = [ dep.name for dep in self.deps if dep.preinstall ]
122         self.runscripts_list = [ dep.name for dep in self.deps if dep.runscripts ]
123
124
125     def has_dep(self, name):
126         for i in self.deps:
127             if i.name == name:
128                 return True
129         return False
130
131     def remove_dep(self, name):
132         for i in self.deps:
133             if i.name == name:
134                 self.deps.remove(i)
135                 return True
136         return False
137
138
139 class Pac:
140     """represent a package to be downloaded
141
142     We build a map that's later used to fill our URL templates
143     """
144     def __init__(self, node, buildarch, pacsuffix, apiurl, localpkgs = []):
145
146         self.mp = {}
147         for i in ['name', 'package',
148                   'version', 'release',
149                   'project', 'repository',
150                   'preinstall', 'vminstall', 'noinstall', 'runscripts',
151                   'cbinstall', 'cbpreinstall',
152                  ]:
153             self.mp[i] = node.get(i)
154
155         self.mp['buildarch']  = buildarch
156         self.mp['pacsuffix']  = pacsuffix
157
158         self.mp['arch'] = node.get('arch') or self.mp['buildarch']
159
160         # this is not the ideal place to check if the package is a localdep or not
161         localdep = self.mp['name'] in localpkgs
162         if not localdep and not (node.get('project') and node.get('repository')):
163             raise oscerr.APIError('incomplete information for package %s, may be caused by a broken project configuration.'
164                                   % self.mp['name'] )
165
166         if not localdep:
167             self.mp['extproject'] = node.get('project').replace(':', ':/')
168             self.mp['extrepository'] = node.get('repository').replace(':', ':/')
169         self.mp['repopackage'] = node.get('package') or '_repository'
170         self.mp['repoarch'] = node.get('repoarch') or self.mp['buildarch']
171
172         if pacsuffix == 'deb' and not (self.mp['name'] and self.mp['arch'] and self.mp['version']):
173             raise oscerr.APIError(
174                 "buildinfo for package %s/%s/%s is incomplete"
175                     % (self.mp['name'], self.mp['arch'], self.mp['version']))
176
177         self.mp['apiurl'] = apiurl
178
179         if pacsuffix == 'deb':
180             self.filename = debquery.DebQuery.filename(self.mp['name'], self.mp['version'], self.mp['release'], self.mp['arch'])
181         else:
182             self.filename = rpmquery.RpmQuery.filename(self.mp['name'], self.mp['version'], self.mp['release'], self.mp['arch'])
183
184         self.mp['filename'] = self.filename
185         if self.mp['repopackage'] == '_repository':
186             self.mp['repofilename'] = self.mp['name']
187         else:
188             self.mp['repofilename'] = self.mp['filename']
189
190         # make the content of the dictionary accessible as class attributes
191         self.__dict__.update(self.mp)
192
193
194     def makeurls(self, cachedir, urllist):
195
196         self.urllist = []
197
198         # build up local URL
199         # by using the urlgrabber with local urls, we basically build up a cache.
200         # the cache has no validation, since the package servers don't support etags,
201         # or if-modified-since, so the caching is simply name-based (on the assumption
202         # that the filename is suitable as identifier)
203         self.localdir = '%s/%s/%s/%s' % (cachedir, self.project, self.repository, self.arch)
204         self.fullfilename = os.path.join(self.localdir, self.filename)
205         self.url_local = 'file://%s' % self.fullfilename
206
207         # first, add the local URL
208         self.urllist.append(self.url_local)
209
210         # remote URLs
211         for url in urllist:
212             self.urllist.append(url % self.mp)
213
214     def __str__(self):
215         return self.name
216
217     def __repr__(self):
218         return "%s" % self.name
219
220
221
222 def get_built_files(pacdir, pactype):
223     if pactype == 'rpm':
224         b_built = subprocess.Popen(['find', os.path.join(pacdir, 'RPMS'),
225                                     '-name', '*.rpm'],
226                                    stdout=subprocess.PIPE).stdout.read().strip()
227         s_built = subprocess.Popen(['find', os.path.join(pacdir, 'SRPMS'),
228                                     '-name', '*.rpm'],
229                                    stdout=subprocess.PIPE).stdout.read().strip()
230     elif pactype == 'kiwi':
231         b_built = subprocess.Popen(['find', os.path.join(pacdir, 'KIWI'),
232                                     '-type', 'f'],
233                                    stdout=subprocess.PIPE).stdout.read().strip()
234     else:
235         b_built = subprocess.Popen(['find', os.path.join(pacdir, 'DEBS'),
236                                     '-name', '*.deb'],
237                                    stdout=subprocess.PIPE).stdout.read().strip()
238         s_built = subprocess.Popen(['find', os.path.join(pacdir, 'SOURCES.DEB'),
239                                     '-type', 'f'],
240                                    stdout=subprocess.PIPE).stdout.read().strip()
241     return s_built, b_built
242
243 def get_repo(path):
244     """Walks up path looking for any repodata directories.
245
246     @param path path to a directory
247     @return str path to repository directory containing repodata directory
248     """
249     oldDirectory = None
250     currentDirectory = os.path.abspath(path)
251     repositoryDirectory = None
252
253     # while there are still parent directories
254     while currentDirectory != oldDirectory:
255         children = os.listdir(currentDirectory)
256
257         if "repodata" in children:
258             repositoryDirectory = currentDirectory
259             break
260
261         # ascend
262         oldDirectory = currentDirectory
263         currentDirectory = os.path.abspath(os.path.join(oldDirectory,
264                                                         os.pardir))
265
266     return repositoryDirectory
267
268 def get_prefer_pkgs(dirs, wanted_arch, type):
269     import glob
270     from util import repodata, packagequery, cpio
271     paths = []
272     repositories = []
273
274     suffix = '*.rpm'
275     if type == 'dsc':
276         suffix = '*.deb'
277
278     for dir in dirs:
279         # check for repodata
280         repository = get_repo(dir)
281         if repository is None:
282             paths += glob.glob(os.path.join(os.path.abspath(dir), suffix))
283         else:
284             repositories.append(repository)
285
286     packageQueries = packagequery.PackageQueries(wanted_arch)
287
288     for repository in repositories:
289         repodataPackageQueries = repodata.queries(repository)
290
291         for packageQuery in repodataPackageQueries:
292             packageQueries.add(packageQuery)
293
294     for path in paths:
295         if path.endswith('src.rpm'):
296             continue
297         if path.find('-debuginfo-') > 0:
298             continue
299         packageQuery = packagequery.PackageQuery.query(path)
300         packageQueries.add(packageQuery)
301
302     prefer_pkgs = dict((name, packageQuery.path())
303                        for name, packageQuery in packageQueries.iteritems())
304
305     depfile = create_deps(packageQueries.values())
306     cpio = cpio.CpioWrite()
307     cpio.add('deps', '\n'.join(depfile))
308     return prefer_pkgs, cpio
309
310
311 def create_deps(pkgqs):
312     """
313     creates a list of requires/provides which corresponds to build's internal
314     dependency file format
315     """
316     depfile = []
317     for p in pkgqs:
318         id = '%s.%s-0/0/0: ' % (p.name(), p.arch())
319         depfile.append('R:%s%s' % (id, ' '.join(p.requires())))
320         depfile.append('P:%s%s' % (id, ' '.join(p.provides())))
321     return depfile
322
323
324 trustprompt = """Would you like to ...
325 0 - quit (default)
326 1 - trust packages from '%(project)s' always
327 2 - trust them just this time
328 ? """
329 def check_trusted_projects(apiurl, projects):
330     trusted = config['api_host_options'][apiurl]['trusted_prj']
331     tlen = len(trusted)
332     for prj in projects:
333         if not prj in trusted:
334             print "\nThe build root needs packages from project '%s'." % prj
335             print "Note that malicious packages can compromise the build result or even your system."
336             r = raw_input(trustprompt % { 'project':prj })
337             if r == '1':
338                 print "adding '%s' to ~/.oscrc: ['%s']['trusted_prj']" % (prj,apiurl)
339                 trusted.append(prj)
340             elif r != '2':
341                 print "Well, good good bye then :-)"
342                 raise oscerr.UserAbort()
343
344     if tlen != len(trusted):
345         config['api_host_options'][apiurl]['trusted_prj'] = trusted
346         conf.config_set_option(apiurl, 'trusted_prj', ' '.join(trusted))
347
348 def main(opts, argv):
349
350     repo = argv[0]
351     arch = argv[1]
352     build_descr = argv[2]
353     xp = []
354     build_root = None
355     build_uid=''
356     vm_type = config['build-type']
357
358     build_descr = os.path.abspath(build_descr)
359     build_type = os.path.splitext(build_descr)[1][1:]
360     if build_type not in ['spec', 'dsc', 'kiwi']:
361         raise oscerr.WrongArgs(
362                 'Unknown build type: \'%s\'. Build description should end in .spec, .dsc or .kiwi.' \
363                         % build_type)
364     if not os.path.isfile(build_descr):
365         raise oscerr.WrongArgs('Error: build description file named \'%s\' does not exist.' % build_descr)
366
367     buildargs = []
368     if not opts.userootforbuild:
369         buildargs.append('--norootforbuild')
370     if opts.clean:
371         buildargs.append('--clean')
372     if opts.noinit:
373         buildargs.append('--noinit')
374     if opts.nochecks:
375         buildargs.append('--no-checks')
376     if not opts.no_changelog:
377         buildargs.append('--changelog')
378     if opts.root:
379         build_root = opts.root
380     if opts.jobs:
381         buildargs.append('--jobs %s' % opts.jobs)
382     elif config['build-jobs'] > 1:
383         buildargs.append('--jobs %s' % config['build-jobs'])
384     if opts.icecream or config['icecream'] != '0':
385         if opts.icecream:
386             num = opts.icecream
387         else:
388             num = config['icecream']
389
390         if int(num) > 0:
391             buildargs.append('--icecream %s' % num)
392             xp.append('icecream')
393             xp.append('gcc-c++')
394     if opts.ccache:
395         buildargs.append('--ccache')
396         xp.append('ccache')
397     if opts.linksources:
398         buildargs.append('--linksources')
399     if opts.baselibs:
400         buildargs.append('--baselibs')
401     if opts.debuginfo:
402         buildargs.append('--debug')
403     if opts._with:
404         for o in opts._with:
405             buildargs.append('--with %s' % o)
406     if opts.without:
407         for o in opts.without:
408             buildargs.append('--without %s' % o)
409 # FIXME: quoting
410 #    if opts.define:
411 #        buildargs.append('--define "%s"' % opts.define)
412     if config['build-uid']:
413         build_uid = config['build-uid']
414     if opts.build_uid:
415         build_uid = opts.build_uid
416     if build_uid:
417         buildidre = re.compile('^[0-9]{1,5}:[0-9]{1,5}$')
418         if build_uid == 'caller':
419             buildargs.append('--uid %s:%s' % (os.getuid(), os.getgid()))
420         elif buildidre.match(build_uid):
421             buildargs.append('--uid %s' % build_uid)
422         else:
423             print >>sys.stderr, 'Error: build-uid arg must be 2 colon separated numerics: "uid:gid" or "caller"'
424             return 1
425     if opts.vm_type:
426         vm_type = opts.vm_type
427     if opts.alternative_project:
428         prj = opts.alternative_project
429         pac = '_repository'
430     else:
431         prj = store_read_project(os.curdir)
432         if opts.local_package:
433             pac = '_repository'
434         else:
435             pac = store_read_package(os.curdir)
436     apiurl = store_read_apiurl(os.curdir)
437
438     # make it possible to override configuration of the rc file
439     for var in ['OSC_PACKAGECACHEDIR', 'OSC_SU_WRAPPER', 'OSC_BUILD_ROOT']:
440         val = os.getenv(var)
441         if val:
442             if var.startswith('OSC_'): var = var[4:]
443             var = var.lower().replace('_', '-')
444             if config.has_key(var):
445                 print 'Overriding config value for %s=\'%s\' with \'%s\'' % (var, config[var], val)
446             config[var] = val
447
448     pacname = pac
449     if pacname == '_repository':
450         if not opts.local_package:
451             try:
452                 pacname = store_read_package(os.curdir)
453             except oscerr.NoWorkingCopy:
454                 opts.local_package = True
455         if opts.local_package:
456             pacname = os.path.splitext(build_descr)[0]
457     if not build_root:
458         build_root = config['build-root'] % { 'repo': repo, 'arch': arch,
459                                                     'project' : prj, 'package' : pacname
460                                                   }
461
462     extra_pkgs = []
463     if not opts.extra_pkgs:
464         extra_pkgs = config['extra-pkgs']
465     elif opts.extra_pkgs != ['']:
466         extra_pkgs = opts.extra_pkgs
467
468     if xp:
469         extra_pkgs += xp
470
471     prefer_pkgs = {}
472     build_descr_data = open(build_descr).read()
473
474     # XXX: dirty hack but there's no api to provide custom defines
475     if opts.without:
476         s = ''
477         for i in opts.without:
478             s += "%%define _without_%s 1\n" % i
479             s += "%%define _with_%s 0\n" % i
480         build_descr_data = s + build_descr_data
481     if opts._with:
482         s = ''
483         for i in opts._with:
484             s += "%%define _without_%s 0\n" % i
485             s += "%%define _with_%s 1\n" % i
486         build_descr_data = s + build_descr_data
487
488     if opts.prefer_pkgs:
489         print 'Scanning the following dirs for local packages: %s' % ', '.join(opts.prefer_pkgs)
490         prefer_pkgs, cpio = get_prefer_pkgs(opts.prefer_pkgs, arch, build_type)
491         cpio.add(os.path.basename(build_descr), build_descr_data)
492         build_descr_data = cpio.get()
493
494     # special handling for overlay and rsync-src/dest
495     specialcmdopts = ''
496     if opts.rsyncsrc or opts.rsyncdest :
497         if not opts.rsyncsrc or not opts.rsyncdest:
498             raise oscerr.WrongOptions('When using --rsync-{src,dest} both parameters have to be specified.')
499         myrsyncsrc = os.path.abspath(os.path.expanduser(os.path.expandvars(opts.rsyncsrc)))
500         if not os.path.isdir(myrsyncsrc):
501             raise oscerr.WrongOptions('--rsync-src %s is no valid directory!' % opts.rsyncsrc)
502         # can't check destination - its in the target chroot ;) - but we can check for sanity
503         myrsyncdest = os.path.expandvars(opts.rsyncdest)
504         if not os.path.isabs(myrsyncdest):
505             raise oscerr.WrongOptions('--rsync-dest %s is no absolute path (starting with \'/\')!' % opts.rsyncdest)
506         specialcmdopts = '--rsync-src="%s" --rsync-dest="%s"' % (myrsyncsrc, myrsyncdest)
507     if opts.overlay:
508         myoverlay = os.path.abspath(os.path.expanduser(os.path.expandvars(opts.overlay)))
509         if not os.path.isdir(myoverlay):
510             raise oscerr.WrongOptions('--overlay %s is no valid directory!' % opts.overlay)
511         specialcmdopts += '--overlay="%s"' % myoverlay
512
513     bi_file = None
514     bc_file = None
515     bi_filename = '_buildinfo-%s-%s.xml' % (repo, arch)
516     bc_filename = '_buildconfig-%s-%s' % (repo, arch)
517     if is_package_dir('.') and os.access(osc.core.store, os.W_OK):
518         bi_filename = os.path.join(os.getcwd(), osc.core.store, bi_filename)
519         bc_filename = os.path.join(os.getcwd(), osc.core.store, bc_filename)
520     elif not os.access('.', os.W_OK):
521         bi_file = NamedTemporaryFile(prefix=bi_filename)
522         bi_filename = bi_file.name
523         bc_file = NamedTemporaryFile(prefix=bc_filename)
524         bc_filename = bc_file.name
525     else:
526         bi_filename = os.path.abspath(bi_filename)
527         bc_filename = os.path.abspath(bc_filename)
528
529     try:
530         if opts.noinit:
531             if not os.path.isfile(bi_filename):
532                 raise oscerr.WrongOptions('--noinit is not possible, no local buildinfo file')
533             print 'Use local \'%s\' file as buildinfo' % bi_filename
534             if not os.path.isfile(bc_filename):
535                 raise oscerr.WrongOptions('--noinit is not possible, no local buildconfig file')
536             print 'Use local \'%s\' file as buildconfig' % bc_filename
537         elif opts.offline:
538             if not os.path.isfile(bi_filename):
539                 raise oscerr.WrongOptions('--offline is not possible, no local buildinfo file')
540             print 'Use local \'%s\' file as buildinfo' % bi_filename
541             if not os.path.isfile(bc_filename):
542                 raise oscerr.WrongOptions('--offline is not possible, no local buildconfig file')
543         else:
544             print 'Getting buildinfo from server and store to %s' % bi_filename
545             if not bi_file:
546                 bi_file = open(bi_filename, 'w')
547             bi_text = ''.join(get_buildinfo(apiurl,
548                                             prj,
549                                             pac,
550                                             repo,
551                                             arch,
552                                             specfile=build_descr_data,
553                                             addlist=extra_pkgs))
554             bi_file.write(bi_text)
555             bi_file.flush()
556             print 'Getting buildconfig from server and store to %s' % bc_filename
557             if not bc_file:
558                 bc_file = open(bc_filename, 'w')
559             bc_file.write(get_buildconfig(apiurl, prj, pac, repo, arch))
560             bc_file.flush()
561     except urllib2.HTTPError, e:
562         if e.code == 404:
563             # check what caused the 404
564             if meta_exists(metatype='prj', path_args=(quote_plus(prj), ),
565                            template_args=None, create_new=False, apiurl=apiurl):
566                 pkg_meta_e = None
567                 try:
568                     # take care, not to run into double trouble.
569                     pkg_meta_e = meta_exists(metatype='pkg', path_args=(quote_plus(prj), 
570                                         quote_plus(pac)), template_args=None, create_new=False, 
571                                         apiurl=apiurl)
572                 except:
573                     pass
574
575                 if pac == '_repository' or pkg_meta_e:
576                     print >>sys.stderr, 'ERROR: Either wrong repo/arch as parameter or a parse error of .spec/.dsc/.kiwi file due to syntax error'
577                 else:
578                     print >>sys.stderr, 'The package \'%s\' does not exists - please ' \
579                                         'rerun with \'--local-package\'' % pac
580             else:
581                 print >>sys.stderr, 'The project \'%s\' does not exists - please ' \
582                                     'rerun with \'--alternative-project <alternative_project>\'' % prj
583             sys.exit(1)
584         else:
585             raise
586
587     bi = Buildinfo(bi_filename, apiurl, build_type, prefer_pkgs.keys())
588
589     if bi.debuginfo and not (opts.disable_debuginfo or '--debug' in buildargs):
590         buildargs.append('--debug')
591
592     if opts.release:
593         bi.release = opts.release
594
595     if bi.release:
596         buildargs.append('--release %s' % bi.release)
597
598     # real arch of this machine
599     # vs.
600     # arch we are supposed to build for
601     if hostarch != bi.buildarch:
602         if not bi.buildarch in can_also_build.get(hostarch, []):
603             print >>sys.stderr, 'Error: hostarch \'%s\' cannot build \'%s\'.' % (hostarch, bi.buildarch)
604             return 1
605
606     rpmlist_prefers = []
607     if prefer_pkgs:
608         print 'Evaluating preferred packages'
609         for name, path in prefer_pkgs.iteritems():
610             if bi.has_dep(name):
611                 # We remove a preferred package from the buildinfo, so that the
612                 # fetcher doesn't take care about them.
613                 # Instead, we put it in a list which is appended to the rpmlist later.
614                 # At the same time, this will make sure that these packages are
615                 # not verified.
616                 bi.remove_dep(name)
617                 rpmlist_prefers.append((name, path))
618                 print ' - %s (%s)' % (name, path)
619                 continue
620
621     print 'Updating cache of required packages'
622
623     urllist = []
624     if not opts.download_api_only:
625         # transform 'url1, url2, url3' form into a list
626         if 'urllist' in config:
627             if type(config['urllist']) == str:
628                 re_clist = re.compile('[, ]+')
629                 urllist = [ i.strip() for i in re_clist.split(config['urllist'].strip()) ]
630             else:
631                 urllist = config['urllist']
632
633         # OBS 1.5 and before has no downloadurl defined in buildinfo
634         if bi.downloadurl:
635             urllist.append(bi.downloadurl + '/%(extproject)s/%(extrepository)s/%(arch)s/%(filename)s')
636     if not opts.cpio_bulk_download:
637         urllist.append( '%(apiurl)s/build/%(project)s/%(repository)s/%(repoarch)s/%(repopackage)s/%(repofilename)s' )
638
639     fetcher = Fetcher(cachedir = config['packagecachedir'],
640                       urllist = urllist,
641                       api_host_options = config['api_host_options'],
642                       offline = opts.noinit or opts.offline,
643                       http_debug = config['http_debug'],
644                       enable_cpio = opts.cpio_bulk_download,
645                       cookiejar=cookiejar)
646
647     # implicitly trust the project we are building for
648     check_trusted_projects(apiurl, [ i for i in bi.projects.keys() if not i == prj ])
649
650     # now update the package cache
651     fetcher.run(bi)
652
653     old_pkg_dir = None
654     if opts.oldpackages:
655         old_pkg_dir = opts.oldpackages
656         if not old_pkg_dir.startswith('/') and not opts.offline:
657             data = [ prj, pacname, repo, arch]
658             if old_pkg_dir == '_link':
659                 p = osc.core.findpacs(os.curdir)[0]
660                 if not p.islink():
661                     raise oscerr.WrongOptions('package is not a link')
662                 data[0] = p.linkinfo.project
663                 data[1] = p.linkinfo.package
664                 repos = osc.core.get_repositories_of_project(apiurl, data[0])
665                 # hack for links to e.g. Factory
666                 if not data[2] in repos and 'standard' in repos:
667                     data[2] = 'standard'
668             elif old_pkg_dir != '' and old_pkg_dir != '_self':
669                 a = old_pkg_dir.split('/')
670                 for i in range(0, len(a)):
671                     data[i] = a[i]
672
673             destdir = os.path.join(config['packagecachedir'], data[0], data[2], data[3])
674             old_pkg_dir = None
675             try:
676                 print "Downloading previous build from %s ..." % '/'.join(data)
677                 binaries = get_binarylist(apiurl, data[0], data[2], data[3], package=data[1], verbose=True)
678             except Exception, e:
679                 print "Error: failed to get binaries: %s" % str(e)
680                 binaries = []
681
682             if binaries:
683                 class mytmpdir:
684                     """ temporary directory that removes itself"""
685                     def __init__(self, *args, **kwargs):
686                         self.name = mkdtemp(*args, **kwargs)
687                     def cleanup(self):
688                         rmtree(self.name)
689                     def __del__(self):
690                         self.cleanup()
691                     def __exit__(self):
692                         self.cleanup()
693                     def __str__(self):
694                         return self.name
695
696                 old_pkg_dir = mytmpdir(prefix='.build.oldpackages', dir=os.path.abspath(os.curdir))
697                 if not os.path.exists(destdir):
698                     os.makedirs(destdir)
699             for i in binaries:
700                 fname = os.path.join(destdir, i.name)
701                 os.symlink(fname, os.path.join(str(old_pkg_dir), i.name))
702                 if os.path.exists(fname):
703                     st = os.stat(fname)
704                     if st.st_mtime == i.mtime and st.st_size == i.size:
705                         continue
706                 get_binary_file(apiurl,
707                                 data[0],
708                                 data[2], data[3],
709                                 i.name,
710                                 package = data[1],
711                                 target_filename = fname,
712                                 target_mtime = i.mtime,
713                                 progress_meter = True)
714
715         if old_pkg_dir != None:
716             buildargs.append('--oldpackages %s' % old_pkg_dir)
717
718     # Make packages from buildinfo available as repos for kiwi
719     if build_type == 'kiwi':
720         if not os.path.exists('repos'):
721             os.mkdir('repos')
722         else:
723             rmtree('repos')
724             os.mkdir('repos')
725         for i in bi.deps:
726             # project
727             pdir = str(i.extproject).replace(':/', ':')
728             # repo
729             rdir = str(i.extrepository).replace(':/', ':')
730             # arch
731             adir = i.repoarch
732             # project/repo
733             prdir = "repos/"+pdir+"/"+rdir
734             # project/repo/arch
735             pradir = prdir+"/"+adir
736             # source fullfilename
737             sffn = i.fullfilename
738             print "Using package: "+sffn
739             # target fullfilename
740             tffn = pradir+"/"+sffn.split("/")[-1]
741             if not os.path.exists(os.path.join(pradir)):
742                 os.makedirs(os.path.join(pradir))
743             if not os.path.exists(tffn):
744                 if opts.linksources:
745                     os.link(sffn, tffn)
746                 else:
747                     os.symlink(sffn, tffn)
748
749     if bi.pacsuffix == 'rpm':
750         if opts.no_verify:
751             print 'Skipping verification of package signatures'
752         else:
753             print 'Verifying integrity of cached packages'
754             verify_pacs([ i.fullfilename for i in bi.deps ], bi.keys)
755
756     elif bi.pacsuffix == 'deb':
757         if vm_type == "xen" or vm_type == "kvm" or vm_type == "lxc":
758             print 'Skipping verification of package signatures due to secure VM build'
759         elif opts.no_verify or opts.noinit:
760             print 'Skipping verification of package signatures'
761         else:
762             print 'WARNING: deb packages get not verified, they can compromise your system !'
763     else:
764         print 'WARNING: unknown packages get not verified, they can compromise your system !'
765
766     print 'Writing build configuration'
767
768     rpmlist = [ '%s %s\n' % (i.name, i.fullfilename) for i in bi.deps if not i.noinstall ]
769     rpmlist += [ '%s %s\n' % (i[0], i[1]) for i in rpmlist_prefers ]
770
771     rpmlist.append('preinstall: ' + ' '.join(bi.preinstall_list) + '\n')
772     rpmlist.append('vminstall: ' + ' '.join(bi.vminstall_list) + '\n')
773     rpmlist.append('cbinstall: ' + ' '.join(bi.cbinstall_list) + '\n')
774     rpmlist.append('cbpreinstall: ' + ' '.join(bi.cbpreinstall_list) + '\n')
775     rpmlist.append('runscripts: ' + ' '.join(bi.runscripts_list) + '\n')
776
777     rpmlist_file = NamedTemporaryFile(prefix='rpmlist.')
778     rpmlist_filename = rpmlist_file.name
779     rpmlist_file.writelines(rpmlist)
780     rpmlist_file.flush()
781
782     subst = { 'repo': repo, 'arch': arch, 'project' : prj, 'package' : pacname }
783     vm_options = ''
784     # XXX check if build-device present
785     my_build_device = ''
786     if config['build-device']:
787         my_build_device = config['build-device'] % subst
788     else:
789         # obs worker uses /root here but that collides with the
790         # /root directory if the build root was used without vm
791         # before
792         my_build_device = build_root + '/img'
793
794     need_root = True
795     if vm_type:
796         if config['build-swap']:
797             my_build_swap = config['build-swap'] % subst
798         else:
799             my_build_swap = build_root + '/swap'
800
801         vm_options = '--vm-type=%s'%vm_type
802         if vm_type != 'lxc':
803             vm_options += ' --vm-disk=' + my_build_device
804             vm_options += ' --vm-swap=' + my_build_swap
805             vm_options += ' --logfile=%s/.build.log' % build_root
806             if vm_type == 'kvm':
807                 if os.access(build_root, os.W_OK) and os.access('/dev/kvm', os.W_OK):
808                     # so let's hope there's also an fstab entry
809                     need_root = False
810             build_root += '/.mount'
811
812         if config['build-memory']:
813             vm_options += ' --memory ' + config['build-memory']
814         if config['build-vmdisk-rootsize']:
815             vm_options += ' --vmdisk-rootsize ' + config['build-vmdisk-rootsize']
816         if config['build-vmdisk-swapsize']:
817             vm_options += ' --vmdisk-swapsize ' + config['build-vmdisk-swapsize']
818
819     if opts.preload:
820         print "Preload done for selected repo/arch."
821         sys.exit(0)
822
823     print 'Running build'
824     cmd = '"%s" --root="%s" --rpmlist="%s" --dist="%s" %s --arch=%s %s %s "%s"' \
825                  % (config['build-cmd'],
826                     build_root,
827                     rpmlist_filename,
828                     bc_filename,
829                     specialcmdopts,
830                     bi.buildarch,
831                     vm_options,
832                     ' '.join(buildargs),
833                     build_descr)
834
835     if need_root:
836         if config['su-wrapper'].startswith('su '):
837             tmpl = '%s \'%s\''
838         else:
839             tmpl = '%s %s'
840         cmd = tmpl % (config['su-wrapper'], cmd)
841
842     # change personality, if needed
843     if hostarch != bi.buildarch:
844         cmd = (change_personality.get(bi.buildarch, '') + ' ' + cmd).strip()
845
846     print cmd
847     rc = subprocess.call(cmd, shell=True)
848     if rc:
849         print
850         print 'The buildroot was:', build_root
851         sys.exit(rc)
852
853     pacdir = os.path.join(build_root, '.build.packages')
854     if os.path.islink(pacdir):
855         pacdir = os.readlink(pacdir)
856         pacdir = os.path.join(build_root, pacdir)
857
858     if os.path.exists(pacdir):
859         (s_built, b_built) = get_built_files(pacdir, bi.pacsuffix)
860
861         print
862         if s_built: print s_built
863         print
864         print b_built
865
866         if opts.keep_pkgs:
867             for i in b_built.splitlines() + s_built.splitlines():
868                 shutil.copy2(i, os.path.join(opts.keep_pkgs, os.path.basename(i)))
869
870     if bi_file:
871         bi_file.close()
872     if bc_file:
873         bc_file.close()
874     rpmlist_file.close()
875
876 # vim: sw=4 et