gitorious.org Git - opensuse:osc.git/blob

2 # This program is free software; it may be used, copied, modified

3 # and distributed under the terms of the GNU General Public Licence,

4 # either version 2, or (at your option) any later version.

5

6

7

8 import os

9 import re

10 import sys

11 from tempfile import NamedTemporaryFile, mkdtemp

12 from shutil import rmtree

13 from osc.fetch import *

14 from osc.core import get_buildinfo, store_read_apiurl, store_read_project, store_read_package, meta_exists, quote_plus, get_buildconfig, is_package_dir

15 from osc.core import get_binarylist, get_binary_file

16 from osc.util import rpmquery, debquery

17 import osc.conf

18 import oscerr

19 import subprocess

20 try:

21 from xml.etree import cElementTree as ET

22 except ImportError:

23 import cElementTree as ET

24

25 from conf import config, cookiejar

26

27 change_personality = {

28 'i686': 'linux32',

29 'i586': 'linux32',

30 'i386': 'linux32',

31 'ppc': 'powerpc32',

32 's390': 's390',

33 'sparc': 'linux32',

34 'sparcv8': 'linux32',

35 }

36

37 can_also_build = {

38 'armv4l': [ 'armv4l' ],

39 'armv5el':[ 'armv4l', 'armv5el' ],

40 'armv6el':[ 'armv4l', 'armv5el', 'armv6el' ],

41 'armv6l' :[ 'armv4l', 'armv5el', 'armv6el' ],

42 'armv7el':[ 'armv4l', 'armv5el', 'armv6el', 'armv7el' ],

43 'armv7l' :[ 'armv4l', 'armv5el', 'armv6el', 'armv7el' ],

44 'armv8el':[ 'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el' ],

45 'armv8l' :[ 'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el' ],

46 's390x': ['s390' ],

47 'ppc64': [ 'ppc', 'ppc64' ],

48 'sh4': [ 'sh4' ],

49 'i386': [ 'i586', 'ppc', 'ppc64', 'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el', 'sh4', 'mips', 'mips64' ],

50 'i586': [ 'i386', 'ppc', 'ppc64', 'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el', 'sh4', 'mips', 'mips64' ],

51 'i686': [ 'i586', 'ppc', 'ppc64', 'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el', 'sh4', 'mips', 'mips64' ],

52 'x86_64': ['i686', 'i586', 'i386', 'ppc', 'ppc64', 'armv4l', 'armv5el', 'armv6el', 'armv7el', 'armv8el', 'sh4', 'mips', 'mips64' ],

53 'sparc64': ['sparc64v', 'sparcv9v', 'sparcv9', 'sparcv8', 'sparc'],

54 }

55

56 # real arch of this machine

57 hostarch = os.uname()[4]

58 if hostarch == 'i686': # FIXME

59 hostarch = 'i586'

60

61 class Buildinfo:

62 """represent the contents of a buildinfo file"""

63

64 def __init__(self, filename, apiurl, buildtype = 'spec', localpkgs = []):

65 try:

66 tree = ET.parse(filename)

67 except:

68 print >>sys.stderr, 'could not parse the buildinfo:'

69 print >>sys.stderr, open(filename).read()

70 sys.exit(1)

71

72 root = tree.getroot()

73

74 self.apiurl = apiurl

75

76 if root.find('error') != None:

77 sys.stderr.write('buildinfo is broken... it says:\n')

78 error = root.find('error').text

79 sys.stderr.write(error + '\n')

80 sys.exit(1)

81

82 if not (apiurl.startswith('https://') or apiurl.startswith('http://')):

83 raise urllib2.URLError('invalid protocol for the apiurl: \'%s\'' % apiurl)

84

85 self.buildtype = buildtype

86 self.apiurl = apiurl

87

88 # are we building .rpm or .deb?

89 # XXX: shouldn't we deliver the type via the buildinfo?

90 self.pacsuffix = 'rpm'

91 if self.buildtype == 'dsc':

92 self.pacsuffix = 'deb'

93

94 self.buildarch = root.find('arch').text

95 if root.find('release') != None:

96 self.release = root.find('release').text

97 else:

98 self.release = None

99 self.downloadurl = root.get('downloadurl')

100 self.debuginfo = 0

101 if root.find('debuginfo') != None:

102 try:

103 self.debuginfo = int(root.find('debuginfo').text)

104 except ValueError:

105 pass

106

107 self.deps = []

108 self.projects = {}

109 self.keys = []

110 self.prjkeys = []

111 for node in root.findall('bdep'):

112 p = Pac(node, self.buildarch, self.pacsuffix,

113 apiurl, localpkgs)

114 if p.project:

115 self.projects[p.project] = 1

116 self.deps.append(p)

117

118 self.vminstall_list = [ dep.name for dep in self.deps if dep.vminstall ]

119 self.cbinstall_list = [ dep.name for dep in self.deps if dep.cbinstall ]

120 self.cbpreinstall_list = [ dep.name for dep in self.deps if dep.cbpreinstall ]

121 self.preinstall_list = [ dep.name for dep in self.deps if dep.preinstall ]

122 self.runscripts_list = [ dep.name for dep in self.deps if dep.runscripts ]

123

124

125 def has_dep(self, name):

126 for i in self.deps:

127 if i.name == name:

128 return True

129 return False

130

131 def remove_dep(self, name):

132 for i in self.deps:

133 if i.name == name:

134 self.deps.remove(i)

135 return True

136 return False

137

138

139 class Pac:

140 """represent a package to be downloaded

141

142 We build a map that's later used to fill our URL templates

143 """

144 def __init__(self, node, buildarch, pacsuffix, apiurl, localpkgs = []):

145

146 self.mp = {}

147 for i in ['name', 'package',

148 'version', 'release',

149 'project', 'repository',

150 'preinstall', 'vminstall', 'noinstall', 'runscripts',

151 'cbinstall', 'cbpreinstall',

152 ]:

153 self.mp[i] = node.get(i)

154

155 self.mp['buildarch'] = buildarch

156 self.mp['pacsuffix'] = pacsuffix

157

158 self.mp['arch'] = node.get('arch') or self.mp['buildarch']

159

160 # this is not the ideal place to check if the package is a localdep or not

161 localdep = self.mp['name'] in localpkgs

162 if not localdep and not (node.get('project') and node.get('repository')):

163 raise oscerr.APIError('incomplete information for package %s, may be caused by a broken project configuration.'

164 % self.mp['name'] )

165

166 if not localdep:

167 self.mp['extproject'] = node.get('project').replace(':', ':/')

168 self.mp['extrepository'] = node.get('repository').replace(':', ':/')

169 self.mp['repopackage'] = node.get('package') or '_repository'

170 self.mp['repoarch'] = node.get('repoarch') or self.mp['buildarch']

171

172 if pacsuffix == 'deb' and not (self.mp['name'] and self.mp['arch'] and self.mp['version']):

173 raise oscerr.APIError(

174 "buildinfo for package %s/%s/%s is incomplete"

175 % (self.mp['name'], self.mp['arch'], self.mp['version']))

176

177 self.mp['apiurl'] = apiurl

178

179 if pacsuffix == 'deb':

180 self.filename = debquery.DebQuery.filename(self.mp['name'], self.mp['version'], self.mp['release'], self.mp['arch'])

181 else:

182 self.filename = rpmquery.RpmQuery.filename(self.mp['name'], self.mp['version'], self.mp['release'], self.mp['arch'])

183

184 self.mp['filename'] = self.filename

185 if self.mp['repopackage'] == '_repository':

186 self.mp['repofilename'] = self.mp['name']

187 else:

188 self.mp['repofilename'] = self.mp['filename']

189

190 # make the content of the dictionary accessible as class attributes

191 self.__dict__.update(self.mp)

192

193

194 def makeurls(self, cachedir, urllist):

195

196 self.urllist = []

197

198 # build up local URL

199 # by using the urlgrabber with local urls, we basically build up a cache.

200 # the cache has no validation, since the package servers don't support etags,

201 # or if-modified-since, so the caching is simply name-based (on the assumption

202 # that the filename is suitable as identifier)

203 self.localdir = '%s/%s/%s/%s' % (cachedir, self.project, self.repository, self.arch)

204 self.fullfilename = os.path.join(self.localdir, self.filename)

205 self.url_local = 'file://%s' % self.fullfilename

206

207 # first, add the local URL

208 self.urllist.append(self.url_local)

209

210 # remote URLs

211 for url in urllist:

212 self.urllist.append(url % self.mp)

213

214 def __str__(self):

215 return self.name

216

217 def __repr__(self):

218 return "%s" % self.name

219

220

221

222 def get_built_files(pacdir, pactype):

223 if pactype == 'rpm':

224 b_built = subprocess.Popen(['find', os.path.join(pacdir, 'RPMS'),

225 '-name', '*.rpm'],

226 stdout=subprocess.PIPE).stdout.read().strip()

227 s_built = subprocess.Popen(['find', os.path.join(pacdir, 'SRPMS'),

228 '-name', '*.rpm'],

229 stdout=subprocess.PIPE).stdout.read().strip()

230 elif pactype == 'kiwi':

231 b_built = subprocess.Popen(['find', os.path.join(pacdir, 'KIWI'),

232 '-type', 'f'],

233 stdout=subprocess.PIPE).stdout.read().strip()

234 else:

235 b_built = subprocess.Popen(['find', os.path.join(pacdir, 'DEBS'),

236 '-name', '*.deb'],

237 stdout=subprocess.PIPE).stdout.read().strip()

238 s_built = subprocess.Popen(['find', os.path.join(pacdir, 'SOURCES.DEB'),

239 '-type', 'f'],

240 stdout=subprocess.PIPE).stdout.read().strip()

241 return s_built, b_built

242

243 def get_repo(path):

244 """Walks up path looking for any repodata directories.

245

246 @param path path to a directory

247 @return str path to repository directory containing repodata directory

248 """

249 oldDirectory = None

250 currentDirectory = os.path.abspath(path)

251 repositoryDirectory = None

252

253 # while there are still parent directories

254 while currentDirectory != oldDirectory:

255 children = os.listdir(currentDirectory)

256

257 if "repodata" in children:

258 repositoryDirectory = currentDirectory

259 break

260

261 # ascend

262 oldDirectory = currentDirectory

263 currentDirectory = os.path.abspath(os.path.join(oldDirectory,

264 os.pardir))

265

266 return repositoryDirectory

267

268 def get_prefer_pkgs(dirs, wanted_arch, type):

269 import glob

270 from util import repodata, packagequery, cpio

271 paths = []

272 repositories = []

273

274 suffix = '*.rpm'

275 if type == 'dsc':

276 suffix = '*.deb'

277

278 for dir in dirs:

279 # check for repodata

280 repository = get_repo(dir)

281 if repository is None:

282 paths += glob.glob(os.path.join(os.path.abspath(dir), suffix))

283 else:

284 repositories.append(repository)

285

286 packageQueries = packagequery.PackageQueries(wanted_arch)

287

288 for repository in repositories:

289 repodataPackageQueries = repodata.queries(repository)

290

291 for packageQuery in repodataPackageQueries:

292 packageQueries.add(packageQuery)

293

294 for path in paths:

295 if path.endswith('src.rpm'):

296 continue

297 if path.find('-debuginfo-') > 0:

298 continue

299 packageQuery = packagequery.PackageQuery.query(path)

300 packageQueries.add(packageQuery)

301

302 prefer_pkgs = dict((name, packageQuery.path())

303 for name, packageQuery in packageQueries.iteritems())

304

305 depfile = create_deps(packageQueries.values())

306 cpio = cpio.CpioWrite()

307 cpio.add('deps', '\n'.join(depfile))

308 return prefer_pkgs, cpio

309

310

311 def create_deps(pkgqs):

312 """

313 creates a list of requires/provides which corresponds to build's internal

314 dependency file format

315 """

316 depfile = []

317 for p in pkgqs:

318 id = '%s.%s-0/0/0: ' % (p.name(), p.arch())

319 depfile.append('R:%s%s' % (id, ' '.join(p.requires())))

320 depfile.append('P:%s%s' % (id, ' '.join(p.provides())))

321 return depfile

322

323

324 trustprompt = """Would you like to ...

325 0 - quit (default)

326 1 - trust packages from '%(project)s' always

327 2 - trust them just this time

328 ? """

329 def check_trusted_projects(apiurl, projects):

330 trusted = config['api_host_options'][apiurl]['trusted_prj']

331 tlen = len(trusted)

332 for prj in projects:

333 if not prj in trusted:

334 print "\nThe build root needs packages from project '%s'." % prj

335 print "Note that malicious packages can compromise the build result or even your system."

336 r = raw_input(trustprompt % { 'project':prj })

337 if r == '1':

338 print "adding '%s' to ~/.oscrc: ['%s']['trusted_prj']" % (prj,apiurl)

339 trusted.append(prj)

340 elif r != '2':

341 print "Well, good good bye then :-)"

342 raise oscerr.UserAbort()

343

344 if tlen != len(trusted):

345 config['api_host_options'][apiurl]['trusted_prj'] = trusted

346 conf.config_set_option(apiurl, 'trusted_prj', ' '.join(trusted))

347

348 def main(opts, argv):

349

350 repo = argv[0]

351 arch = argv[1]

352 build_descr = argv[2]

353 xp = []

354 build_root = None

355 build_uid=''

356 vm_type = config['build-type']

357

358 build_descr = os.path.abspath(build_descr)

359 build_type = os.path.splitext(build_descr)[1][1:]

360 if build_type not in ['spec', 'dsc', 'kiwi']:

361 raise oscerr.WrongArgs(

362 'Unknown build type: \'%s\'. Build description should end in .spec, .dsc or .kiwi.' \

363 % build_type)

364 if not os.path.isfile(build_descr):

365 raise oscerr.WrongArgs('Error: build description file named \'%s\' does not exist.' % build_descr)

366

367 buildargs = []

368 if not opts.userootforbuild:

369 buildargs.append('--norootforbuild')

370 if opts.clean:

371 buildargs.append('--clean')

372 if opts.noinit:

373 buildargs.append('--noinit')

374 if opts.nochecks:

375 buildargs.append('--no-checks')

376 if not opts.no_changelog:

377 buildargs.append('--changelog')

378 if opts.root:

379 build_root = opts.root

380 if opts.jobs:

381 buildargs.append('--jobs %s' % opts.jobs)

382 elif config['build-jobs'] > 1:

383 buildargs.append('--jobs %s' % config['build-jobs'])

384 if opts.icecream or config['icecream'] != '0':

385 if opts.icecream:

386 num = opts.icecream

387 else:

388 num = config['icecream']

389

390 if int(num) > 0:

391 buildargs.append('--icecream %s' % num)

392 xp.append('icecream')

393 xp.append('gcc-c++')

394 if opts.ccache:

395 buildargs.append('--ccache')

396 xp.append('ccache')

397 if opts.linksources:

398 buildargs.append('--linksources')

399 if opts.baselibs:

400 buildargs.append('--baselibs')

401 if opts.debuginfo:

402 buildargs.append('--debug')

403 if opts._with:

404 for o in opts._with:

405 buildargs.append('--with %s' % o)

406 if opts.without:

407 for o in opts.without:

408 buildargs.append('--without %s' % o)

409 # FIXME: quoting

410 # if opts.define:

411 # buildargs.append('--define "%s"' % opts.define)

412 if config['build-uid']:

413 build_uid = config['build-uid']

414 if opts.build_uid:

415 build_uid = opts.build_uid

416 if build_uid:

417 buildidre = re.compile('^[0-9]{1,5}:[0-9]{1,5}$')

418 if build_uid == 'caller':

419 buildargs.append('--uid %s:%s' % (os.getuid(), os.getgid()))

420 elif buildidre.match(build_uid):

421 buildargs.append('--uid %s' % build_uid)

422 else:

423 print >>sys.stderr, 'Error: build-uid arg must be 2 colon separated numerics: "uid:gid" or "caller"'

424 return 1

425 if opts.vm_type:

426 vm_type = opts.vm_type

427 if opts.alternative_project:

428 prj = opts.alternative_project

429 pac = '_repository'

430 else:

431 prj = store_read_project(os.curdir)

432 if opts.local_package:

433 pac = '_repository'

434 else:

435 pac = store_read_package(os.curdir)

436 apiurl = store_read_apiurl(os.curdir)

437

438 # make it possible to override configuration of the rc file

439 for var in ['OSC_PACKAGECACHEDIR', 'OSC_SU_WRAPPER', 'OSC_BUILD_ROOT']:

440 val = os.getenv(var)

441 if val:

442 if var.startswith('OSC_'): var = var[4:]

443 var = var.lower().replace('_', '-')

444 if config.has_key(var):

445 print 'Overriding config value for %s=\'%s\' with \'%s\'' % (var, config[var], val)

446 config[var] = val

447

448 pacname = pac

449 if pacname == '_repository':

450 if not opts.local_package:

451 try:

452 pacname = store_read_package(os.curdir)

453 except oscerr.NoWorkingCopy:

454 opts.local_package = True

455 if opts.local_package:

456 pacname = os.path.splitext(build_descr)[0]

457 if not build_root:

458 build_root = config['build-root'] % { 'repo': repo, 'arch': arch,

459 'project' : prj, 'package' : pacname

460 }

461

462 extra_pkgs = []

463 if not opts.extra_pkgs:

464 extra_pkgs = config['extra-pkgs']

465 elif opts.extra_pkgs != ['']:

466 extra_pkgs = opts.extra_pkgs

467

468 if xp:

469 extra_pkgs += xp

470

471 prefer_pkgs = {}

472 build_descr_data = open(build_descr).read()

473

474 # XXX: dirty hack but there's no api to provide custom defines

475 if opts.without:

476 s = ''

477 for i in opts.without:

478 s += "%%define _without_%s 1\n" % i

479 s += "%%define _with_%s 0\n" % i

480 build_descr_data = s + build_descr_data

481 if opts._with:

482 s = ''

483 for i in opts._with:

484 s += "%%define _without_%s 0\n" % i

485 s += "%%define _with_%s 1\n" % i

486 build_descr_data = s + build_descr_data

487

488 if opts.prefer_pkgs:

489 print 'Scanning the following dirs for local packages: %s' % ', '.join(opts.prefer_pkgs)

490 prefer_pkgs, cpio = get_prefer_pkgs(opts.prefer_pkgs, arch, build_type)

491 cpio.add(os.path.basename(build_descr), build_descr_data)

492 build_descr_data = cpio.get()

493

494 # special handling for overlay and rsync-src/dest

495 specialcmdopts = ''

496 if opts.rsyncsrc or opts.rsyncdest :

497 if not opts.rsyncsrc or not opts.rsyncdest:

498 raise oscerr.WrongOptions('When using --rsync-{src,dest} both parameters have to be specified.')

499 myrsyncsrc = os.path.abspath(os.path.expanduser(os.path.expandvars(opts.rsyncsrc)))

500 if not os.path.isdir(myrsyncsrc):

501 raise oscerr.WrongOptions('--rsync-src %s is no valid directory!' % opts.rsyncsrc)

502 # can't check destination - its in the target chroot ;) - but we can check for sanity

503 myrsyncdest = os.path.expandvars(opts.rsyncdest)

504 if not os.path.isabs(myrsyncdest):

505 raise oscerr.WrongOptions('--rsync-dest %s is no absolute path (starting with \'/\')!' % opts.rsyncdest)

506 specialcmdopts = '--rsync-src="%s" --rsync-dest="%s"' % (myrsyncsrc, myrsyncdest)

507 if opts.overlay:

508 myoverlay = os.path.abspath(os.path.expanduser(os.path.expandvars(opts.overlay)))

509 if not os.path.isdir(myoverlay):

510 raise oscerr.WrongOptions('--overlay %s is no valid directory!' % opts.overlay)

511 specialcmdopts += '--overlay="%s"' % myoverlay

512

513 bi_file = None

514 bc_file = None

515 bi_filename = '_buildinfo-%s-%s.xml' % (repo, arch)

516 bc_filename = '_buildconfig-%s-%s' % (repo, arch)

517 if is_package_dir('.') and os.access(osc.core.store, os.W_OK):

518 bi_filename = os.path.join(os.getcwd(), osc.core.store, bi_filename)

519 bc_filename = os.path.join(os.getcwd(), osc.core.store, bc_filename)

520 elif not os.access('.', os.W_OK):

521 bi_file = NamedTemporaryFile(prefix=bi_filename)

522 bi_filename = bi_file.name

523 bc_file = NamedTemporaryFile(prefix=bc_filename)

524 bc_filename = bc_file.name

525 else:

526 bi_filename = os.path.abspath(bi_filename)

527 bc_filename = os.path.abspath(bc_filename)

528

529 try:

530 if opts.noinit:

531 if not os.path.isfile(bi_filename):

532 raise oscerr.WrongOptions('--noinit is not possible, no local buildinfo file')

533 print 'Use local \'%s\' file as buildinfo' % bi_filename

534 if not os.path.isfile(bc_filename):

535 raise oscerr.WrongOptions('--noinit is not possible, no local buildconfig file')

536 print 'Use local \'%s\' file as buildconfig' % bc_filename

537 elif opts.offline:

538 if not os.path.isfile(bi_filename):

539 raise oscerr.WrongOptions('--offline is not possible, no local buildinfo file')

540 print 'Use local \'%s\' file as buildinfo' % bi_filename

541 if not os.path.isfile(bc_filename):

542 raise oscerr.WrongOptions('--offline is not possible, no local buildconfig file')

543 else:

544 print 'Getting buildinfo from server and store to %s' % bi_filename

545 if not bi_file:

546 bi_file = open(bi_filename, 'w')

547 bi_text = ''.join(get_buildinfo(apiurl,

548 prj,

549 pac,

550 repo,

551 arch,

552 specfile=build_descr_data,

553 addlist=extra_pkgs))

554 bi_file.write(bi_text)

555 bi_file.flush()

556 print 'Getting buildconfig from server and store to %s' % bc_filename

557 if not bc_file:

558 bc_file = open(bc_filename, 'w')

559 bc_file.write(get_buildconfig(apiurl, prj, pac, repo, arch))

560 bc_file.flush()

561 except urllib2.HTTPError, e:

562 if e.code == 404:

563 # check what caused the 404

564 if meta_exists(metatype='prj', path_args=(quote_plus(prj), ),

565 template_args=None, create_new=False, apiurl=apiurl):

566 pkg_meta_e = None

567 try:

568 # take care, not to run into double trouble.

569 pkg_meta_e = meta_exists(metatype='pkg', path_args=(quote_plus(prj),

570 quote_plus(pac)), template_args=None, create_new=False,

571 apiurl=apiurl)

572 except:

573 pass

574

575 if pac == '_repository' or pkg_meta_e:

576 print >>sys.stderr, 'ERROR: Either wrong repo/arch as parameter or a parse error of .spec/.dsc/.kiwi file due to syntax error'

577 else:

578 print >>sys.stderr, 'The package \'%s\' does not exists - please ' \

579 'rerun with \'--local-package\'' % pac

580 else:

581 print >>sys.stderr, 'The project \'%s\' does not exists - please ' \

582 'rerun with \'--alternative-project <alternative_project>\'' % prj

583 sys.exit(1)

584 else:

585 raise

586

587 bi = Buildinfo(bi_filename, apiurl, build_type, prefer_pkgs.keys())

588

589 if bi.debuginfo and not (opts.disable_debuginfo or '--debug' in buildargs):

590 buildargs.append('--debug')

591

592 if opts.release:

593 bi.release = opts.release

594

595 if bi.release:

596 buildargs.append('--release %s' % bi.release)

597

598 # real arch of this machine

599 # vs.

600 # arch we are supposed to build for

601 if hostarch != bi.buildarch:

602 if not bi.buildarch in can_also_build.get(hostarch, []):

603 print >>sys.stderr, 'Error: hostarch \'%s\' cannot build \'%s\'.' % (hostarch, bi.buildarch)

604 return 1

605

606 rpmlist_prefers = []

607 if prefer_pkgs:

608 print 'Evaluating preferred packages'

609 for name, path in prefer_pkgs.iteritems():

610 if bi.has_dep(name):

611 # We remove a preferred package from the buildinfo, so that the

612 # fetcher doesn't take care about them.

613 # Instead, we put it in a list which is appended to the rpmlist later.

614 # At the same time, this will make sure that these packages are

615 # not verified.

616 bi.remove_dep(name)

617 rpmlist_prefers.append((name, path))

618 print ' - %s (%s)' % (name, path)

619 continue

620

621 print 'Updating cache of required packages'

622

623 urllist = []

624 if not opts.download_api_only:

625 # transform 'url1, url2, url3' form into a list

626 if 'urllist' in config:

627 if type(config['urllist']) == str:

628 re_clist = re.compile('[, ]+')

629 urllist = [ i.strip() for i in re_clist.split(config['urllist'].strip()) ]

630 else:

631 urllist = config['urllist']

632

633 # OBS 1.5 and before has no downloadurl defined in buildinfo

634 if bi.downloadurl:

635 urllist.append(bi.downloadurl + '/%(extproject)s/%(extrepository)s/%(arch)s/%(filename)s')

636 if not opts.cpio_bulk_download:

637 urllist.append( '%(apiurl)s/build/%(project)s/%(repository)s/%(repoarch)s/%(repopackage)s/%(repofilename)s' )

638

639 fetcher = Fetcher(cachedir = config['packagecachedir'],

640 urllist = urllist,

641 api_host_options = config['api_host_options'],

642 offline = opts.noinit or opts.offline,

643 http_debug = config['http_debug'],

644 enable_cpio = opts.cpio_bulk_download,

645 cookiejar=cookiejar)

646

647 # implicitly trust the project we are building for

648 check_trusted_projects(apiurl, [ i for i in bi.projects.keys() if not i == prj ])

649

650 # now update the package cache

651 fetcher.run(bi)

652

653 old_pkg_dir = None

654 if opts.oldpackages:

655 old_pkg_dir = opts.oldpackages

656 if not old_pkg_dir.startswith('/') and not opts.offline:

657 data = [ prj, pacname, repo, arch]

658 if old_pkg_dir == '_link':

659 p = osc.core.findpacs(os.curdir)[0]

660 if not p.islink():

661 raise oscerr.WrongOptions('package is not a link')

662 data[0] = p.linkinfo.project

663 data[1] = p.linkinfo.package

664 repos = osc.core.get_repositories_of_project(apiurl, data[0])

665 # hack for links to e.g. Factory

666 if not data[2] in repos and 'standard' in repos:

667 data[2] = 'standard'

668 elif old_pkg_dir != '' and old_pkg_dir != '_self':

669 a = old_pkg_dir.split('/')

670 for i in range(0, len(a)):

671 data[i] = a[i]

672

673 destdir = os.path.join(config['packagecachedir'], data[0], data[2], data[3])

674 old_pkg_dir = None

675 try:

676 print "Downloading previous build from %s ..." % '/'.join(data)

677 binaries = get_binarylist(apiurl, data[0], data[2], data[3], package=data[1], verbose=True)

678 except Exception, e:

679 print "Error: failed to get binaries: %s" % str(e)

680 binaries = []

681

682 if binaries:

683 class mytmpdir:

684 """ temporary directory that removes itself"""

685 def __init__(self, *args, **kwargs):

686 self.name = mkdtemp(*args, **kwargs)

687 def cleanup(self):

688 rmtree(self.name)

689 def __del__(self):

690 self.cleanup()

691 def __exit__(self):

692 self.cleanup()

693 def __str__(self):

694 return self.name

695

696 old_pkg_dir = mytmpdir(prefix='.build.oldpackages', dir=os.path.abspath(os.curdir))

697 if not os.path.exists(destdir):

698 os.makedirs(destdir)

699 for i in binaries:

700 fname = os.path.join(destdir, i.name)

701 os.symlink(fname, os.path.join(str(old_pkg_dir), i.name))

702 if os.path.exists(fname):

703 st = os.stat(fname)

704 if st.st_mtime == i.mtime and st.st_size == i.size:

705 continue

706 get_binary_file(apiurl,

707 data[0],

708 data[2], data[3],

709 i.name,

710 package = data[1],

711 target_filename = fname,

712 target_mtime = i.mtime,

713 progress_meter = True)

714

715 if old_pkg_dir != None:

716 buildargs.append('--oldpackages %s' % old_pkg_dir)

717

718 # Make packages from buildinfo available as repos for kiwi

719 if build_type == 'kiwi':

720 if not os.path.exists('repos'):

721 os.mkdir('repos')

722 else:

723 rmtree('repos')

724 os.mkdir('repos')

725 for i in bi.deps:

726 # project

727 pdir = str(i.extproject).replace(':/', ':')

728 # repo

729 rdir = str(i.extrepository).replace(':/', ':')

730 # arch

731 adir = i.repoarch

732 # project/repo

733 prdir = "repos/"+pdir+"/"+rdir

734 # project/repo/arch

735 pradir = prdir+"/"+adir

736 # source fullfilename

737 sffn = i.fullfilename

738 print "Using package: "+sffn

739 # target fullfilename

740 tffn = pradir+"/"+sffn.split("/")[-1]

741 if not os.path.exists(os.path.join(pradir)):

742 os.makedirs(os.path.join(pradir))

743 if not os.path.exists(tffn):

744 if opts.linksources:

745 os.link(sffn, tffn)

746 else:

747 os.symlink(sffn, tffn)

748

749 if bi.pacsuffix == 'rpm':

750 if opts.no_verify:

751 print 'Skipping verification of package signatures'

752 else:

753 print 'Verifying integrity of cached packages'

754 verify_pacs([ i.fullfilename for i in bi.deps ], bi.keys)

755

756 elif bi.pacsuffix == 'deb':

757 if vm_type == "xen" or vm_type == "kvm" or vm_type == "lxc":

758 print 'Skipping verification of package signatures due to secure VM build'

759 elif opts.no_verify or opts.noinit:

760 print 'Skipping verification of package signatures'

761 else:

762 print 'WARNING: deb packages get not verified, they can compromise your system !'

763 else:

764 print 'WARNING: unknown packages get not verified, they can compromise your system !'

765

766 print 'Writing build configuration'

767

768 rpmlist = [ '%s %s\n' % (i.name, i.fullfilename) for i in bi.deps if not i.noinstall ]

769 rpmlist += [ '%s %s\n' % (i[0], i[1]) for i in rpmlist_prefers ]

770

771 rpmlist.append('preinstall: ' + ' '.join(bi.preinstall_list) + '\n')

772 rpmlist.append('vminstall: ' + ' '.join(bi.vminstall_list) + '\n')

773 rpmlist.append('cbinstall: ' + ' '.join(bi.cbinstall_list) + '\n')

774 rpmlist.append('cbpreinstall: ' + ' '.join(bi.cbpreinstall_list) + '\n')

775 rpmlist.append('runscripts: ' + ' '.join(bi.runscripts_list) + '\n')

776

777 rpmlist_file = NamedTemporaryFile(prefix='rpmlist.')

778 rpmlist_filename = rpmlist_file.name

779 rpmlist_file.writelines(rpmlist)

780 rpmlist_file.flush()

781

782 subst = { 'repo': repo, 'arch': arch, 'project' : prj, 'package' : pacname }

783 vm_options = ''

784 # XXX check if build-device present

785 my_build_device = ''

786 if config['build-device']:

787 my_build_device = config['build-device'] % subst

788 else:

789 # obs worker uses /root here but that collides with the

790 # /root directory if the build root was used without vm

791 # before

792 my_build_device = build_root + '/img'

793

794 need_root = True

795 if vm_type:

796 if config['build-swap']:

797 my_build_swap = config['build-swap'] % subst

798 else:

799 my_build_swap = build_root + '/swap'

800

801 vm_options = '--vm-type=%s'%vm_type

802 if vm_type != 'lxc':

803 vm_options += ' --vm-disk=' + my_build_device

804 vm_options += ' --vm-swap=' + my_build_swap

805 vm_options += ' --logfile=%s/.build.log' % build_root

806 if vm_type == 'kvm':

807 if os.access(build_root, os.W_OK) and os.access('/dev/kvm', os.W_OK):

808 # so let's hope there's also an fstab entry

809 need_root = False

810 build_root += '/.mount'

811

812 if config['build-memory']:

813 vm_options += ' --memory ' + config['build-memory']

814 if config['build-vmdisk-rootsize']:

815 vm_options += ' --vmdisk-rootsize ' + config['build-vmdisk-rootsize']

816 if config['build-vmdisk-swapsize']:

817 vm_options += ' --vmdisk-swapsize ' + config['build-vmdisk-swapsize']

818

819 if opts.preload:

820 print "Preload done for selected repo/arch."

821 sys.exit(0)

822

823 print 'Running build'

824 cmd = '"%s" --root="%s" --rpmlist="%s" --dist="%s" %s --arch=%s %s %s "%s"' \

825 % (config['build-cmd'],

826 build_root,

827 rpmlist_filename,

828 bc_filename,

829 specialcmdopts,

830 bi.buildarch,

831 vm_options,

832 ' '.join(buildargs),

833 build_descr)

834

835 if need_root:

836 if config['su-wrapper'].startswith('su '):

837 tmpl = '%s \'%s\''

838 else:

839 tmpl = '%s %s'

840 cmd = tmpl % (config['su-wrapper'], cmd)

841

842 # change personality, if needed

843 if hostarch != bi.buildarch:

844 cmd = (change_personality.get(bi.buildarch, '') + ' ' + cmd).strip()

845

846 print cmd

847 rc = subprocess.call(cmd, shell=True)

848 if rc:

849 print

850 print 'The buildroot was:', build_root

851 sys.exit(rc)

852

853 pacdir = os.path.join(build_root, '.build.packages')

854 if os.path.islink(pacdir):

855 pacdir = os.readlink(pacdir)

856 pacdir = os.path.join(build_root, pacdir)

857

858 if os.path.exists(pacdir):

859 (s_built, b_built) = get_built_files(pacdir, bi.pacsuffix)

860

861 print

862 if s_built: print s_built

863 print

864 print b_built

865

866 if opts.keep_pkgs:

867 for i in b_built.splitlines() + s_built.splitlines():

868 shutil.copy2(i, os.path.join(opts.keep_pkgs, os.path.basename(i)))

869

870 if bi_file:

871 bi_file.close()

872 if bc_file:

873 bc_file.close()

874 rpmlist_file.close()

875

876 # vim: sw=4 et