gitorious.org Git - opensuse:osc.git/blob

2 # This program is free software; it may be used, copied, modified

3 # and distributed under the terms of the GNU General Public Licence,

4 # either version 2, or (at your option) any later version.

5

6 import sys, os

7 import urllib2

8 from urlgrabber.grabber import URLGrabError

9 from urlgrabber.mirror import MirrorGroup

10 from core import makeurl, streamfile

11 from util import packagequery, cpio

12 import conf

13 import oscerr

14 import tempfile

15 try:

16 from meter import TextMeter

17 except:

18 TextMeter = None

19

20

21 def join_url(self, base_url, rel_url):

22 """to override _join_url of MirrorGroup, because we want to

23 pass full URLs instead of base URL where relative_url is added later...

24 IOW, we make MirrorGroup ignore relative_url"""

25 return base_url

26

27 class OscFileGrabber:

28 def __init__(self, progress_obj = None):

29 self.progress_obj = progress_obj

30

31 def urlgrab(self, url, filename, text = None, **kwargs):

32 if url.startswith('file://'):

33 file = url.replace('file://', '', 1)

34 if os.path.isfile(file):

35 return file

36 else:

37 raise URLGrabError(2, 'Local file \'%s\' does not exist' % file)

38 f = open(filename, 'wb')

39 try:

40 try:

41 for i in streamfile(url, progress_obj=self.progress_obj, text=text):

42 f.write(i)

43 except urllib2.HTTPError, e:

44 exc = URLGrabError(14, str(e))

45 exc.url = url

46 exc.exception = e

47 exc.code = e.code

48 raise exc

49 except IOError, e:

50 raise URLGrabError(4, str(e))

51 finally:

52 f.close()

53 return filename

54

55 class Fetcher:

56 def __init__(self, cachedir = '/tmp', api_host_options = {}, urllist = [], http_debug = False,

57 cookiejar = None, offline = False, enable_cpio = True):

58 # set up progress bar callback

59 if sys.stdout.isatty() and TextMeter:

60 self.progress_obj = TextMeter(fo=sys.stdout)

61 else:

62 self.progress_obj = None

63

64 self.cachedir = cachedir

65 self.urllist = urllist

66 self.http_debug = http_debug

67 self.offline = offline

68 self.cpio = {}

69 self.enable_cpio = enable_cpio

70

71 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()

72 for host in api_host_options.keys():

73 passmgr.add_password(None, host, api_host_options[host]['user'], api_host_options[host]['pass'])

74 openers = (urllib2.HTTPBasicAuthHandler(passmgr), )

75 if cookiejar:

76 openers += (urllib2.HTTPCookieProcessor(cookiejar), )

77 self.gr = OscFileGrabber(progress_obj=self.progress_obj)

78

79 def failureReport(self, errobj):

80 """failure output for failovers from urlgrabber"""

81 if errobj.url.startswith('file://'):

82 return {}

83 print 'Trying openSUSE Build Service server for %s (%s), not found at %s.' \

84 % (self.curpac, self.curpac.project, errobj.url.split('/')[2])

85 return {}

86

87 def __add_cpio(self, pac):

88 prpap = '%s/%s/%s/%s' % (pac.project, pac.repository, pac.repoarch, pac.repopackage)

89 self.cpio.setdefault(prpap, {})[pac.repofilename] = pac

90

91 def __fetch_cpio(self, apiurl):

92 from urllib import quote_plus

93 for prpap, pkgs in self.cpio.iteritems():

94 project, repo, arch, package = prpap.split('/', 3)

95 query = ['binary=%s' % quote_plus(i) for i in pkgs.keys()]

96 query.append('view=cpio')

97 tmparchive = tmpfile = None

98 try:

99 (fd, tmparchive) = tempfile.mkstemp(prefix='osc_build_cpio')

100 (fd, tmpfile) = tempfile.mkstemp(prefix='osc_build')

101 url = makeurl(apiurl, ['build', project, repo, arch, package], query=query)

102 sys.stdout.write("preparing download ...\r")

103 sys.stdout.flush()

104 self.gr.urlgrab(url, filename = tmparchive, text = 'fetching cpio for \'%s\'' % project)

105 archive = cpio.CpioRead(tmparchive)

106 archive.read()

107 for hdr in archive:

108 # XXX: we won't have an .errors file because we're using

109 # getbinarylist instead of the public/... route (which is

110 # routed to getbinaries (but that won't work for kiwi products))

111 if hdr.filename == '.errors':

112 archive.copyin_file(hdr.filename)

113 raise oscerr.APIError('CPIO archive is incomplete (see .errors file)')

114 if package == '_repository':

115 pac = pkgs[hdr.filename.rsplit('.', 1)[0]]

116 else:

117 # this is a kiwi product

118 pac = pkgs[hdr.filename]

119 archive.copyin_file(hdr.filename, os.path.dirname(tmpfile), os.path.basename(tmpfile))

120 self.move_package(tmpfile, pac.localdir, pac)

121 # check if we got all packages... (because we've no .errors file)

122 for pac in pkgs.itervalues():

123 if not os.path.isfile(pac.fullfilename):

124 raise oscerr.APIError('failed to fetch file \'%s\': ' \

125 'does not exist in CPIO archive' % pac.repofilename)

126 finally:

127 if not tmparchive is None and os.path.exists(tmparchive):

128 os.unlink(tmparchive)

129 if not tmpfile is None and os.path.exists(tmpfile):

130 os.unlink(tmpfile)

131

132 def fetch(self, pac, prefix=''):

133 # for use by the failure callback

134 self.curpac = pac

135

136 MirrorGroup._join_url = join_url

137 mg = MirrorGroup(self.gr, pac.urllist, failure_callback=(self.failureReport,(),{}))

138

139 if self.http_debug:

140 print >>sys.stderr, '\nURLs to try for package \'%s\':' % pac

141 print >>sys.stderr, '\n'.join(pac.urllist)

142 print >>sys.stderr

143

144 (fd, tmpfile) = tempfile.mkstemp(prefix='osc_build')

145 try:

146 try:

147 mg.urlgrab(pac.filename,

148 filename = tmpfile,

149 text = '%s(%s) %s' %(prefix, pac.project, pac.filename))

150 self.move_package(tmpfile, pac.localdir, pac)

151 except URLGrabError, e:

152 if self.enable_cpio and e.errno == 256:

153 self.__add_cpio(pac)

154 return

155 print

156 print >>sys.stderr, 'Error:', e.strerror

157 print >>sys.stderr, 'Failed to retrieve %s from the following locations (in order):' % pac.filename

158 print >>sys.stderr, '\n'.join(pac.urllist)

159 sys.exit(1)

160 finally:

161 os.close(fd)

162 if os.path.exists(tmpfile):

163 os.unlink(tmpfile)

164

165 def move_package(self, tmpfile, destdir, pac_obj = None):

166 import shutil

167 pkgq = packagequery.PackageQuery.query(tmpfile, extra_rpmtags=(1044, 1051, 1052))

168 canonname = pkgq.canonname()

169 fullfilename = os.path.join(destdir, canonname)

170 if pac_obj is not None:

171 pac_obj.filename = canonname

172 pac_obj.fullfilename = fullfilename

173 shutil.move(tmpfile, fullfilename)

174

175 def dirSetup(self, pac):

176 dir = os.path.join(self.cachedir, pac.localdir)

177 if not os.path.exists(dir):

178 try:

179 os.makedirs(dir, mode=0755)

180 except OSError, e:

181 print >>sys.stderr, 'packagecachedir is not writable for you?'

182 print >>sys.stderr, e

183 sys.exit(1)

184

185 def run(self, buildinfo):

186 cached = 0

187 all = len(buildinfo.deps)

188 for i in buildinfo.deps:

189 i.makeurls(self.cachedir, self.urllist)

190 if os.path.exists(i.fullfilename):

191 cached += 1

192 miss = 0

193 needed = all - cached

194 if all:

195 miss = 100.0 * needed / all

196 print "%.1f%% cache miss. %d/%d dependencies cached.\n" % (miss, cached, all)

197 done = 1

198 for i in buildinfo.deps:

199 i.makeurls(self.cachedir, self.urllist)

200 if not os.path.exists(i.fullfilename):

201 if self.offline:

202 raise oscerr.OscIOError(None, 'Missing package \'%s\' in cache: --offline not possible.' % i.fullfilename)

203 self.dirSetup(i)

204 try:

205 # if there isn't a progress bar, there is no output at all

206 if not self.progress_obj:

207 print '%d/%d (%s) %s' % (done, needed, i.project, i.filename)

208 self.fetch(i)

209 if self.progress_obj:

210 print " %d/%d\r" % (done, needed),

211 sys.stdout.flush()

212

213 except KeyboardInterrupt:

214 print 'Cancelled by user (ctrl-c)'

215 print 'Exiting.'

216 sys.exit(0)

217 done += 1

218

219 self.__fetch_cpio(buildinfo.apiurl)

220

221 prjs = buildinfo.projects.keys()

222 for i in prjs:

223 dest = "%s/%s" % (self.cachedir, i)

224 if not os.path.exists(dest):

225 os.makedirs(dest, mode=0755)

226 dest += '/_pubkey'

227

228 url = "%s/source/%s/_pubkey" % (buildinfo.apiurl, i)

229 try:

230 OscFileGrabber().urlgrab(url, dest)

231 if not i in buildinfo.prjkeys: # not that many keys usually

232 buildinfo.keys.append(dest)

233 buildinfo.prjkeys.append(i)

234 except KeyboardInterrupt:

235 print 'Cancelled by user (ctrl-c)'

236 print 'Exiting.'

237 if os.path.exists(dest):

238 os.unlink(dest)

239 sys.exit(0)

240 except URLGrabError, e:

241 if self.http_debug:

242 print >>sys.stderr, "can't fetch key for %s: %s" %(i, e.strerror)

243 print >>sys.stderr, "url: %s" % url

244

245 if os.path.exists(dest):

246 os.unlink(dest)

247

248 l = i.rsplit(':', 1)

249 # try key from parent project

250 if len(l) > 1 and l[1] and not l[0] in buildinfo.projects:

251 prjs.append(l[0])

252

253 def verify_pacs_old(pac_list):

254 """Take a list of rpm filenames and run rpm -K on them.

255

256 In case of failure, exit.

257

258 Check all packages in one go, since this takes only 6 seconds on my Athlon 700

259 instead of 20 when calling 'rpm -K' for each of them.

260 """

261 import subprocess

262

263 if not pac_list:

264 return

265

266 # don't care about the return value because we check the

267 # output anyway, and rpm always writes to stdout.

268

269 # save locale first (we rely on English rpm output here)

270 saved_LC_ALL = os.environ.get('LC_ALL')

271 os.environ['LC_ALL'] = 'en_EN'

272

273 o = subprocess.Popen(['rpm', '-K'] + pac_list, stdout=subprocess.PIPE,

274 stderr=subprocess.STDOUT, close_fds=True).stdout

275

276 # restore locale

277 if saved_LC_ALL: os.environ['LC_ALL'] = saved_LC_ALL

278 else: os.environ.pop('LC_ALL')

279

280 for line in o.readlines():

281

282 if not 'OK' in line:

283 print

284 print >>sys.stderr, 'The following package could not be verified:'

285 print >>sys.stderr, line

286 sys.exit(1)

287

288 if 'NOT OK' in line:

289 print

290 print >>sys.stderr, 'The following package could not be verified:'

291 print >>sys.stderr, line

292

293 if 'MISSING KEYS' in line:

294 missing_key = line.split('#')[-1].split(')')[0]

295

296 print >>sys.stderr, """

297 - If the key (%(name)s) is missing, install it first.

298 For example, do the following:

299 osc signkey PROJECT > file

300 and, as root:

301 rpm --import %(dir)s/keyfile-%(name)s

302

303 Then, just start the build again.

304

305 - If you do not trust the packages, you should configure osc build for XEN or KVM

306

307 - You may use --no-verify to skip the verification (which is a risk for your system).

308 """ % {'name': missing_key,

309 'dir': os.path.expanduser('~')}

310

311 else:

312 print >>sys.stderr, """

313 - If the signature is wrong, you may try deleting the package manually

314 and re-run this program, so it is fetched again.

315 """

316

317 sys.exit(1)

318

319

320 def verify_pacs(bi):

321 """Take a list of rpm filenames and verify their signatures.

322

323 In case of failure, exit.

324 """

325

326 pac_list = [ i.fullfilename for i in bi.deps ]

327 if not conf.config['builtin_signature_check']:

328 return verify_pacs_old(pac_list)

329

330 if not pac_list:

331 return

332

333 if not bi.keys:

334 raise oscerr.APIError("can't verify packages due to lack of GPG keys")

335

336 print "using keys from", ', '.join(bi.prjkeys)

337

338 import checker

339 failed = False

340 checker = checker.Checker()

341 try:

342 checker.readkeys(bi.keys)

343 for pkg in pac_list:

344 try:

345 checker.check(pkg)

346 except Exception, e:

347 failed = True

348 print pkg, ':', e

349 except:

350 checker.cleanup()

351 raise

352

353 if failed:

354 checker.cleanup()

355 sys.exit(1)

356

357 checker.cleanup()

358

359 # vim: sw=4 et