gitorious.org Git - opensuse:osc.git/blob

2 # This program is free software; it may be used, copied, modified

3 # and distributed under the terms of the GNU General Public Licence,

4 # either version 2, or (at your option) any later version.

5

6 import sys, os

7 import urllib2

8 from urlgrabber.grabber import URLGrabError

9 from urlgrabber.mirror import MirrorGroup

10 from core import makeurl, streamfile

11 from util import packagequery, cpio

12 import conf

13 import oscerr

14 import tempfile

15 try:

16 from meter import TextMeter

17 except:

18 TextMeter = None

19

20

21 def join_url(self, base_url, rel_url):

22 """to override _join_url of MirrorGroup, because we want to

23 pass full URLs instead of base URL where relative_url is added later...

24 IOW, we make MirrorGroup ignore relative_url"""

25 return base_url

26

27 class OscFileGrabber:

28 def __init__(self, progress_obj = None):

29 self.progress_obj = progress_obj

30

31 def urlgrab(self, url, filename, text = None, **kwargs):

32 if url.startswith('file://'):

33 file = url.replace('file://', '', 1)

34 if os.path.isfile(file):

35 return file

36 else:

37 raise URLGrabError(2, 'Local file \'%s\' does not exist' % file)

38 f = open(filename, 'wb')

39 try:

40 try:

41 for i in streamfile(url, progress_obj=self.progress_obj, text=text):

42 f.write(i)

43 except urllib2.HTTPError, e:

44 exc = URLGrabError(14, str(e))

45 exc.url = url

46 exc.exception = e

47 exc.code = e.code

48 raise exc

49 except IOError, e:

50 raise URLGrabError(4, str(e))

51 finally:

52 f.close()

53 return filename

54

55 class Fetcher:

56 def __init__(self, cachedir = '/tmp', api_host_options = {}, urllist = [], http_debug = False,

57 cookiejar = None, offline = False, enable_cpio = False):

58 # set up progress bar callback

59 if sys.stdout.isatty() and TextMeter:

60 self.progress_obj = TextMeter(fo=sys.stdout)

61 else:

62 self.progress_obj = None

63

64 self.cachedir = cachedir

65 self.urllist = urllist

66 self.http_debug = http_debug

67 self.offline = offline

68 self.cpio = {}

69 self.enable_cpio = enable_cpio

70

71 passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()

72 for host in api_host_options.keys():

73 passmgr.add_password(None, host, api_host_options[host]['user'], api_host_options[host]['pass'])

74 openers = (urllib2.HTTPBasicAuthHandler(passmgr), )

75 if cookiejar:

76 openers += (urllib2.HTTPCookieProcessor(cookiejar), )

77 self.gr = OscFileGrabber(progress_obj=self.progress_obj)

78

79 def failureReport(self, errobj):

80 """failure output for failovers from urlgrabber"""

81 if errobj.url.startswith('file://'):

82 return {}

83 print 'Trying openSUSE Build Service server for %s (%s), not found at %s.' \

84 % (self.curpac, self.curpac.project, errobj.url.split('/')[2])

85 return {}

86

87 def fetch(self, pac, prefix=''):

88 # for use by the failure callback

89 self.curpac = pac

90

91 MirrorGroup._join_url = join_url

92 mg = MirrorGroup(self.gr, pac.urllist, failure_callback=(self.failureReport,(),{}))

93

94 if self.http_debug:

95 print

96 print 'URLs to try for package \'%s\':' % pac

97 print '\n'.join(pac.urllist)

98 print

99

100 (fd, tmpfile) = tempfile.mkstemp(prefix='osc_build')

101 try:

102 try:

103 mg.urlgrab(pac.filename,

104 filename = tmpfile,

105 text = '%s(%s) %s' %(prefix, pac.project, pac.filename))

106 self.move_package(tmpfile, pac.localdir, pac)

107 except URLGrabError, e:

108 if self.enable_cpio and e.errno == 256:

109 self.cpio.setdefault(pac.project, {})[pac.name] = pac

110 return

111 print

112 print >>sys.stderr, 'Error:', e.strerror

113 print >>sys.stderr, 'Failed to retrieve %s from the following locations (in order):' % pac.filename

114 print >>sys.stderr, '\n'.join(pac.urllist)

115 sys.exit(1)

116 finally:

117 os.close(fd)

118 if os.path.exists(tmpfile):

119 os.unlink(tmpfile)

120

121 def move_package(self, tmpfile, destdir, pac_obj = None):

122 import shutil

123 pkgq = packagequery.PackageQuery.query(tmpfile, extra_rpmtags=(1044, 1051, 1052))

124 arch = pkgq.arch()

125 # SOURCERPM = 1044

126 if pkgq.filename_suffix == 'rpm' and not pkgq.gettag(1044):

127 # NOSOURCE = 1051, NOPATCH = 1052

128 if pkgq.gettag(1051) or pkgq.gettag(1052):

129 arch = "nosrc"

130 else:

131 arch = "src"

132 canonname = pkgq.canonname()

133 fullfilename = os.path.join(destdir, canonname)

134 if pac_obj is not None:

135 pac_obj.filename = canonname

136 pac_obj.fullfilename = fullfilename

137 shutil.move(tmpfile, fullfilename)

138

139 def dirSetup(self, pac):

140 dir = os.path.join(self.cachedir, pac.localdir)

141 if not os.path.exists(dir):

142 try:

143 os.makedirs(dir, mode=0755)

144 except OSError, e:

145 print >>sys.stderr, 'packagecachedir is not writable for you?'

146 print >>sys.stderr, e

147 sys.exit(1)

148

149 def run(self, buildinfo):

150 from urllib import quote_plus

151 cached = 0

152 all = len(buildinfo.deps)

153 for i in buildinfo.deps:

154 i.makeurls(self.cachedir, self.urllist)

155 if os.path.exists(i.fullfilename):

156 cached += 1

157 miss = 0

158 needed = all - cached

159 if all:

160 miss = 100.0 * needed / all

161 print "%.1f%% cache miss. %d/%d dependencies cached.\n" % (miss, cached, all)

162 done = 1

163 for i in buildinfo.deps:

164 i.makeurls(self.cachedir, self.urllist)

165 if not os.path.exists(i.fullfilename):

166 if self.offline:

167 raise oscerr.OscIOError(None, 'Missing package \'%s\' in cache: --offline not possible.' % i.fullfilename)

168 self.dirSetup(i)

169 try:

170 # if there isn't a progress bar, there is no output at all

171 if not self.progress_obj:

172 print '%d/%d (%s) %s' % (done, needed, i.project, i.filename)

173 self.fetch(i)

174 if self.progress_obj:

175 print " %d/%d\r" % (done, needed),

176 sys.stdout.flush()

177

178 except KeyboardInterrupt:

179 print 'Cancelled by user (ctrl-c)'

180 print 'Exiting.'

181 sys.exit(0)

182 done += 1

183 for project, pkgs in self.cpio.iteritems():

184 repo = pkgs.values()[0].repository

185 query = [ 'binary=%s' % quote_plus(i) for i in pkgs.keys() ]

186 query.append('view=cpio')

187 try:

188 (fd, tmparchive) = tempfile.mkstemp(prefix='osc_build_cpio')

189 (fd, tmpfile) = tempfile.mkstemp(prefix='osc_build')

190 url = makeurl(buildinfo.apiurl,

191 ['public/build', project, repo, buildinfo.buildarch, '_repository'],

192 query=query)

193 sys.stdout.write("preparing download ...\r")

194 sys.stdout.flush()

195 self.gr.urlgrab(url, filename = tmparchive, text = 'fetching cpio for \'%s\'' % project)

196 archive = cpio.CpioRead(tmparchive)

197 archive.read()

198 for hdr in archive:

199 if hdr.filename == '.errors':

200 archive.copyin_file(hdr.filename)

201 raise oscerr.APIError('CPIO archive is incomplete (see .errors file)')

202 pac = pkgs[hdr.filename.rsplit('.', 1)[0]]

203 archive.copyin_file(hdr.filename, os.path.dirname(tmpfile), os.path.basename(tmpfile))

204 self.move_package(tmpfile, pac.localdir, pac)

205 finally:

206 if os.path.exists(tmparchive):

207 os.unlink(tmparchive)

208 if os.path.exists(tmpfile):

209 os.unlink(tmpfile)

210

211 prjs = buildinfo.projects.keys()

212 for i in prjs:

213 dest = "%s/%s" % (self.cachedir, i)

214 if not os.path.exists(dest):

215 os.makedirs(dest, mode=0755)

216 dest += '/_pubkey'

217

218 if os.path.exists(dest):

219 buildinfo.keys.append(dest)

220 buildinfo.prjkeys.append(i)

221 else:

222 url = "%s/source/%s/_pubkey" % (buildinfo.apiurl, i)

223 try:

224 self.gr.urlgrab(url, dest, text="fetching key for %s" % i)

225 buildinfo.keys.append(dest)

226 buildinfo.prjkeys.append(i)

227 except KeyboardInterrupt:

228 print 'Cancelled by user (ctrl-c)'

229 print 'Exiting.'

230 if os.path.exists(dest):

231 os.unlink(dest)

232 sys.exit(0)

233 except URLGrabError, e:

234 if self.http_debug:

235 print "can't fetch key for %s: %s" %(i, e.strerror)

236 print "url: %s" % url

237 else:

238 print "%s doesn't have a gpg key" % i

239

240 if os.path.exists(dest):

241 os.unlink(dest)

242

243 l = i.rsplit(':', 1)

244 # try key from parent project

245 if len(l) > 1 and l[1] and not l[0] in buildinfo.projects:

246 prjs.append(l[0])

247

248 def verify_pacs_old(pac_list):

249 """Take a list of rpm filenames and run rpm -K on them.

250

251 In case of failure, exit.

252

253 Check all packages in one go, since this takes only 6 seconds on my Athlon 700

254 instead of 20 when calling 'rpm -K' for each of them.

255 """

256 import subprocess

257

258 if not pac_list:

259 return

260

261 # don't care about the return value because we check the

262 # output anyway, and rpm always writes to stdout.

263

264 # save locale first (we rely on English rpm output here)

265 saved_LC_ALL = os.environ.get('LC_ALL')

266 os.environ['LC_ALL'] = 'en_EN'

267

268 o = subprocess.Popen(['rpm', '-K'] + pac_list, stdout=subprocess.PIPE,

269 stderr=subprocess.STDOUT, close_fds=True).stdout

270

271 # restore locale

272 if saved_LC_ALL: os.environ['LC_ALL'] = saved_LC_ALL

273 else: os.environ.pop('LC_ALL')

274

275 for line in o.readlines():

276

277 if not 'OK' in line:

278 print

279 print >>sys.stderr, 'The following package could not be verified:'

280 print >>sys.stderr, line

281 sys.exit(1)

282

283 if 'NOT OK' in line:

284 print

285 print >>sys.stderr, 'The following package could not be verified:'

286 print >>sys.stderr, line

287

288 if 'MISSING KEYS' in line:

289 missing_key = line.split('#')[-1].split(')')[0]

290

291 print >>sys.stderr, """

292 - If the key is missing, install it first.

293 For example, do the following:

294 osc signkey PROJECT > file

295 and, as root:

296 rpm --import %(dir)s/keyfile-%(name)s

297

298 Then, just start the build again.

299

300 - If you do not trust the packages, you should configure osc build for XEN or KVM

301

302 - You may use --no-verify to skip the verification (which is a risk for your system).

303 """ % {'name': missing_key,

304 'dir': os.path.expanduser('~')}

305

306 else:

307 print >>sys.stderr, """

308 - If the signature is wrong, you may try deleting the package manually

309 and re-run this program, so it is fetched again.

310 """

311

312 sys.exit(1)

313

314

315 def verify_pacs(pac_list, key_list):

316 """Take a list of rpm filenames and verify their signatures.

317

318 In case of failure, exit.

319 """

320

321 # XXX: remove if new code stable

322 if not conf.config.get('builtin_signature_check', False):

323 return verify_pacs_old(pac_list)

324

325 if not pac_list:

326 return

327

328 if not key_list:

329 print "no keys"

330 sys.exit(1)

331 return

332

333 print key_list

334

335 import checker

336 failed = False

337 checker = checker.Checker()

338 try:

339 checker.readkeys(key_list)

340 for pkg in pac_list:

341 try:

342 checker.check(pkg)

343 except Exception, e:

344 failed = True

345 print pkg, ':', e

346 except:

347 checker.cleanup()

348 raise

349

350 if failed:

351 checker.cleanup()

352 sys.exit(1)

353

354 checker.cleanup()

355

356 # vim: sw=4 et