Refactoring of stage and *DEPENDS handling and add fstage/FDEPENDS
[oe-lite:cbss-base.git] / lib / oelite / fetch / fetch.py
1 import re
2 import os
3 import hashlib
4 import shutil
5 import string
6
7 import bb.utils
8 import oe.process
9
10 import oelite.fetch
11 import local
12 import url
13 import git
14 import hg
15
16 FETCHERS = {
17     "file"      : local.LocalFetcher,
18     "http"      : url.UrlFetcher,
19     "https"     : url.UrlFetcher,
20     "ftp"       : url.UrlFetcher,
21     "git"       : git.GitFetcher,
22     "hg"        : hg.HgFetcher,
23 }
24
25 uri_pattern = re.compile("(?P<scheme>[^:]*)://(?P<location>[^;]+)(;(?P<params>.*))?")
26
27 unpack_ext = (
28     ("tar_gz",  (".tar.gz", ".tgz", ".tar.Z")), 
29     ("tar_bz2", (".tar.bz2", ".tbz", ".tbz2")),
30     ("tar_xz",  (".tar.xz", ".txz")),
31     ("tar_lz",  (".tar.lz", ".tlz")),
32     ("zip",     (".zip", ".jar")),
33     ("gz",      (".gz", ".Z", ".z")),
34     ("bz2",     (".bz2",)),
35     ("xz",      (".xz",)),
36     ("lz",      (".lz",)),
37     )
38
39 class OEliteUri:
40
41     def __init__(self, uri, d):
42         # Note, do not store reference to meta
43         self.recipe = "%s:%s_%s"%(d.get("RECIPE_TYPE"),
44                                   d.get("PN"), d.get("PV"))
45         # Try to avoid readinng recipe meta-data here, as it might
46         # change later on, so better to read the meta-data when
47         # needed.  This is especially problematic with variables
48         # depending on ${EXTRA_ARCH} which might be changed after all
49         # parse when resolving task dependencies.
50         self.ingredients = d.get("INGREDIENTS")
51         self.isubdir = (d.get("INGREDIENTS_SUBDIR") or
52                         os.path.basename(d.get("FILE_DIRNAME")))
53         self.strict = d.get("SRC_URI_STRICT") or False
54         if self.strict == "0":
55             self.strict = False
56         m = uri_pattern.match(uri)
57         if not m:
58             raise oelite.fetch.InvalidURI(uri, "not an URI at all")
59         self.scheme = m.group("scheme")
60         self.location = m.group("location")
61         if not self.scheme:
62             raise oelite.fetch.InvalidURI(uri, "no URI scheme")
63         if not self.location:
64             raise oelite.fetch.InvalidURI(uri, "no URI location")
65         self.params = {}
66         if m.group("params"):
67             for param in (m.group("params") or "").split(";"):
68                 try:
69                     name, value = param.split("=")
70                 except ValueError:
71                     raise oelite.fetch.InvalidURI(
72                         uri, "bad parameter: %s"%param)
73                 self.params[name] = value
74         if not self.scheme in FETCHERS:
75             raise oelite.fetch.InvalidURI(
76                 uri, "unsupported URI scheme: %s"%(self.scheme))
77         self.fdepends = []
78         self.fetcher = FETCHERS[self.scheme](self, d)
79         self.init_unpack_params()
80         self.add_unpack_fdepends(d)
81         self.init_patch_params(d)
82         self.add_patch_fdepends()
83         self.mirrors = d.get("MIRRORS") or None
84         return
85
86     def alternative_mirror(self):
87         if self.mirrors is None:
88             return None
89         if isinstance(self.mirrors, str):
90             url = "%s://%s"%(self.scheme, self.location)
91             mirrors = self.mirrors.split("\n")
92             mirrors = map(string.strip, mirrors)
93             mirrors = filter(None, mirrors)
94             mirrors = map(string.split, mirrors)
95             if not mirrors:
96                 self.mirrors = None
97                 return None
98             self.mirrors = []
99             for mirror in mirrors:
100                 (src_uri, mirror_uri) = tuple(mirror)
101                 m = re.match(src_uri, url)
102                 if m:
103                     self.mirrors.append(mirror_uri + url[m.end():])
104             self.next_mirror = 0
105         if self.next_mirror == len(self.mirrors):
106             return None
107         mirror = self.mirrors[self.next_mirror]
108         self.next_mirror += 1
109         return mirror
110
111     def __str__(self):
112         return "%s://%s"%(self.scheme, self.location)
113
114     def init_unpack_params(self):
115         if not "localpath" in dir(self.fetcher):
116             return
117         if not "unpack" in self.params:
118             for (unpack, exts) in unpack_ext:
119                 assert isinstance(exts, tuple)
120                 for ext in exts:
121                     if self.fetcher.localpath.endswith(ext):
122                         self.params["unpack"] = unpack
123                         return
124         elif self.params["unpack"] == "0":
125             del self.params["unpack"]
126         if not "unpack" in self.params:
127             return
128         if self.params["unpack"] == "zip":
129             if "dos" in self.params and self.params["dos"] != "0":
130                 self.params["unpack"] += "_dos"
131         return
132
133     def add_unpack_fdepends(self, d):
134         if not "unpack" in self.params or not self.params["unpack"]:
135             return
136         fdepends = d.get("UNPACK_CMD_FDEPENDS_" + self.params["unpack"])
137         if fdepends:
138             self.fdepends.extend(fdepends.split())
139         return
140
141     def init_patch_params(self, d):
142         if not "localpath" in dir(self.fetcher):
143             return
144         if not "apply" in self.params:
145             patchfile = self.fetcher.localpath
146             try:
147                 unpack = self.params["unpack"] or None
148                 if unpack == "0":
149                     unpack = None
150             except KeyError:
151                 unpack = None
152             if unpack and self.fetcher.localpath.endswith(unpack):
153                 patchfile = self.fetcher.localpath[-len(unpack):]
154             if patchfile.endswith(".patch") or patchfile.endswith(".diff"):
155                 self.params["apply"] = "yes"
156         elif not self.params["apply"] in ["yes", "y", "1"]:
157             del self.params["apply"]
158         if "apply" in self.params:
159             patchsubdir = d.get("PATCHSUBDIR")
160             if "subdir" in self.params:
161                 subdir = self.params["subdir"]
162                 if (subdir != patchsubdir and
163                     not subdir.startswith(patchsubdir + "")):
164                     subdir = os.path.join(patchsubdir, subdir)
165             else:
166                 subdir = patchsubdir
167             self.params["subdir"] = subdir
168         if not "striplevel" in self.params:
169             self.params["striplevel"] = 1
170         if "patchdir" in self.params:
171             raise Exception("patchdir URI parameter support not implemented")
172         return
173
174     def add_patch_fdepends(self):
175         if not "apply" in self.params or not self.params["apply"]:
176             return
177         self.fdepends.append("native:quilt")
178         return
179
180     def signature(self):
181         try:
182             return self.fetcher.signature()
183         except oelite.fetch.NoSignature as e:
184             if self.strict:
185                 raise e
186             try:
187                 url = self.fetcher.url
188             except AttributeError:
189                 url = self.uri
190             print "%s: no checksum known for %s"%(self.recipe, url)
191             return ""
192
193     def cache(self):
194         if not "cache" in dir(self.fetcher):
195             return True
196         return self.fetcher.cache()
197
198     def write_checksum(self, filepath):
199         md5path = filepath + ".md5"
200         checksum = hashlib.md5()
201         with open(filepath) as f:
202             checksum.update(f.read())
203         with open(md5path, "w") as f:
204             f.write(checksum.digest())
205
206     def verify_checksum(self, filepath):
207         md5path = filepath + ".md5"
208         if not os.path.exists(md5path):
209             return None
210         checksum = hashlib.md5()
211         with open(filepath) as f:
212             checksum.update(f.read())
213         with open(md5path) as f:
214             return f.readline().strip() == checksum.digest()
215
216     def fetch(self):
217         if not "fetch" in dir(self.fetcher):
218             return True
219         print "Fetching", str(self)
220         return self.fetcher.fetch()
221
222     def unpack(self, d, cmd):
223         if "unpack" in dir(self.fetcher):
224             return self.fetcher.unpack(d)
225         print "Unpacking", self.fetcher.localpath
226         srcpath = os.getcwd()
227         self.srcfile = None
228         if "subdir" in self.params:
229             srcpath = os.path.join(srcpath, self.params["subdir"])
230             bb.utils.mkdirhier(srcpath)
231             os.chdir(srcpath)
232         if not cmd or not "unpack" in self.params:
233             if os.path.isdir(self.fetcher.localpath):
234                 shutil.rmtree(srcpath, True)
235                 shutil.copytree(self.fetcher.localpath, self.srcpath(d))
236                 return True
237             else:
238                 shutil.copy2(self.fetcher.localpath, self.srcpath(d))
239                 return True
240         if "unpack_to" in self.params:
241             cmd = cmd%(self.fetcher.localpath, self.srcpath(d))
242         else:
243             cmd = cmd%(self.fetcher.localpath)
244         rc = oe.process.run(cmd)
245         return rc == 0
246
247     def srcpath(self, d):
248         srcdir = d.get("SRCDIR")
249         if "subdir" in self.params:
250             srcdir = os.path.join(srcdir, self.params["subdir"])
251         if "unpack_to" in self.params:
252             return os.path.join(srcdir, self.params["unpack_to"])
253         else:
254             return os.path.join(srcdir,
255                                 os.path.basename(self.fetcher.localpath))
256
257     def patchpath(self, d):
258         srcpath = self.srcpath(d)
259         patchdir = d.get("PATCHDIR")
260         assert srcpath.startswith(patchdir + "/")
261         return srcpath[len(patchdir) + 1:]
262
263     def patch(self, d):
264         with open("%s/series"%(d.get("PATCHDIR")), "a") as series:
265             series.write("%s -p%s\n"%(
266                     self.patchpath(d), self.params["striplevel"]))
267
268         rc = oe.process.run("quilt -v --quiltrc %s push"%(d.get("QUILTRC")))
269         if rc != 0:
270             # FIXME: proper error handling
271             raise Exception("quilt push failed: %d"%(rc))
272         return True
273
274
275 def patch_init(d):
276     quiltrc = d.get("QUILTRC")
277     patchdir = d.get("PATCHDIR")
278     with open(quiltrc, "w") as quiltrcfile:
279         quiltrcfile.write("QUILT_PATCHES=%s\n"%(patchdir))
280     series = os.path.join(patchdir, "series")
281     if os.path.exists(series):
282         os.remove(series)
283     s = d.get("S")
284     os.chdir(s)
285     if os.path.exists(".pc"):
286         while os.path.exists(".pc/applied-patches"):
287             rc = oe.process.run("quilt -v --quiltrc %s pop"%(quiltrc))
288             if rc != 0:
289                 # FIXME: proper error handling
290                 raise Exception("quilt pop failed")
291         if not os.path.exists(".pc/series") and not os.path.exists(".pc/.quilt_series"):
292             # FIXME: proper error handling
293             raise Exception("Bad quilt .pc dir")