Refactoring of stage and *DEPENDS handling and add fstage/FDEPENDS
[oe-lite:cbss-base.git] / lib / oelite / task.py
1 import oebakery
2 from oebakery import die, err, warn, info, debug
3 import oelite.meta
4 import oelite.recipe
5 from oelite.dbutil import *
6 import oelite.function
7 import bb.utils
8
9 import sys
10 import os
11 import warnings
12 import shutil
13 import re
14
15 def task_name(name):
16     if name.startswith("do_"):
17         return name
18     return "do_" + name
19
20
21 class OEliteTask:
22
23     TASKFUNC_RE = re.compile(r"^do_([a-z]+).*?")
24
25     def __init__(self, id, recipe, name, nostamp, cookbook):
26         self.id = id
27         self.recipe = cookbook.get_recipe(id=recipe)
28         self.name = name
29         self.nostamp = nostamp
30         self.cookbook = cookbook
31         self._meta = None
32         return
33
34     def __str__(self):
35         return "%s:%s"%(self.recipe, self.name)
36
37
38     def get_parents(self):
39         parents = flatten_single_column_rows(self.cookbook.dbc.execute(
40             "SELECT parent FROM task_parent WHERE recipe=? AND task=?",
41             (self.recipe.id, self.name,)))
42         if not parents:
43             return []
44         return self.cookbook.get_tasks(recipe=self.recipe, name=parents)
45
46     def get_deptasks(self, deptypes):
47         assert isinstance(deptypes, list) and len(deptypes) > 0
48         return flatten_single_column_rows(self.cookbook.dbc.execute(
49             "SELECT deptask FROM task_deptask "
50             "WHERE deptype IN (%s) "%(",".join("?" for i in deptypes)) +
51             "AND task=?", deptypes + [self.id]))
52
53     def get_recdeptasks(self, deptypes):
54         return flatten_single_column_rows(self.cookbook.dbc.execute(
55             "SELECT recdeptask FROM task_recdeptask "
56             "WHERE deptype IN (%s) "%(",".join("?" for i in deptypes)) +
57             "AND task=?", deptypes + [self.id]))
58
59     def get_taskdepends(self):
60         return flatten_single_column_rows(self.cookbook.dbc.execute(
61             "SELECT parent_item, parent_task FROM task_depend WHERE task=?",
62             (self.id,)))
63
64
65     def stampfile_path(self):
66         try:
67             return self._stampfile
68         except AttributeError:
69             stampdir = self.recipe.get("STAMPDIR")
70             self._stampfile = (stampdir, os.path.join(stampdir, self.name))
71         return self._stampfile
72
73
74     # return (mtime, hash) from stamp file
75     def read_stamp(self):
76         stampfile = self.stampfile_path()[1]
77         if not os.path.exists(stampfile):
78             return (None, None)
79         if not os.path.isfile(stampfile):
80             die("bad hash file: %s"%(stampfile))
81         if os.path.getsize(stampfile) == 0:
82             return (None, None)
83         mtime = os.stat(stampfile).st_mtime
84         with open(stampfile, "r") as stampfile:
85             tmphash = stampfile.read()
86         return (mtime, tmphash)
87
88
89     def build_started(self):
90         if self.nostamp:
91             return
92         (stampdir, stampfile) = self.stampfile_path()
93         if not os.path.exists(stampdir):
94             os.makedirs(stampdir)
95         open(stampfile, "w").close()
96         return
97
98
99     def build_done(self, buildhash):
100         if self.nostamp:
101             return
102         (stampdir, stampfile) = self.stampfile_path()
103         if not os.path.exists(stampdir):
104             os.makedirs(stampdir)
105         with open(stampfile, "w") as _stampfile:
106             _stampfile.write(buildhash)
107         return
108
109
110     def build_failed(self):
111         return
112
113
114     def task_cleaned(self):
115         hashpath = self.stampfile_path()[1]
116         if os.path.exists(hashpath):
117             os.remove(hashpath)
118
119
120     def prepare(self, runq):
121         meta = self.meta()
122
123         buildhash = self.cookbook.baker.runq.get_task_buildhash(self)
124         debug("buildhash=%s"%(repr(buildhash)))
125         meta.set("TASK_BUILDHASH", buildhash)
126
127         def prepare_stage(deptype):
128             stage = {}
129             recdepends = []
130             get_package_filename = self.cookbook.baker.runq.get_package_filename
131             packages = self.cookbook.baker.runq.get_depend_packages(
132                 self, deptype) or []
133             for package in packages:
134                 package = self.cookbook.get_package(id=package)
135                 filename = get_package_filename(package)
136                 if not filename in stage:
137                     stage[filename] = package
138             return stage
139
140         meta["__stage"] = prepare_stage("DEPENDS")
141         meta["__rstage"] = prepare_stage("RDEPENDS")
142         meta["__fstage"] = prepare_stage("FDEPENDS")
143         meta.set_flag("__stage", "nohash", True)
144         meta.set_flag("__rstage", "nohash", True)
145         meta.set_flag("__fstage", "nohash", True)
146
147         return
148
149
150     def meta(self):
151         if self._meta is not None:
152             return self._meta
153         meta = self.recipe.meta.copy()
154         # Filter meta-data, enforcing restrictions on which tasks to
155         # emit vars to and not including other task functions.
156         emit_prefixes = (meta.get("META_EMIT_PREFIX") or "").split()
157         def colon_split(s):
158             import string
159             return string.split(s, ":", 1)
160         emit_prefixes = map(colon_split, emit_prefixes)
161         for var in meta.keys():
162             emit_flag = meta.get_flag(var, "emit")
163             emit = (emit_flag or "").split()
164             taskfunc_match = self.TASKFUNC_RE.match(var)
165             if taskfunc_match:
166                 if taskfunc_match.group(0) not in emit:
167                     emit.append(taskfunc_match.group(0))
168             for emit_task, emit_prefix in emit_prefixes:
169                 if not var.startswith(emit_prefix):
170                     continue
171                 if emit_task == "":
172                     if emit_flag is None:
173                         emit_flag = ""
174                     continue
175                 if not emit_task.startswith("do_"):
176                     emit_task = "do_" + emit_task
177                 if not emit_task in emit:
178                     emit.append(emit_task)
179             if (emit or emit_flag == "") and not self.name in emit:
180                 del meta[var]
181                 continue
182             omit = meta.get_flag(var, "omit")
183             if omit is not None and self.name in omit.split():
184                 del meta[var]
185                 continue
186
187         self._meta = meta
188         return meta
189
190
191     def run(self):
192         meta = self.meta()
193         function = meta.get_function(self.name)
194
195         self.do_cleandirs()
196         cwd = self.do_dirs() or meta.get("B")
197
198         # Setup stdin, stdout and stderr redirection
199         stdin = open("/dev/null", "r")
200         logfn = "%s/%s.%s.log"%(function.tmpdir, self.name, str(os.getpid()))
201         logsymlink = "%s/%s.log"%(function.tmpdir, self.name)
202         bb.utils.mkdirhier(os.path.dirname(logfn))
203         try:
204             if oebakery.DEBUG:
205                 logfile = os.popen("tee %s"%logfn, "w")
206             else:
207                 logfile = open(logfn, "w")
208         except OSError:
209             print "Opening log file failed: %s"%(logfn)
210             raise
211
212         if os.path.exists(logsymlink) or os.path.islink(logsymlink):
213             os.remove(logsymlink)
214         os.symlink(logfn, logsymlink)
215
216         real_stdin = os.dup(sys.stdin.fileno())
217         real_stdout = os.dup(sys.stdout.fileno())
218         real_stderr = os.dup(sys.stderr.fileno())
219         os.dup2(stdin.fileno(), sys.stdin.fileno())
220         os.dup2(logfile.fileno(), sys.stdout.fileno())
221         os.dup2(logfile.fileno(), sys.stderr.fileno())
222
223         try:
224             for prefunc in self.get_prefuncs():
225                 print "running prefunc", prefunc
226                 self.do_cleandirs(prefunc)
227                 wd = self.do_dirs(prefunc)
228                 if not prefunc.run(wd or cwd):
229                     return False
230             try:
231                 if not function.run(cwd):
232                     return False
233             except oebakery.FatalError:
234                 return False
235             for postfunc in self.get_postfuncs():
236                 print "running postfunc", postfunc
237                 self.do_cleandirs(postfunc)
238                 wd = self.do_dirs(postfunc)
239                 if not postfunc.run(wd or cwd):
240                     return False
241             return True
242
243         finally:
244             # Cleanup stdin, stdout and stderr redirection
245             os.dup2(real_stdin, sys.stdin.fileno())
246             os.dup2(real_stdout, sys.stdout.fileno())
247             os.dup2(real_stderr, sys.stderr.fileno())
248             stdin.close()
249             logfile.close()
250             os.close(real_stdin)
251             os.close(real_stdout)
252             os.close(real_stderr)
253             if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
254                 os.remove(logsymlink)
255                 os.remove(logfn) # prune empty logfiles
256
257
258     def do_cleandirs(self, name=None):
259         if not name:
260             name = self.name
261         cleandirs = (self.meta().get_flag(name, "cleandirs",
262                                           oelite.meta.FULL_EXPANSION))
263         if cleandirs:
264             for cleandir in cleandirs.split():
265                 if not os.path.exists(cleandir):
266                     continue
267                 try:
268                     #print "cleandir %s"%(cleandir)
269                     shutil.rmtree(cleandir)
270                 except Exception, e:
271                     err("cleandir %s failed: %s"%(cleandir, e))
272                     raise
273
274     def do_dirs(self, name=None):
275         if not name:
276             name = self.name
277         # Create directories and find directory to execute in
278         dirs = (self.meta().get_flag(name, "dirs",
279                                      oelite.meta.FULL_EXPANSION))
280         if dirs:
281             dirs = dirs.split()
282             for dir in dirs:
283                 bb.utils.mkdirhier(dir)
284             return dir
285
286     def get_postfuncs(self):
287         postfuncs = []
288         for name in (self.meta().get_flag(self.name, "postfuncs", 1)
289                      or "").split():
290             postfuncs.append(self.meta().get_function(name))
291         return postfuncs
292
293     def get_prefuncs(self):
294         prefuncs = []
295         for name in (self.meta().get_flag(self.name, "prefuncs", 1)
296                      or "").split():
297             prefuncs.append(self.meta().get_function(name))
298         return prefuncs