Refactoring of stage and *DEPENDS handling and add fstage/FDEPENDS
[oe-lite:cbss-base.git] / lib / oelite / baker.py
1 import oebakery
2 from oebakery import die, err, warn, info, debug, FatalError
3 from oelite import *
4 from recipe import OEliteRecipe
5 from runq import OEliteRunQueue
6 import oelite.meta
7 import oelite.util
8 import oelite.arch
9 import oelite.parse
10 import oelite.task
11 import oelite.item
12 from oelite.parse import *
13 from oelite.cookbook import CookBook
14
15 import bb.utils
16
17 import sys
18 import os
19 import glob
20 import shutil
21 import datetime
22
23 OE_ENV_WHITELIST = [
24     "PATH",
25     "PWD",
26     "SHELL",
27     "TERM",
28 ]
29
30 #INITIAL_OE_IMPORTS = "oe.path oe.utils sys os time"
31 INITIAL_OE_IMPORTS = "sys os time"
32
33 def add_bake_parser_options(parser):
34     parser.add_option("-t", "--task",
35                       action="store", type="str", default=None,
36                       help="task(s) to do")
37
38     parser.add_option("--rebuild",
39                       action="append_const", dest="rebuild", const=1,
40                       help="rebuild specified recipes")
41     parser.add_option("--rebuildall",
42                       action="append_const", dest="rebuild", const=2,
43                       help="rebuild specified recipes and all dependencies (except cross and native)")
44     parser.add_option("--reallyrebuildall",
45                       action="append_const", dest="rebuild", const=3,
46                       help="rebuild specified recipes and all dependencies")
47
48     parser.add_option("--relaxed",
49                       action="append_const", dest="relax", const=1,
50                       help="don't rebuild ${RELAXED} recipes because of metadata changes")
51     parser.add_option("--sloppy",
52                       action="append_const", dest="relax", const=2,
53                       help="don't rebuild dependencies because of metadata changes")
54
55     parser.add_option("-y", "--yes",
56                       action="store_true", default=False,
57                       help="assume 'y' response to trivial questions")
58
59     parser.add_option("--rmwork",
60                       action="store_true", default=None,
61                       help="clean workdir for all recipes being built")
62     parser.add_option("--no-rmwork",
63                       action="store_false", dest="rmwork", default=None,
64                       help="do not clean workdir for all recipes being built")
65
66     parser.add_option("--no-prebake",
67                       action="store_false", dest="prebake", default=True,
68                       help="do not use prebaked packages")
69
70     return
71
72
73 def add_show_parser_options(parser):
74     parser.add_option("--nohash",
75                       action="store_true",
76                       help="don't show variables that will be ignored when computing data hash")
77     parser.add_option("-t", "--task",
78                       action="store", type="str", default=None,
79                       metavar="TASK",
80                       help="prepare recipe for TASK before showing")
81
82     return
83
84
85 class OEliteBaker:
86
87     def __init__(self, options, args, config):
88         self.options = options
89
90         self.config = oelite.meta.DictMeta(meta=config)
91         self.config["OE_IMPORTS"] = INITIAL_OE_IMPORTS
92         self.import_env()
93         self.config.pythonfunc_init()
94         self.topdir = self.config.get("TOPDIR", True)
95         # FIXME: self.config.freeze("TOPDIR")
96
97         self.confparser = confparse.ConfParser(self.config)
98         self.confparser.parse("conf/oe-lite.conf")
99
100         oelite.pyexec.exechooks(self.config, "post_conf_parse")
101
102         # FIXME: refactor oelite.arch.init to a post_conf_parse hook
103         oelite.arch.init(self.config)
104
105         # Handle any INHERITs and inherit the base class
106         inherits  = ["core"] + (self.config.get("INHERIT", 1) or "").split()
107         # and inherit rmwork when needed
108         try:
109             rmwork = self.options.rmwork
110             inherits.append("rmwork")
111             if rmwork is None:
112                 rmwork = self.config.get("RMWORK", True)
113                 if rmwork == "0":
114                     rmwork = False
115             if rmwork:
116                 debug("rmwork")
117                 self.options.rmwork = True
118         except AttributeError:
119             pass
120         self.oeparser = oeparse.OEParser(self.config)
121         for inherit in inherits:
122             self.oeparser.reset_lexstate()
123             self.oeparser.parse("classes/%s.oeclass"%(inherit), require=True)
124
125         oelite.pyexec.exechooks(self.config, "post_common_inherits")
126
127         self.cookbook = CookBook(self)
128
129         # things (ritem, item, recipe, or package) to do
130         if args:
131             self.things_todo = args
132         elif "OE_DEFAULT_THING" in self.config:
133             self.things_todo = self.config.get("OE_DEFAULT_THING", 1).split()
134         else:
135             self.things_todo = [ "world" ]
136
137         recipe_types = ("machine", "native", "sdk",
138                         "cross", "sdk-cross", "canadian-cross")
139         def thing_todo(thing):
140             if not thing in self.things_todo:
141                 self.things_todo.append(thing)
142         def dont_do_thing(thing):
143             while thing in self.things_todo:
144                 self.things_todo.remove(thing)
145         self.recipes_todo = set()
146         if "universe" in self.things_todo:
147             dont_do_thing("universe")
148             for recipe_type in recipe_types:
149                 thing_todo(recipe_type + ":world")
150         if "world" in self.things_todo:
151             dont_do_thing("world")
152             thing_todo("machine:world")
153         for recipe_type in recipe_types:
154             world = recipe_type + ":world"
155             if world in self.things_todo:
156                 dont_do_thing(world)
157                 for recipe in self.cookbook.get_recipes(type=recipe_type):
158                     self.recipes_todo.add(recipe)
159
160         return
161
162
163     def __del__(self):
164         return
165
166
167     def import_env(self):
168         whitelist = OE_ENV_WHITELIST
169         if "OE_ENV_WHITELIST" in os.environ:
170             whitelist += os.environ["OE_ENV_WHITELIST"].split()
171         if "OE_ENV_WHITELIST" in self.config:
172             whitelist += self.config.get("OE_ENV_WHITELIST", True).split()
173         debug("whitelist=%s"%(whitelist))
174         for var in set(os.environ).difference(whitelist):
175             del os.environ[var]
176         if oebakery.DEBUG:
177             debug("Whitelist filtered shell environment:")
178             for var in os.environ:
179                 debug("> %s=%s"%(var, os.environ[var]))
180         for var in whitelist:
181             if not var in self.config and var in os.environ:
182                 self.config[var] = os.environ[var]
183                 debug("importing %s=%s"%(var, os.environ[var]))
184         return
185
186
187     def show(self):
188
189         if len(self.things_todo) == 0:
190             die("you must specify something to show")
191         if len(self.recipes_todo) > 0:
192             die("you cannot show world")
193
194         thing = oelite.item.OEliteItem(self.things_todo[0])
195         recipe = self.cookbook.get_recipe(
196             type=thing.type, name=thing.name, version=thing.version,
197             strict=False)
198         if not recipe:
199             die("Cannot find %s"%(thing))
200
201         if self.options.task:
202             if self.options.task.startswith("do_"):
203                 task = self.options.task
204             else:
205                 task = "do_" + self.options.task
206             self.runq = OEliteRunQueue(self.config, self.cookbook)
207             self.runq._add_recipe(recipe, task)
208             task = self.cookbook.get_task(recipe=recipe, name=task)
209             task.prepare(self.runq)
210             meta = task.meta()
211         else:
212             meta = recipe.meta
213
214         #meta.dump(pretty=False, nohash=False, flags=True,
215         #          ignore_flags=("filename", "lineno"),
216         meta.dump(pretty=True, nohash=(not self.options.nohash),
217                   only=(self.things_todo[1:] or None))
218
219         return 0
220
221
222     def bake(self):
223
224         self.setup_tmpdir()
225
226         # task(s) to do
227         if self.options.task:
228             tasks_todo = self.options.task
229         elif "OE_DEFAULT_TASK" in self.config:
230             tasks_todo = self.config.get("OE_DEFAULT_TASK", 1)
231         else:
232             #tasks_todo = "all"
233             tasks_todo = "build"
234         self.tasks_todo = tasks_todo.split(",")
235
236         if self.options.rebuild:
237             self.options.rebuild = max(self.options.rebuild)
238             self.options.prebake = False
239         else:
240             self.options.rebuild = None
241         if self.options.relax:
242             self.options.relax = max(self.options.relax)
243         else:
244             default_relax = self.config.get("DEFAULT_RELAX", 1)
245             if default_relax and default_relax != "0":
246                 self.options.relax = int(default_relax)
247             else:
248                 self.options.relax = None
249
250         # init build quue
251         self.runq = OEliteRunQueue(self.config, self.cookbook,
252                                    self.options.rebuild, self.options.relax)
253
254         # first, add complete dependency tree, with complete
255         # task-to-task and task-to-package/task dependency information
256         debug("Building dependency tree")
257         start = datetime.datetime.now()
258         for task in self.tasks_todo:
259             task = oelite.task.task_name(task)
260             try:
261                 for thing in self.recipes_todo:
262                     if not self.runq.add_recipe(thing, task):
263                         die("No such recipe: %s"%(thing))
264                 for thing in self.things_todo:
265                     thing = oelite.item.OEliteItem(thing)
266                     if not self.runq.add_something(thing, task):
267                         die("No such thing: %s"%(thing))
268             except RecursiveDepends, e:
269                 die("dependency loop: %s\n\t--> %s"%(
270                         e.args[1], "\n\t--> ".join(e.args[0])))
271             except NoSuchTask, e:
272                 die("No such task: %s: %s"%(thing, e.__str__()))
273             except FatalError, e:
274                 die("Failed to add %s:%s to runqueue"%(thing, task))
275         if oebakery.DEBUG:
276             timing_info("Building dependency tree", start)
277
278         # Generate recipe dependency graph
279         recipes = set([])
280         for task in self.runq.get_tasks():
281             task_deps = self.runq.task_dependencies(task, flatten=True)
282             recipe = task.recipe
283             recipe.add_task(task, task_deps)
284             recipes.add(recipe)
285         unresolved_recipes = []
286         for recipe in recipes:
287             unresolved_recipes.append((recipe, list(recipe.recipe_deps)))
288
289         # Traverse recipe dependency graph, propagating EXTRA_ARCH on
290         # recipe level.
291         resolved_recipes = set([])
292         while len(unresolved_recipes) > 0:
293             progress = False
294             for i in xrange(len(unresolved_recipes)-1, -1, -1):
295                 recipe, unresolved_deps = unresolved_recipes[i]
296                 resolved = True
297                 for j in xrange(len(unresolved_deps)-1, -1, -1):
298                     recipe_dep = unresolved_deps[j]
299                     if not recipe_dep in resolved_recipes:
300                         continue
301                     recipe_dep_extra_arch = recipe_dep.meta.get("EXTRA_ARCH")
302                     if recipe_dep_extra_arch:
303                         # FIXME: sanity check for inconsistent EXTRA_ARCH here
304                         recipe.meta.set("EXTRA_ARCH", recipe_dep_extra_arch)
305                     del unresolved_deps[j]
306                 if len(unresolved_deps) == 0:
307                     resolved_recipes.add(recipe)
308                     del unresolved_recipes[i]
309                     progress = True
310             if not progress:
311                 bb.fatal("recipe EXTRA_ARCH resolving deadlocked!")
312
313         # update runq task list, checking recipe and src hashes and
314         # determining which tasks needs to be run
315         # examing each task, computing it's hash, and checking if the
316         # task has already been built, and with the same hash.
317         task = self.runq.get_metahashable_task()
318         total = self.runq.number_of_runq_tasks()
319         count = 0
320         start = datetime.datetime.now()
321         while task:
322             oelite.util.progress_info("Calculating task metadata hashes",
323                                       total, count)
324             recipe = task.recipe
325
326             if task.nostamp:
327                 self.runq.set_task_metahash(task, "0")
328                 task = self.runq.get_metahashable_task()
329                 count += 1
330                 continue
331
332             dephashes = {}
333             for depend in self.runq.task_dependencies(task, flatten=True):
334                 dephashes[depend] = self.runq.get_task_metahash(depend)
335             recipe_extra_arch = recipe.meta.get("EXTRA_ARCH")
336             task_meta = task.meta()
337             if (recipe_extra_arch and
338                 task_meta.get("EXTRA_ARCH") != recipe_extra_arch):
339                 task_meta.set("EXTRA_ARCH", recipe_extra_arch)
340             try:
341                 datahash = task_meta.signature()
342             except oelite.meta.ExpansionError as e:
343                 e.msg += " in %s"%(task)
344                 raise
345
346             import hashlib
347
348             hasher = hashlib.md5()
349             hasher.update(str(sorted(dephashes.values())))
350             dephash = hasher.hexdigest()
351
352             hasher = hashlib.md5()
353             hasher.update(datahash)
354             hasher.update(dephash)
355             metahash = hasher.hexdigest()
356
357             # FIXME: instad of all of the above
358             # metasig = task.get_meta_signature()
359
360             #if oebakery.DEBUG:
361             #    recipe_name = self.db.get_recipe(recipe_id)
362             #    task_name = self.db.get_task(task=task)
363             #    debug(" %d %s:%s data=%s dep=%s meta=%s"%(
364             #            task, "_".join(recipe_name), task_name,
365             #            datahash, dephash, metahash))
366
367             self.runq.set_task_metahash(task, metahash)
368
369             (stamp_mtime, stamp_signature) = task.read_stamp()
370             if not stamp_mtime:
371                 self.runq.set_task_build(task)
372             else:
373                 self.runq.set_task_stamp(task, stamp_mtime, stamp_signature)
374
375             task = self.runq.get_metahashable_task()
376             count += 1
377             continue
378
379         oelite.util.progress_info("Calculating task metadata hashes",
380                                   total, count)
381
382         if oebakery.DEBUG:
383             timing_info("Calculation task metadata hashes", start)
384
385         if count != total:
386             print ""
387             self.runq.print_metahashable_tasks()
388             print "count=%s total=%s"%(count, total)
389             die("Circular dependencies I presume.  Add more debug info!")
390
391         self.runq.set_task_build_on_nostamp_tasks()
392         self.runq.set_task_build_on_retired_tasks()
393         self.runq.set_task_build_on_hashdiff()
394
395         # check for availability of prebaked packages, and set package
396         # filename for all packages.
397         depend_packages = self.runq.get_depend_packages()
398         for package in depend_packages:
399             # FIXME: skip this package if it is to be rebuild
400             prebake = self.find_prebaked_package(package)
401             if prebake:
402                 self.runq.set_package_filename(package, prebake,
403                                                prebake=True)
404
405         # clear parent_task for all runq_depends where all runq_depend
406         # rows with the same parent_task has prebake flag set
407         self.runq.prune_prebaked_runq_depends()
408
409         # FIXME: this might prune to much. If fx. A depends on B and
410         # C, and B depends on C, and all A->B dependencies are
411         # prebaked, but not all A->C dependencies, B will be used
412         # prebaked, and A will build with a freshly built C, which
413         # might be different from the C used in B.  This is especially
414         # risky when manually fidling with content of WORKDIR manually
415         # (fx. manually fixing something to get do_compile to
416         # complete, and then wanting to test the result before
417         # actually integrating it in the recipe).  Hmm....  Why not
418         # just add a --no-prebake option, so when developer is
419         # touching WORKDIR manually, this should be used to avoid
420         # strange prebake issues.  The mtime / retired task stuff
421         # should guarantee that consistency is kept then.
422
423         # Argh! if prebake is to work with rmwork, we might have to do
424         # the above after all :-( We will now have som self.runq_depends
425         # with parent_task.prebake flag set, but when we follow its
426         # dependencies, we will find one or more recipes that has to
427         # be rebuilt, fx. because of a --rebuild flag.
428
429         self.runq.propagate_runq_task_build()
430
431         build_count = self.runq.set_buildhash_for_build_tasks()
432         nobuild_count = self.runq.set_buildhash_for_nobuild_tasks()
433         if (build_count + nobuild_count) != total:
434             die("build_count + nobuild_count != total")
435
436         deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR", True)
437         packages = self.runq.get_packages_to_build()
438         for package in packages:
439             package = self.cookbook.get_package(id=package)
440             recipe = self.cookbook.get_recipe(package=package.id)
441             buildhash = self.runq.get_package_buildhash(package.id)
442             filename = os.path.join(
443                 deploy_dir, package.type,
444                 package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
445                 "%s_%s_%s.tar"%(package.name, recipe.version, buildhash))
446             debug("will use from build: %s"%(filename))
447             self.runq.set_package_filename(package.id, filename)
448
449         self.runq.mark_primary_runq_depends()
450         self.runq.prune_runq_depends_nobuild()
451         self.runq.prune_runq_depends_with_nobody_depending_on_it()
452         self.runq.prune_runq_tasks()
453
454         remaining = self.runq.number_of_tasks_to_build()
455         debug("%d tasks remains"%remaining)
456
457         recipes = self.runq.get_recipes_with_tasks_to_build()
458         if not recipes:
459             info("Nothing to do")
460             return 0
461
462         if self.options.rmwork:
463             for recipe in recipes:
464                 if (tasks_todo != ["build"]
465                     and self.runq.is_recipe_primary(recipe[0])):
466                     debug("skipping...")
467                     continue
468                 debug("adding %s:do_rmwork"%(recipe[1]))
469                 recipe = self.cookbook.get_recipe(recipe[0])
470                 self.runq._add_recipe(recipe, "do_rmwork")
471                 task = self.cookbook.get_task(recipe=recipe, name="do_rmwork")
472                 self.runq.set_task_build(task)
473             self.runq.propagate_runq_task_build()
474             remaining = self.runq.number_of_tasks_to_build()
475             debug("%d tasks remains after adding rmwork"%remaining)
476             recipes = self.runq.get_recipes_with_tasks_to_build()
477
478         print "The following will be build:"
479         text = []
480         for recipe in recipes:
481             if recipe[1] == "machine":
482                 text.append("%s(%d)"%(recipe[2], recipe[4]))
483             else:
484                 text.append("%s:%s(%d)"%(recipe[1], recipe[2], recipe[4]))
485         print oelite.util.format_textblock(" ".join(text))
486
487         if os.isatty(sys.stdin.fileno()) and not self.options.yes:
488             while True:
489                 try:
490                     response = raw_input("Do you want to continue [Y/n/?/??]? ")
491                 except KeyboardInterrupt:
492                     response = "n"
493                     print ""
494                 if response == "" or response[0] in ("y", "Y"):
495                     break
496                 elif response == "?":
497                     tasks = self.runq.get_tasks_to_build_description()
498                     for task in tasks:
499                         print "  " + task
500                     continue
501                 elif response == "??":
502                     tasks = self.runq.get_tasks_to_build_description(hashinfo=True)
503                     for task in tasks:
504                         print "  " + task
505                     continue
506                 else:
507                     info("Maybe next time")
508                     return 0
509
510         # FIXME: add some kind of statistics, with total_tasks,
511         # prebaked_tasks, running_tasks, failed_tasks, done_tasks
512         task = self.runq.get_runabletask()
513         start = datetime.datetime.now()
514         total = self.runq.number_of_tasks_to_build()
515         count = 0
516         exitcode = 0
517         failed_task_list = ""
518         while task:
519             count += 1
520             debug("")
521             debug("Preparing %s"%(task))
522             task.prepare(self.runq)
523             meta = task.meta()
524             info("Running %d / %d %s"%(count, total, task))
525             task.build_started()
526             if task.run():
527                 task.build_done(self.runq.get_task_buildhash(task))
528                 self.runq.mark_done(task)
529             else:
530                 err("%s failed"%(task))
531                 exitcode = 1
532                 failed_task_list += "\nERROR: %s failed"%(task)
533                 task.build_failed()
534                 # FIXME: support command-line option to abort on first
535                 # failed task
536             task = self.runq.get_runabletask()
537         timing_info("Build", start)
538
539         if exitcode:
540             print failed_task_list
541         return exitcode
542
543
544     def setup_tmpdir(self):
545
546         tmpdir = os.path.realpath(self.config.get("TMPDIR", 1) or "tmp")
547         #debug("TMPDIR = %s"%tmpdir)
548
549         try:
550
551             if not os.path.exists(tmpdir):
552                 os.makedirs(tmpdir)
553
554             if (os.path.islink(tmpdir) and
555                 not os.path.exists(os.path.realpath(tmpdir))):
556                 os.makedirs(os.path.realpath(tmpdir))
557
558         except Exception, e:
559             die("failed to setup TMPDIR: %s"%e)
560             import traceback
561             e.print_exception(type(e), e, True)
562
563         return
564
565
566     def find_prebaked_package(self, package):
567         """return full-path filename string or None"""
568         package_deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR")
569         if not package_deploy_dir:
570             die("PACKAGE_DEPLOY_DIR not defined")
571         if self.options.prebake:
572             prebake_path = self.config.get("PREBAKE_PATH") or []
573             if prebake_path:
574                 prebake_path = prebake_path.split(":")
575             prebake_path.insert(0, package_deploy_dir)
576         else:
577             prebake_path = [package_deploy_dir]
578         debug("package=%s"%(repr(package)))
579         recipe = self.cookbook.get_recipe(package=package)
580         if not recipe:
581             raise NoSuchRecipe()
582         metahash = self.runq.get_package_metahash(package)
583         debug("got metahash=%s"%(metahash))
584         package = self.cookbook.get_package(id=package)
585         if not package:
586             raise NoSuchPackage()
587         filename = "%s_%s_%s.tar"%(package.name, recipe.version, metahash)
588         debug("prebake_path=%s"%(prebake_path))
589         for base_dir in prebake_path:
590             path = os.path.join(
591                 base_dir,
592                 package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
593                 filename)
594             debug("checking for prebake: %s"%(path))
595             if os.path.exists(path):
596                 debug("found prebake: %s"%(path))
597                 return path
598         return None
599
600
601
602
603 def timing_info(msg, start):
604     msg += " time "
605     delta = datetime.datetime.now() - start
606     hours = delta.seconds // 3600
607     minutes = delta.seconds // 60 % 60
608     seconds = delta.seconds % 60
609     milliseconds = delta.microseconds // 1000
610     if hours:
611         msg += "%dh%02dm%02ds"%(hours, minutes, seconds)
612     elif minutes:
613         msg += "%dm%02ds"%(minutes, seconds)
614     else:
615         msg += "%d.%03d seconds"%(seconds, milliseconds)
616     info(msg)
617     return