Minor fixes for new usecases.
[image-building-tools:daily-image-builder.git] / builder.py
1 #!/usr/bin/python -t
2
3 #
4 # Daily Image Builder
5 # Copyright (C) 2010 Nokia Corporation.
6 #
7 # Contact: Marko Saukko <marko.saukko@cybercom.com>
8 #
9 # This program is free software; you can redistribute it and/or
10 # modify it under the terms of the GNU General Public License
11 # as published by the Free Software Foundation; either version 2
12 # of the License, or (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program; if not, write to the Free Software
21 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
22 #
23
24 import sys
25 import os
26 import subprocess
27 import shutil
28 import glob
29 import re
30 import time
31 import fnmatch
32 from datetime import date
33
34 from optparse import OptionParser
35
36 import modules.misc as misc
37
38 CADAVER_SCRIPT_NAME = "cadaver_script"
39
40 parser = OptionParser()
41 parser.add_option("","--use-ks-from-git",dest="use_ks_from_git", action="store_true",
42                   help="Use .ks files from gitorious")
43 parser.add_option("","--use-ks-dir", dest="use_ks_dir", default=None,
44                   help="Use ks files from specific dir.")
45 parser.add_option("-r","--release-id",dest="releaseid", default=None,
46                   help="Release ID for the image. If not entered the id will "
47                   "be retrieved based on url in config file."
48                   "Special IDs:"
49                   " - NEMO : gets the latest release based on URL provided"
50                   " - NEMO,ID : uses the NEMO id string with provied ID.")
51 parser.add_option("","--run-mic",dest="runmic", action="store_true",
52                   default=False,
53                   help="Execute mic-image-creator for each found .ks.")
54 parser.add_option("","--architectures",dest="architectures", default=None,
55                   help="The architectures that are build. If not set then all "
56                   "found architectures are build. This is the same value that "
57                   "is set in the kickstart filename. Must be a comma ',' "
58                   "separated list.")
59 parser.add_option("","--verticals",dest="verticals", default=None,
60                   help="The vertical that are build. If not set then all "
61                   "found verticals are build. This is the same value that "
62                   "is set in the kickstart filename. Must be a comma ',' "
63                   "separated list.")
64 parser.add_option("","--extra-filters",dest="extra_filters", default=None,
65                   help="The extra filters for the kickstart filtering. "
66                   "With this you can define extra filters to be used with the"
67                   "kickstart filename. Must be a comma ',' "
68                   "separated list.")
69 parser.add_option("","--extra-dir-filters",dest="extra_dir_filters", default=None,
70                   help="The extra directory filters for the kickstart filtering. "
71                   "With this you can define extra filters to be used with the"
72                   "kickstart directoryname. Must be a comma ',' "
73                   "separated list.")             
74 parser.add_option("","--branch",dest="branch",
75                   default=None,
76                   help="Branch filter for branches in image-config dir."
77                   "Must be a comma ',' separated list.")
78 parser.add_option("","--upload-to-webdav",dest="upload_webdav",action="store_true", 
79                   default=False,
80                   help="Upload packages with the cadaver to webdav addresss "
81                   "provided by config.")
82 parser.add_option("","--upload-with-rsync",dest="upload_rsync",action="store_true",
83                   default=False,
84                   help="Upload packages with rsync to rsync address provided by config.")
85 parser.add_option("","--send-release-note",dest="send_release_note", 
86                   action="store_true", default=False,
87                   help="Send release notes to the email given in conf.")
88 parser.add_option("","--configfile",dest="configfile",
89                   default="builder.conf",
90                   help="Config file where the configs are read.")
91 parser.add_option("","--erase-local-images",dest="erase", 
92                   action="store_true",default=False,
93                   help="Erase the image after uploading.")
94 parser.add_option("","--outputdir",dest="outputdir",
95                   default="meego-images",
96                   help="Directory where mic outputs are stored.")
97 parser.add_option("","--clean-cache",dest="clean_cache", 
98                   action="store_true",default=False,
99                   help="Erase the mic cache before uploading.")
100 parser.add_option("","--remove-old", dest="remove_old",
101                   action="store_true",default=False,
102                   help="Remove old releases with rsync.")
103 parser.add_option("","--releases-to-keep", dest="releases_to_keep",
104                   type="int", default=7,
105                   help="How many old releases are kept when removing old releases.")
106 parser.add_option("","--old-mic2-api",dest="mic2cmdline", 
107                   action="store_true",default=False,
108                   help="Use old mic2 cmdline.")
109
110 (options,args) = parser.parse_args()
111
112 DEFAULT_PREFIX="mg"
113
114 TRUE_VALUES = ["true","yes","1"]
115 KS_TMP = "ks-tmp/"
116 GIT_KS_TMP = "git-ks-tmp/"
117
118 def calc_release_id(options,configs,branch,image_config_dir):
119     release_id_tmp = None
120     if options.releaseid and options.releaseid.startswith("NEMO"):
121         # For Nemo Mobile we have specific release ID's.
122
123         idarray = options.releaseid.split(',',2)
124         if len(idarray) > 1:
125             options.releaseid_base = idarray[1]
126         else:
127             options.releaseid_base = misc.get_weekly_release_id(configs)
128         release_id_tmp = "%s.NEMO.%s" % (options.releaseid_base,date.today())
129     elif options.releaseid:
130         options.releaseid_new = options.releaseid
131         return True
132     else:
133         # If id not set lets see if there is csv file we can use.
134         release_id_file = "%s/%s/releases.csv" % (image_config_dir,branch)
135         if not os.path.exists(release_id_file):
136             print "Release id file '%s' not available, skipping." % (release_id_file)
137             return False
138         
139         release_id_tmp = misc.parse_release_id(release_id_file)
140         
141         if release_id_tmp == None:
142             print "Could not resolve release id for branch '%s' from file %s." % (branch, release_id_file)
143             return False
144         
145     release_number = 1
146     """ Increase the release number if multiple releases are done during same day. """
147     while release_number < 100:
148         options.releaseid_new = "%s.%d" % (release_id_tmp,release_number)
149         # TODO: Here we should really check from upload URL if the release is actually made already.
150         if not os.path.exists("%s/%s" % (options.outputdir, options.releaseid_new)):
151             break
152         release_number += 1
153     
154     print "\nNo release id given, using %s\n" % (options.releaseid_new)
155     return True
156
157 def is_filtered(filters, values):
158     """ Extra filters """
159     if not filters:
160         return False
161         
162     skip = False
163     for filter_to_test in filters:
164         if filter_to_test.startswith("-"):
165             filter_to_test = filter_to_test.replace("-","",1)
166             if filter_to_test in values:
167                 skip = True
168                 break
169         elif filter_to_test not in values:
170             skip = True
171             break
172             
173     return skip
174
175 def parse_mic_cmdline(options,kickstart,siteconfig,cachedir,configs):
176     kstmp = kickstart.split("/")[-1]
177     # Remove trailing .ks
178     kstmp = kstmp.replace(".ks","")
179     ksvalues = kstmp.split("-")
180         
181     """ Architecture filter """
182     if options.architectures and ksvalues[1] not in options.architectures:
183         return True,None,None,None,None
184     
185     """ Vertical filter """
186     if options.verticals and ksvalues[0] not in options.verticals:
187         return True,None,None,None,None
188     
189     """ Extra filters """
190     if is_filtered(options.extra_filters, ksvalues):
191         return True,None,None,None,None
192
193     """ When device is not set use architecture as device indicator. """
194     if len(ksvalues) < 3:
195         ksvalues.append(ksvalues[1])
196
197     print "Using '%s' as kickstart with vertical %s and architecture %s for device %s.\n" % (kickstart,ksvalues[0],ksvalues[1],ksvalues[2])
198     
199     path = "%s/%s/images/" % (options.outputdir, options.releaseid_new)
200     
201     # Creating last directory name.
202     name_prefix = misc.get_config(configs,'name_prefix')
203     if not name_prefix:
204         name_prefix = DEFAULT_PREFIX
205     
206     common_part = name_prefix
207     
208     for ksvalue in ksvalues:
209         common_part += "-%s" % (ksvalue)
210     
211     path += common_part
212     
213     logfile = "%s/%s-%s-mic.log" % (path, common_part,options.releaseid_new)
214     
215     miccmdline = ""
216     
217     if options.mic2cmdline:    
218         miccmdline = [ "mic-image-creator", "--cache="+cachedir,
219                     "--config="+kickstart, "--release="+options.releaseid_new,
220                     "--outdir="+options.outputdir, "--logfile="+logfile,
221                     "--prefix="+name_prefix ]
222         if siteconfig:
223             miccmdline.append("--siteconf="+siteconfig)
224     else:
225         ks_f = open( kickstart, 'r' )
226         mic_options = ks_f.readline()
227         ks_f.close()
228         m = re.match(r".* -f (\w+)", mic_options)
229         mic_format = m.group(1)
230         m = re.match(r".* --arch=(\w+)", mic_options)
231         architecture = m.group(1)
232
233         miccmdline = [ "mic", "cr", mic_format, kickstart, 
234                     "--logfile="+logfile,
235                     "--cachedir="+cachedir,
236                     "--outdir="+options.outputdir, 
237                     "--arch="+architecture,
238                     "--release="+options.releaseid_new,
239                     "--record-pkgs=name",
240                     "--pkgmgr=yum"
241                     #"--prefix="+name_prefix 
242                     ]
243         if siteconfig:
244             miccmdline.append("--config="+siteconfig)
245         if mic_format == "fs":
246             miccmdline.append("--compress-disk-image=tar.bz2")
247         elif mic_format == "raw":
248             miccmdline.append("--compress-disk-image=bz2")
249         if "--save-kernel" in mic_options:
250             miccmdline.append("--save-kernel")
251
252     print "Commandline: %s\n" % (" ".join(miccmdline))
253     
254     return (False,path,logfile,ksvalues,miccmdline)
255
256 def upload_with_cadaver(options,path,releasefiles,upload_url,ksvalues):
257     cadaver_handle = open(CADAVER_SCRIPT_NAME,"w")
258     cadaver_handle.write("lcd %s\n" % (path))
259     
260     for releasefile in releasefiles:
261         if ".tar" in releasefile:
262             """ upload tar file """
263             uploaded_files.append(releasefile)
264             cadaver_handle.write("mput %s\n" % (releasefile))
265         elif "-mic.log" in releasefile:
266             """ upload mic log """
267             uploaded_files.append(releasefile)
268             cadaver_handle.write("mput %s\n" % (releasefile))
269         else:
270             print "File '%s' should not exist in release dir." % (releasefile)
271
272     cadaver_handle.write("quit\n")
273     cadaver_handle.close()
274     return_code = subprocess.call(["cadaver","-r","cadaver_script",upload_url])
275     # TODO: Cadaver returns 0 always so we can not base decision on that.
276     # TODO: if cadaver upload is not successfull how to handle the data?
277     
278     try:
279         os.remove(CADAVER_SCRIPT_NAME)
280     except:
281         pass
282
283 def dir_list(path,leaf_dir = None):
284     html = ""
285     if not os.path.exists(path):
286         return html
287
288     dir_entries = os.listdir(path)
289     dir_entries.sort()
290     files_in_dir = []
291     for dir_entry in dir_entries:
292         # No need to list index.html file.
293         if dir_entry == "index.html":
294             continue
295         full_path = "%s/%s" % (path, dir_entry)
296         # Recursion for directories
297         if os.path.isdir(full_path):
298             leaf_dir = full_path
299             html += dir_list(full_path,leaf_dir)
300             continue
301         files_in_dir.append(full_path)
302     
303     if leaf_dir and leaf_dir == path:
304         # Print the title for each leaf dir.
305         html += "<h3>%s</h3>\n" % (leaf_dir)
306     
307     if files_in_dir:
308         # Print files in the dir.
309         html += "<ul>\n"
310         for f in files_in_dir:
311             html += "<li><a href='%s'>%s</a></li>\n" % (f,os.path.basename(f))
312         html += "</ul>\n"
313     
314     return html
315
316 def run_rsync(source, destination, use_delete = False):
317     rsync_cmdline = ["rsync", "-aHxl", "--progress" ]
318     
319     if use_delete:
320         rsync_cmdline.append("--delete")
321     
322     rsync_cmdline.append("%s/" % (source))
323     rsync_cmdline.append("%s/" % (destination))
324     
325     print "Calling: %s" % (" ".join(rsync_cmdline))
326     
327     return_code = subprocess.call(rsync_cmdline)
328     
329     print "rsync from '%s' to '%s' returned with code '%s'." % ( source, destination, return_code )
330     return
331
332 def main():
333     configs = misc.read_configs(options.configfile)
334
335     if options.upload_webdav:
336         # TODO: Check that cadaver version is 0.23.3 or later.
337         misc.check_progs(["cadaver"])
338         upload_url_webdav = misc.get_config(configs,'webdav_upload_url')
339         if not upload_url_webdav:
340             print "No webdav_upload_url provided with --upload-to-webdav option." 
341
342     if options.upload_rsync:
343         misc.check_progs(["rsync"])
344         upload_url_rsync = misc.get_config(configs,'rsync_upload_url')
345         if not upload_url_rsync:
346             print "No rsync_upload_url provided with --upload-with-rsync option." 
347             sys.exit(1)
348         
349         """ Sync the remote url to our local disk first to make sure we have the
350             latest versions locally as well to build proper release ID's. 
351             So delete is used for a reason here. """
352         run_rsync(upload_url_rsync,options.outputdir,True)
353
354     if not configs:
355         print "Config file '%s' could not be read." % (options.configfile)
356         sys.exit(1)
357     
358     siteconfig = None
359     
360     if not options.outputdir:
361         print "Output dir can not be empty."
362         sys.exit(1)
363     
364     if options.architectures:
365         options.architectures = options.architectures.split(',')
366     
367     if options.verticals:
368         options.verticals = options.verticals.split(',')
369     
370     if options.extra_filters:
371         options.extra_filters = options.extra_filters.split(',')
372
373     if options.extra_dir_filters:
374         options.extra_dir_filters = options.extra_dir_filters.split(',')
375         
376     if options.branch:
377         options.branch = options.branch.split(',')
378         
379     if options.runmic:
380         if os.geteuid() != 0:
381             print "In order to run mic you need to have root privileges."
382             sys.exit(1)
383     
384     ks_search_dir = "image_configs/"
385     
386     if options.use_ks_from_git:
387         misc.check_progs(["git"])
388             
389         git_ks_url = misc.get_config(configs,'git_ks_url')
390         if not git_ks_url:
391             print "No git URL for .ks files given in conf."
392             sys.exit(1)
393
394         if os.path.isdir(GIT_KS_TMP):
395             shutil.rmtree(GIT_KS_TMP)
396             
397         print "Getting the image configurations from gitorious..."
398         retcode = subprocess.call(["git", "clone", git_ks_url, GIT_KS_TMP])
399         
400         if retcode:
401             print "Failed to retrieve image configuratios from gitorious."
402             sys.exit(1)
403                 
404         ks_search_dir = GIT_KS_TMP
405
406     elif options.use_ks_dir:
407         ks_search_dir = options.use_ks_dir
408
409     image_config_dirs = []
410
411     for root, dirnames, filenames in os.walk(ks_search_dir):
412         for filename in fnmatch.filter(filenames, '*.ks'):
413             image_config_dirs.append(root)
414
415     image_config_dirs = list(set(image_config_dirs))
416
417     image_config_dirs.sort()
418     #image_config_dirs.reverse()
419     
420     configs_global = configs
421     
422     for image_config_dir in image_config_dirs:
423         
424         """ Return the global configs for each dir. """
425         configs = configs_global
426         
427         print "Checking images for dir '%s'..." % (image_config_dir)
428         branch = image_config_dir.split("/")[1]
429         
430         # HACK: To skip one dir with git things.
431         if branch == "kickstarts":
432             branch = ""
433         
434         """ Extra filters """
435         if options.branch and branch not in options.branch:
436             print "Skipping branch '%s' because of filters." % (branch)
437             continue
438         
439         if is_filtered(options.extra_dir_filters,image_config_dir.split("/")):
440             print "Skipping because of dir filters %s." % (options.extra_dir_filters)
441             continue
442         
443         """ Check if we have options provided with .ks file. """
444         dir_options = "%s/options.conf" % (image_config_dir)
445         
446         if os.path.exists(dir_options):
447             print "Reading dir specific configs: %s" % (dir_options)
448             configs = misc.read_configs(dir_options,configs)
449         
450         # Use the base dir by default.
451         kspath = image_config_dir
452         
453         # If kickstarts subdir is found use that instead.
454         if os.path.exists(image_config_dir+"/kickstarts/"):
455             kspath += "/kickstarts/"
456         
457         # Config dirs may have their specific site configs use those if available.
458         if os.path.exists(image_config_dir+"/mic-site.conf"):
459             siteconfig = image_config_dir+"/mic-site.conf"
460         
461         if os.path.exists(kspath):
462             kickstarts = glob.glob("%s/*.ks" % (kspath))
463             kickstarts.sort()
464             kickstarts.reverse()
465         else:
466             print "The '%s' directory does not exist." % ( kspath )
467             sys.exit(1)
468             
469         if siteconfig and not os.path.exists(siteconfig):
470             print "Site config file '%s' does not exist." % (siteconfig)
471             sys.exit(1)
472         
473         if not calc_release_id(options,configs,branch,image_config_dir):
474             print "Skipping '%s' dir because release id could not be calculated." % (image_config_dir)
475             continue
476
477         cachedir="mic-cache/%s" % (branch)
478         
479         if options.clean_cache:
480             shutil.rmtree(cachedir,ignore_errors=True)
481         
482         replace_base_url = misc.get_config(configs,"replace_base_url")
483         if not os.path.exists(KS_TMP):
484             os.makedirs("%s/" % (KS_TMP))
485         
486         for kickstart in kickstarts:
487             """ If we have releaseid_base present then we want to replace some parts of the URLS """
488             if options.releaseid_base and replace_base_url and replace_base_url.lower() in TRUE_VALUES:
489                 ks_fd = open(kickstart, "r")
490                 ks_cont = ks_fd.read()
491                 ks_fd.close()
492                 
493                 ks_cont = ks_cont.replace("/latest/repos/","/%s/repos/" % (options.releaseid_base))
494                 new_ks_file = "%s/%s" % (KS_TMP,os.path.basename(kickstart))
495                 
496                 # There might be .ks file with same name in different directories so remove old first.
497                 if os.path.lexists(new_ks_file):
498                     os.unlink(new_ks_file)
499                 
500                 ks_fd = open(new_ks_file, "w")
501                 ks_fd.write(ks_cont)
502                 ks_fd.close()
503                 
504                 kickstart = new_ks_file
505
506             (skip,path,logfile,ksvalues,miccmdline) = parse_mic_cmdline(options,kickstart,siteconfig,cachedir,configs)
507             if skip:
508                 print "Skipping kickstart '%s'." % (kickstart)
509                 continue
510
511             (p_out, p_err, error_txt,creation_success) = (None, None, None, False)
512             
513             releasefiles = []
514             
515             if options.runmic:
516                 print "Creating image with mic. Please wait..."
517
518                 if not os.path.exists(path):
519                     os.makedirs(path)
520
521                 try:
522                     rc = subprocess.call(miccmdline)
523                     if rc == 0:
524                         creation_success = True
525                 except Exception, e:
526                     error_txt = "%s" % (e)
527                     print "Failed to create image: %s" % (error_txt)
528
529                 if error_txt != None:
530                     """ Lets add the error message to the end if there is one """
531                     log_file_handle = open(logfile,"a")
532                     log_file_handle.write("\n------------------ error_txt ------------------\n")
533                     log_file_handle.write(error_txt)
534                     log_file_handle.close()
535                     
536                 releasefiles = os.listdir(path)
537             
538             uploaded_files = []
539             
540             """ Upload files with the selected upload methods. """
541             if options.upload_webdav:
542                 upload_with_cadaver(options,path,releasefiles,upload_url_webdav,ksvalues)
543                 
544             if options.erase:
545                 """ Remove the release after creation to make sure we have enough space for next release.
546                 We do not remove the first dir because that indicates what versions has been build already. """
547                 shutil.rmtree("%s/%s" % (options.outputdir,options.releaseid_new,ksvalues[0]),ignore_errors=True)
548                 
549             if options.send_release_note:
550                 misc.send_release_note(kickstart,ksvalues[0],options.releaseid_new,miccmdline,uploaded_files,creation_success,configs)
551             
552             print "\nFinished kickstart '%s'.\n" % (kickstart)
553     
554     """ With rsync we can do everything at the very end. """
555     if options.upload_rsync:
556         
557         """ Remove the latest link first so it is not included to the other checks. """
558         latest_link = "%s/latest" % (options.outputdir)
559         if os.path.lexists(latest_link):
560             print "Removing old link %s" % (latest_link)
561             os.remove(latest_link)
562         
563         source_dir_list = os.listdir(options.outputdir)
564         """ Sort the list so that latest are first..."""
565         source_dir_list.sort()
566         source_dir_list.reverse()
567         
568         latest_release = None
569         
570         """ Lets remove the old releases to keep the space usage in control. """
571         i = 0
572         for entry in source_dir_list:
573             i += 1
574             """ Store the latest release for later use. """
575             if latest_release == None:
576                 latest_release = entry
577             
578             if i <= options.releases_to_keep:
579                 print "%s <= %s" % (i, options.releases_to_keep)
580                 continue
581             release_dir = "%s/%s" % (options.outputdir, entry)
582             print "Removing old release dir: '%s'" % (release_dir)
583             shutil.rmtree(release_dir)
584         
585         """ If wanted add the latest link to the rsync dir. """
586         add_latest_link = misc.get_config(configs,"add_latest_link")
587         if add_latest_link and add_latest_link.lower() in TRUE_VALUES:
588             print "Creating new link from %s to %s." % (latest_release, latest_link)
589             os.symlink(latest_release,latest_link)
590         
591         run_rsync(options.outputdir,upload_url_rsync,options.remove_old)
592
593     # Remove old dirs to help running test runs as non root user.
594     if os.path.isdir(GIT_KS_TMP):
595         shutil.rmtree(GIT_KS_TMP)
596     if os.path.isdir(KS_TMP):
597         shutil.rmtree(KS_TMP)
598
599     print "\nImage creation process completed.\n"
600
601
602 if __name__ == "__main__":
603     main()
604