Cleanup some tmp dirs.
[image-building-tools:daily-image-builder.git] / builder.py
1 #!/usr/bin/python -t
2
3 #
4 # Daily Image Builder
5 # Copyright (C) 2010 Nokia Corporation.
6 #
7 # Contact: Marko Saukko <marko.saukko@cybercom.com>
8 #
9 # This program is free software; you can redistribute it and/or
10 # modify it under the terms of the GNU General Public License
11 # as published by the Free Software Foundation; either version 2
12 # of the License, or (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program; if not, write to the Free Software
21 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
22 #
23
24 import sys
25 import os
26 import subprocess
27 import shutil
28 import glob
29 import re
30 import time
31 import fnmatch
32 from datetime import date
33
34 from optparse import OptionParser
35
36 import modules.misc as misc
37
38 CADAVER_SCRIPT_NAME = "cadaver_script"
39
40 parser = OptionParser()
41 parser.add_option("","--use-ks-from-git",dest="use_ks_from_git", action="store_true",
42                   help="Use .ks files from gitorious")
43 parser.add_option("","--use-ks-dir", dest="use_ks_dir", default=None,
44                   help="Use ks files from specific dir.")
45 parser.add_option("-r","--release-id",dest="releaseid", default=None,
46                   help="Release ID for the image. If not entered the id will "
47                   "be retrieved based on url in config file.")
48 parser.add_option("","--run-mic",dest="runmic", action="store_true",
49                   default=False,
50                   help="Execute mic-image-creator for each found .ks.")
51 parser.add_option("","--architectures",dest="architectures", default=None,
52                   help="The architectures that are build. If not set then all "
53                   "found architectures are build. This is the same value that "
54                   "is set in the kickstart filename. Must be a comma ',' "
55                   "separated list.")
56 parser.add_option("","--verticals",dest="verticals", default=None,
57                   help="The vertical that are build. If not set then all "
58                   "found verticals are build. This is the same value that "
59                   "is set in the kickstart filename. Must be a comma ',' "
60                   "separated list.")
61 parser.add_option("","--extra-filters",dest="extra_filters", default=None,
62                   help="The extra filters for the kickstart filtering. "
63                   "With this you can define extra filters to be used with the"
64                   "kickstart filename. Must be a comma ',' "
65                   "separated list.")
66 parser.add_option("","--branch",dest="branch",
67                   default=None,
68                   help="Branch filter for branches in image-config dir."
69                   "Must be a comma ',' separated list.")
70 parser.add_option("","--upload-to-webdav",dest="upload_webdav",action="store_true", 
71                   default=False,
72                   help="Upload packages with the cadaver to webdav addresss "
73                   "provided by config.")
74 parser.add_option("","--upload-with-rsync",dest="upload_rsync",action="store_true",
75                   default=False,
76                   help="Upload packages with rsync to rsync address provided by config.")
77 parser.add_option("","--send-release-note",dest="send_release_note", 
78                   action="store_true", default=False,
79                   help="Send release notes to the email given in conf.")
80 parser.add_option("","--configfile",dest="configfile",
81                   default="builder.conf",
82                   help="Config file where the configs are read.")
83 parser.add_option("","--erase-local-images",dest="erase", 
84                   action="store_true",default=False,
85                   help="Erase the image after uploading.")
86 parser.add_option("","--outputdir",dest="outputdir",
87                   default="meego-images",
88                   help="Directory where mic outputs are stored.")
89 parser.add_option("","--clean-cache",dest="clean_cache", 
90                   action="store_true",default=False,
91                   help="Erase the mic cache before uploading.")
92 parser.add_option("","--remove-old", dest="remove_old",
93                   action="store_true",default=False,
94                   help="Remove old releases with rsync.")
95 parser.add_option("","--releases-to-keep", dest="releases_to_keep",
96                   type="int", default=7,
97                   help="How many old releases are kept when removing old releases.")
98
99 (options,args) = parser.parse_args()
100
101 DEFAULT_PREFIX="mg"
102
103 TRUE_VALUES = ["true","yes","1"]
104 KS_TMP = "ks-tmp/"
105 GIT_KS_TMP = "git-ks-tmp/"
106
107 def calc_release_id(options,configs,branch,image_config_dir):
108     release_id_tmp = None
109     if options.releaseid and options.releaseid in ("CE"):
110         # For community edition we have specific release ID's.
111         options.releaseid_base = misc.get_weekly_release_id(configs)
112         release_id_tmp = "%s.CE.%s" % (options.releaseid_base,date.today())
113     elif options.releaseid:
114         options.releaseid_new = options.releaseid
115         return True
116     else:
117         # If id not set lets see if there is csv file we can use.
118         release_id_file = "%s/%s/releases.csv" % (image_config_dir,branch)
119         if not os.path.exists(release_id_file):
120             print "Release id file '%s' not available, skipping." % (release_id_file)
121             return False
122         
123         release_id_tmp = misc.parse_release_id(release_id_file)
124         
125         if release_id_tmp == None:
126             print "Could not resolve release id for branch '%s' from file %s." % (branch, release_id_file)
127             return False
128         
129     release_number = 1
130     """ Increase the release number if multiple releases are done during same day. """
131     while release_number < 100:
132         options.releaseid_new = "%s.%d" % (release_id_tmp,release_number)
133         # TODO: Here we should really check from upload URL if the release is actually made already.
134         if not os.path.exists("%s/%s" % (options.outputdir, options.releaseid_new)):
135             break
136         release_number += 1
137     
138     print "\nNo release id given, using %s\n" % (options.releaseid_new)
139     return True
140
141
142 def parse_mic_cmdline(options,kickstart,siteconfig,cachedir):
143     kstmp = kickstart.split("/")[-1]
144     # Remove trailing .ks
145     kstmp = kstmp.replace(".ks","")
146     ksvalues = kstmp.split("-")
147     
148     """ Architecture filter """
149     if options.architectures and ksvalues[1] not in options.architectures:
150         return True,None,None,None,None
151     
152     """ Vertical filter """
153     if options.verticals and ksvalues[0] not in options.verticals:
154         return True,None,None,None,None
155     
156     """ Extra filters """
157     if options.extra_filters:
158         filter_match = True
159         for extra_filter in options.extra_filters:
160             if extra_filter.startswith("-"):
161                 extra_filter = extra_filter.replace("-","",1)
162                 if extra_filter in ksvalues:
163                     filter_match = False
164                     break
165             elif extra_filter not in ksvalues:
166                 filter_match = False
167                 break
168         if not filter_match:
169             return True,None,None,None,None
170     
171     print "Using '%s' as kickstart with vertical %s and architecture %s for device %s.\n" % (kickstart,ksvalues[0],ksvalues[1],ksvalues[2])
172     
173     path = "%s/%s/images/" % (options.outputdir, options.releaseid_new)
174     
175     # Creating last directory name.
176     common_part = DEFAULT_PREFIX
177     
178     for ksvalue in ksvalues:
179         common_part += "-%s" % (ksvalue)
180     
181     path += common_part
182     
183     logfile = "%s/%s-%s-mic.log" % (path, common_part,options.releaseid_new)
184     
185     miccmdline = [ "mic-image-creator", "--cache="+cachedir,
186                 "--config="+kickstart, "--release="+options.releaseid_new,
187                 "--outdir="+options.outputdir, "--logfile="+logfile,
188                 "--prefix="+DEFAULT_PREFIX ]
189     
190     if siteconfig:
191         miccmdline.append("--siteconf="+siteconfig)
192
193     print "Commandline: %s\n" % (" ".join(miccmdline))
194     
195     return (False,path,logfile,ksvalues,miccmdline)
196
197 def upload_with_cadaver(options,path,releasefiles,upload_url,ksvalues):
198     cadaver_handle = open(CADAVER_SCRIPT_NAME,"w")
199     cadaver_handle.write("lcd %s\n" % (path))
200     
201     for releasefile in releasefiles:
202         if ".tar" in releasefile:
203             """ upload tar file """
204             uploaded_files.append(releasefile)
205             cadaver_handle.write("mput %s\n" % (releasefile))
206         elif "-mic.log" in releasefile:
207             """ upload mic log """
208             uploaded_files.append(releasefile)
209             cadaver_handle.write("mput %s\n" % (releasefile))
210         else:
211             print "File '%s' should not exist in release dir." % (releasefile)
212
213     cadaver_handle.write("quit\n")
214     cadaver_handle.close()
215     return_code = subprocess.call(["cadaver","-r","cadaver_script",upload_url])
216     # TODO: Cadaver returns 0 always so we can not base decision on that.
217     # TODO: if cadaver upload is not successfull how to handle the data?
218     
219     try:
220         os.remove(CADAVER_SCRIPT_NAME)
221     except:
222         pass
223
224 def dir_list(path,leaf_dir = None):
225     html = ""
226     if not os.path.exists(path):
227         return html
228
229     dir_entries = os.listdir(path)
230     dir_entries.sort()
231     files_in_dir = []
232     for dir_entry in dir_entries:
233         # No need to list index.html file.
234         if dir_entry == "index.html":
235             continue
236         full_path = "%s/%s" % (path, dir_entry)
237         # Recursion for directories
238         if os.path.isdir(full_path):
239             leaf_dir = full_path
240             html += dir_list(full_path,leaf_dir)
241             continue
242         files_in_dir.append(full_path)
243     
244     if leaf_dir and leaf_dir == path:
245         # Print the title for each leaf dir.
246         html += "<h3>%s</h3>\n" % (leaf_dir)
247     
248     if files_in_dir:
249         # Print files in the dir.
250         html += "<ul>\n"
251         for f in files_in_dir:
252             html += "<li><a href='%s'>%s</a></li>\n" % (f,os.path.basename(f))
253         html += "</ul>\n"
254     
255     return html
256
257 def run_rsync(source, destination, use_delete = False):
258     rsync_cmdline = ["rsync", "-aHxl", "--progress" ]
259     
260     if use_delete:
261         rsync_cmdline.append("--delete")
262     
263     rsync_cmdline.append("%s/" % (source))
264     rsync_cmdline.append("%s/" % (destination))
265     
266     print "Calling: %s" % (" ".join(rsync_cmdline))
267     
268     return_code = subprocess.call(rsync_cmdline)
269     
270     print "rsync from '%s' to '%s' returned with code '%s'." % ( source, destination, return_code )
271     return
272
273 def main():
274     misc.check_progs(["mic-image-creator"])
275
276     configs = misc.read_configs(options.configfile)
277
278     if options.upload_webdav:
279         # TODO: Check that cadaver version is 0.23.3 or later.
280         misc.check_progs(["cadaver"])
281         upload_url_webdav = misc.get_config(configs,'webdav_upload_url')
282         if not upload_url_webdav:
283             print "No webdav_upload_url provided with --upload-to-webdav option." 
284
285     if options.upload_rsync:
286         misc.check_progs(["rsync"])
287         upload_url_rsync = misc.get_config(configs,'rsync_upload_url')
288         if not upload_url_rsync:
289             print "No rsync_upload_url provided with --upload-with-rsync option." 
290             sys.exit(1)
291         
292         """ Sync the remote url to our local disk first to make sure we have the
293             latest versions locally as well to build proper release ID's. 
294             So delete is used for a reason here. """
295         run_rsync(upload_url_rsync,options.outputdir,True)
296
297     if not configs:
298         print "Config file '%s' could not be read." % (options.configfile)
299         sys.exit(1)
300     
301     siteconfig = None
302     
303     if not options.outputdir:
304         print "Output dir can not be empty."
305         sys.exit(1)
306     
307     if options.architectures:
308         options.architectures = options.architectures.split(',')
309     
310     if options.verticals:
311         options.verticals = options.verticals.split(',')
312     
313     if options.extra_filters:
314         options.extra_filters = options.extra_filters.split(',')
315         
316     if options.branch:
317         options.branch = options.branch.split(',')
318         
319     if options.runmic:
320         if os.geteuid() != 0:
321             print "In order to run mic you need to have root privileges."
322             sys.exit(1)
323     
324     ks_search_dir = "image_configs/"
325     
326     if options.use_ks_from_git:
327         misc.check_progs(["git"])
328             
329         git_ks_url = misc.get_config(configs,'git_ks_url')
330         if not git_ks_url:
331             print "No git URL for .ks files given in conf."
332             sys.exit(1)
333
334         if os.path.isdir(GIT_KS_TMP):
335             shutil.rmtree(GIT_KS_TMP)
336             
337         print "Getting the image configurations from gitorious..."
338         retcode = subprocess.call(["git", "clone", git_ks_url, GIT_KS_TMP])
339         
340         if retcode:
341             print "Failed to retrieve image configuratios from gitorious."
342             sys.exit(1)
343                 
344         ks_search_dir = GIT_KS_TMP
345
346     elif options.use_ks_dir:
347         ks_search_dir = options.use_ks_dir
348
349     image_config_dirs = []
350
351     for root, dirnames, filenames in os.walk(ks_search_dir):
352         for filename in fnmatch.filter(filenames, '*.ks'):
353             image_config_dirs.append(root)
354
355     image_config_dirs = list(set(image_config_dirs))
356
357     image_config_dirs.sort()
358     #image_config_dirs.reverse()
359     
360     configs_global = configs
361     
362     for image_config_dir in image_config_dirs:
363         
364         """ Return the global configs for each dir. """
365         configs = configs_global
366         
367         print "Checking images for dir '%s'..." % (image_config_dir)
368         branch = image_config_dir.split("/")[1]
369         
370         # HACK: To skip one dir with git things.
371         if branch == "kickstarts":
372             branch = ""
373         
374         """ Extra filters """
375         if options.branch and branch not in options.branch:
376             print "Skipping branch '%s' because of filters." % (branch)
377             continue
378         
379         """ Check if we have options provided with .ks file. """
380         dir_options = "%s/options.conf" % (image_config_dir)
381         
382         if os.path.exists(dir_options):
383             print "Reading dir specific configs: %s" % (dir_options)
384             configs = misc.read_configs(dir_options,configs)
385         
386         # Use the base dir by default.
387         kspath = image_config_dir
388         
389         # If kickstarts subdir is found use that instead.
390         if os.path.exists(image_config_dir+"/kickstarts/"):
391             kspath += "/kickstarts/"
392         
393         # Config dirs may have their specific site configs use those if available.
394         if os.path.exists(image_config_dir+"/mic-site.conf"):
395             siteconfig = image_config_dir+"/mic-site.conf"
396         
397         if os.path.exists(kspath):
398             kickstarts = glob.glob("%s/*.ks" % (kspath))
399             kickstarts.sort()
400             kickstarts.reverse()
401         else:
402             print "The '%s' directory does not exist." % ( kspath )
403             sys.exit(1)
404             
405         if siteconfig and not os.path.exists(siteconfig):
406             print "Site config file '%s' does not exist." % (siteconfig)
407             sys.exit(1)
408         
409         if not calc_release_id(options,configs,branch,image_config_dir):
410             print "Skipping '%s' dir because release id could not be calculated." % (image_config_dir)
411             continue
412
413         cachedir="mic-cache/%s" % (branch)
414         
415         if options.clean_cache:
416             shutil.rmtree(cachedir,ignore_errors=True)
417         
418         replace_base_url = misc.get_config(configs,"replace_base_url")
419         if not os.path.exists(KS_TMP):
420             os.makedirs("%s/" % (KS_TMP))
421         
422         for kickstart in kickstarts:
423             """ If we have releaseid_base present then we want to replace some parts of the URLS """
424             if options.releaseid_base and replace_base_url and replace_base_url.lower() in TRUE_VALUES:
425                 ks_fd = open(kickstart, "r")
426                 ks_cont = ks_fd.read()
427                 ks_fd.close()
428                 
429                 ks_cont = ks_cont.replace("/latest/repos/","/%s/repos/" % (options.releaseid_base))
430                 new_ks_file = "%s/%s" % (KS_TMP,os.path.basename(kickstart))
431                 
432                 if os.path.lexists(new_ks_file):
433                     os.unlink(new_ks_file)
434                 
435                 ks_fd = open(new_ks_file, "w")
436                 ks_fd.write(ks_cont)
437                 ks_fd.close()
438                 
439                 kickstart = new_ks_file
440
441             (skip,path,logfile,ksvalues,miccmdline) = parse_mic_cmdline(options,kickstart,siteconfig,cachedir)
442             if skip:
443                 print "Skipping kickstart '%s'." % (kickstart)
444                 continue
445
446             (p_out, p_err, error_txt,creation_success) = (None, None, None, False)
447             
448             releasefiles = []
449             
450             if options.runmic:
451                 print "Creating image with mic. Please wait..."
452
453                 if not os.path.exists(path):
454                     os.makedirs(path)
455
456                 try:
457                     rc = subprocess.call(miccmdline)
458                     if rc == 0:
459                         creation_success = True
460                 except Exception, e:
461                     error_txt = "%s" % (e)
462                     print "Failed to create image: %s" % (error_txt)
463
464                 if error_txt != None:
465                     """ Lets add the error message to the end if there is one """
466                     log_file_handle = open(logfile,"a")
467                     log_file_handle.write("\n------------------ error_txt ------------------\n")
468                     log_file_handle.write(error_txt)
469                     log_file_handle.close()
470                     
471                 releasefiles = os.listdir(path)
472             
473             uploaded_files = []
474             
475             """ Upload files with the selected upload methods. """
476             if options.upload_webdav:
477                 upload_with_cadaver(options,path,releasefiles,upload_url_webdav,ksvalues)
478                 
479             if options.erase:
480                 """ Remove the release after creation to make sure we have enough space for next release.
481                 We do not remove the first dir because that indicates what versions has been build already. """
482                 shutil.rmtree("%s/%s" % (options.outputdir,options.releaseid_new,ksvalues[0]),ignore_errors=True)
483                 
484             if options.send_release_note:
485                 misc.send_release_note(kickstart,ksvalues[0],options.releaseid_new,miccmdline,uploaded_files,creation_success,configs)
486             
487             print "\nFinished kickstart '%s'.\n" % (kickstart)
488     
489     """ With rsync we can do everything at the very end. """
490     if options.upload_rsync:
491         
492         """ Remove the latest link first so it is not included to the other checks. """
493         latest_link = "%s/latest" % (options.outputdir)
494         if os.path.lexists(latest_link):
495             print "Removing old link %s" % (latest_link)
496             os.remove(latest_link)
497         
498         source_dir_list = os.listdir(options.outputdir)
499         """ Sort the list so that latest are first..."""
500         source_dir_list.sort()
501         source_dir_list.reverse()
502         
503         latest_release = None
504         
505         """ Lets remove the old releases to keep the space usage in control. """
506         i = 0
507         for entry in source_dir_list:
508             i += 1
509             """ Store the latest release for later use. """
510             if latest_release == None:
511                 latest_release = entry
512             
513             if i <= options.releases_to_keep:
514                 print "%s <= %s" % (i, options.releases_to_keep)
515                 continue
516             release_dir = "%s/%s" % (options.outputdir, entry)
517             print "Removing old release dir: '%s'" % (release_dir)
518             shutil.rmtree(release_dir)
519         
520         """ If wanted add the latest link to the rsync dir. """
521         add_latest_link = misc.get_config(configs,"add_latest_link")
522         if add_latest_link and add_latest_link.lower() in TRUE_VALUES:
523             print "Creating new link from %s to %s." % (latest_release, latest_link)
524             os.symlink(latest_release,latest_link)
525         
526         run_rsync(options.outputdir,upload_url_rsync,options.remove_old)
527
528     # Remove old dirs to help running test runs as non root user.
529     if os.path.isdir(GIT_KS_TMP):
530         shutil.rmtree(GIT_KS_TMP)
531     if os.path.isdir(KS_TMP):
532         shutil.rmtree(KS_TMP)
533
534     print "\nImage creation process completed.\n"
535
536
537 if __name__ == "__main__":
538     main()
539