Put market version info into index, and warn if we don't have the market version...
[f-droid:fdroidserver.git] / update.py
1 # -*- coding: UTF-8 -*-
2 #
3 # update.py - part of the FDroid server tools
4 # Copyright (C) 2010, Ciaran Gultnieks, ciaran@ciarang.com
5 #
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
10 #
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 # GNU Affero General Public License for more details.
15 #
16 # You should have received a copy of the GNU Affero General Public License
17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
18
19 import sys
20 import os
21 import shutil
22 import glob
23 import subprocess
24 import re
25 import zipfile
26 import md5
27 from xml.dom.minidom import Document
28 from optparse import OptionParser
29
30 #Read configuration...
31 execfile('config.py')
32
33 # Parse command line...
34 parser = OptionParser()
35 parser.add_option("-c", "--createmeta", action="store_true", default=False,
36                   help="Create skeleton metadata files that are missing")
37 parser.add_option("-v", "--verbose", action="store_true", default=False,
38                   help="Spew out even more information than normal")
39 (options, args) = parser.parse_args()
40
41
42 icon_dir=os.path.join('repo','icons')
43
44 # Delete and re-create the icon directory...
45 if os.path.exists(icon_dir):
46     shutil.rmtree(icon_dir)
47 os.mkdir(icon_dir)
48
49 # Gather information about all the apk files in the repo directory...
50 apks = []
51 for apkfile in glob.glob(os.path.join('repo','*.apk')):
52
53     apkfilename = apkfile[5:]
54
55     print "Processing " + apkfilename
56     thisinfo = {}
57     thisinfo['apkname'] = apkfilename
58     thisinfo['size'] = os.path.getsize(apkfile)
59     thisinfo['permissions'] = []
60     thisinfo['features'] = []
61     p = subprocess.Popen([aapt_path,'dump','badging',
62        apkfile], stdout=subprocess.PIPE)
63     output = p.communicate()[0]
64     if options.verbose:
65         print output
66     if p.returncode != 0:
67         print "ERROR: Failed to get apk information"
68         sys.exit(1)
69     for line in output.splitlines():
70         if line.startswith("package:"):
71             pat = re.compile(".*name='([a-z0-9.]*)'.*")
72             thisinfo['id'] = re.match(pat, line).group(1)
73             pat = re.compile(".*versionCode='([0-9]*)'.*")
74             thisinfo['versioncode'] = re.match(pat, line).group(1)
75             pat = re.compile(".*versionName='([^']*)'.*")
76             thisinfo['version'] = re.match(pat, line).group(1)
77         if line.startswith("application:"):
78             pat = re.compile(".*label='([^']*)'.*")
79             thisinfo['name'] = re.match(pat, line).group(1)
80             pat = re.compile(".*icon='([^']*)'.*")
81             thisinfo['iconsrc'] = re.match(pat, line).group(1)
82         if line.startswith("sdkVersion:"):
83             pat = re.compile(".*'([0-9]*)'.*")
84             thisinfo['sdkversion'] = re.match(pat, line).group(1)
85         if line.startswith("native-code:"):
86             pat = re.compile(".*'([^']*)'.*")
87             thisinfo['nativecode'] = re.match(pat, line).group(1)
88         if line.startswith("uses-permission:"):
89             pat = re.compile(".*'([^']*)'.*")
90             perm = re.match(pat, line).group(1)
91             if perm.startswith("android.permission."):
92                 perm = perm[19:]
93             thisinfo['permissions'].append(perm)
94         if line.startswith("uses-feature:"):
95             pat = re.compile(".*'([^']*)'.*")
96             perm = re.match(pat, line).group(1)
97             if perm.startswith("android.feature."):
98                 perm = perm[16:]
99             thisinfo['features'].append(perm)
100
101     if not thisinfo.has_key('sdkversion'):
102         print "  WARNING: no SDK version information found"
103         thisinfo['sdkversion'] = 0
104
105     # Calculate the md5...
106     m = md5.new()
107     f = open(apkfile, 'rb')
108     while True:
109         t = f.read(1024)
110         if len(t) == 0:
111             break
112         m.update(t)
113     thisinfo['md5'] = m.hexdigest()
114     f.close()
115
116     # Extract the icon file...
117     apk = zipfile.ZipFile(apkfile, 'r')
118     thisinfo['icon'] = (thisinfo['id'] + '.' +
119         thisinfo['versioncode'] + '.png')
120     iconfilename = os.path.join(icon_dir, thisinfo['icon'])
121     iconfile = open(iconfilename, 'wb')
122     iconfile.write(apk.read(thisinfo['iconsrc']))
123     iconfile.close()
124     apk.close()
125
126     apks.append(thisinfo)
127
128 # Get all apps...
129 apps = []
130
131 for metafile in glob.glob(os.path.join('metadata','*.txt')):
132
133     thisinfo = {}
134
135     # Get metadata...
136     thisinfo['id'] = metafile[9:-4]
137     print "Reading metadata for " + thisinfo['id']
138     thisinfo['description'] = ''
139     thisinfo['summary'] = ''
140     thisinfo['license'] = 'Unknown'
141     thisinfo['web'] = ''
142     thisinfo['source'] = ''
143     thisinfo['tracker'] = ''
144     thisinfo['disabled'] = None
145     thisinfo['marketversion'] = ''
146     thisinfo['marketvercode'] = '0'
147     f = open(metafile, 'r')
148     mode = 0
149     for line in f.readlines():
150         line = line.rstrip('\r\n')
151         if len(line) == 0:
152             pass
153         elif mode == 0:
154             index = line.find(':')
155             if index == -1:
156                 print "Invalid metadata in " + metafile + " at:" + line
157                 sys.exit(1)
158             field = line[:index]
159             value = line[index+1:]
160             if field == 'Description':
161                 mode = 1
162             elif field == 'Summary':
163                 thisinfo['summary'] = value
164             elif field == 'Source Code':
165                 thisinfo['source'] = value
166             elif field == 'License':
167                 thisinfo['license'] = value
168             elif field == 'Web Site':
169                 thisinfo['web'] = value
170             elif field == 'Issue Tracker':
171                 thisinfo['tracker'] = value
172             elif field == 'Disabled':
173                 thisinfo['disabled'] = value
174             elif field == 'Market Version':
175                 thisinfo['marketversion'] = value
176             elif field == 'Market Version Code':
177                 thisinfo['marketvercode'] = value
178             else:
179                 print "Unrecognised field " + field
180                 sys.exit(1)
181         elif mode == 1:
182             if line == '.':
183                 mode = 0
184             else:
185                 if len(line) == 0:
186                     thisinfo['description'] += '\n\n'
187                 else:
188                     if (not thisinfo['description'].endswith('\n') and
189                         len(thisinfo['description']) > 0):
190                         thisinfo['description'] += ' '
191                     thisinfo['description'] += line
192     if len(thisinfo['description']) == 0:
193         thisinfo['description'] = 'No description available'
194
195     apps.append(thisinfo)
196
197 # Some information from the apks needs to be applied up to the application
198 # level. When doing this, we use the info from the most recent version's apk.
199 for app in apps:
200     bestver = 0 
201     for apk in apks:
202         if apk['id'] == app['id']:
203             if apk['versioncode'] > bestver:
204                 bestver = apk['versioncode']
205                 bestapk = apk
206
207     if bestver == 0:
208         app['name'] = app['id']
209         app['icon'] = ''
210         print "WARNING: Application " + app['id'] + " has no packages"
211     else:
212         app['name'] = bestapk['name']
213         app['icon'] = bestapk['icon']
214
215 # Generate warnings for apk's with no metadata (or create skeleton
216 # metadata files, if requested on the command line)
217 for apk in apks:
218     found = False
219     for app in apps:
220         if app['id'] == apk['id']:
221             found = True
222             break
223     if not found:
224         if options.createmeta:
225             f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
226             f.write("License:Unknown\n")
227             f.write("Web Site:\n")
228             f.write("Source Code:\n")
229             f.write("Issue Tracker:\n")
230             f.write("Summary:" + apk['name'] + "\n")
231             f.write("Description:\n")
232             f.write(apk['name'] + "\n")
233             f.write(".\n")
234             f.close()
235             print "Generated skeleton metadata for " + apk['id']
236         else:
237             print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
238             print "       " + apk['name'] + " - " + apk['version']  
239
240 # Create the index
241 doc = Document()
242
243 def addElement(name, value, doc, parent):
244     el = doc.createElement(name)
245     el.appendChild(doc.createTextNode(value))
246     parent.appendChild(el)
247
248 root = doc.createElement("fdroid")
249 doc.appendChild(root)
250
251 apps_inrepo = 0
252 apps_disabled = 0
253
254 for app in apps:
255
256     if app['disabled'] is None:
257         apps_inrepo += 1
258         apel = doc.createElement("application")
259         root.appendChild(apel)
260
261         addElement('id', app['id'], doc, apel)
262         addElement('name', app['name'], doc, apel)
263         addElement('summary', app['summary'], doc, apel)
264         addElement('icon', app['icon'], doc, apel)
265         addElement('description', app['description'], doc, apel)
266         addElement('license', app['license'], doc, apel)
267         addElement('web', app['web'], doc, apel)
268         addElement('source', app['source'], doc, apel)
269         addElement('tracker', app['tracker'], doc, apel)
270         addElement('marketversion', app['marketversion'], doc, apel)
271         addElement('marketvercode', app['marketvercode'], doc, apel)
272
273         gotmarketver = False
274
275         for apk in apks:
276             if apk['id'] == app['id']:
277                 if apk['versioncode'] == app['marketvercode']:
278                     gotmarketver = True
279                 apkel = doc.createElement("package")
280                 apel.appendChild(apkel)
281                 addElement('version', apk['version'], doc, apkel)
282                 addElement('versioncode', apk['versioncode'], doc, apkel)
283                 addElement('apkname', apk['apkname'], doc, apkel)
284                 addElement('hash', apk['md5'], doc, apkel)
285                 addElement('size', str(apk['size']), doc, apkel)
286                 addElement('sdkver', str(apk['sdkversion']), doc, apkel)
287                 perms = ""
288                 for p in apk['permissions']:
289                     if len(perms) > 0:
290                         perms += ","
291                     perms += p
292                 if len(perms) > 0:
293                     addElement('permissions', perms, doc, apkel)
294                 features = ""
295                 for f in apk['features']:
296                     if len(features) > 0:
297                         features += ","
298                     features += f
299                 if len(features) > 0:
300                     addElement('features', features, doc, apkel)
301
302         if not gotmarketver and app['marketvercode'] != '0':
303             print "WARNING: Don't have market version (" + app['marketversion'] + ") of " + app['name']
304
305     else:
306         apps_disabled += 1
307
308 of = open(os.path.join('repo','index.xml'), 'wb')
309 output = doc.toxml()
310 of.write(output)
311 of.close()
312
313 print "Finished."
314 print str(apps_inrepo) + " apps in repo"
315 print str(apps_disabled) + " disabled"
316