[mirror] Add utility to write content + url to disk
[qtwebkit:performance.git] / host-tools / mirror / store_all.py
1 #!/usr/bin/env python
2
3 import sqlite3, sys, optparse
4
5 def parse():
6     parser = optparse.OptionParser(version = "Store all data to disk",
7                                    usage = "%prog [options]")
8     parser.add_option("-d", "--db", help = "The crawl database to be used",
9                       action = "store", dest = "db_name", default="crawl_db.db")
10     parser.add_option("-o", "--output", help = "Basename of the files..",
11                       action = "store", dest = "output", default="dump-")
12     return parser.parse_args(sys.argv)
13
14 (opts, args) = parse()
15 connection = sqlite3.connect(opts.db_name)
16
17 i = 0
18 cursor = connection.execute("SELECT url, data FROM responses")
19 for row in cursor:
20     out = open("%s-%.12d" % (opts.output, i), "w")
21     out.write(row[0])
22     out.write("\n")
23     if row[1]:
24         out.write(row[1])
25     i = i + 1