[mirror] Add script to update the http header
[qtwebkit:performance.git] / host-tools / mirror / update_content_length.py
1 #!/usr/bin/env python
2
3 import sqlite3, sys, optparse
4
5 def parse():
6     parser = optparse.OptionParser(version = "Update the HTTP headers",
7                                    usage = "%prog [options] URL")
8     parser.add_option("-d", "--db", help = "The crawl database to be used",
9                       action = "store", dest = "db_name", default="crawl_db.db")
10     return parser.parse_args(sys.argv)
11
12 (opts, _) = parse()
13 connection = sqlite3.connect(opts.db_name)
14
15
16 def to_num(length):
17     if not length:
18         return 0
19     return int(length)
20
21 length = len("content-length: ")
22 cur = connection.execute("SELECT url, header, LENGTH(data) from responses")
23 for row in cur:
24     old_header = str(row[1])
25     start = old_header.find("content-length: ")
26     end = old_header.find("\r", start)
27
28     old_length = old_header[start + length:end]
29     new_length = to_num(row[2])
30
31     if int(old_length) == new_length:
32         continue
33
34     new_entry = "content-length: %d" % new_length
35     new_header = old_header.replace(old_header[start:end], new_entry, 1)
36     connection.execute("UPDATE responses SET header = ? WHERE url like ?", [new_header, row[0]])
37 connection.commit()