Use multi-argument system()
[opensuse:expand-kernel-source.git] / expand-kernel-source.pl
1 #!/usr/bin/perl
2
3 use strict;
4 use warnings;
5 no warnings 'recursion';
6
7 my $USAGE = "Usage: $0 --mainline <git> --suse <git> [--incremental] [[from]..[to:]]ref...\n";
8
9 BEGIN {
10     if ($0 =~ /^(.*)\/[^\/]*/) {
11         push @INC, "$1/lib";
12     } else {
13         push @INC,  "./lib";
14     }
15 }
16
17 use Getopt::Long;
18 use SUSE::Kernel::Git;
19
20 my ($mainline_path, $suse_path, $force, $incremental, $expand_all, $jobs);
21 my ($mainline_git, $suse_git);
22
23 # ({ id, fullname, type }, ...)
24 my @refs;
25
26 # commit => 1
27 my %exclude_commits;
28
29 # suse rev => { author, tree, ... expanded => { tree, version, broken} }
30 my %revs;
31
32 # suse tree => expanded tree
33 my %expanded_trees;
34
35 # suse commit => expanded commit
36 my $expanded_commits = GitTable->new("expanded-commits");
37
38 # mainline version => mainline commit
39 my $mainline_commits = GitTable->new("mainline-commits");
40
41 # generic way of storing hashes in named blobs in git
42 package GitTable;
43
44 sub new {
45         my ($class, $name) = @_;
46
47         my $self = { _name => $name };
48         bless ($self, $class);
49         return $self;
50 }
51
52 sub load {
53         my $self = shift;
54
55         my $fd = $mainline_git->popen("cat-file", "blob", $self->{_name});
56         while (<$fd>) {
57                 chomp;
58                 next if /^#/;
59                 my ($key, $value) = split;
60                 if ($key =~ /^_/) {
61                         print STDERR "warning: invalid key $key in table $self->{_name}\n";
62                         next;
63                 }
64                 $self->{$key} = $value;
65         }
66         close($fd);
67 }
68
69 sub save {
70         my $self = shift;
71
72         my ($pid, $in, $out) = $mainline_git->popen2("hash-object", "-w",
73                 "--stdin");
74         for my $key (sort(keys(%$self))) {
75                 next if $key =~ /^_/;
76                 printf $out "%s %s\n", $key, $self->{$key};
77         }
78         close($out);
79         my $hash = <$in>;
80         chomp $hash;
81         close($in);
82         waitpid($pid, 0);
83         die "Error saving $self->{_name} table\n" unless $? == 0 && $hash;
84         $mainline_git->read_cmd("tag", "-f", $self->{_name}, $hash);
85 }
86
87 package main;
88
89 sub load_suse_rev_data {
90         print STDERR "Loading commit data from $suse_path...\n";
91         my $total = scalar(keys(%revs));
92         my ($i, $percent_last) = (0, 0);
93         for my $rev (keys(%revs)) {
94                 my $data = $suse_git->commit_data($rev);
95                 for my $key (keys(%$data)) {
96                         $revs{$rev}->{$key} = $data->{$key};
97                 }
98                 if ($suse_git->writable) {
99                         $revs{$rev}->{tree} = patches_subtree($revs{$rev}->{tree});
100                 }
101                 $expanded_trees{$revs{$rev}->{tree}} ||= {};
102                 $revs{$rev}->{expanded} = $expanded_trees{$revs{$rev}->{tree}};
103                 $i++;
104                 my $percent = int($i / $total * 100 + 0.5);
105                 if ($percent != $percent_last) {
106                         printf "%d%%\n", $percent;
107                 }
108                 $percent_last = $percent;
109         }
110 }
111
112 # load suse-tree -> mainline-tree mapping from refs/heads/expanded-trees
113 sub load_expanded_trees {
114         my ($suse, $expanded, $version, $broken);
115
116         print STDERR "Loading expanded tree map from $mainline_path...\n";
117         my $fd = $mainline_git->popen("rev-list", "--no-merges",
118                 "--pretty=raw", "expanded-trees", "--");
119         while (<$fd>) {
120                 chomp;
121                 if (s/^tree //) {
122                         if ($suse) {
123                                 $expanded_trees{$suse}{tree} = $expanded;
124                                 $expanded_trees{$suse}{version} = $version;
125                                 $expanded_trees{$suse}{broken} = $broken;
126                         }
127                         $expanded = $_;
128                         ($suse, $version, $broken) = (undef, undef, undef);
129                 } elsif (s/^ *suse-tree: //) {
130                         $suse = $_;
131                         if (!exists($expanded_trees{$suse})) {
132                                 # not needed
133                                 $suse = undef;
134                         }
135                 } elsif (s/^ *version: //) {
136                         $version = $_;
137                 } elsif (/^ *BROKEN/) {
138                         $broken = 1;
139                 }
140         }
141         close($fd);
142         if ($suse) {
143                 $expanded_trees{$suse}{tree} = $expanded;
144                 $expanded_trees{$suse}{version} = $version;
145                 $expanded_trees{$suse}{broken} = $broken;
146         }
147 }
148
149 # Replace, add or delete items in a given tree and return the new tree id
150 sub filter_tree {
151         my ($tree, $filter, $git) = @_;
152         $git ||= $suse_git;
153
154         my ($pid, $in, $out) = $git->popen2("mktree");
155         if ($tree) {
156                 my $ls_tree = $git->popen("ls-tree", $tree);
157                 while (<$ls_tree>) {
158                         chomp;
159                         my @old = split;
160                         my @new = &$filter(@old);
161                         next if !@new;
162                         # If the filter returns an empty list, the entry is deleted.
163                         # If the filter returns a list of four items (mode, type, hash
164                         # name), the item is either modified or left intact.
165                         # 8, 12, etc. returned items add entries to the tree.
166                         while (@new) {
167                                 printf $out "%s %s %s\t%s\n", splice(@new, 0, 4);
168                         }
169                 }
170                 close($ls_tree);
171         } else {
172                 my @new = &$filter();
173                 while (@new) {
174                         printf $out "%s %s %s\t%s\n", splice(@new, 0, 4);
175                 }
176         }
177         close($out);
178         my $res = <$in>;
179         chomp $res;
180         close($in);
181         waitpid($pid, 0);
182         die "Could not shrink tree $tree\n" unless $? == 0 && $res;
183         return $res;
184 }
185
186 # returns a tree that only contains series.conf, the patches.* directories
187 # and {scripts,rpm}/config.sh (to determine $SRCVERSION)
188 sub patches_subtree {
189         my $tree = shift;
190
191         # first, filter scripts/ and rpm/
192         my $filter_config_sh = sub {
193                 return unless $_[3] eq "config.sh";
194                 return @_;
195         };
196         my $rpm_tree = filter_tree("$tree:rpm", $filter_config_sh);
197         my $scripts_tree = filter_tree("$tree:scripts", $filter_config_sh);
198         # now pick series.conf, patches.*/, replace scripts/ and rpm/
199         # and linux-*.tar.bz2 if present
200         my $filter = sub {
201                 my ($mode, $type, $hash, $name) = @_;
202
203                 if ($type eq "tree" && $name eq "rpm") {
204                         return ($mode, "tree", $rpm_tree, "rpm");
205                 } elsif ($type eq "tree" && $name eq "scripts") {
206                         return ($mode, "tree", $scripts_tree, "scripts");
207                 } elsif ($name =~ /^(series\.conf$|patches\.|linux-.*\.tar\.bz2$)/) {
208                         return ($mode, $type, $hash, $name);
209                 }
210                 return;
211         };
212         return filter_tree($tree, $filter);
213 }
214
215 {
216 my $broken_file;
217 sub mark_tree_broken {
218         my $tree = shift;
219
220         if (!$broken_file) {
221                 my ($pid, $in, $out) = $mainline_git->popen2("hash-object",
222                         "-w", "--stdin");
223                 print $out "This patch series did not apply, a later commit will contain the changes.\n";
224                 close($out);
225                 $broken_file = <$in>;
226                 chomp $broken_file;
227                 close($in);
228                 waitpid($pid, 0);
229                 die "Error creating the BROKEN marker\n" unless $? == 0 &&
230                         $broken_file;
231         }
232         my $added;
233         my $filter = sub {
234                 my @entry = @_;
235                 my @res;
236                 my $cmp = 1;
237                 if (@entry) {
238                         $cmp = "BROKEN" cmp $entry[3];
239                 }
240                 if ($cmp == 0) {
241                         $added = 1;
242                 } elsif ($cmp == 1 && !$added) {
243                         $added = 1;
244                         @res = ("100644", "blob", $broken_file, "BROKEN");
245                 }
246                 push(@res, @entry);
247                 return @res;
248         };
249         return filter_tree($tree, $filter, $mainline_git);
250 }
251 }
252
253 sub do_expand_trees {
254         my $repository = shift;
255         my $append = shift;
256         my @missing = @_;
257
258         my @cmd = ("./expand-trees", "--mainline=$repository",
259                 "--suse=$suse_path");
260         push(@cmd, "--append") if $append;
261         push(@cmd, "--force") if $force;
262         system(@cmd, @missing);
263         if ($? != 0) {
264                 die "expand-trees failed\n";
265         }
266 }
267
268 # fill the %expanded_trees array
269 sub expand_trees {
270         load_expanded_trees();
271         my @missing = sort(grep { !defined($expanded_trees{$_}{tree}) }
272                 keys(%expanded_trees));
273         return unless @missing;
274         print STDERR "Need to expand ", scalar(@missing), " trees\n";
275         if (!$jobs || $jobs == 1) {
276                 do_expand_trees($mainline_path, 1, @missing);
277         } else {
278                 for (my $i = 0; $i < $jobs; $i++) {
279                         my $clone = "$mainline_path-$i";
280                         system("rm", "-rf", "$clone");
281                         my $start = int($i * $#missing / $jobs);
282                         if ($start) {
283                                 $start++;
284                         }
285                         my $stop = int(($i + 1) * $#missing / $jobs);
286                         next if $stop < $start;
287                         my $pid = fork();
288                         die "Can't fork: $!\n" unless defined($pid);
289                         next if $pid > 0;
290                         system("git", "clone", "-s", "$mainline_path", "$clone");
291                         do_expand_trees($clone, 0, @missing[$start..$stop]);
292                         exit 0;
293                 }
294                 my $pid;
295                 do {
296                         $pid = wait();
297                         if ($pid == 0 && $? != 0) {
298                                 die "job failed\n";
299                         }
300                 } while ($pid > 0);
301                 # merge the expanded-trees branches together
302                 my @ids;
303                 my $id = $mainline_git->read_cmd("rev-parse", "expanded-trees");
304                 if ($id) {
305                         chomp($id);
306                         push(@ids, $id);
307                 }
308                 for (my $i = 0; $i < $jobs; $i++) {
309                         my $clone = "$mainline_path-$i";
310                         next unless -d $clone;
311                         $mainline_git->read_cmd("fetch", $clone, "expanded-trees");
312                         my $id = $mainline_git->read_cmd("rev-parse", "FETCH_HEAD");
313                         next unless $id;
314                         chomp($id);
315                         push(@ids, $id);
316                         print STDERR "job$i: $id\n";
317                 }
318                 chomp(@ids);
319                 my ($in, $out);
320                 ($pid, $in, $out) = $mainline_git->popen2("hash-object",
321                         "-t", "commit", "-w", "--stdin");
322                 # use the empty tree for the merge commit
323                 print $out "tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n";
324                 foreach $id (@ids) {
325                         print $out "parent $id\n";
326                 }
327                 print $out "author x <y\@z> 1273225547 +0200\n";
328                 print $out "committer x <y\@z> 1273225547 +0200\n";
329                 print $out "\n";
330                 print $out "merge\n";
331                 close($out);
332                 my $res = <$in>;
333                 chomp($res);
334                 close($in);
335                 waitpid($pid, 0);
336                 die "Error merging expanded-trees branches\n" unless $? == 0 && $res;
337                 $mainline_git->read_cmd("update-ref", "refs/heads/expanded-trees", $res);
338         }
339         load_expanded_trees();
340         @missing = grep { !defined($expanded_trees{$_}{tree}) }
341                 keys(%expanded_trees);
342         if (@missing) {
343                 die "Internal error, following trees were not expanded: ",
344                         join(" ", @missing);
345         }
346 }
347
348 sub mainline_commit {
349         my $version = shift;
350
351         if (!$mainline_commits->{$version}) {
352                 $mainline_commits->{$version} =
353                         qx(./mainline-commit --git $mainline_path $version);
354                 die "Unable to determine commit id of v$version"
355                         unless $? == 0 && $mainline_commits->{$version};
356                 chomp $mainline_commits->{$version};
357         }
358         return $mainline_commits->{$version};
359 }
360
361 sub expand_commit {
362         my ($commit) = @_;
363
364         if (exists($expanded_commits->{$commit})) {
365                 return $expanded_commits->{$commit};
366         }
367         return if $exclude_commits{$commit};
368         my $data = $revs{$commit};
369         my @expanded_parents;
370         my ($version_changed, $version_same);
371         my $broken = $data->{expanded}{broken};
372         my $this_version = $data->{expanded}{version};
373         for my $parent (@{$data->{parent}}) {
374                 next if $exclude_commits{$parent};
375                 push(@expanded_parents, expand_commit($parent));
376                 next if $broken or $revs{$parent}->{expanded}{broken};
377                 if ($revs{$parent}->{expanded}{version} eq $this_version) {
378                         $version_same = 1;
379                 } else {
380                         $version_changed = 1;
381                 }
382         }
383         # Consider a commit a version bump iff it is the root commit or
384         # there is a parent with a different version and no parent with
385         # the same version (i.e. broken commits are not taken into account).
386         if ($version_changed && !$version_same ||
387                                         !@expanded_parents && !$broken)  {
388                 my $mainline = mainline_commit($this_version);
389                 if ($mainline !~ /^0*$/) {
390                         push(@expanded_parents, $mainline);
391                 }
392         }
393         my ($pid, $in, $out) = $mainline_git->popen2("hash-object", "-t",
394                 "commit", "-w", "--stdin");
395         die "Could not expand tree $data->{tree} (commit $commit)\n"
396                 unless $data->{expanded}{tree};
397         if ($broken) {
398                 my ($parent_tree, $parent_broken);
399                 # take the tree of the first parent
400                 for my $parent (@{$data->{parent}}) {
401                         $parent_tree = $revs{$parent}->{final_tree};
402                         $parent_broken = $revs{$parent}->{expanded}{broken};
403                         last;
404                 }
405                 if (!$parent_broken) {
406                         $parent_tree = mark_tree_broken($parent_tree);
407                 }
408                 # We store the final tree in each member of the %revs hash,
409                 # because for broken trees, the final tree is dependent on the
410                 # parent (transitively), so we can't use ->{expanded}{tree}
411                 $data->{final_tree} = $parent_tree;
412         } else {
413                 $data->{final_tree} = $data->{expanded}{tree};
414         }
415         sub translate_id {
416                 my $id = shift;
417
418                 my @match = grep { /^$id/ } keys(%$expanded_commits);
419                 return $id unless @match == 1;
420                 return substr($expanded_commits->{$match[0]}, 0, length($id));
421         }
422         my $message = $data->{message};
423         $message =~ s/\b([0-9a-f]{7,})\b/translate_id($1)/eg;
424         print $out "tree $data->{final_tree}\n";
425         for my $parent (@expanded_parents) {
426                 print $out "parent $parent\n";
427         }
428         print $out "author $data->{author}\n";
429         print $out "committer $data->{committer}\n";
430         print $out "\n";
431         print $out $message;
432         print $out "\n";
433         print $out "suse-commit: $commit\n";
434         if ($broken) {
435                 print $out "Note: This patch series did not apply\n";
436         }
437         close($out);
438         my $res = <$in>;
439         close($in);
440         chomp $res;
441         waitpid($pid, 0);
442         die "Could not expand commit $commit\n" unless $? == 0 && $res;
443         $expanded_commits->{$commit} = $res;
444         return $res;
445 }
446
447 sub expand_tag {
448         my ($id) = @_;
449
450         my $data = $suse_git->tag_data($id);
451         my $expanded = expand_object($data->{object}, $data->{type});
452         return unless $expanded;
453         my ($pid, $in, $out) = $mainline_git->popen2("hash-object", "-t",
454                 "tag", "-w", "--stdin");
455         print $out "object $expanded\n";
456         print $out "type $data->{type}\n";
457         print $out "tag $data->{tag}\n";
458         if ($data->{tagger}) {
459                 print $out "tagger $data->{tagger}\n";
460         }
461         print $out "\n";
462         print $out $data->{message};
463         close($out);
464         my $res = <$in>;
465         close($in);
466         chomp $res;
467         waitpid($pid, 0);
468         die "Could not expand tag $id\n" unless $? == 0 && $res;
469         return $res;
470 }
471
472 sub expand_object {
473         my ($id, $type) = @_;
474
475         return expand_commit($id) if $type eq "commit";
476         return expand_tag($id) if $type eq "tag";
477         die "Cannot expand object $id of type $type\n";
478 }
479
480 sub expand_refs {
481         for my $ref (@refs) {
482                 print STDERR "Expanding $ref->{fullname}... ";
483                 my $mainline_id = expand_object($ref->{id}, $ref->{type});
484                 if (!$mainline_id) {
485                         print STDERR "skipped\n";
486                         next;
487                 }
488                 $mainline_git->read_cmd("update-ref", $ref->{fullname},
489                         $mainline_id);
490                 print STDERR "done\n";
491         }
492 }
493
494 GetOptions(
495         "m|mainline=s" => \$mainline_path,
496         "s|suse=s" => \$suse_path,
497         "f|force" => \$force,
498         "i|incremental" => \$incremental,
499         "a|all" => \$expand_all,
500         "j|jobs=n" => \$jobs,
501         "h|help" => sub { print $USAGE; exit; },
502 ) or die $USAGE;
503
504 if (!$mainline_path || !$suse_path || (!@ARGV && !$expand_all)) {
505         die $USAGE;
506 }
507 $mainline_path =~ s/\/*$//;
508 $mainline_git = SUSE::Kernel::Git->new($mainline_path);
509 $suse_git = SUSE::Kernel::Git->new($suse_path);
510 if (!$suse_git->writable) {
511         print STDERR "Warning: $suse_path is not writable, this will slow down the process.\n";
512 }
513
514 if ($incremental) {
515         $expanded_commits->load();
516 }
517 if ($expand_all) {
518         push(@ARGV, $suse_git->read_cmd("for-each-ref", "--format=%(refname)",
519                         "refs/heads", "refs/tags"));
520 }
521
522 for my $spec (@ARGV) {
523         if ($spec =~ /^\^([^:]*)$/) {
524                 # ^foo exclude
525                 my @exclude = $suse_git->read_cmd("rev-list", $1, "--");
526                 $exclude_commits{$_} = 1 for @exclude;
527                 next;
528         }
529         # [[from]..[to:]]<ref>
530         if ($spec !~ /^(?:([^\.:]+)?\.\.([^\.:]+:)?)?([^:]+)$/) {
531                 die $USAGE;
532         }
533         my ($bottom, $top, $ref) = ($1, $2, $3);
534         if (!$top) {
535                 $top = $suse_git->read_cmd("rev-parse", "--verify", $ref);
536         } else {
537                 $top =~ s/:$//;
538         }
539         my $fullname = $suse_git->read_cmd("rev-parse", "--symbolic-full-name",
540                 $ref);
541         my $type = $suse_git->read_cmd("cat-file", "-t", $ref);
542         if (!$top || !$fullname || !$type) {
543                 die "Ref $ref does not exist\n";
544         }
545         push(@refs, { id => $top, fullname => $fullname, type => $type });
546         if ($bottom) {
547                 my @exclude = $suse_git->read_cmd("rev-list", $bottom, "--");
548                 $exclude_commits{$_} = 1 for @exclude;
549         }
550 }
551
552 print STDERR "Determining commit ids to expand...\n";
553         for my $ref (@refs) {
554                 for my $rev ($suse_git->read_cmd("rev-list", $ref->{id}, "--")) {
555                 next if exists($expanded_commits->{$rev});
556                 next if exists($exclude_commits{$rev});
557                 $revs{$rev} = {};
558         }
559 }
560
561 load_suse_rev_data();
562 $mainline_commits->load();
563 expand_trees();
564 expand_refs();
565 if ($incremental) {
566         $expanded_commits->save();
567 }
568 $mainline_commits->save();
569 $mainline_git->read_cmd("gc", "--auto");
570