mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-13 13:25:37 +00:00
add a new report type that generates stats for all files in a given directory
This commit is contained in:
myk%mozilla.org
parent
10b39fef0c
commit
c7539d43a0
@ -34,6 +34,9 @@ our ($start_date, $end_date);
|
||||
# DNS lookups, which we aren't doing for performance reasons.
|
||||
our $do_segment_count = 0;
|
||||
|
||||
# Do a platform-breakout report by default; this can get overridden by the defs file.
|
||||
our $report_type = "platform_build";
|
||||
|
||||
# Parse the definition file and make sure it defines some stats.
|
||||
my $stats_defs = $ARGV[0];
|
||||
defined($stats_defs) or die "You didn't reference a stats definition file.\n";
|
||||
@ -87,57 +90,106 @@ my $done_in_segments =
|
||||
"WHERE $date_clause AND files.path = ? AND files.name = ? " .
|
||||
"GROUP BY client HAVING SUM(bytes) = ? AND COUNT(bytes) > 1");
|
||||
|
||||
# Grab a list of files in a given directory, recursing infinitely.
|
||||
# The first condition matches the directory itself, while the second
|
||||
# matches subdirectories and should be passed the value '<dir>/%'.
|
||||
my $files_in_directory =
|
||||
$dbh->prepare("SELECT path, name FROM files WHERE files.path = ? OR files.path LIKE ?");
|
||||
|
||||
|
||||
################################################################################
|
||||
# Stats Retrieval
|
||||
|
||||
foreach my $stat (@stats) {
|
||||
next if !$stat->{isactive};
|
||||
sub retrieve_by_file {
|
||||
foreach my $stat (@stats) {
|
||||
next if !$stat->{isactive};
|
||||
print STDERR "$stat->{name} $stat->{version}...\n" if $VERBOSE;
|
||||
|
||||
print STDERR "$stat->{name} $stat->{version}...\n" if $VERBOSE;
|
||||
|
||||
my $platforms = $stat->{platforms};
|
||||
|
||||
foreach my $platform (keys %$platforms) {
|
||||
print STDERR " $platform\n" if $VERBOSE;
|
||||
|
||||
my $files = $platforms->{$platform};
|
||||
|
||||
foreach my $type (keys %$files) {
|
||||
print STDERR " $type: " if $VERBOSE;
|
||||
|
||||
my $file = $files->{$type};
|
||||
|
||||
my (undef, $file_size) = head("http://ftp.mozilla.org$stat->{path}/$file->{name}");
|
||||
$file_size ||= $file->{size}
|
||||
or die "Can't figure out the size of $stat->{path}/$file->{name}.";
|
||||
|
||||
$done->execute($stat->{path}, $file->{name}, $file_size);
|
||||
my ($done_count) = $done->fetchrow_array();
|
||||
|
||||
my $done_in_segments_count = "N/A";
|
||||
my $total_done = $done_count;
|
||||
if ($do_segment_count) {
|
||||
$done_in_segments->execute($stat->{path}, $file->{name}, $file_size);
|
||||
$done_in_segments_count = $done_in_segments->fetchall_arrayref();
|
||||
$done_in_segments_count = scalar(@$done_in_segments_count);
|
||||
$total_done += $done_in_segments_count;
|
||||
# Expand directories in the file list.
|
||||
my @files;
|
||||
foreach my $file (@{$stat->{files}}) {
|
||||
if ($file->{name}) {
|
||||
print STDERR "$file->{name}\n" if $VERBOSE;
|
||||
push(@files, $file);
|
||||
}
|
||||
elsif (!$file->{path}) {
|
||||
print STDERR "file has no name or path\n" if $VERBOSE;
|
||||
next;
|
||||
}
|
||||
else {
|
||||
print STDERR "expanding $file->{path}\n" if $VERBOSE;
|
||||
$files_in_directory->execute($file->{path}, "$file->{path}/%");
|
||||
while (my $file = $files_in_directory->fetchrow_hashref()) {
|
||||
print STDERR "expanding to include $file->{path}/$file->{name}\n" if $VERBOSE;
|
||||
push(@files, $file);
|
||||
}
|
||||
}
|
||||
}
|
||||
$stat->{files} = \@files;
|
||||
|
||||
$not_done->execute($stat->{path}, $file->{name}, $file_size);
|
||||
my ($not_done_count) = $not_done->fetchrow_array();
|
||||
foreach my $file (@files) {
|
||||
(undef, $file->{size}) = head("http://ftp.mozilla.org$file->{path}/$file->{name}");
|
||||
next if !$file->{size};
|
||||
$done->execute($file->{path}, $file->{name}, $file->{size});
|
||||
$file->{counts} = {};
|
||||
($file->{counts}->{done}) = $done->fetchrow_array();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$may_be_done->execute($stat->{path}, $file->{name});
|
||||
my ($may_be_done_count) = $may_be_done->fetchrow_array();
|
||||
|
||||
$file->{counts} = {
|
||||
complete_uni => $done_count,
|
||||
#complete_multi => $done_in_segments_count,
|
||||
incomplete => $not_done_count,
|
||||
partial => $may_be_done_count,
|
||||
};
|
||||
sub retrieve_by_platform_build {
|
||||
foreach my $stat (@stats) {
|
||||
next if !$stat->{isactive};
|
||||
|
||||
print STDERR "$done_count / $not_done_count / $may_be_done_count / $done_in_segments_count / $total_done\n" if $VERBOSE;
|
||||
print STDERR "$stat->{name} $stat->{version}...\n" if $VERBOSE;
|
||||
|
||||
my $platforms = $stat->{platforms};
|
||||
|
||||
foreach my $platform (keys %$platforms) {
|
||||
print STDERR " $platform\n" if $VERBOSE;
|
||||
|
||||
my $files = $platforms->{$platform};
|
||||
|
||||
foreach my $type (keys %$files) {
|
||||
print STDERR " $type: " if $VERBOSE;
|
||||
|
||||
my $file = $files->{$type};
|
||||
|
||||
my (undef, $file_size) = head("http://ftp.mozilla.org$stat->{path}/$file->{name}");
|
||||
$file_size ||= $file->{size}
|
||||
or die "Can't figure out the size of $stat->{path}/$file->{name}.";
|
||||
|
||||
$done->execute($stat->{path}, $file->{name}, $file_size);
|
||||
my ($done_count) = $done->fetchrow_array();
|
||||
|
||||
my $done_in_segments_count = "N/A";
|
||||
my $total_done = $done_count;
|
||||
if ($do_segment_count) {
|
||||
$done_in_segments->execute($stat->{path}, $file->{name}, $file_size);
|
||||
$done_in_segments_count = $done_in_segments->fetchall_arrayref();
|
||||
$done_in_segments_count = scalar(@$done_in_segments_count);
|
||||
$total_done += $done_in_segments_count;
|
||||
}
|
||||
|
||||
# commenting these out for speed -bryner
|
||||
#$not_done->execute($stat->{path}, $file->{name}, $file_size);
|
||||
#my ($not_done_count) = $not_done->fetchrow_array();
|
||||
my ($not_done_count) = 0;
|
||||
|
||||
#$may_be_done->execute($stat->{path}, $file->{name});
|
||||
#my ($may_be_done_count) = $may_be_done->fetchrow_array();
|
||||
my ($may_be_done_count) = 0;
|
||||
|
||||
$file->{counts} = {
|
||||
complete_uni => $done_count,
|
||||
#complete_multi => $done_in_segments_count,
|
||||
#incomplete => $not_done_count,
|
||||
#partial => $may_be_done_count,
|
||||
};
|
||||
|
||||
print STDERR "$done_count / $not_done_count / $may_be_done_count / $done_in_segments_count / $total_done\n" if $VERBOSE;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -145,7 +197,7 @@ foreach my $stat (@stats) {
|
||||
################################################################################
|
||||
# Output
|
||||
|
||||
my $template = <<'EOF';
|
||||
my $platform_build_template = <<'EOF';
|
||||
<html>
|
||||
<head>
|
||||
<title></title>
|
||||
@ -203,6 +255,53 @@ my $template = <<'EOF';
|
||||
</html>
|
||||
EOF
|
||||
|
||||
my $file_template = <<'EOF';
|
||||
<html>
|
||||
<head>
|
||||
<title></title>
|
||||
<style type="text/css">
|
||||
th { text-align: left; }
|
||||
th, td { border: solid 1px black; }
|
||||
table { border-collapse: collapse;
|
||||
border: solid 1px black; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
[% FOREACH stat = stats %]
|
||||
[% NEXT IF !stat.isactive %]
|
||||
<h2>[% stat.name %] [%+ stat.version %] Download Stats</h2>
|
||||
|
||||
<p>[% start_date || "the beginning of time" %] to [% end_date || "the end of time" %]</p>
|
||||
|
||||
<table summary="[% stat.name %] [%+ stat.version %] Downloads">
|
||||
<tr>
|
||||
<th>File</th>
|
||||
<th>Downloads</th>
|
||||
</tr>
|
||||
[% FOREACH file = stat.files %]
|
||||
<tr>
|
||||
<td>[% file.path %]/[% file.name %]</td>
|
||||
<td>[% file.counts.done %]</td>
|
||||
</tr>
|
||||
[% END %]
|
||||
</table>
|
||||
[% END %]
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
|
||||
|
||||
my $template;
|
||||
if ($report_type eq "file") {
|
||||
retrieve_by_file();
|
||||
$template = $file_template;
|
||||
}
|
||||
elsif ($report_type eq "platform_build") {
|
||||
retrieve_by_platform_build();
|
||||
$template = $platform_build_template;
|
||||
}
|
||||
|
||||
my $tt = new Template({ PRE_CHOMP => 1, POST_CHOMP => 1});
|
||||
|
||||
$tt->process(\$template, {stats => \@stats,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user