mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-05 00:25:27 +00:00
607 lines
17 KiB
Perl
Executable File
607 lines
17 KiB
Perl
Executable File
#!/usr/bin/perl
|
|
# -*- Mode: Perl; tab-width: 4; indent-tabs-mode: nil; -*-
|
|
|
|
# ***** BEGIN LICENSE BLOCK *****
|
|
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
|
|
#
|
|
# The contents of this file are subject to the Mozilla Public License Version
|
|
# 1.1 (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
# http://www.mozilla.org/MPL/
|
|
#
|
|
# Software distributed under the License is distributed on an "AS IS" basis,
|
|
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
|
|
# for the specific language governing rights and limitations under the
|
|
# License.
|
|
#
|
|
# The Original Code is Mozilla JavaScript Testing Utilities
|
|
#
|
|
# The Initial Developer of the Original Code is
|
|
# Mozilla Corporation.
|
|
# Portions created by the Initial Developer are Copyright (C) 2007
|
|
# the Initial Developer. All Rights Reserved.
|
|
#
|
|
# Contributor(s): Bob Clary <bclary@bclary.com>
|
|
#
|
|
# Alternatively, the contents of this file may be used under the terms of
|
|
# either the GNU General Public License Version 2 or later (the "GPL"), or
|
|
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
|
|
# in which case the provisions of the GPL or the LGPL are applicable instead
|
|
# of those above. If you wish to allow use of your version of this file only
|
|
# under the terms of either the GPL or the LGPL, and not to allow others to
|
|
# use your version of this file under the terms of the MPL, indicate your
|
|
# decision by deleting the provisions above and replace them with the notice
|
|
# and other provisions required by the GPL or the LGPL. If you do not delete
|
|
# the provisions above, a recipient may use your version of this file under
|
|
# the terms of any one of the MPL, the GPL or the LGPL.
|
|
#
|
|
# ***** END LICENSE BLOCK *****
|
|
|
|
use strict;
|
|
use Getopt::Mixed "nextOption";
|
|
|
|
# predeclarations
|
|
sub debug;
|
|
sub usage;
|
|
sub parse_options;
|
|
sub escape_pattern;
|
|
sub unescape_pattern;
|
|
|
|
# option arguments
|
|
|
|
my $option_desc = "b=s branch>b T=s buildtype>T t=s testtype>t l=s rawlogfile>l f=s failurelogfile>f o=s os>o r=s patterns>r z=s timezone>z O=s outputprefix>O A=s arch>A K=s kernel>K D debug>D";
|
|
|
|
my $testid;
|
|
my $branch;
|
|
my $buildtype;
|
|
my $testtype;
|
|
my $rawlogfile;
|
|
my $failurelogfile;
|
|
my $os;
|
|
my $patterns;
|
|
my $timezone;
|
|
my $outputprefix;
|
|
my $arch;
|
|
my $kernel;
|
|
my $debug = $ENV{DEBUG};
|
|
|
|
# pattern variables
|
|
|
|
my $knownfailurebranchpattern;
|
|
my $failurebranchpattern;
|
|
my $knownfailureospattern;
|
|
my $failureospattern;
|
|
my $knownfailurebuildtypepattern;
|
|
my $failurebuildtypepattern;
|
|
my $knownfailuretesttypepattern;
|
|
my $failuretesttypepattern;
|
|
my $knownfailuretimezonepattern;
|
|
my $failuretimezonepattern;
|
|
my $knownfailurearchpattern;
|
|
my $failurearchpattern;
|
|
my $knownfailurekernelpattern;
|
|
my $failurekernelpattern;
|
|
|
|
my @patterns;
|
|
my $pattern;
|
|
my @failures;
|
|
my @fixes;
|
|
my @excludedtests;
|
|
my $excludedtest;
|
|
my $excludedfile;
|
|
my %includedtests = {};
|
|
my $includedfile;
|
|
my @results;
|
|
|
|
my $regchars = '\[\^\-\]\|\{\}\?\*\+\.\<\>\$\(\)';
|
|
|
|
|
|
&parse_options;
|
|
|
|
my $jsdir = $ENV{TEST_JSDIR};
|
|
|
|
if (!defined($jsdir)) {
|
|
$jsdir = "/work/mozilla/mozilla.com/test.mozilla.com/www/tests/mozilla.org/js";
|
|
}
|
|
|
|
my @excludedfiles = ("excluded-$branch-$testtype-$buildtype.tests");
|
|
my @includedfiles = ("included-$branch-$testtype-$buildtype.tests");
|
|
|
|
# create working patterns file consisting of matches to users selection
|
|
# and which has the test description patterns escaped
|
|
|
|
# remove the excluded tests from the possible fixes log
|
|
|
|
|
|
foreach $excludedfile ( @excludedfiles ) {
|
|
open EXCLUDED, "<$jsdir/$excludedfile" or die "Unable to open excluded file $jsdir/$excludedfile: $!\n";
|
|
while (<EXCLUDED>) {
|
|
chomp;
|
|
|
|
next if ($_ =~ /^\#/);
|
|
|
|
s/\s+$//;
|
|
|
|
push @excludedtests, ($_);
|
|
}
|
|
close EXCLUDED;
|
|
}
|
|
|
|
@excludedtests = sort @excludedtests;
|
|
|
|
foreach $includedfile ( @includedfiles ) {
|
|
open INCLUDED, "<$jsdir/$includedfile" or die "Unable to open included file $jsdir/$includedfile: $!\n";
|
|
while (<INCLUDED>) {
|
|
chomp;
|
|
|
|
next if ($_ =~ /^\#/);
|
|
|
|
s/\s+$//;
|
|
|
|
$includedtests{$_} = 1;
|
|
}
|
|
close INCLUDED;
|
|
}
|
|
|
|
debug "loading patterns $patterns";
|
|
debug "pattern filter: /^TEST_ID=[^,]*, TEST_BRANCH=$knownfailurebranchpattern, TEST_RESULT=[^,]*, TEST_BUILDTYPE=$knownfailurebuildtypepattern, TEST_TYPE=$knownfailuretesttypepattern, TEST_OS=$knownfailureospattern, TEST_MACHINE=[^,]*, TEST_PROCESSORTYPE=$knownfailurearchpattern, TEST_KERNEL=$knownfailurekernelpattern, TEST_DATE=[^,]*, TEST_TIMEZONE=$knownfailuretimezonepattern,/\n";
|
|
|
|
open PATTERNS, "<$patterns" or die "Unable to open known failure patterns file $patterns: $!\n";
|
|
while (<PATTERNS>) {
|
|
chomp;
|
|
|
|
s/\s+$//;
|
|
|
|
($testid) = $_ =~ /^TEST_ID=([^,]*),/;
|
|
|
|
if (!$includedtests{$testid})
|
|
{
|
|
debug "test $testid was not included during this run";
|
|
}
|
|
elsif ($_ =~ /^TEST_ID=[^,]*, TEST_BRANCH=$knownfailurebranchpattern, TEST_RESULT=[^,]*, TEST_BUILDTYPE=$knownfailurebuildtypepattern, TEST_TYPE=$knownfailuretesttypepattern, TEST_OS=$knownfailureospattern, TEST_MACHINE=[^,]*, TEST_PROCESSORTYPE=$knownfailurearchpattern, TEST_KERNEL=$knownfailurekernelpattern, TEST_DATE=[^,]*, TEST_TIMEZONE=$knownfailuretimezonepattern,/) {
|
|
debug "adding pattern : $_";
|
|
push @patterns, (escape_pattern($_));
|
|
}
|
|
else {
|
|
debug "skipping pattern: $_";
|
|
}
|
|
|
|
}
|
|
close PATTERNS;
|
|
|
|
# create a working copy of the current failures which match the users selection
|
|
|
|
debug "failure filter: ^TEST_ID=[^,]*, TEST_BRANCH=$failurebranchpattern, TEST_RESULT=FAIL[^,]*, TEST_BUILDTYPE=$failurebuildtypepattern, TEST_TYPE=$failuretesttypepattern, TEST_OS=$failureospattern, TEST_MACHINE=[^,]*, TEST_PROCESSORTYPE=$failurearchpattern, TEST_KERNEL=$failurekernelpattern, TEST_DATE=[^,]*, TEST_TIMEZONE=$failuretimezonepattern,";
|
|
|
|
if (defined($rawlogfile)) {
|
|
|
|
$failurelogfile = "$outputprefix-results-failures.log";
|
|
my $alllog = "$outputprefix-results-all.log";
|
|
|
|
debug "writing failures $failurelogfile";
|
|
|
|
open INPUTLOG, "$jsdir/post-process-logs.pl $rawlogfile |" or die "Unable to open $rawlogfile $!\n";
|
|
open ALLLOG, ">$alllog" or die "Unable to open $alllog $!\n";
|
|
open FAILURELOG, ">$failurelogfile" or die "Unable to open $failurelogfile $!\n";
|
|
|
|
while (<INPUTLOG>) {
|
|
chomp;
|
|
|
|
print ALLLOG "$_\n";
|
|
|
|
if ($_ =~ /^TEST_ID=[^,]*, TEST_BRANCH=$failurebranchpattern, TEST_RESULT=FAIL[^,]*, TEST_BUILDTYPE=$failurebuildtypepattern, TEST_TYPE=$failuretesttypepattern, TEST_OS=$failureospattern, TEST_MACHINE=[^,]*, TEST_PROCESSORTYPE=$failurearchpattern, TEST_KERNEL=$failurekernelpattern, TEST_DATE=[^,]*, TEST_TIMEZONE=$failuretimezonepattern,/) {
|
|
debug "failure: $_";
|
|
push @failures, ($_);
|
|
print FAILURELOG "$_\n";
|
|
}
|
|
}
|
|
close INPUTLOG;
|
|
my $inputrc = $?;
|
|
close ALLLOG;
|
|
close FAILURELOG;
|
|
|
|
die "FATAL ERROR in post-process-logs.pl" if $inputrc != 0;
|
|
|
|
}
|
|
else {
|
|
|
|
debug "loading failures $failurelogfile";
|
|
|
|
open FAILURES, "<$failurelogfile" or die "Unable to open current failure log $failurelogfile: $!\n";
|
|
while (<FAILURES>) {
|
|
chomp;
|
|
|
|
if ($_ =~ /^TEST_ID=[^,]*, TEST_BRANCH=$failurebranchpattern, TEST_RESULT=FAIL[^,]*, TEST_BUILDTYPE=$failurebuildtypepattern, TEST_TYPE=$failuretesttypepattern, TEST_OS=$failureospattern, TEST_MACHINE=[^,]*, TEST_PROCESSORTYPE=$failurearchpattern, TEST_KERNEL=$failurekernelpattern, TEST_DATE=[^,]*, TEST_TIMEZONE=$failuretimezonepattern,/) {
|
|
debug "failure: $_";
|
|
push @failures, ($_);
|
|
}
|
|
}
|
|
close FAILURES;
|
|
}
|
|
|
|
debug "finding fixed bugs";
|
|
|
|
unlink "$outputprefix-results-possible-fixes.log";
|
|
|
|
foreach $pattern (@patterns) {
|
|
# look for known failure patterns that don't have matches in the
|
|
# the current failures selected by the user.
|
|
|
|
@results = grep m@^$pattern@, @failures;
|
|
|
|
if ($#results == -1) {
|
|
debug "fix: $pattern";
|
|
push @fixes, ($pattern)
|
|
}
|
|
}
|
|
|
|
foreach $excludedtest ( @excludedtests ) {
|
|
# remove any potential fixes which are due to the test being excluded
|
|
|
|
if ($debug) {
|
|
@results = grep m@$excludedtest@, @fixes;
|
|
if ($#results > -1) {
|
|
print "excluding: " . (join ', ', @results) . "\n";
|
|
}
|
|
}
|
|
|
|
@results = grep !m@$excludedtest@, @fixes;
|
|
|
|
@fixes = @results;
|
|
}
|
|
|
|
my $fix;
|
|
open OUTPUT, ">$outputprefix-results-possible-fixes.log" or die "Unable to open $outputprefix-results-possible-fixes.log: $!";
|
|
foreach $fix (@fixes) {
|
|
print OUTPUT unescape_pattern($fix) . "\n";
|
|
if ($debug) {
|
|
debug "fix: $fix";
|
|
}
|
|
}
|
|
close OUTPUT;
|
|
|
|
print STDOUT "log: $outputprefix-results-possible-fixes.log\n";
|
|
|
|
debug "finding regressions";
|
|
|
|
my $pass = 0;
|
|
my $changed = ($#patterns != -1);
|
|
|
|
while ($changed) {
|
|
|
|
$pass = $pass + 1;
|
|
|
|
$changed = 0;
|
|
|
|
debug "pass $pass";
|
|
|
|
foreach $pattern (@patterns) {
|
|
|
|
debug "Pattern: $pattern";
|
|
|
|
my @nomatches = grep !m@^$pattern@, @failures;
|
|
my @matches = grep m@^$pattern@, @failures;
|
|
|
|
if ($debug) {
|
|
my $temp = join ', ', @nomatches;
|
|
debug "nomatches: $#nomatches $temp";
|
|
$temp = join ', ', @matches;
|
|
debug "matches: $#matches $temp";
|
|
}
|
|
|
|
@failures = @nomatches;
|
|
|
|
if ($#matches > -1) {
|
|
$changed = 1;
|
|
}
|
|
|
|
debug "*****************************************";
|
|
}
|
|
|
|
}
|
|
|
|
foreach $excludedtest ( @excludedtests ) {
|
|
|
|
if ($debug) {
|
|
@results = grep m@$excludedtest@, @failures;
|
|
if ($#results > -1) {
|
|
print "excluding: " . (join ', ', @results) . "\n";
|
|
}
|
|
}
|
|
|
|
@results = grep !m@$excludedtest@, @failures;
|
|
|
|
@failures = @results;
|
|
}
|
|
|
|
open OUTPUT, ">$outputprefix-results-possible-regressions.log" or die "Unable to open $outputprefix-results-possible-regressions.log: $!";
|
|
|
|
my $failure;
|
|
foreach $failure (@failures) {
|
|
print OUTPUT "$failure\n";
|
|
if ($debug) {
|
|
debug "regression: $failure";
|
|
}
|
|
}
|
|
close OUTPUT;
|
|
|
|
print STDOUT "log: $outputprefix-results-possible-regressions.log\n";
|
|
|
|
|
|
sub debug {
|
|
if ($debug) {
|
|
my $msg = shift;
|
|
print STDERR "DEBUG: $msg\n";
|
|
}
|
|
}
|
|
|
|
sub usage {
|
|
|
|
my $msg = shift;
|
|
|
|
print STDERR <<EOF;
|
|
|
|
usage: $msg
|
|
|
|
known-failures.pl [-b|--branch] branch [-T|--buildtype] buildtype
|
|
[-t|--testtype] testtype [-o os|--os]
|
|
([-f|--failurelogfile] failurelogfile|[-l|--logfile] rawlogfile])
|
|
[-r|--patterns] patterns [-z|--timezone] timezone
|
|
[-O|--outputprefix] outputprefix
|
|
|
|
variable description
|
|
=============== ============================================================
|
|
-b branch branch 1.8.0, 1.8.1, 1.9.0, all
|
|
-T buildtype build type opt, debug, all
|
|
-t testtype test type browser, shell, all
|
|
-l rawlogfile raw logfile
|
|
-f failurelogfile failure logfile
|
|
-o os operating system win32, mac, linux, all
|
|
-r patterns known failure patterns
|
|
-z timezone -0400, -0700, etc. default to user\'s zone
|
|
-O outputprefix output files will be generated with this prefix
|
|
-A arch architecture, all or a specific pattern
|
|
-K kernel kernel, all or a specific pattern
|
|
-D turn on debugging output
|
|
EOF
|
|
|
|
exit(2);
|
|
}
|
|
|
|
sub parse_options {
|
|
my ($option, $value);
|
|
|
|
Getopt::Mixed::init ($option_desc);
|
|
$Getopt::Mixed::order = $Getopt::Mixed::RETURN_IN_ORDER;
|
|
|
|
while (($option, $value) = nextOption()) {
|
|
|
|
if ($option eq "b") {
|
|
$branch = $value;
|
|
}
|
|
elsif ($option eq "T") {
|
|
$buildtype = $value;
|
|
}
|
|
elsif ($option eq "t") {
|
|
$testtype = $value;
|
|
}
|
|
elsif ($option eq "l") {
|
|
$rawlogfile = $value;
|
|
}
|
|
elsif ($option eq "f") {
|
|
$failurelogfile = $value;
|
|
}
|
|
elsif ($option eq "o") {
|
|
$os = $value;
|
|
}
|
|
elsif ($option eq "r") {
|
|
$patterns = $value;
|
|
}
|
|
elsif ($option eq "z") {
|
|
$timezone = $value;
|
|
}
|
|
elsif ($option eq "O") {
|
|
$outputprefix = $value;
|
|
}
|
|
elsif ($option eq "A") {
|
|
$arch = $value;
|
|
}
|
|
elsif ($option eq "K") {
|
|
$kernel = $value;
|
|
}
|
|
elsif ($option eq "D") {
|
|
$debug = 1;
|
|
}
|
|
|
|
}
|
|
|
|
if ($debug) {
|
|
print "branch=$branch, rawlogfile=$rawlogfile failurelogfile=$failurelogfile, os=$os, buildtype=$buildtype, testtype=$testtype, patterns=$patterns, timezone=$timezone, outputprefix=$outputprefix\n";
|
|
}
|
|
Getopt::Mixed::cleanup();
|
|
|
|
if ( !defined($branch) ) {
|
|
usage "missing branch";
|
|
}
|
|
|
|
if (!defined($rawlogfile) && !defined($failurelogfile)) {
|
|
usage "missing logfile";
|
|
}
|
|
|
|
|
|
if (!defined($os)) {
|
|
usage "missing os";
|
|
}
|
|
|
|
if (!defined($buildtype)) {
|
|
usage "missing buildtype";
|
|
}
|
|
|
|
if (!defined($testtype)) {
|
|
usage "missing testtype";
|
|
}
|
|
|
|
if (!defined($patterns)) {
|
|
usage "missing patterns";
|
|
}
|
|
|
|
|
|
if (!defined($timezone)) {
|
|
usage "missing timezone";
|
|
}
|
|
|
|
|
|
if (!defined($outputprefix)) {
|
|
usage "missing outputprefix";
|
|
}
|
|
|
|
if ($branch eq "1.8.0") {
|
|
$knownfailurebranchpattern = "([^,]*1\\.8\\.0[^,]*|\\.\\*)";
|
|
$failurebranchpattern = "1\\.8\\.0";
|
|
}
|
|
if ($branch eq "1.8.1") {
|
|
$knownfailurebranchpattern = "([^,]*1\\.8\\.1[^,]*|\\.\\*)";
|
|
$failurebranchpattern = "1\\.8\\.1";
|
|
}
|
|
elsif ($branch eq "1.9.0") {
|
|
$knownfailurebranchpattern = "([^,]*1\\.9\\.0[^,]*|\\.\\*)";
|
|
$failurebranchpattern = "1\\.9\\.0";
|
|
}
|
|
elsif ($branch eq "all") {
|
|
$knownfailurebranchpattern = "[^,]*";
|
|
$failurebranchpattern = "[^,]*";
|
|
}
|
|
|
|
if ($os eq "win32") {
|
|
$knownfailureospattern = "([^,]*win32[^,]*|\\.\\*)";
|
|
$failureospattern = "win32";
|
|
}
|
|
elsif ($os eq "mac") {
|
|
$knownfailureospattern = "([^,]*mac[^,]*|\\.\\*)";
|
|
$failureospattern = "mac";
|
|
}
|
|
elsif ($os eq "linux") {
|
|
$knownfailureospattern = "([^,]*linux[^,]*|\\.\\*)";
|
|
$failureospattern = "linux";
|
|
}
|
|
elsif ($os eq "all") {
|
|
$knownfailureospattern = "[^,]*";
|
|
$failureospattern = "[^,]*";
|
|
}
|
|
|
|
if ($buildtype eq "opt") {
|
|
$knownfailurebuildtypepattern = "([^,]*opt[^,]*|\\.\\*)";
|
|
$failurebuildtypepattern = "opt";
|
|
}
|
|
elsif ($buildtype eq "debug") {
|
|
$knownfailurebuildtypepattern = "([^,]*debug[^,]*|\\.\\*)";
|
|
$failurebuildtypepattern = "debug";
|
|
}
|
|
elsif ($buildtype eq "all") {
|
|
$knownfailurebuildtypepattern = "[^,]*";
|
|
$failurebuildtypepattern = "[^,]*";
|
|
}
|
|
|
|
if ($testtype eq "shell") {
|
|
$knownfailuretesttypepattern = "([^,]*shell[^,]*|\\.\\*)";
|
|
$failuretesttypepattern = "shell";
|
|
}
|
|
elsif ($testtype eq "browser") {
|
|
$knownfailuretesttypepattern = "([^,]*browser[^,]*|\\.\\*)";
|
|
$failuretesttypepattern = "browser";
|
|
}
|
|
elsif ($testtype eq "all") {
|
|
$knownfailuretesttypepattern = "[^,]*";
|
|
$failuretesttypepattern = "[^,]*";
|
|
}
|
|
|
|
if ($timezone eq "all") {
|
|
$knownfailuretimezonepattern = "[^,]*";
|
|
$failuretimezonepattern = "[^,]*";
|
|
}
|
|
else {
|
|
$knownfailuretimezonepattern = "([^,]*" . $timezone . "[^,]*|\\.\\*)";
|
|
$failuretimezonepattern = "$timezone";
|
|
}
|
|
|
|
if ($arch ne "all") {
|
|
$knownfailurearchpattern = "([^,]*" . $arch . "[^,]*|\\.\\*)";
|
|
$failurearchpattern = "$arch";
|
|
}
|
|
else {
|
|
$knownfailurearchpattern = "[^,]*";
|
|
$failurearchpattern = "[^,]*";
|
|
}
|
|
|
|
if ($kernel ne "all") {
|
|
$knownfailurekernelpattern = "([^,]*" . $kernel . "[^,]*|\\.\\*)";
|
|
$failurekernelpattern = "$kernel";
|
|
}
|
|
else {
|
|
$knownfailurekernelpattern = "[^,]*";
|
|
$failurekernelpattern = "[^,]*";
|
|
}
|
|
|
|
|
|
}
|
|
|
|
sub escape_pattern {
|
|
|
|
my $line = shift;
|
|
|
|
chomp;
|
|
|
|
my ($leading, $trailing) = $line =~ /(.*TEST_DESCRIPTION=)(.*)/;
|
|
|
|
# debug "escape_pattern: before: $leading$trailing";
|
|
|
|
# replace unescaped regular expression characters in the
|
|
# description so they are not interpreted as regexp chars
|
|
# when matching descriptions. leave the escaped regexp chars
|
|
# `regexp alone so they can be unescaped later and used in
|
|
# pattern matching.
|
|
|
|
# see perldoc perlre
|
|
|
|
$trailing =~ s/\\/\\\\/g;
|
|
|
|
# escape non word chars that aren't surrounded by ``
|
|
$trailing =~ s/(?<!`)([$regchars])(?!`)/\\$1/g;
|
|
$trailing =~ s/(?<!`)([$regchars])(?=`)/\\$1/g;
|
|
$trailing =~ s/(?<=`)([$regchars])(?!`)/\\$1/g;
|
|
# $trailing =~ s/(?<!`)(\W)(?!`)/\\$1/g;
|
|
|
|
# unquote the regchars
|
|
$trailing =~ s/\`([^\`])\`/$1/g;
|
|
|
|
debug "escape_pattern : $leading$trailing";
|
|
|
|
return "$leading$trailing";
|
|
|
|
}
|
|
|
|
sub unescape_pattern {
|
|
my $line = shift;
|
|
|
|
chomp;
|
|
|
|
my ($leading, $trailing) = $line =~ /(.*TEST_DESCRIPTION=)(.*)/;
|
|
|
|
# quote the unescaped non word chars
|
|
$trailing =~ s/(?<!\\)([$regchars])/`$1`/g;
|
|
|
|
# unescape the escaped non word chars
|
|
$trailing =~ s/\\([$regchars])/$1/g;
|
|
|
|
$trailing =~ s/\\\\/\\/g;
|
|
|
|
debug "unescape_pattern: after: $leading$trailing";
|
|
|
|
return "$leading$trailing";
|
|
}
|
|
|
|
####
|
|
|
|
|
|
1;
|