From eb138baf54d202b7025edf6fa34deddc99ecb181 Mon Sep 17 00:00:00 2001 From: Chris Lattner Date: Wed, 22 Jan 2003 20:35:59 +0000 Subject: [PATCH] Implement "new/removed/broken/fixed" tests llvm-svn: 5399 --- utils/NightlyTest.pl | 86 +++++++++++++++++++++++++++------- utils/NightlyTestTemplate.html | 11 +++-- 2 files changed, 74 insertions(+), 23 deletions(-) diff --git a/utils/NightlyTest.pl b/utils/NightlyTest.pl index 33c20356431..4d67f9d75df 100755 --- a/utils/NightlyTest.pl +++ b/utils/NightlyTest.pl @@ -39,7 +39,7 @@ sub WriteFile { # (filename, contents) } sub GetRegex { # (Regex with ()'s, value) - $_[1] =~ /$_[0]/; + $_[1] =~ /$_[0]/m; return $1; } @@ -134,6 +134,8 @@ system "(time -p cvs -d $CVSRootDir co llvm) > $Prefix-CVS-Log.txt 2>&1" chdir "llvm" or die "Could not change into llvm directory!"; +system "cvs up -P -d > /dev/null 2>&1" if (!$NOCHECKOUT); + # Read in the HTML template file... undef $/; my $TemplateContents = ReadFile $Template; @@ -172,7 +174,10 @@ my @Linked = split '\n', `grep Linking $Prefix-Build-Log.txt`; my $NumExecutables = scalar(grep(/executable/, @Linked)); my $NumLibraries = scalar(grep(!/executable/, @Linked)); my $NumObjects = `grep '^Compiling' $Prefix-Build-Log.txt | wc -l` + 0; -my $BuildTime = GetRegex "([0-9.]+)", `grep '^real' $Prefix-Build-Log.txt`; +my $BuildTimeU = GetRegex "([0-9.]+)", `grep '^user' $Prefix-Build-Log.txt`; +my $BuildTimeS = GetRegex "([0-9.]+)", `grep '^sys' $Prefix-Build-Log.txt`; +my $BuildWallTime = GetRegex "([0-9.]+)", `grep '^real' $Prefix-Build-Log.txt`; +my $BuildTime = $BuildTimeU+$BuildTimeS; # BuildTime = User+System my $BuildError = ""; if (`grep '^gmake: .*Error' $Prefix-Build-Log.txt | wc -l` + 0) { $BuildError = "

Build error: compilation " @@ -219,27 +224,35 @@ $WarningsRemoved = AddPreTag $WarningsRemoved; my (%AddedFiles, %ModifiedFiles, %RemovedFiles, %UsersCommitted, %UsersUpdated); +my $DateRE = "[-:0-9 ]+\\+[0-9]+"; + # Loop over every record from the CVS history, filling in the hashes. foreach $File (@CVSHistory) { my ($Type, $Date, $UID, $Rev, $Filename); - if ($File =~ /([AMR]) ([-:0-9 ]+\+[0-9]+) ([^ ]+) ([0-9.]+) +([^ ]+) +([^ ]+)/) { + if ($File =~ /([AMRUGC])\s($DateRE)\s([^\s]+)\s+([0-9.]+)\s+([^\s]+)\s+([^\s]+)/) { ($Type, $Date, $UID, $Rev, $Filename) = ($1, $2, $3, $4, "$6/$5"); - } elsif ($File =~ /([OCGUW]) ([-:0-9 ]+\+[0-9]+) ([^ ]+)/) { - ($Type, $Date, $UID, $Rev, $Filename) = ($1, $2, $3, "", ""); - } - #print "Ty = $Type Date = '$Date' UID=$UID Rev=$Rev File = '$Filename'\n"; - - if ($Type eq 'M') { # Modified - $ModifiedFiles{$Filename} = 1; - $UsersCommitted{$UID} = 1; - } elsif ($Type eq 'A') { # Added - $AddedFiles{$Filename} = 1; - $UsersCommitted{$UID} = 1; - } elsif ($Type eq 'R') { # Removed - $RemovedFiles{$Filename} = 1; - $UsersCommitted{$UID} = 1; + } elsif ($File =~ /([W])\s($DateRE)\s([^\s]+) +([^\s]+)\s+([^\s]+)/) { + ($Type, $Date, $UID, $Rev, $Filename) = ($1, $2, $3, $4, "$6/$5"); + } elsif ($File =~ /([O]) ($DateRE) ([^ ]+) +([^\s]+)/) { + ($Type, $Date, $UID, $Rev, $Filename) = ($1, $2, $3, "", "$4/"); } else { - $UsersUpdated{$UID} = 1; + print "UNMATCHABLE: $File\n"; + } + # print "$File\nTy = $Type Date = '$Date' UID=$UID Rev=$Rev File = '$Filename'\n"; + + if ($Filename =~ /^llvm/) { + if ($Type eq 'M') { # Modified + $ModifiedFiles{$Filename} = 1; + $UsersCommitted{$UID} = 1; + } elsif ($Type eq 'A') { # Added + $AddedFiles{$Filename} = 1; + $UsersCommitted{$UID} = 1; + } elsif ($Type eq 'R') { # Removed + $RemovedFiles{$Filename} = 1; + $UsersCommitted{$UID} = 1; + } else { + $UsersUpdated{$UID} = 1; + } } } @@ -260,6 +273,43 @@ system "gmake $MAKEOPTS report.nightly.html TEST=nightly " my $ProgramsTable = ReadFile "report.nightly.html"; +# +# Create a list of the tests which were run... +# +system "grep -E 'TEST-(PASS|FAIL)' < $Prefix-ProgramTest.txt " + . "| sort --key=3 > $Prefix-Tests.txt"; + +my ($RTestsAdded, $RTestsRemoved) = DiffFiles "-Tests.txt"; + +my @RawTestsAddedArray = split '\n', $RTestsAdded; +my @RawTestsRemovedArray = split '\n', $RTestsRemoved; + +my %OldTests = map {GetRegex('TEST-....: (.+)', $_)=>$_} @RawTestsRemovedArray; +my %NewTests = map {GetRegex('TEST-....: (.+)', $_)=>$_} @RawTestsAddedArray; + +my ($TestsAdded, $TestsRemoved, $TestsFixed, $TestsBroken) = ("","","",""); + +foreach $Test (keys %NewTests) { + if (!exists $OldTests{$Test}) { # TestAdded if in New but not old + $TestsAdded = "$TestsAdded$Test\n"; + } else { + if ($OldTests{$Test} =~ /TEST-PASS/) { # Was the old one a pass? + $TestsBroken = "$TestsBroken$Test\n"; # New one must be a failure + } else { + $TestsFixed = "$TestsFixed$Test\n"; # No, new one is a pass. + } + } +} +foreach $Test (keys %OldTests) { # TestRemoved if in Old but not New + $TestsRemoved = "$TestsRemoved$Test\n" if (!exists $NewTests{$Test}); +} + +$TestsAdded = AddPreTag $TestsAdded; +$TestsRemoved = AddPreTag $TestsRemoved; +$TestsFixed = AddPreTag $TestsFixed; +$TestsBroken = AddPreTag $TestsBroken; + + # # Get a list of the previous days that we can link to... # diff --git a/utils/NightlyTestTemplate.html b/utils/NightlyTestTemplate.html index 3b8ea2c3692..4703d92059d 100644 --- a/utils/NightlyTestTemplate.html +++ b/utils/NightlyTestTemplate.html @@ -52,7 +52,8 @@
  • Compilation Log
      $BuildError - Time to build CVS tree: $BuildTime seconds
      + Time to build CVS tree: $BuildTime seconds + ($BuildWallTime seconds wall time)
      Number of object files compiled: $NumObjects
      Number of libraries linked: $NumLibraries
      Number of executables linked: $NumExecutables
      @@ -85,10 +86,10 @@ $WarningsList

      Changes in the test suite:

        -
      • New Tests: ... -
      • Removed Tests: ... -
      • Newly passing tests: ... -
      • Newly failing tests: ... +
      • New Tests: $TestsAdded +
      • Removed Tests: $TestsRemoved +
      • Newly passing tests: $TestsFixed +
      • Newly failing tests: $TestsBroken