Add support for compressing & encoding logs when sending them to the server.

Thanks to Philip K. Warren <pkw@us.ibm.com> for the patch.
Bug #113212 r=leaf

tools/tinderbox/build-seamonkey-util.pl

@@ -23,7 +23,7 @@ use Config;         # for $Config{sig_name} and $Config{sig_num}
 use File::Find ();
 use File::Copy;
 
-$::UtilsVersion = '$Revision: 1.252 $ ';
+$::UtilsVersion = '$Revision: 1.253 $ ';
 
 package TinderUtils;
 
@@ -63,6 +63,7 @@ sub Setup {
     GetSystemInfo();
     SetupEnv();
     SetupPath();
+    ValidateSettings(); # Perform some basic validation on settings
 }
 
 
@@ -147,6 +148,41 @@ sub ApplyArgs {
     }
 }
 
+sub ValidateSettings {
+    # Lowercase the LogCompression and LogEncoding variables for convenience.
+    $Settings::LogCompression = lc $Settings::LogCompression;
+    $Settings::LogEncoding = lc $Settings::LogEncoding;
+
+    # Make sure LogCompression and LogEncoding are set to valid values.
+    if ($Settings::LogCompression !~ /^(bzip2|gzip)?$/) {
+        warn "Invalid value for LogCompression: $Settings::LogCompression.\n";
+        exit;
+    }
+    if ($Settings::LogEncoding !~ /^(base64|uuencode)?$/) {
+        warn "Invalid value for LogEncoding: $Settings::LogEncoding.\n";
+        exit;
+    }
+
+    # If LogEncoding is set to 'base64', ensure we have the MIME::Base64
+    # module before we go through the entire build.
+    if ($Settings::LogEncoding eq 'base64') {
+        eval "use MIME::Base64 ();";
+        if ($@) {
+            warn "LogEncoding set to base64 but the MIME::Base64 module could not be loaded.\n";
+            warn "The error message was:\n\n";
+            warn $@;
+            exit;
+        }
+    }
+
+    # If LogCompression is set, make sure LogEncoding is set or else the log
+    # will not be transferred properly.
+    if ($Settings::LogCompression ne '' && $Settings::LogEncoding eq '') {
+        warn "LogEncoding must be set if LogCompression is set.\n";
+        exit;
+    }
+}
+
 
 my $tinder_defaults = "tinder-defaults.pl";
 
@@ -610,6 +646,44 @@ sub mail_build_started_message {
     unlink "$msg_log";
 }
 
+sub encode_log {
+    my $input_file = shift;
+    my $output_file = shift;
+    my $buf;
+    if($Settings::LogEncoding eq 'base64') {
+        eval "use MIME::Base64 ();";
+        while(read($input_file, $buf, 60*57)) {
+            print $output_file MIME::Base64::encode($buf);
+        }
+    }
+    elsif($Settings::LogEncoding eq 'uuencode') {
+        while(read($input_file, $buf, 45)) {
+            print $output_file pack("u*", $buf);
+        }
+    }
+    else {
+        # Make sendmail happy.
+        # Split lines longer than 1000 charaters into 1000 character lines.
+        # If any line is a dot on a line by itself, replace it with a blank
+        # line. This prevents cases where a <cr>.<cr> occurs in the log file.
+        # Sendmail interprets that as the end of the mail, and truncates the
+        # log before it gets to Tinderbox.  (terry weismann, chris yeh)
+        
+        while (<$input_file>) {
+            my $length = length($_);
+            my $offset;
+            for ($offset = 0; $offset < $length ; $offset += 1000) {
+                my $chars_left = $length - $offset;
+                my $output_length = $chars_left < 1000 ? $chars_left : 1000;
+                my $output = substr $_, $offset, $output_length;
+                $output =~ s/^\.$//g;
+                $output =~ s/\n//g;
+                print $output_file "$output\n";
+            }
+        }
+    }
+}
+
 sub mail_build_finished_message {
     my ($start_time, $build_status, $binary_url, $logfile) = @_;
 
@@ -630,30 +704,26 @@ sub mail_build_finished_message {
     print OUTLOG "tinderbox: buildfamily: $platform\n";
     print OUTLOG "tinderbox: version: $::Version\n";
     print OUTLOG "tinderbox: utilsversion: $::UtilsVersion\n";
+    print OUTLOG "tinderbox: logcompression: $Settings::LogCompression\n";
+    print OUTLOG "tinderbox: logencoding: $Settings::LogEncoding\n";
     print OUTLOG "tinderbox: END\n";
 
-    # Make sendmail happy.
-    # Split lines longer than 1000 charaters into 1000 character lines.
-    # If any line is a dot on a line by itself, replace it with a blank
-    # line. This prevents cases where a <cr>.<cr> occurs in the log file.
-    # Sendmail interprets that as the end of the mail, and truncates the
-    # log before it gets to Tinderbox.  (terry weismann, chris yeh)
-
-    open LOG, "$logfile" or die "Couldn't open logfile, $logfile: $!";
-    while (<LOG>) {
-        my $length = length($_);
-        my $offset;
-        for ($offset = 0; $offset < $length ; $offset += 1000) {
-            my $chars_left = $length - $offset;
-            my $output_length = $chars_left < 1000 ? $chars_left : 1000;
-            my $output = substr $_, $offset, $output_length;
-            $output =~ s/^\.$//g;
-            $output =~ s/\n//g;
-            print OUTLOG "$output\n";
-        }
+    if ($Settings::LogCompression eq 'gzip') {
+        open GZIPLOG, "gzip -c $logfile |" or die "Couldn't open gzip'd logfile: $!\n";
+        encode_log(\*GZIPLOG, \*OUTLOG);
+        close GZIPLOG;
     }
+    elsif ($Settings::LogCompression eq 'bzip2') {
+        open BZ2LOG, "bzip2 -c $logfile |" or die "Couldn't open bzip2'd logfile: $!\n";
+        encode_log(\*BZ2LOG, \*OUTLOG);
+        close BZ2LOG;
+    }
+    else {
+        open LOG, "$logfile" or die "Couldn't open logfile, $logfile: $!";
+        encode_log(\*LOG, \*OUTLOG);
+        close LOG;
+    }    
     close OUTLOG;
-    close LOG;
     unlink($logfile);
 
     # If on Windows, make sure the log mail has unix lineendings, or

tools/tinderbox/tinder-defaults.pl

@@ -152,3 +152,15 @@ $Timezone = '';
 # as an exercise to the reader. 
 $RebootSystem = 0;
 
+# LogCompression specifies the type of compression used on the log file.
+# Valid options are 'gzip', and 'bzip2'. Please make sure the binaries
+# for 'gzip' or 'bzip2' are in the user's path before setting this
+# option.
+$LogCompression = '';
+
+# LogEncoding specifies the encoding format used for the logs. Valid
+# options are 'base64', and 'uuencode'. If $LogCompression is set above,
+# this needs to be set to 'base64' or 'uuencode' to ensure that the
+# binary data is transferred properly.
+$LogEncoding = '';
+

webtools/tinderbox/processbuild.pl

@@ -179,6 +179,17 @@ sub check_required_variables {
     $err_string .= "Variable 'tinderbox:status' must be 'success', 'busted', 'testfailed', or 'building'\n";
   }
 
+  # Log compression
+  #
+  if ($tbx->{logcompression} !~ /^(bzip2|gzip)?$/) {
+    $err_string .= "Variable 'tinderbox:logcompression' must be '', 'bzip2' or 'gzip'\n";
+  }
+
+  # Log encoding
+  if ($tbx->{logencoding} !~ /^(base64|uuencode)?$/) {
+    $err_string .= "Variable 'tinderbox:logencoding' must be '', 'base64' or 'uuencode'\n";
+  }
+
   # Report errors
   #
   die $err_string unless $err_string eq '';
@@ -208,16 +219,65 @@ sub compress_log_file {
 
   open ZIPLOG, "| $gzip -c > $tbx->{tree}/$tbx->{logfile}"
     or die "can't open $! for writing";
-  my $inBinary = 0;
-  while (<LOG2>) {
-    unless ($inBinary) {
+
+  # If this log is compressed, we need to decode it and decompress
+  # it before storing its contents into ZIPLOG.
+  if($tbx->{logcompression} ne '') {
+
+    # tinderbox variables are not compressed
+    # write them directly to the gzip'd log
+    while(<LOG2>) {
       print ZIPLOG $_;
-      if ($hasBinary) {
-        $inBinary = (/^begin [0-7][0-7][0-7] /);
-      }
+      last if(m/^tinderbox: END/);
     }
-    elsif (/^end\n/) {
-      $inBinary = 0;
+
+    # Decode the log using the logencoding variable to determine
+    # the type of encoding.
+    my $decoded = "$tbx->{tree}/$tbx->{logfile}.uncomp";
+    if ($tbx->{logencoding} eq 'base64') {
+      eval "use MIME::Base64 ();";
+      open DECODED, ">$decoded"
+        or die "Can't open $decoded for writing: $!";
+      while (<LOG2>) {
+        print DECODED MIME::Base64::decode($_);
+      }
+      close DECODED;
+    }
+    elsif ($tbx->{logencoding} eq 'uuencode') {
+      open DECODED, ">$decoded"
+        or die "Can't open $decoded for writing: $!";
+      while (<LOG2>) {
+        print DECODED unpack("u*", $_);
+      }
+      close DECODED;
+    }
+
+    # Decompress the log using the logcompression variable to determine
+    # the type of compression used.
+    my $cmd = undef;
+    if ($tbx->{logcompression} eq 'gzip') {
+      $cmd = $gzip;
+    }
+    elsif ($tbx->{logcompression} eq 'bzip2') {
+      $cmd = $bzip2;
+    }
+    if (defined $cmd) {
+      open UNCOMP, "$cmd -dc $decoded |"
+        or die "Can't open $! for reading";
+      while (<UNCOMP>) {
+        print ZIPLOG $_;
+      }
+      close UNCOMP;
+    }
+
+    # Remove our temporary decoded file
+    unlink($decoded) if -f $decoded;
+  }
+  # This log is not compressed/encoded so we can simply write out
+  # it's contents to the gzip'd log file.
+  else {
+    while (<LOG2>) {
+      print ZIPLOG $_;
     }
   }
   close ZIPLOG;

webtools/tinderbox/tbglobals.pl

@@ -49,7 +49,8 @@ $build_table = [];
 $who_list = [];
 @note_array = ();
 
-$gzip = '/usr/local/bin/gzip';
+$gzip = '/usr/bin/gzip';
+$bzip2 = '/usr/bin/bzip2';
 
 $data_dir='data';