Bug 371350: Fix various Bootstrap regressions:

In Step.pm
-- Use RunShellCommand() for chdir() handling.
-- Use RunShellCommand() for timeout handling.
-- Fix the signal name/number madness (fallout from the patch for bug 372583).
-- Re-ordered the use statements
-- add a "dumpLogs" config variable that will cause shell statements to be echo'd this is in prep for buildbot deployment of automation.

In Stage.pm:
-- Clean up/add a bunch of comments
-- Cleanup prestage-trimmed using shipped-locales as a basis for the cleanup
-- Run groom-files for mars afterwards
-- Create a stage-unsigned and stage-signed directory
-- Remove unknown directories (that we don't know how to handle) from prestage-trimmed
-- chown() all deliverables to the right group
-- Point the Verify() function at the copy of shipped-locales that the step checked out
-- Handle all of the ja/ja-JP-mac via the shipped-locales logic

Config.pm
-- Make sure Get() and Exists() assert if var is not passed
-- Fix a Get() bug where it would die with no such config variables if the actual variable evaluated to false

r=rhelmer.
This commit is contained in:
preed%mozilla.com 2007-03-09 21:20:05 +00:00
parent edd5e50080
commit 7bbacc8855
3 changed files with 386 additions and 63 deletions

View File

@ -50,7 +50,10 @@ sub Get {
my %args = @_;
my $var = $args{'var'};
if ($config{$var}) {
die "ASSERT: Bootstep::Config::Get(): null var requested" if (not
defined($args{'var'}));
if ($this->Exists(var => $var)) {
return $config{$var};
} else {
die("No such config variable: $var\n");
@ -61,6 +64,9 @@ sub Exists {
my $this = shift;
my %args = @_;
die "ASSERT: Bootstep::Config::Exists(): null var requested" if (not
defined($args{'var'}));
my $var = $args{'var'};
return exists($config{$var});
}

View File

@ -3,12 +3,16 @@
#
package Bootstrap::Step;
use IO::Handle;
use File::Spec::Functions;
use POSIX qw(strftime);
use Bootstrap::Config;
use MozBuild::Util qw(RunShellCommand Email);
use POSIX qw(strftime);
use base 'Exporter';
our @EXPORT = qw(catfile);
my $DEFAULT_TIMEOUT = 3600;
@ -31,17 +35,29 @@ sub Shell {
my $logFile = $args{'logFile'};
my $ignoreExitValue = $args{'ignoreExitValue'};
my $rv = '';
my $config = new Bootstrap::Config();
if (ref($cmdArgs) ne 'ARRAY') {
die("ASSERT: Bootstrap::Step(): cmdArgs is not an array ref\n");
die("ASSERT: Bootstrap::Step::Shell(): cmdArgs is not an array ref\n");
}
my %runShellCommandArgs = (command => $cmd,
args => $cmdArgs,
timeout => $timeout,
logfile => $logFile);
if ($config->Exists(var => 'dumpLogs')) {
if ($config->Get(var => 'dumpLogs')) {
$runShellCommandArgs{'output'} = 1;
}
}
if ($dir) {
$this->Log(msg => 'Changing directory to ' . $dir);
chdir($dir) or die("Cannot chdir to $dir: $!");
$runShellCommandArgs{'dir'} = $dir;
}
$this->Log(msg => 'Running shell command:');
$this->Log(msg => 'Running shell command' .
(defined($dir) ? " in $dir" : '') . ':');
$this->Log(msg => ' arg0: ' . $cmd);
my $argNum = 1;
foreach my $arg (@{$cmdArgs}) {
@ -53,25 +69,18 @@ sub Shell {
$this->Log(msg => 'Timeout: ' . $timeout);
if ($timeout) {
$rv = RunShellCommand(
command => $cmd,
args => $cmdArgs,
timeout => $timeout,
logfile => $logFile,
);
}
$rv = RunShellCommand(%runShellCommandArgs);
my $exitValue = $rv->{'exitValue'};
my $timedOut = $rv->{'timedOut'};
my $signalName = $rv->{'signalName'};
my $signalNum = $rv->{'signalNum'};
my $dumpedCore = $rv->{'dumpedCore'};
if ($timedOut) {
$this->Log(msg => "output: $rv->{'output'}") if $rv->{'output'};
die('FAIL shell call timed out after' . $timeout . 'seconds');
die('FAIL shell call timed out after ' . $timeout . ' seconds');
}
if ($signalName) {
$this->Log(msg => 'WARNING shell recieved signal' . $signalName);
if ($signalNum) {
$this->Log(msg => 'WARNING shell recieved signal ' . $signalNum);
}
if ($dumpedCore) {
$this->Log(msg => "output: $rv->{'output'}") if $rv->{'output'};
@ -79,7 +88,7 @@ sub Shell {
}
if ($ignoreExitValue) {
$this->Log(msg => "Exit value $rv->{'output'}, but ignoring as told");
}elsif ($exitValue) {
} elsif ($exitValue) {
if ($exitValue != 0) {
$this->Log(msg => "output: $rv->{'output'}") if $rv->{'output'};
die("shell call returned bad exit code: $exitValue");

View File

@ -3,15 +3,107 @@
# structure.
#
package Bootstrap::Step::Stage;
use Bootstrap::Step;
use Bootstrap::Config;
use File::Copy qw(copy move);
use File::Find qw(find);
use File::Path qw(rmtree);
use File::Basename;
use MozBuild::Util qw(MkdirWithPath);
@ISA = ("Bootstrap::Step");
use strict;
#
# List of directories that are allowed to be in the prestage-trimmed directory,
# theoretically because TrimCallback() will know what to do with them.
#
my @ALLOWED_DELIVERABLE_DIRECTORIES = qw(windows-xpi mac-xpi linux-xpi);
# List of bouncer platforms; also used in shipped-locales files...
my @ALL_PLATFORMS = qw(win32 linux osx osxppc);
# Various platform maps that map things to to the platform name used in the
# shipped-locale files (which happen to be bouncer OSes, for some reason).
my %DELIVERABLE_TO_PLATFORM = ('win32' => 'win32',
'mac' => 'osx',
'linux-i686' => 'linux');
my %XPIDIR_TO_PLATFORM = ('windows-xpi' => 'win32',
'mac-xpi' => 'osx',
'linux-xpi' => 'linux');
# Loads and parses the shipped-locales manifest file, so get hash of
# locale -> [bouncer] platform mappings; returns success/failure in
# reading/parsing the locales file.
sub LoadLocaleManifest {
my $this = shift;
my %args = @_;
die "ASSERT: LoadLocaleManifest(): needs a HASH ref" if
(not exists($args{'localeHashRef'}) or
ref($args{'localeHashRef'}) ne 'HASH');
my $localeHash = $args{'localeHashRef'};
my $manifestFile = $args{'manifest'};
if (not -e $manifestFile) {
$this->Log(msg => "Can't find manifest $manifestFile");
return 0;
}
open(MANIFEST, "<$manifestFile") or return 0;
my @manifestLines = <MANIFEST>;
close(MANIFEST);
foreach my $line (@manifestLines) {
my @elements = split(/\s+/, $line);
# Grab the locale; we do it this way, so we can use the rest of the
# array as a platform list, if we need it...
my $locale = shift(@elements);
# We don't add a $ on the end, because of things like ja-JP-mac
if ($locale !~ /^[a-z]{2}(\-[A-Z]{2})?/) {
die "ASSERT: invalid locale in manifest file: $locale";
}
# So this is kinda weird; if the locales are followed by a platform,
# then they don't exist for all the platforms; if they're all by their
# lonesome, then they exist for all platforms. So, since we shifted off
# the locale above, if there's anything left in the array, it's the
# platforms that are valid for this locale; if there isn't, then that
# platform is valid for all locales.
$localeHash->{$locale} = scalar(@elements) ? \@elements :
\@ALL_PLATFORMS;
foreach my $platform (@{$localeHash->{$locale}}) {
die "ASSERT: invalid platform: $platform" if
(not grep($platform eq $_, @ALL_PLATFORMS));
}
}
# Add en-US, which isn't in shipped-locales, because it's assumed that
# we always need en-US, for all platforms, to ship.
$localeHash->{'en-US'} = \@ALL_PLATFORMS;
return 1;
}
sub GetStageDir {
my $this = shift;
my $config = new Bootstrap::Config();
my $stageHome = $config->Get(var => 'stageHome');
my $product = $config->Get(var => 'product');
my $version = $config->Get(var => 'version');
return catfile($stageHome, $product . '-' . $version);
}
sub Execute {
my $this = shift;
@ -21,11 +113,14 @@ sub Execute {
my $rc = $config->Get(var => 'rc');
my $logDir = $config->Get(var => 'logDir');
my $stageHome = $config->Get(var => 'stageHome');
my $appName = $config->Get(var => 'appName');
my $mozillaCvsroot = $config->Get(var => 'mozillaCvsroot');
my $releaseTag = $config->Get(var => 'productTag') . '_RELEASE';
## Prepare the staging directory for the release.
# Create the staging directory.
my $stageDir = catfile($stageHome, $product . '-' . $version);
my $stageDir = $this->GetStageDir();
my $mergeDir = catfile($stageDir, 'stage-merged');
if (not -d $stageDir) {
@ -63,7 +158,7 @@ sub Execute {
$this->Log(msg => "Changed group of $fullDir to $product");
}
# NOTE - should have a standard "master" copy somewhere else
# TODO - should have a standard "master" copy somewhere else
# Copy the KEY file from the previous release directory.
my $keyFile = catfile('/home', 'ftp', 'pub', $product, 'releases', '1.5',
'KEY');
@ -77,8 +172,8 @@ sub Execute {
dir => $stageDir,
);
# Collect the release files onto stage.mozilla.org.
# Collect the release files from the candidates directory into a prestage
# directory.
my $prestageDir = catfile($stageDir, 'batch1', 'prestage');
if (not -d $prestageDir) {
MkdirWithPath(dir => $prestageDir)
@ -96,7 +191,8 @@ sub Execute {
dir => catfile($stageDir, 'batch1', 'prestage'),
);
# Remove unreleased builds
# Create a pruning/"trimmed" area; this area will be used to remove
# locales and deliverables we don't ship.
$this->Shell(
cmd => 'rsync',
cmdArgs => ['-av', 'prestage/', 'prestage-trimmed/'],
@ -104,17 +200,58 @@ sub Execute {
dir => catfile($stageDir, 'batch1'),
);
# Remove unshipped files and set proper mode on dirs
# Remove unknown/unrecognized directories from the -candidates dir; after
# this, the only directories that should be in the prestage-trimmed
# directory are directories that we expliciately handle below, to prep
# for groom-files.
$this->{'scrubTrimmedDirDeleteList'} = [];
find(sub { return $this->ScrubTrimmedDirCallback(); },
catfile($stageDir, 'batch1', 'prestage-trimmed'));
foreach my $delDir (@{$this->{'scrubTrimmedDirDeleteList'}}) {
if (-e $delDir && -d $delDir) {
$this->Log(msg => "rmtree() ing $delDir");
if (rmtree($delDir, 1, 1) <= 0) {
die("ASSERT: rmtree() called on $delDir, but nothing deleted.");
}
}
}
# Remove unshipped files/locales and set proper mode on dirs; start
# by checking out the shipped-locales file
$ENV{'CVS_RSH'} = 'ssh';
$this->Shell(cmd => 'cvs',
cmdArgs => ['-d', $mozillaCvsroot,
'co', '-dconfig',
'-r', $releaseTag,
catfile('mozilla', $appName, 'locales', 'shipped-locales')],
dir => catfile($stageDir, 'batch1'),
logFile => catfile($logDir, 'stage-shipped-locales_checkout.log'));
$this->{'localeManifest'} = {};
if (not $this->LoadLocaleManifest(localeHashRef =>
$this->{'localeManifest'},
manifest => catfile($stageDir, 'batch1',
'config', 'shipped-locales'))) {
die "Failed to load locale manifest\n";
}
# All the magic happens here; we remove unshipped deliverables and cross-
# check the locales we do ship in this callback.
find(sub { return $this->TrimCallback(); },
catfile($stageDir, 'batch1', 'prestage-trimmed'));
# Create a stage-unsigned directory to run groom-files in.
$this->Shell(
cmd => 'rsync',
cmdArgs => ['-Lav', 'prestage-trimmed/', 'stage/'],
cmdArgs => ['-av', 'prestage-trimmed/', 'stage-unsigned/'],
logFile => catfile($logDir, 'stage_collect_stage.log'),
dir => catfile($stageDir, 'batch1'),
);
find(sub { return $this->RemoveMarsCallback(); },
catfile($stageDir, 'batch1', 'stage-unsigned'));
# Nightly builds using a different naming scheme than production.
# Rename the files.
# TODO should support --long filenames, for e.g. Alpha and Beta
@ -122,26 +259,71 @@ sub Execute {
cmd => catfile($stageHome, 'bin', 'groom-files'),
cmdArgs => ['--short=' . $version, '.'],
logFile => catfile($logDir, 'stage_groom_files.log'),
dir => catfile($stageDir, 'batch1', 'stage'),
dir => catfile($stageDir, 'batch1', 'stage-unsigned'),
);
# fix xpi dir names
# fix xpi dir names - This is a hash of directory names in the pre-stage
# dir -> directories under which those directories should be moved to;
# the name will be "xpi", so windows-xpi becomes win32/xpi, etc.
my %xpiDirs = ('windows-xpi' => 'win32',
'linux-xpi' => 'linux-i686',
'mac-xpi' => 'mac');
foreach my $xpiDir (keys(%xpiDirs)) {
my $fromDir = catfile($stageDir, 'batch1', 'stage', $xpiDir);
my $toDir = catfile($stageDir, 'batch1', 'stage', $xpiDirs{$xpiDir},
'xpi');
my $fromDir = catfile($stageDir, 'batch1', 'stage-unsigned', $xpiDir);
my $toDir = catfile($stageDir, 'batch1', 'stage-unsigned',
$xpiDirs{$xpiDir}, 'xpi');
if (-e $fromDir) {
move($fromDir, $toDir)
or die(msg => "Cannot rename $fromDir $toDir: $!");
$this->Log(msg => "Moved $fromDir $toDir");
$this->Log(msg => "Moved $fromDir -> $toDir");
} else {
$this->Log(msg => "Couldn't find $fromDir; not moving to $toDir");
}
}
# Create an rsync'ed stage-signed directory to rsync over to the
# signing server.
$this->Shell(
cmd => 'rsync',
cmdArgs => ['-av', 'stage-unsigned/', 'stage-signed/'],
logFile => catfile($logDir, 'stage_unsigned_to_sign.log'),
dir => catfile($stageDir, 'batch1'),
);
# Process the update mars; we copy everything from the trimmed directory
# that we created above; this will have only the locales/deliverables
# we actually ship; then, remove everything but the mars, including
# the [empty] directories; then, run groom-files in the directory that
# has only updates now.
$this->Shell(
cmd => 'rsync',
cmdArgs => ['-av', 'prestage-trimmed/', 'mar/'],
logFile => catfile($logDir, 'stage_trimmed_to_mars.log'),
dir => catfile($stageDir, 'batch1'),
);
$this->{'leaveOnlyMarsDirDeleteList'} = [];
find(sub { return $this->LeaveOnlyUpdateMarsCallback(); },
catfile($stageDir, 'batch1', 'mar'));
foreach my $delDir (@{$this->{'leaveOnlyUpdateMarsDirDeleteList'}}) {
if (-e $delDir && -d $delDir) {
$this->Log(msg => "rmtree() ing $delDir");
if (rmtree($delDir, 1, 1) <= 0) {
die("ASSERT: rmtree() called on $delDir, but nothing deleted.");
}
}
}
$this->Shell(
cmd => catfile($stageHome, 'bin', 'groom-files'),
cmdArgs => ['--short=' . $version, '.'],
logFile => catfile($logDir, 'stage_groom_files_updates.log'),
dir => catfile($stageDir, 'batch1', 'mar'),
);
}
sub Verify {
@ -158,18 +340,88 @@ sub Verify {
## Prepare the staging directory for the release.
# Create the staging directory.
my $stageDir = catfile($stageHome, $product . '-' . $version);
my $stageDir = $this->GetStageDir();
# Verify locales
$this->Shell(
cmd => catfile($stageHome, 'bin', 'verify-locales.pl'),
cmdArgs => ['-m', catfile($stageDir, 'batch-source', 'rc' . $rc,
'mozilla', $appName, 'locales', 'shipped-locales')],
cmdArgs => ['-m', catfile($stageDir, 'batch1', 'config',
'shipped-locales')],
logFile => catfile($logDir, 'stage_verify_l10n.log'),
dir => catfile($stageDir, 'batch1', 'stage'),
dir => catfile($stageDir, 'batch1', 'stage-signed'),
);
}
sub LeaveOnlyUpdateMarsCallback {
my $this = shift;
my $dirent = $File::Find::name;
if (-f $dirent) {
if ($dirent !~ /\.mar$/) {
$this->Log(msg => "Unlinking non-mar deliverable: $dirent");
unlink($dirent) or die("Couldn't unlink $dirent");
}
} elsif (-d $dirent) {
push(@{$this->{'leaveOnlyMarsDirDeleteList'}}, $dirent);
} else {
$this->Log(msg => 'WARNING: LeaveOnlyUpdateMarsCallback(): '.
"Unknown dirent type: $dirent");
}
}
sub RemoveMarsCallback {
my $this = shift;
my $dirent = $File::Find::name;
if (-f $dirent) {
if ($dirent =~ /\.mar$/) {
$this->Log(msg => "Unlinking mar: $dirent");
unlink($dirent) or die("Couldn't unlink $dirent");
}
} elsif (-d $dirent) {
# do nothing
} else {
$this->Log(msg => 'WARNING: RemoveMarsCallback(): '.
"Unknown dirent type: $dirent");
}
}
#
# This is a bit of an odd callback; the idea is to emulate find's -maxdepth
# option with an argument of 1; unfortunately, find2perl barfs on this option.
# Also, File::Find::find() is an annoying combination of depth and breadth-first
# search, depending on which version is installed (find() used to be different
# than finddepth(), but now they're the same?).
#
# Anyway, what this does is add an entry in the object to add the list of
# directories that should be deleted; if we rmtree() those directories here,
# find() will go bonkers, because we removed things out from under it. So,
# the next step after find() is called with this callback is to rmtree()
# all the directories on the list that is populated in this callback.
#
sub ScrubTrimmedDirCallback {
my $this = shift;
my $config = new Bootstrap::Config();
my $dirent = $File::Find::name;
my $trimmedDir = catfile($this->GetStageDir(), 'batch1',
'prestage-trimmed');
# if $dirent is a directory and is a direct child of the prestage-trimmed
# directory (a hacky attempt at the equivalent of find's maxdepth 1 option);
if (-d $dirent && dirname($dirent) eq $trimmedDir) {
foreach my $allowedDir (@ALLOWED_DELIVERABLE_DIRECTORIES) {
return if (basename($dirent) eq $allowedDir);
}
$this->Log(msg => "Adding extra RC directory entry for deletion: " .
$dirent);
push(@{$this->{'scrubTrimmedDirDeleteList'}}, $dirent);
}
}
sub TrimCallback {
my $this = shift;
@ -177,40 +429,96 @@ sub TrimCallback {
my $dirent = $File::Find::name;
if (-f $dirent) {
# Don't ship xforms in the release area
if (($dirent =~ /xforms\.xpi/) ||
# ja-JP-mac is the JA locale for mac, do not ship ja
($dirent =~ /ja\.mac/) ||
($dirent =~ /mac\-xpi\/ja\.xpi$/) ||
# ja is the JA locale for win32/linux, do not ship ja-JP-mac
($dirent =~ /ja\-JP\-mac\.win32/) ||
($dirent =~ /ja\-JP\-mac\.linux/) ||
($dirent =~ /windows\-xpi\/ja\-JP\-mac\.xpi$/) ||
($dirent =~ /linux\-xpi\/ja\-JP\-mac\.xpi$/) ||
# MAR files are merged in later
($dirent =~ /\.mar$/) ||
# ZIP files are not shipped
($dirent =~ /\.zip$/) ||
($dirent =~ /en-US\.xpi$/)) {
unlink($dirent) || die "Could not unlink $dirent: $!";
# ZIP files are not shipped; neither are en-US lang packs
($dirent =~ /\.zip$/) ||
($dirent =~ /en-US\.xpi$/)) {
unlink($dirent) || die "Could not unlink $dirent: $!";
$this->Log(msg => "Unlinked $dirent");
} else {
chmod(0644, $dirent)
|| die "Could not chmod $dirent to 0644: $!";
$this->Log(msg => "Changed mode of $dirent to 0644");
}
return;
}
# source tarballs don't have a locale, so don't check them for one;
# all other deliverables need to be checked to make sure they should
# be in prestage-trimmed, i.e. if their locale shipped.
if ($dirent !~ /\-source\.tar\.bz2$/) {
my $validDeliverable = $this->IsValidLocaleDeliverable();
if (not $validDeliverable) {
$this->Log(msg => "Deleting unwanted locale deliverable: " .
$dirent);
unlink($dirent) or die("Couldn't unlink() $dirent\n");
return;
}
}
chmod(0644, $dirent)
|| die "Could not chmod $dirent to 0644: $!";
$this->Log(msg => "Changed mode of $dirent to 0644");
} elsif (-d $dirent) {
my $product = $config->Get(var => 'product');
my (undef, undef, $gid) = getgrnam($product)
or die "Could not getgrname for $product: $!";
chown(-1, $gid, $dirent)
or die "Cannot chgrp $dirent to $product: $!";
$this->Log(msg => "Changed group of $dirent to $product");
chmod(0755, $dirent)
or die "Could not chmod $dirent to 0755: $!";
$this->Log(msg => "Changed mode of $dirent to 0755");
} else {
die("Unexpected non-file/non-dir directory entry: $dirent");
}
my $product = $config->Get(var => 'product');
my (undef, undef, $gid) = getgrnam($product)
or die "Could not getgrname for $product: $!";
chown(-1, $gid, $dirent)
or die "Cannot chgrp $dirent to $product: $!";
$this->Log(msg => "Changed group of $dirent to $product");
}
sub IsValidLocaleDeliverable {
my $this = shift;
my %args = @_;
my $dirent = $File::Find::name;
my ($locale, $platform);
my @parts = split(/\./, basename($dirent));
my $partsCount = scalar(@parts);
if ($dirent =~ /\.tar\.gz/) {
# e.g. firefox-2.0.0.2.sk.linux-i686.tar.gz
$locale = $parts[$partsCount - 4];
$platform = 'linux';
} elsif ($dirent =~ /\.exe/) {
# e.g. firefox-2.0.0.2.zh-TW.win32.installer.exe
$locale = $parts[$partsCount - 4];
$platform = 'win32';
} elsif ($dirent =~ /\.dmg/) {
# e.g. firefox-2.0.0.2.tr.mac.dmg
$locale = $parts[$partsCount - 3];
$platform = 'osx';
} elsif ($dirent =~ /\.xpi/) {
# e.g. en-GB.xpi
$locale = basename($File::Find::name);
$locale =~ s/\.xpi$//;
my $parentDir = basename($File::Find::dir);
if (exists($XPIDIR_TO_PLATFORM{$parentDir})) {
$platform = $XPIDIR_TO_PLATFORM{$parentDir};
} else {
die 'ASSERT: IsValidLocaleDeliverable(): xpis found in an ' .
'unexpected directory';
}
} elsif ($dirent =~ /\.mar/) {
# e.g. firefox-2.0.0.2.tr.win32.[partial,complete].mar
$locale = $parts[$partsCount - 4];
$platform = $DELIVERABLE_TO_PLATFORM{$parts[$partsCount - 3]};
} else {
$this->Log(msg => "WARNING: Unknown file type in tree: $dirent");
}
foreach my $allowedPlatform (@{$this->{'localeManifest'}->{$locale}}) {
return 1 if ($allowedPlatform eq $platform);
}
return 0;
}
sub Announce {