mirror of
https://github.com/mozilla/gecko-dev.git
synced 2025-01-26 23:23:33 +00:00
Bug 180870 - Remove old shadowdb manual replication code
r, a=myk
This commit is contained in:
bbaetz%student.usyd.edu.au
parent
9011f9db22
commit
a3c47abf4b
@ -820,7 +820,7 @@ if ($my_create_htaccess) {
|
||||
open HTACCESS, ">.htaccess";
|
||||
print HTACCESS <<'END';
|
||||
# don't allow people to retrieve non-cgi executable files or our private data
|
||||
<FilesMatch ^(.*\.pl|localconfig|processmail|syncshadowdb|runtests.sh)$>
|
||||
<FilesMatch ^(.*\.pl|localconfig|processmail|runtests.sh)$>
|
||||
deny from all
|
||||
</FilesMatch>
|
||||
END
|
||||
@ -1085,7 +1085,7 @@ WriteParams();
|
||||
|
||||
# These are the files which need to be marked executable
|
||||
my @executable_files = ('processmail', 'whineatnews.pl', 'collectstats.pl',
|
||||
'checksetup.pl', 'syncshadowdb', 'importxml.pl', 'runtests.sh');
|
||||
'checksetup.pl', 'importxml.pl', 'runtests.sh');
|
||||
|
||||
# tell me if a file is executable. All CGI files and those in @executable_files
|
||||
# are executable
|
||||
@ -1648,13 +1648,6 @@ $table{milestones} =
|
||||
sortkey smallint not null,
|
||||
unique (product_id, value)';
|
||||
|
||||
$table{shadowlog} =
|
||||
'id int not null auto_increment primary key,
|
||||
ts timestamp,
|
||||
reflected tinyint not null,
|
||||
command mediumtext not null,
|
||||
index(reflected)';
|
||||
|
||||
# GRM
|
||||
$table{duplicates} =
|
||||
'dupe_of mediumint(9) not null,
|
||||
@ -3828,6 +3821,11 @@ if ($sth->rows == 0) {
|
||||
}
|
||||
}
|
||||
|
||||
# 2002-11-XX Bug 180870 - remove manual shadowdb replication code
|
||||
if (TableExists('shadowlog')) {
|
||||
print "Removing shadowlog table\n";
|
||||
$dbh->do("DROP TABLE shadowlog");
|
||||
}
|
||||
|
||||
#
|
||||
# Final checks...
|
||||
|
||||
@ -67,30 +67,14 @@ sub check_shadowdb {
|
||||
if ($value eq "") {
|
||||
return "";
|
||||
}
|
||||
if (!Param("updateshadowdb")) {
|
||||
# Can't test this, because ConnectToDatabase uses the param, but
|
||||
# we can't set this before testing....
|
||||
return "";
|
||||
}
|
||||
&::SendSQL("SHOW DATABASES");
|
||||
while (&::MoreSQLData()) {
|
||||
my $n = &::FetchOneColumn();
|
||||
if (lc($n) eq lc($value)) {
|
||||
return "The $n database already exists. If that's really the name you want to use for the backup, please CAREFULLY make the existing database go away somehow, and then try again.";
|
||||
}
|
||||
}
|
||||
# We trust the admin....
|
||||
trick_taint($value);
|
||||
&::SendSQL("CREATE DATABASE $value");
|
||||
&::SendSQL("INSERT INTO shadowlog (command) VALUES ('SYNCUP')", 1);
|
||||
return "";
|
||||
}
|
||||
|
||||
sub check_shadowdbhost {
|
||||
my ($value) = (@_);
|
||||
if ($value && Param("updateshadowdb")) {
|
||||
return "Sorry, you can't have the shadowdb on a different connection to the main database if you want Bugzilla to handle the replication for you.";
|
||||
if (!Param('shadowdbhost')) {
|
||||
return "You need to specify a host when using a shadow database";
|
||||
}
|
||||
|
||||
# Can't test existance of this because ConnectToDatabase uses the param,
|
||||
# but we can't set this before testing....
|
||||
# This can really only be fixed after we can use the DBI more openly
|
||||
return "";
|
||||
}
|
||||
|
||||
@ -270,51 +254,11 @@ sub check_netmask {
|
||||
default => 0
|
||||
},
|
||||
|
||||
{
|
||||
name => 'queryagainstshadowdb',
|
||||
desc => 'If this is on, and the <tt>shadowdb</tt> parameter is set, then ' .
|
||||
'certain queries will happen against the shadow database.',
|
||||
type => 'b',
|
||||
default => 0,
|
||||
},
|
||||
|
||||
{
|
||||
name => 'updateshadowdb',
|
||||
desc => 'If this is on, and the <tt>shadowdb</tt> parameter is set, then ' .
|
||||
'Bugzilla will use the old style of shadow database in which it ' .
|
||||
'manually propogates changes to the shadow database. Otherwise, ' .
|
||||
'Bugzilla will assume that the <tt>shadowdb</tt> database (if ' .
|
||||
'any) is being updated via replication. <b>WARNING! This ' .
|
||||
'manual replication is deprecated and is going away soon ' .
|
||||
'(<u>BEFORE</u> the next stable Bugzilla release).</b> It has ' .
|
||||
'several problems with data consistency, and replication is the ' .
|
||||
'preferred option. If this parameter is on, and you disable it, ' .
|
||||
'make sure that the shadow database is already set up for ' .
|
||||
'replication, or queries will return stale data.',
|
||||
type => 'b',
|
||||
default => 1,
|
||||
},
|
||||
|
||||
# This entry must be _after_ updateshadowdb, because check_shadowdbhost uses
|
||||
# that
|
||||
{
|
||||
name => 'shadowdbhost',
|
||||
desc => 'The host the shadow database is on. If blank, then then we ' .
|
||||
'assume it\'s on the main database host (as defined in ' .
|
||||
'localconfig) and ingore the <tt>shadowdbport</tt> and ' .
|
||||
'<tt>shadowdbsock</tt> parameters below, which means that this ' .
|
||||
'parameter <em>must be filled in</em> if your shadow database is ' .
|
||||
'on a different instance of the mysql server, even if that ' .
|
||||
'instance runs on the same machine as the main database. Note ' .
|
||||
'that <tt>updateshadowdb</tt> must be off if the shadow database ' .
|
||||
'is on a difference mysql instance, since Bugzilla can\'t ' .
|
||||
'propogate changes between instances itself, and this should be ' .
|
||||
'left blank if the shadow database is on the same instance, ' .
|
||||
'since Bugzilla can then reuse the same database connection for '.
|
||||
'better performance.',
|
||||
desc => 'The host the shadow database is on.',
|
||||
type => 't',
|
||||
default => '',
|
||||
checker => \&check_shadowdbhost,
|
||||
},
|
||||
|
||||
{
|
||||
@ -346,11 +290,12 @@ sub check_netmask {
|
||||
{
|
||||
name => 'shadowdb',
|
||||
desc => 'If non-empty, then this is the name of another database in ' .
|
||||
'which Bugzilla will keep a shadow read-only copy of everything. ' .
|
||||
'which Bugzilla will use as a read-only copy of everything. ' .
|
||||
'This is done so that long slow read-only operations can be used ' .
|
||||
'against this db, and not lock up things for everyone else. ' .
|
||||
'Turning on this parameter will create the given database ; be ' .
|
||||
'careful not to use the name of an existing database with useful ' . 'data in it!',
|
||||
'against this db, and not lock up things for everyone else. This ' .
|
||||
'database is on the <tt>shadowdbhost</tt>, and must exist. ' .
|
||||
'Bugzilla does not update it, if you use this paramater, then ' .
|
||||
'you need to set up replication for your database',
|
||||
type => 't',
|
||||
default => '',
|
||||
checker => \&check_shadowdb
|
||||
|
||||
@ -69,8 +69,14 @@
|
||||
write locking. What this means is that if someone needs to make a
|
||||
change to a bug, they will lock the entire table until the operation
|
||||
is complete. Locking for write also blocks reads until the write is
|
||||
complete. The
|
||||
<quote>shadowdb</quote>
|
||||
complete. Note that more recent versions of mysql support row level
|
||||
locking using different table types. These types are slower than the
|
||||
standard type, and Bugzilla does not yet take advantage of features
|
||||
such as transactions which would justify this speed decrease. The
|
||||
Bugzilla team are, however, happy to hear about any experiences with
|
||||
row level locking and Bugzilla</para>
|
||||
|
||||
<para>The <quote>shadowdb</quote>
|
||||
parameter was designed to get around this limitation. While only a
|
||||
single user is allowed to write to a table at a time, reads can
|
||||
continue unimpeded on a read-only shadow copy of the database.
|
||||
@ -85,23 +91,10 @@
|
||||
Bugzilla bug changes and comments per day.</para>
|
||||
|
||||
<para>The value of the parameter defines the name of the
|
||||
shadow bug database.
|
||||
Set "shadowdb" to e.g. "bug_shadowdb" if you will be running a
|
||||
*very* large installation of Bugzilla.
|
||||
<note>
|
||||
<para>Enabling "shadowdb" can adversely affect the stability of
|
||||
your installation of Bugzilla. You should regularly check that your
|
||||
database is in sync. It is often advisable to force a shadow
|
||||
database sync nightly via
|
||||
<quote>cron</quote>.
|
||||
</para>
|
||||
</note>
|
||||
</para>
|
||||
|
||||
<para>If you use the "shadowdb" option, it is only natural that you
|
||||
should turn the "queryagainstshadowdb" option on as well. Otherwise
|
||||
you are replicating data into a shadow database for no reason!</para>
|
||||
|
||||
shadow bug database. You will need to set the host and port settings
|
||||
from the params page, and set up replication in your database server
|
||||
so that updates reach this readonly mirror. Consult your database
|
||||
documentation for more detail.</para>
|
||||
</step>
|
||||
|
||||
<step>
|
||||
|
||||
@ -198,7 +198,6 @@
|
||||
| products |
|
||||
| profiles |
|
||||
| profiles_activity |
|
||||
| shadowlog |
|
||||
| tokens |
|
||||
| versions |
|
||||
| votes |
|
||||
@ -290,10 +289,6 @@ sshh... don't tell your users!)
|
||||
profiles_activity: Need to know who did what when to who's profile? This'll
|
||||
tell you, it's a pretty complete history.
|
||||
|
||||
shadowlog: I could be mistaken here, but I believe this table tells you when
|
||||
your shadow database is updated and what commands were used to update it. We
|
||||
don't use a shadow database at our site yet, so it's pretty empty for us.
|
||||
|
||||
versions: Version information for every product
|
||||
|
||||
votes: Who voted for what when
|
||||
|
||||
@ -559,9 +559,7 @@ AllowOverride Limit
|
||||
<para>There are important files and directories that should not be a
|
||||
served by the HTTP server - most files in the
|
||||
<quote>data</quote>
|
||||
and
|
||||
<quote>shadow</quote>
|
||||
directories and the
|
||||
directory and the
|
||||
<quote>localconfig</quote>
|
||||
file. You should configure your HTTP server to not serve
|
||||
these files. Failure to do so will expose critical passwords and
|
||||
|
||||
@ -69,8 +69,14 @@
|
||||
write locking. What this means is that if someone needs to make a
|
||||
change to a bug, they will lock the entire table until the operation
|
||||
is complete. Locking for write also blocks reads until the write is
|
||||
complete. The
|
||||
<quote>shadowdb</quote>
|
||||
complete. Note that more recent versions of mysql support row level
|
||||
locking using different table types. These types are slower than the
|
||||
standard type, and Bugzilla does not yet take advantage of features
|
||||
such as transactions which would justify this speed decrease. The
|
||||
Bugzilla team are, however, happy to hear about any experiences with
|
||||
row level locking and Bugzilla</para>
|
||||
|
||||
<para>The <quote>shadowdb</quote>
|
||||
parameter was designed to get around this limitation. While only a
|
||||
single user is allowed to write to a table at a time, reads can
|
||||
continue unimpeded on a read-only shadow copy of the database.
|
||||
@ -85,23 +91,10 @@
|
||||
Bugzilla bug changes and comments per day.</para>
|
||||
|
||||
<para>The value of the parameter defines the name of the
|
||||
shadow bug database.
|
||||
Set "shadowdb" to e.g. "bug_shadowdb" if you will be running a
|
||||
*very* large installation of Bugzilla.
|
||||
<note>
|
||||
<para>Enabling "shadowdb" can adversely affect the stability of
|
||||
your installation of Bugzilla. You should regularly check that your
|
||||
database is in sync. It is often advisable to force a shadow
|
||||
database sync nightly via
|
||||
<quote>cron</quote>.
|
||||
</para>
|
||||
</note>
|
||||
</para>
|
||||
|
||||
<para>If you use the "shadowdb" option, it is only natural that you
|
||||
should turn the "queryagainstshadowdb" option on as well. Otherwise
|
||||
you are replicating data into a shadow database for no reason!</para>
|
||||
|
||||
shadow bug database. You will need to set the host and port settings
|
||||
from the params page, and set up replication in your database server
|
||||
so that updates reach this readonly mirror. Consult your database
|
||||
documentation for more detail.</para>
|
||||
</step>
|
||||
|
||||
<step>
|
||||
|
||||
@ -198,7 +198,6 @@
|
||||
| products |
|
||||
| profiles |
|
||||
| profiles_activity |
|
||||
| shadowlog |
|
||||
| tokens |
|
||||
| versions |
|
||||
| votes |
|
||||
@ -290,10 +289,6 @@ sshh... don't tell your users!)
|
||||
profiles_activity: Need to know who did what when to who's profile? This'll
|
||||
tell you, it's a pretty complete history.
|
||||
|
||||
shadowlog: I could be mistaken here, but I believe this table tells you when
|
||||
your shadow database is updated and what commands were used to update it. We
|
||||
don't use a shadow database at our site yet, so it's pretty empty for us.
|
||||
|
||||
versions: Version information for every product
|
||||
|
||||
votes: Who voted for what when
|
||||
|
||||
@ -559,9 +559,7 @@ AllowOverride Limit
|
||||
<para>There are important files and directories that should not be a
|
||||
served by the HTTP server - most files in the
|
||||
<quote>data</quote>
|
||||
and
|
||||
<quote>shadow</quote>
|
||||
directories and the
|
||||
directory and the
|
||||
<quote>localconfig</quote>
|
||||
file. You should configure your HTTP server to not serve
|
||||
these files. Failure to do so will expose critical passwords and
|
||||
|
||||
@ -101,11 +101,6 @@ foreach my $i (GetParamList()) {
|
||||
WriteParams();
|
||||
|
||||
unlink "data/versioncache";
|
||||
if (Param("updateshadowdb")) {
|
||||
print "<PRE>";
|
||||
system("./syncshadowdb", "-v");
|
||||
print "</PRE>";
|
||||
}
|
||||
|
||||
print "OK, done.<p>\n";
|
||||
print "<a href=editparams.cgi>Edit the params some more.</a><p>\n";
|
||||
|
||||
@ -109,19 +109,16 @@ $::dbwritesallowed = 1;
|
||||
sub ConnectToDatabase {
|
||||
my ($useshadow) = (@_);
|
||||
$::dbwritesallowed = !$useshadow;
|
||||
$useshadow = ($useshadow && Param("shadowdb") &&
|
||||
Param("queryagainstshadowdb"));
|
||||
my $useshadow_dbh = ($useshadow && Param("shadowdbhost") ne "");
|
||||
my $name = $useshadow ? Param("shadowdb") : $::db_name;
|
||||
$useshadow &&= Param("shadowdb");
|
||||
my $connectstring;
|
||||
|
||||
if ($useshadow_dbh) {
|
||||
if ($useshadow) {
|
||||
if (defined $::shadow_dbh) {
|
||||
$::db = $::shadow_dbh;
|
||||
return;
|
||||
}
|
||||
$connectstring="DBI:mysql:host=" . Param("shadowdbhost") .
|
||||
";database=$name;port=" . Param("shadowdbport");
|
||||
";database=" . Param('shadowdb') . ";port=" . Param("shadowdbport");
|
||||
if (Param("shadowdbsock") ne "") {
|
||||
$connectstring .= ";mysql_socket=" . Param("shadowdbsock");
|
||||
}
|
||||
@ -130,7 +127,7 @@ sub ConnectToDatabase {
|
||||
$::db = $::main_dbh;
|
||||
return;
|
||||
}
|
||||
$connectstring="DBI:mysql:host=$::db_host;database=$name;port=$::db_port";
|
||||
$connectstring="DBI:mysql:host=$::db_host;database=$::db_name;port=$::db_port";
|
||||
if ($::db_sock ne "") {
|
||||
$connectstring .= ";mysql_socket=$::db_sock";
|
||||
}
|
||||
@ -141,7 +138,7 @@ sub ConnectToDatabase {
|
||||
Param("maintainer") . ". The error you should quote is: " .
|
||||
$DBI::errstr;
|
||||
|
||||
if ($useshadow_dbh) {
|
||||
if ($useshadow) {
|
||||
$::shadow_dbh = $::db;
|
||||
} else {
|
||||
$::main_dbh = $::db;
|
||||
@ -149,58 +146,17 @@ sub ConnectToDatabase {
|
||||
}
|
||||
|
||||
sub ReconnectToShadowDatabase {
|
||||
# This will connect us to the shadowdb if we're not already connected,
|
||||
# but if we're using the same dbh for both the main db and the shadowdb,
|
||||
# be sure to USE the correct db
|
||||
if (Param("shadowdb") && Param("queryagainstshadowdb")) {
|
||||
if (Param("shadowdb")) {
|
||||
ConnectToDatabase(1);
|
||||
if (!Param("shadowdbhost")) {
|
||||
SendSQL("USE " . Param("shadowdb"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub ReconnectToMainDatabase {
|
||||
if (Param("shadowdb") && Param("queryagainstshadowdb")) {
|
||||
if (Param("shadowdb")) {
|
||||
ConnectToDatabase();
|
||||
if (!Param("shadowdbhost")) {
|
||||
SendSQL("USE $::db_name");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
my $shadowchanges = 0;
|
||||
sub SyncAnyPendingShadowChanges {
|
||||
if ($shadowchanges && Param("updateshadowdb")) {
|
||||
my $pid;
|
||||
FORK: {
|
||||
if ($pid = fork) { # create a fork
|
||||
# parent code runs here
|
||||
$shadowchanges = 0;
|
||||
return;
|
||||
} elsif (defined $pid) {
|
||||
# child process code runs here
|
||||
my $redir = ($^O =~ /MSWin32/i) ? "NUL" : "/dev/null";
|
||||
open STDOUT,">$redir";
|
||||
open STDERR,">$redir";
|
||||
exec("./syncshadowdb","--") or die "Unable to exec syncshadowdb: $!";
|
||||
# the idea was that passing the second parameter tricks it into
|
||||
# using execvp instead of running a shell. Not really necessary since
|
||||
# there are no shell meta-characters, but it passes our tinderbox
|
||||
# test that way. :) http://bugzilla.mozilla.org/show_bug.cgi?id=21253
|
||||
} elsif ($! =~ /No more process/) {
|
||||
# recoverable fork error, try again in 5 seconds
|
||||
sleep 5;
|
||||
redo FORK;
|
||||
} else {
|
||||
# something weird went wrong
|
||||
die "Can't create background process to run syncshadowdb: $!";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# This is used to manipulate global state used by SendSQL(),
|
||||
# MoreSQLData() and FetchSQLData(). It provides a way to do another
|
||||
# SQL query without losing any as-yet-unfetched data from an existing
|
||||
@ -248,7 +204,7 @@ sub SqlLog {
|
||||
}
|
||||
|
||||
sub SendSQL {
|
||||
my ($str, $dontshadow) = (@_);
|
||||
my ($str) = (@_);
|
||||
|
||||
# Don't use DBI's taint stuff yet, because:
|
||||
# a) We don't want out vars to be tainted (yet)
|
||||
@ -262,12 +218,10 @@ sub SendSQL {
|
||||
if ($iswrite && !$::dbwritesallowed) {
|
||||
die "Evil code attempted to write '$str' to the shadow database";
|
||||
}
|
||||
if ($str =~ /^LOCK TABLES/i && $str !~ /shadowlog/ && $::dbwritesallowed) {
|
||||
$str =~ s/^LOCK TABLES/LOCK TABLES shadowlog WRITE, /i;
|
||||
}
|
||||
|
||||
# If we are shutdown, we don't want to run queries except in special cases
|
||||
if (Param('shutdownhtml')) {
|
||||
if ($0 =~ m:[\\/]((do)?editparams.cgi|syncshadowdb)$:) {
|
||||
if ($0 =~ m:[\\/]((do)?editparams.cgi)$:) {
|
||||
$::ignorequery = 0;
|
||||
} else {
|
||||
$::ignorequery = 1;
|
||||
@ -284,19 +238,6 @@ sub SendSQL {
|
||||
die "$str: " . $errstr;
|
||||
}
|
||||
SqlLog("Done");
|
||||
if (!$dontshadow && $iswrite && Param("shadowdb") && Param("updateshadowdb")) {
|
||||
my $q = SqlQuote($str);
|
||||
my $insertid;
|
||||
if ($str =~ /^(INSERT|REPLACE)/i) {
|
||||
SendSQL("SELECT LAST_INSERT_ID()");
|
||||
$insertid = FetchOneColumn();
|
||||
}
|
||||
SendSQL("INSERT INTO shadowlog (command) VALUES ($q)", 1);
|
||||
if ($insertid) {
|
||||
SendSQL("SET LAST_INSERT_ID = $insertid");
|
||||
}
|
||||
$shadowchanges++;
|
||||
}
|
||||
}
|
||||
|
||||
sub MoreSQLData {
|
||||
|
||||
@ -395,14 +395,6 @@ while (MoreSQLData()) {
|
||||
push(@groupstoadd, $id)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# Lock tables before inserting records for the new bug into the database
|
||||
# if we are using a shadow database to prevent shadow database corruption
|
||||
# when two bugs get created at the same time.
|
||||
SendSQL("LOCK TABLES bugs WRITE, bug_group_map WRITE, longdescs WRITE, " .
|
||||
"cc WRITE, keywords WRITE, dependencies WRITE, bugs_activity WRITE, " .
|
||||
"fielddefs READ, profiles READ, keyworddefs READ") if Param("shadowdb");
|
||||
|
||||
# Add the bug report to the DB.
|
||||
SendSQL($sql);
|
||||
@ -465,8 +457,6 @@ if (UserInGroup("editbugs")) {
|
||||
}
|
||||
}
|
||||
|
||||
SendSQL("UNLOCK TABLES") if Param("shadowdb");
|
||||
|
||||
# Assemble the -force* strings so this counts as "Added to this capacity"
|
||||
my @ARGLIST = ();
|
||||
if (@cc) {
|
||||
|
||||
@ -1,327 +0,0 @@
|
||||
#!/usr/bonsaitools/bin/perl -w
|
||||
# -*- Mode: perl; indent-tabs-mode: nil -*-
|
||||
#
|
||||
# The contents of this file are subject to the Mozilla Public
|
||||
# License Version 1.1 (the "License"); you may not use this file
|
||||
# except in compliance with the License. You may obtain a copy of
|
||||
# the License at http://www.mozilla.org/MPL/
|
||||
#
|
||||
# Software distributed under the License is distributed on an "AS
|
||||
# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
|
||||
# implied. See the License for the specific language governing
|
||||
# rights and limitations under the License.
|
||||
#
|
||||
# The Original Code is the Bugzilla Bug Tracking System.
|
||||
#
|
||||
# The Initial Developer of the Original Code is Netscape Communications
|
||||
# Corporation. Portions created by Netscape are
|
||||
# Copyright (C) 1998 Netscape Communications Corporation. All
|
||||
# Rights Reserved.
|
||||
#
|
||||
# Contributor(s): Terry Weissman <terry@mozilla.org>
|
||||
# David Gardiner <david.gardiner@unisa.edu.au>
|
||||
|
||||
use strict;
|
||||
|
||||
use lib qw(.);
|
||||
|
||||
use Bugzilla::Config qw(:DEFAULT :admin);
|
||||
|
||||
require "globals.pl";
|
||||
require "defparams.pl";
|
||||
|
||||
# Shut up misguided -w warnings about "used only once". "use vars" just
|
||||
# doesn't work for me.
|
||||
|
||||
sub sillyness {
|
||||
my $zz;
|
||||
open SAVEOUT,">/dev/null";
|
||||
$zz = $::db;
|
||||
$zz = $::dbwritesallowed;
|
||||
$zz = $::db_host;
|
||||
$zz = $::db_port;
|
||||
}
|
||||
|
||||
my $verbose = 0;
|
||||
my $syncall = 0;
|
||||
my $shutdown = 0;
|
||||
my $tempdir = "data";
|
||||
my $force = 0;
|
||||
|
||||
my $shutdown_msg = "Bugzilla is temporarily disabled while the database is backed up. Try again in a few minutes.";
|
||||
|
||||
sub Usage {
|
||||
print "Usage: syncshadowdb [-v] [-syncall] [-shutdown] [-tempdir dirname] [-force]\n";
|
||||
exit;
|
||||
}
|
||||
|
||||
while (my $opt = shift @ARGV) {
|
||||
if ($opt eq '-v') {
|
||||
$verbose = 1;
|
||||
} elsif ($opt eq '-syncall') {
|
||||
$syncall = 1;
|
||||
$verbose = 1;
|
||||
} elsif ($opt eq '-shutdown') {
|
||||
$shutdown = 1;
|
||||
} elsif ($opt eq '-tempdir') {
|
||||
my $dir = shift @ARGV;
|
||||
if (-d $dir) {
|
||||
$tempdir = $dir;
|
||||
} else {
|
||||
print "$dir does not exist or is not a directory. No syncing performed";
|
||||
exit;
|
||||
}
|
||||
} elsif ($opt eq '-force') {
|
||||
$force = 1;
|
||||
} elsif ($opt eq '--') {
|
||||
# do nothing - null parameter so we can use
|
||||
# multi-param system() call in globals.pl
|
||||
} else {
|
||||
Usage();
|
||||
}
|
||||
}
|
||||
$| = 1;
|
||||
|
||||
my $logtostderr = 0;
|
||||
|
||||
sub Verbose ($) {
|
||||
my ($str) = (@_);
|
||||
if ($verbose) {
|
||||
if ($logtostderr) {
|
||||
print STDERR $str, "\n";
|
||||
} else {
|
||||
print $str, "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!Param("shadowdb")) {
|
||||
Verbose("We don't have shadow databases turned on; no syncing performed.");
|
||||
exit;
|
||||
}
|
||||
|
||||
if (!Param("updateshadowdb")) {
|
||||
Verbose("This shadow database is not set to be updated by Bugzilla.\nSee the mysql replication FAQ if you want to pause the main db until the\nshadowdb catches up");
|
||||
# I could run the commands here, but that involves keeping a connection
|
||||
# open to the main db and the shadowdb at the same time, and our current
|
||||
# db stuff doesn't support that. Its not sufficient to reconnect, because
|
||||
# the lock on the main db will be dropped when the connection closes...
|
||||
exit 1;
|
||||
}
|
||||
|
||||
if (Param("shutdownhtml") && ! $force) {
|
||||
Verbose("Bugzilla was shutdown prior to running syncshadowdb. \n" .
|
||||
" If you wish to sync anyway, use the -force command line option");
|
||||
exit;
|
||||
}
|
||||
|
||||
my $wasshutdown = "";
|
||||
if ($shutdown) {
|
||||
Verbose ("Shutting down bugzilla and waiting for connections to clear");
|
||||
# Record the old shutdownhtml so it can be restored at the end (this will
|
||||
# only be an issue if we are called using the -force command line param)
|
||||
$wasshutdown = Param("shutdownhtml");
|
||||
SetParam('shutdownhtml', $shutdown_msg);
|
||||
WriteParams();
|
||||
# Now we need to wait for existing connections to this database to clear. We
|
||||
# do this by looking for connections to the main or shadow database using
|
||||
# 'mysqladmin processlist'
|
||||
my $cmd = "$::mysqlpath/mysqladmin -u $::db_user -h $::db_host -P $::db_port";
|
||||
if ($::db_pass) { $cmd .= " -p$::db_pass"; }
|
||||
if ($::db_sock) { $cmd .= " -S$::db_sock"; }
|
||||
$cmd .= " processlist";
|
||||
my $found_proc = 1;
|
||||
# We need to put together a nice little regular expression to use in the
|
||||
# following loop that'll tell us if the return from mysqladmin contains
|
||||
# either the main or shadow database.
|
||||
my @dbs = ($::db_name, Param("shadowdb"));
|
||||
my $db_expr = "^\\s*(" . join ("\|", @dbs) . ")\\s*\$";
|
||||
# Don't let this thing wait forever...
|
||||
my $starttime = time();
|
||||
while ($found_proc) {
|
||||
$found_proc = 0;
|
||||
open (PROC, $cmd . "|");
|
||||
my @output = <PROC>;
|
||||
close (PROC);
|
||||
foreach my $line(@output) {
|
||||
my @info = split (/\|/, $line);
|
||||
# Ignore any line that doesn't have 9 pieces of info
|
||||
# or contain Id (pretty printing crap)
|
||||
if ($#info != 9 || $line =~ /Id/) { next }
|
||||
if ($info[4] =~ m/$db_expr/) {
|
||||
$found_proc = 1;
|
||||
}
|
||||
}
|
||||
# If there are still active connections to Bugzilla 10 minutes after
|
||||
# shutting it down, then something is wrong.
|
||||
if ((time() - $starttime) > 600) {
|
||||
# There should be a better way to notify the admin of something bad like
|
||||
# this happening.
|
||||
Verbose ("*** Waited for 10 minutes and there were still active \n" .
|
||||
" connections to the bugzilla database. Giving up.");
|
||||
SetParam('shutdownhtml', $wasshutdown);
|
||||
WriteParams();
|
||||
exit;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
my $wasusing = Param("queryagainstshadowdb");
|
||||
|
||||
SetParam('queryagainstshadowdb', 1); # Force us to be able to use the
|
||||
# shadowdb, even if other processes
|
||||
# are not supposed to.
|
||||
|
||||
|
||||
ConnectToDatabase(1);
|
||||
|
||||
Verbose("Acquiring lock");
|
||||
if ( $syncall == 1) {
|
||||
SendSQL("SELECT GET_LOCK('synclock', 2700)");
|
||||
} else {
|
||||
SendSQL("SELECT GET_LOCK('synclock', 1)");
|
||||
}
|
||||
if (!FetchOneColumn()) {
|
||||
Verbose("Couldn't get the lock to do the shadow database syncing.");
|
||||
exit;
|
||||
}
|
||||
|
||||
my $shadowtable = "$::db_name.shadowlog";
|
||||
|
||||
if (!$syncall) {
|
||||
Verbose("Looking for requests to sync the whole database.");
|
||||
SendSQL("SELECT id FROM $shadowtable " .
|
||||
"WHERE reflected = 0 AND command = 'SYNCUP'");
|
||||
if (FetchOneColumn()) {
|
||||
$syncall = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if ($syncall) {
|
||||
Verbose("Syncing up the shadow database by copying entire database in.");
|
||||
if ($wasusing) {
|
||||
SetParam('queryagainstshadowdb',0);
|
||||
WriteParams();
|
||||
if (! $shutdown) {
|
||||
Verbose("Disabled reading from the shadowdb. Sleeping 10 seconds to let other procs catch up.");
|
||||
sleep(10);
|
||||
}
|
||||
SetParam('queryagainstshadowdb', 1);
|
||||
}
|
||||
my @tables;
|
||||
SendSQL("SHOW TABLES");
|
||||
my $query = "";
|
||||
while (MoreSQLData()) {
|
||||
my $table = FetchOneColumn();
|
||||
push(@tables, $table);
|
||||
if ($query) {
|
||||
$query .= ", $table WRITE";
|
||||
} else {
|
||||
$query = "LOCK TABLES $table WRITE";
|
||||
}
|
||||
}
|
||||
if (@tables) {
|
||||
Verbose("Locking entire shadow database");
|
||||
SendSQL($query);
|
||||
foreach my $table (@tables) {
|
||||
Verbose("Dropping old shadow table $table");
|
||||
SendSQL("DROP TABLE $table");
|
||||
}
|
||||
SendSQL("UNLOCK TABLES");
|
||||
}
|
||||
# Carefully lock the whole real database for reading, except for the
|
||||
# shadowlog table, which we lock for writing. Then dump everything
|
||||
# into the shadowdb database. Then mark everything in the shadowlog
|
||||
# as reflected. Only then unlock everything. This sequence causes
|
||||
# us to be sure not to miss anything or get something twice.
|
||||
SendSQL("USE $::db_name");
|
||||
SendSQL("SHOW TABLES");
|
||||
@tables = ();
|
||||
$query = "LOCK TABLES shadowlog WRITE";
|
||||
while (MoreSQLData()) {
|
||||
my $table = FetchOneColumn();
|
||||
if ($table ne "shadowlog") {
|
||||
$query .= ", $table READ";
|
||||
push(@tables, $table);
|
||||
}
|
||||
}
|
||||
Verbose("Locking entire database");
|
||||
SendSQL($query);
|
||||
my $tempfile = "$tempdir/tmpsyncshadow.$$";
|
||||
Verbose("Dumping database to a temp file ($tempfile).");
|
||||
my @ARGS = ("-u", $::db_user);
|
||||
if ($::db_pass) { push @ARGS, "-p$::db_pass" }
|
||||
if ($::db_sock) { push @ARGS, "-S$::db_sock" }
|
||||
push @ARGS, "-l", "-e", $::db_name, @tables;
|
||||
open SAVEOUT, ">&STDOUT"; # stash the original output stream
|
||||
open STDOUT, ">$tempfile"; # redirect to file
|
||||
select STDOUT; $| = 1; # disable buffering
|
||||
system("$::mysqlpath/mysqldump", @ARGS);
|
||||
open STDOUT, ">&SAVEOUT"; # redirect back to original stream
|
||||
Verbose("Restoring from tempfile into shadowdb");
|
||||
my $extra = "-u $::db_user";
|
||||
if ($::db_pass) {
|
||||
$extra .= " -p$::db_pass";
|
||||
}
|
||||
if ($::db_sock) {
|
||||
$extra .= " -S$::db_sock";
|
||||
}
|
||||
if ($verbose) {
|
||||
$extra .= " -v";
|
||||
}
|
||||
open(MYSQL, "/bin/cat $tempfile | $::mysqlpath/mysql $extra " .
|
||||
Param("shadowdb") . "|") || die "Couldn't do db copy";
|
||||
my $count = 0;
|
||||
while (<MYSQL>) {
|
||||
print ".";
|
||||
$count++;
|
||||
if ($count % 70 == 0) {
|
||||
print "\n";
|
||||
}
|
||||
}
|
||||
close(MYSQL);
|
||||
unlink($tempfile);
|
||||
Verbose("");
|
||||
|
||||
|
||||
$::dbwritesallowed = 1;
|
||||
# SendSQL("UPDATE shadowlog SET reflected = 1 WHERE reflected = 0", 1);
|
||||
SendSQL("DELETE FROM shadowlog", 1);
|
||||
SendSQL("UNLOCK TABLES");
|
||||
if ($wasusing) {
|
||||
Verbose("Reenabling other processes to read from the shadow db");
|
||||
SetParam('queryagainstshadowdb', 1);
|
||||
WriteParams();
|
||||
}
|
||||
if ($shutdown) {
|
||||
Verbose("Restoring the original shutdown message (if any)");
|
||||
SetParam('shutdownhtml', $wasshutdown);
|
||||
WriteParams();
|
||||
}
|
||||
Verbose("OK, done.");
|
||||
}
|
||||
|
||||
Verbose("Looking for commands to execute.");
|
||||
$::dbwritesallowed = 1;
|
||||
|
||||
# Make us low priority, to not block anyone who is trying to actually use
|
||||
# the shadowdb. Note that this is carefully coded to ignore errors; we want
|
||||
# to keep going even on older mysqld's that don't have the
|
||||
# SQL_LOW_PRIORITY_UPDATES option.
|
||||
$::db->do("SET OPTION SQL_LOW_PRIORITY_UPDATES = 1");
|
||||
|
||||
while (1) {
|
||||
SendSQL("SELECT id, command FROM $shadowtable WHERE reflected = 0 " .
|
||||
"ORDER BY id LIMIT 1");
|
||||
my ($id, $command) = (FetchSQLData());
|
||||
if (!$id) {
|
||||
last;
|
||||
}
|
||||
Verbose("Executing command in shadow db: $command");
|
||||
SendSQL($command, 1);
|
||||
SendSQL("UPDATE $shadowtable SET reflected = 1 WHERE id = $id", 1);
|
||||
}
|
||||
|
||||
Verbose("Releasing lock.");
|
||||
SendSQL("SELECT RELEASE_LOCK('synclock')");
|
||||
@ -53,5 +53,3 @@
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
[% CALL SyncAnyPendingShadowChanges() IF SyncAnyPendingShadowChanges %]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user