Skip to content

bugzilla
bugzilla
Commit a9fd0d7e authored by mkanat%bugzilla.org's avatar mkanat%bugzilla.org
Browse files
Options

Bug 442882: Populating bugs_fulltext can be very slow on large installations

Patch By Max Kanat-Alexander <mkanat@bugzilla.org> r=dkl, a=mkanat
parent 09a46476
Hide whitespace changes
Inline Side-by-side
Showing with 59 additions and 19 deletions
Bugzilla/DB/Mysql.pm
View file @ a9fd0d7e
...@@ -106,6 +106,15 @@ sub bz_last_key { ...@@ -106,6 +106,15 @@ sub bz_last_key {
return $last_insert_id; return $last_insert_id;
} }
sub sql_group_concat {
my ($self, $column, $separator) = @_;
my $sep_sql;
if ($separator) {
$sep_sql = " SEPARATOR $separator";
}
return "GROUP_CONCAT($column$sep_sql)";
}
sub sql_regexp { sub sql_regexp {
my ($self, $expr, $pattern, $nocheck) = @_; my ($self, $expr, $pattern, $nocheck) = @_;
......
Bugzilla/Install/DB.pm
View file @ a9fd0d7e
...@@ -3029,26 +3029,57 @@ sub _populate_bugs_fulltext { ...@@ -3029,26 +3029,57 @@ sub _populate_bugs_fulltext {
my $bug_ids = $dbh->selectcol_arrayref('SELECT bug_id FROM bugs'); my $bug_ids = $dbh->selectcol_arrayref('SELECT bug_id FROM bugs');
return if !@$bug_ids; return if !@$bug_ids;
print "Populating bugs_fulltext.short_desc...\n"; # Populating bugs_fulltext can be very slow for large installs,
$dbh->do('INSERT INTO bugs_fulltext (bug_id, short_desc) # so we special-case any DB that supports GROUP_CONCAT, which is
SELECT bug_id, short_desc FROM bugs'); # a much faster way to do things.
print "Populating bugs_fulltext comments fields...\n"; if (UNIVERSAL::can($dbh, 'sql_group_concat')) {
my $count = 1; print "Populating bugs_fulltext...";
my $sth_all = $dbh->prepare('SELECT thetext FROM longdescs print " (this can take a long time.)\n";
WHERE bug_id = ?'); # XXX This hack should probably be moved elsewhere.
my $sth_nopriv = $dbh->prepare('SELECT thetext FROM longdescs if ($dbh->isa('Bugzilla::DB::Mysql')) {
WHERE bug_id = ? AND isprivate = 0'); $dbh->do('SET SESSION group_concat_max_len = 128000000');
my $sth_update = $dbh->prepare( $dbh->do('SET SESSION max_allowed_packet = 128000000');
'UPDATE bugs_fulltext SET comments = ?, comments_noprivate = ? }
WHERE bug_id = ?'); $dbh->do(
foreach my $id (@$bug_ids) { q{INSERT INTO bugs_fulltext (bug_id, short_desc, comments,
my $all = $dbh->selectcol_arrayref($sth_all, undef, $id); comments_noprivate)
my $nopriv = $dbh->selectcol_arrayref($sth_nopriv, undef, $id); SELECT bugs.bug_id, bugs.short_desc, }
$sth_update->execute(join("\n", @$all), join("\n", @$nopriv), $id); . $dbh->sql_group_concat('longdescs.thetext', '\'\n\'')
indicate_progress({ total => scalar @$bug_ids, every => 100, . ', ' . $dbh->sql_group_concat('nopriv.thetext', '\'\n\'') .
current => $count++ }); q{ FROM bugs
LEFT JOIN longdescs
ON bugs.bug_id = longdescs.bug_id
LEFT JOIN longdescs AS nopriv
ON longdescs.comment_id = nopriv.comment_id
AND nopriv.isprivate = 0 }
. $dbh->sql_group_by('bugs.bug_id', 'bugs.short_desc'));
}
# The slow way, without group_concat.
else {
print "Populating bugs_fulltext.short_desc...\n";
$dbh->do('INSERT INTO bugs_fulltext (bug_id, short_desc)
SELECT bug_id, short_desc FROM bugs');
my $count = 1;
my $sth_all = $dbh->prepare('SELECT thetext FROM longdescs
WHERE bug_id = ?');
my $sth_nopriv = $dbh->prepare(
'SELECT thetext FROM longdescs
WHERE bug_id = ? AND isprivate = 0');
my $sth_update = $dbh->prepare(
'UPDATE bugs_fulltext SET comments = ?, comments_noprivate = ?
WHERE bug_id = ?');
print "Populating bugs_fulltext comment fields...\n";
foreach my $id (@$bug_ids) {
my $all = $dbh->selectcol_arrayref($sth_all, undef, $id);
my $nopriv = $dbh->selectcol_arrayref($sth_nopriv, undef, $id);
$sth_update->execute(join("\n", @$all), join("\n", @$nopriv), $id);
indicate_progress({ total => scalar @$bug_ids, every => 100,
current => $count++ });
}
print "\n";
} }
print "\n";
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment