Skip to content

bugzilla
bugzilla
Commit 0becc20d authored by gerv%gerv.net's avatar gerv%gerv.net
Browse files
Options

Bug 80157 - Add "regenerate" option to collectstats.pl. Patch by…

Bug 80157 - Add "regenerate" option to collectstats.pl. Patch by rwalters@qualcomm.com, with tweaks by jean_seb@hybride.com, dberlin@dberlin.org and gerv@mozilla.org; r=gerv, a=justdave.
parent b0b9a238
Hide whitespace changes
Inline Side-by-side
Showing with 202 additions and 5 deletions
collectstats.pl
View file @ 0becc20d
......@@ -20,20 +20,27 @@
#
# Contributor(s): Terry Weissman <terry@mozilla.org>,
# Harrison Page <harrison@netscape.com>
# Gervase Markham <gerv@gerv.net>
# Gervase Markham <gerv@gerv.net>
# Richard Walters <rwalters@qualcomm.com>
# Jean-Sebastien Guay <jean_seb@hybride.com>
# Run me out of cron at midnight to collect Bugzilla statistics.
use AnyDBM_File;
use strict;
use IO::Handle;
use vars @::legal_product;
require "globals.pl";
use Bugzilla;
# tidy up after graphing module
# Turn off output buffering (probably needed when displaying output feedback
# in the regenerate mode.)
$| = 1;
# Tidy up after graphing module
if (chdir("graphs")) {
unlink <./*.gif>;
unlink <./*.png>;
......@@ -45,15 +52,29 @@ GetVersionTable();
Bugzilla->switch_to_shadow_db();
# To recreate the daily statistics, run "collectstats.pl --regenerate" .
my $regenerate = 0;
if ($#ARGV >= 0 && $ARGV[0] eq "--regenerate") {
$regenerate = 1;
}
my @myproducts;
push( @myproducts, "-All-", @::legal_product );
my $tstart = time;
foreach (@myproducts) {
my $dir = "data/mining";
&check_data_dir ($dir);
&collect_stats ($dir, $_);
if ($regenerate) {
&regenerate_stats($dir, $_);
} else {
&collect_stats($dir, $_);
}
}
my $tend = time;
print "Total time taken " . delta_time($tstart, $tend) . "\n";
&calculate_dupes();
......@@ -78,8 +99,8 @@ sub check_data_dir {
my $dir = shift;
if (! -d $dir) {
mkdir $dir, 0777;
chmod 0777, $dir;
mkdir $dir, 0755;
chmod 0755, $dir;
}
}
......@@ -135,6 +156,7 @@ FIN
print DATA (join '|', @row) . "\n";
close DATA;
chmod 0644, $file;
} else {
print "$0: $file, $!";
}
......@@ -213,6 +235,165 @@ sub calculate_dupes {
dbmclose(%count);
}
# This regenerates all statistics from the database.
sub regenerate_stats {
my $dir = shift;
my $product = shift;
my $when = localtime(time());
my $tstart = time();
# NB: Need to mangle the product for the filename, but use the real
# product name in the query
my $file_product = $product;
$file_product =~ s/\//-/gs;
my $file = join '/', $dir, $file_product;
my @bugs;
my $and_product = "";
my $from_product = "";
if ($product ne '-All-') {
$and_product = "AND bugs.product_id = products.id " .
"AND products.name = " . SqlQuote($product) . " ";
$from_product = ", products";
}
# Determine the start date from the date the first bug in the
# database was created, and the end date from the current day.
# If there were no bugs in the search, return early.
SendSQL("SELECT to_days(creation_ts) AS start, " .
"to_days(current_date) AS end, " .
"to_days('1970-01-01') " .
"FROM bugs $from_product WHERE to_days(creation_ts) != 'NULL' " .
$and_product .
"ORDER BY start LIMIT 1");
my ($start, $end, $base) = FetchSQLData();
if (!defined $start) {
return;
}
if (open DATA, ">$file") {
DATA->autoflush(1);
print DATA <<FIN;
# Bugzilla Daily Bug Stats
#
# Do not edit me! This file is generated.
#
# fields: DATE|NEW|ASSIGNED|REOPENED|UNCONFIRMED|RESOLVED|VERIFIED|CLOSED|FIXED|INVALID|WONTFIX|LATER|REMIND|DUPLICATE|WORKSFORME|MOVED
# Product: $product
# Created: $when
FIN
# For each day, generate a line of statistics.
my $total_days = $end - $start;
for (my $day = $start + 1; $day <= $end; $day++) {
# Some output feedback
my $percent_done = ($day - $start - 1) * 100 / $total_days;
printf "\rRegenerating $product \[\%.1f\%\%]", $percent_done;
# Get a list of bugs that were created the previous day, and
# add those bugs to the list of bugs for this product.
SendSQL("SELECT bug_id FROM bugs $from_product " .
"WHERE bugs.creation_ts < from_days(" . ($day - 1) . ") " .
"AND bugs.creation_ts >= from_days(" . ($day - 2) . ") " .
$and_product .
" ORDER BY bug_id");
my @row;
while (@row = FetchSQLData()) {
push @bugs, $row[0];
}
# For each bug that existed on that day, determine its status
# at the beginning of the day. If there were no status
# changes on or after that day, the status was the same as it
# is today, which can be found in the bugs table. Otherwise,
# the status was equal to the first "previous value" entry in
# the bugs_activity table for that bug made on or after that
# day.
my %bugcount;
my @logstates = qw(NEW ASSIGNED REOPENED UNCONFIRMED RESOLVED
VERIFIED CLOSED);
my @logresolutions = qw(FIXED INVALID WONTFIX LATER REMIND
DUPLICATE WORKSFORME MOVED);
foreach (@logstates) {
$bugcount{$_} = 0;
}
foreach (@logresolutions) {
$bugcount{$_} = 0;
}
for my $bug (@bugs) {
# First, get information on various bug states.
SendSQL("SELECT bugs_activity.removed " .
"FROM bugs_activity,fielddefs " .
"WHERE bugs_activity.fieldid = fielddefs.fieldid " .
"AND fielddefs.name = 'bug_status' " .
"AND bugs_activity.bug_id = $bug " .
"AND bugs_activity.bug_when >= from_days($day) " .
"ORDER BY bugs_activity.bug_when LIMIT 1");
my $status;
if (@row = FetchSQLData()) {
$status = $row[0];
} else {
SendSQL("SELECT bug_status FROM bugs WHERE bug_id = $bug");
$status = FetchOneColumn();
}
if (defined $bugcount{$status}) {
$bugcount{$status}++;
}
# Next, get information on various bug resolutions.
SendSQL("SELECT bugs_activity.removed " .
"FROM bugs_activity,fielddefs " .
"WHERE bugs_activity.fieldid = fielddefs.fieldid " .
"AND fielddefs.name = 'resolution' " .
"AND bugs_activity.bug_id = $bug " .
"AND bugs_activity.bug_when >= from_days($day) " .
"ORDER BY bugs_activity.bug_when LIMIT 1");
if (@row = FetchSQLData()) {
$status = $row[0];
} else {
SendSQL("SELECT resolution FROM bugs WHERE bug_id = $bug");
$status = FetchOneColumn();
}
if (defined $bugcount{$status}) {
$bugcount{$status}++;
}
}
# Generate a line of output containing the date and counts
# of bugs in each state.
my $date = sqlday($day, $base);
print DATA "$date";
foreach (@logstates) {
print DATA "|$bugcount{$_}";
}
foreach (@logresolutions) {
print DATA "|$bugcount{$_}";
}
print DATA "\n";
}
# Finish up output feedback for this product.
my $tend = time;
print "\rRegenerating $product \[100.0\%] - " .
delta_time($tstart, $tend) . "\n";
close DATA;
chmod 0640, $file;
}
}
sub today {
my ($dom, $mon, $year) = (localtime(time))[3, 4, 5];
return sprintf "%04d%02d%02d", 1900 + $year, ++$mon, $dom;
......@@ -223,3 +404,19 @@ sub today_dash {
return sprintf "%04d-%02d-%02d", 1900 + $year, ++$mon, $dom;
}
sub sqlday {
my ($day, $base) = @_;
$day = ($day - $base) * 86400;
my ($dom, $mon, $year) = (gmtime($day))[3, 4, 5];
return sprintf "%04d%02d%02d", 1900 + $year, ++$mon, $dom;
}
sub delta_time {
my $tstart = shift;
my $tend = shift;
my $delta = $tend - $tstart;
my $hours = int($delta/3600);
my $minutes = int($delta/60) - ($hours * 60);
my $seconds = $delta - ($minutes * 60) - ($hours * 3600);
return sprintf("%02d:%02d:%02d" , $hours, $minutes, $seconds);
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment