Bug 252272: Allow extremely large attachments to be stored locally

Bugzilla/Attachment.pm

@@ -33,6 +33,7 @@ package Bugzilla::Attachment;
 
 # Use the Flag module to handle flags.
 use Bugzilla::Flag;
+use Bugzilla::Config qw(:locations);
 
 ############################################################################
 # Functions
@@ -93,6 +94,17 @@ sub query
     $a{'flags'} = Bugzilla::Flag::match({ 'attach_id' => $a{'attachid'},
                                           'is_active' => 1 });
 
+    # A zero size indicates that the attachment is stored locally.
+    if ($a{'datasize'} == 0) {
+        my $attachid = $a{'attachid'};
+        my $hash = ($attachid % 100) + 100;
+        $hash =~ s/.*(\d\d)$/group.$1/;
+        if (open(AH, "$attachdir/$hash/attachment.$attachid")) {
+            $a{'datasize'} = (stat(AH))[7];
+            close(AH);
+        }
+    }
+    
     # We will display the edit link if the user can edit the attachment;
     # ie the are the submitter, or they have canedit.
     # Also show the link if the user is not logged in - in that cae,

Bugzilla/Config.pm

@@ -55,6 +55,7 @@ use Bugzilla::Util;
 our $libpath = '.';
 our $localconfig = "$libpath/localconfig";
 our $datadir = "$libpath/data";
+our $attachdir = "$datadir/attachments";
 our $templatedir = "$libpath/template";
 our $webdotdir = "$datadir/webdot";
 
@@ -72,7 +73,8 @@ our $webdotdir = "$datadir/webdot";
   (
    admin => [qw(GetParamList UpdateParams SetParam WriteParams)],
    db => [qw($db_driver $db_host $db_port $db_name $db_user $db_pass $db_sock)],
-   locations => [qw($libpath $localconfig $datadir $templatedir $webdotdir)],
+   locations => [qw($libpath $localconfig $attachdir
+                    $datadir $templatedir $webdotdir)],
   );
 Exporter::export_ok_tags('admin', 'db', 'locations');
 

attachment.cgi

@@ -40,6 +40,7 @@ use vars qw(
 
 # Include the Bugzilla CGI and general utility library.
 require "CGI.pl";
+use Bugzilla::Config qw(:locations);
 
 # Use these modules to handle flags.
 use Bugzilla::Constants;
@@ -360,12 +361,18 @@ sub validateData
 {
   my $maxsize = $::FORM{'ispatch'} ? Param('maxpatchsize') : Param('maxattachmentsize');
   $maxsize *= 1024; # Convert from K
-
-  my $fh = $cgi->upload('data');
+  my $fh;
+  # Skip uploading into a local variable if the user wants to upload huge
+  # attachments into local files.
+  if (!$::FORM{'bigfile'})
+  {
+    $fh = $cgi->upload('data');
+  }
   my $data;
 
   # We could get away with reading only as much as required, except that then
   # we wouldn't have a size to print to the error handler below.
+  if (!$::FORM{'bigfile'})
   {
       # enable 'slurp' mode
       local $/;
@@ -373,10 +380,11 @@ sub validateData
   }
 
   $data
+    || ($::FORM{'bigfile'})
     || ThrowUserError("zero_length_file");
 
   # Make sure the attachment does not exceed the maximum permitted size
-  my $len = length($data);
+  my $len = $data ? length($data) : 0;
   if ($maxsize && $len > $maxsize) {
       my $vars = { filesize => sprintf("%.0f", $len/1024) };
       if ( $::FORM{'ispatch'} ) {
@@ -504,6 +512,23 @@ sub view
     # Return the appropriate HTTP response headers.
     $filename =~ s/^.*[\/\\]//;
     my $filesize = length($thedata);
+    # A zero length attachment in the database means the attachment is 
+    # stored in a local file
+    if ($filesize == 0)
+    {
+        my $attachid = $::FORM{'id'};
+        my $hash = ($attachid % 100) + 100;
+        $hash =~ s/.*(\d\d)$/group.$1/;
+        if (open(AH, "$attachdir/$hash/attachment.$attachid")) {
+            binmode AH;
+            $filesize = (stat(AH))[7];
+        }
+    }
+    if ($filesize == 0)
+    {
+        ThrowUserError("attachment_removed");
+    }
+
 
     # escape quotes and backslashes in the filename, per RFCs 2045/822
     $filename =~ s/\\/\\\\/g; # escape backslashes
@@ -513,7 +538,15 @@ sub view
                                 -content_disposition=> "inline; filename=\"$filename\"",
                                 -content_length => $filesize);
 
+    if ($thedata) {
         print $thedata;
+    } else {
+        while (<AH>) {
+            print $_;
+        }
+        close(AH);
+    }
+
 }
 
 sub interdiff
@@ -889,6 +922,34 @@ sub insert
   # Retrieve the ID of the newly created attachment record.
   my $attachid = $dbh->bz_last_key('attachments', 'attach_id');
 
+  # If the file is to be stored locally, stream the file from the webserver
+  # to the local file without reading it into a local variable.
+  if ($::FORM{'bigfile'})
+  {
+    my $fh = $cgi->upload('data');
+    my $hash = ($attachid % 100) + 100;
+    $hash =~ s/.*(\d\d)$/group.$1/;
+    mkdir "$attachdir/$hash", 0770;
+    chmod 0770, "$attachdir/$hash";
+    open(AH, ">$attachdir/$hash/attachment.$attachid");
+    binmode AH;
+    my $sizecount = 0;
+    my $limit = (Param("maxlocalattachment") * 1048576);
+    while (<$fh>) {
+        print AH $_;
+        $sizecount += length($_);
+        if ($sizecount > $limit) {
+            close AH;
+            close $fh;
+            unlink "$attachdir/$hash/attachment.$attachid";
+            ThrowUserError("local_file_too_large");
+        }
+    }
+    close AH;
+    close $fh;
+  }
+
+
   # Insert a comment about the new attachment into the database.
   my $comment = "Created an attachment (id=$attachid)\n$::FORM{'description'}\n";
   $comment .= ("\n" . $::FORM{'comment'}) if $::FORM{'comment'};
@@ -1090,7 +1151,7 @@ sub update
            SET     description = $quoteddescription ,
                    mimetype = $quotedcontenttype ,
                    filename = $quotedfilename ,
-                   ispatch = $::FORM{'ispatch'} , 
+                   ispatch = $::FORM{'ispatch'},
                    isobsolete = $::FORM{'isobsolete'} ,
                    isprivate = $::FORM{'isprivate'} 
            WHERE   attach_id = $::FORM{'id'}

checksetup.pl

@@ -904,6 +904,14 @@ unless (-d $datadir && -e "$datadir/nomail") {
     open FILE, '>>', "$datadir/mail"; close FILE;
 }
 
+
+ unless (-d $attachdir) {
+     print "Creating local attachments directory ...\n";
+     # permissions for non-webservergroup are fixed later on
+     mkdir $attachdir, 0770;
+ }
+
+
 # 2000-12-14 New graphing system requires a directory to put the graphs in
 # This code copied from what happens for the data dir above.
 # If the graphs dir is not present, we assume that they have been using
@@ -1088,6 +1096,17 @@ END
     }
 
   }
+  if (!-e "$attachdir/.htaccess") {
+    print "Creating $attachdir/.htaccess...\n";
+    open HTACCESS, ">$attachdir/.htaccess";
+    print HTACCESS <<'END';
+# nothing in this directory is retrievable unless overriden by an .htaccess
+# in a subdirectory;
+deny from all
+END
+    close HTACCESS;
+    chmod $fileperm, "$attachdir/.htaccess";
+  }
   if (!-e "Bugzilla/.htaccess") {
     print "Creating Bugzilla/.htaccess...\n";
     open HTACCESS, '>', 'Bugzilla/.htaccess';
@@ -1428,6 +1447,7 @@ if ($^O !~ /MSWin32/i) {
         fixPerms("$datadir/duplicates", $<, $webservergid, 027, 1);
         fixPerms("$datadir/mining", $<, $webservergid, 027, 1);
         fixPerms("$datadir/template", $<, $webservergid, 007, 1); # webserver will write to these
+        fixPerms($attachdir, $<, $webservergid, 007, 1); # webserver will write to these
         fixPerms($webdotdir, $<, $webservergid, 007, 1);
         fixPerms("$webdotdir/.htaccess", $<, $webservergid, 027);
         fixPerms("$datadir/params", $<, $webservergid, 017);

defparams.pl

@@ -1270,6 +1270,17 @@ Reason: %reason%
   },
 
   {
+   name => 'maxlocalattachment',
+   desc => 'The maximum size (in Megabytes) of attachments identified by ' .
+           'the user as "Big Files" to be stored locally on the webserver. ' .
+           'If set to zero, attachments will never be kept on the local ' .
+           'filesystem.',
+   type => 't',
+   default => '0',
+   checker => \&check_numeric
+  },
+
+  {
    name => 'chartgroup',
    desc => 'The name of the group of users who can use the "New Charts" ' .
            'feature. Administrators should ensure that the public categories ' .

template/en/default/attachment/create.html.tmpl

@@ -65,6 +65,18 @@
         <input type="file" id="data" name="data" size="50">
       </td>
     </tr>
+    [% IF Param("maxlocalattachment") %]
+    <tr>
+      <th>BigFile:</th>
+      <td>
+        <input type="checkbox" id="bigfile"
+               name="bigfile" value="bigfile">
+        <label for="bigfile">
+          Big File - Stored locally and may be purged
+        </label>
+      </td>
+    </tr>
+    [% END %]
     <tr>
       <th><label for="description">Description:</label></th>
       <td>

template/en/default/global/user-error.html.tmpl

@@ -156,6 +156,10 @@
     [% title = "Access Denied" %]
     You are not authorized to access this attachment.
 
+  [% ELSIF error == "attachment_removed" %]
+    [% title = "Attachment Removed" %]
+    The attachment you are attempting to access has been removed.
+
   [% ELSIF error == "bug_access_denied" %]
     [% title = "Access Denied" %]
     You are not authorized to access [% terms.bug %] #[% bug_id FILTER html %].
@@ -604,6 +608,11 @@
     [% title = "Invalid Keyword Name" %]
     You may not use commas or whitespace in a keyword name.
      
+  [% ELSIF error == "local_file_too_large" %]
+    [% title = "Local File Too Large" %]
+    Local file uploads must not exceed 
+    [% Param('maxlocalattachment') %] MB in size.
+
   [% ELSIF error == "login_needed_for_password_change" %]
     [% title = "Login Name Required" %]
     You must enter a login name when requesting to change your password.