gnatsparse.py

try:
# Using Psyco makes it about 25% faster, but there's a bug in psyco in
# handling of eval causing it to use unlimited memory with the magic
# file enabled.
#    import psyco
#    psyco.full()
#    from psyco.classes import *
    pass
except:
    pass
import re
import base64
import cStringIO
import specialuu
import array
import email.Utils
import zlib
import magic

# Comment out if you don't want magic detection
magicf = magic.MagicFile()

# Open our output file
outfile = open("gnats2bz_data.sql", "w")

# List of GNATS fields
fieldnames = ("Number", "Category", "Synopsis", "Confidential", "Severity",
              "Priority", "Responsible", "State", "Quarter", "Keywords",
              "Date-Required", "Class", "Submitter-Id", "Arrival-Date",
              "Closed-Date", "Last-Modified", "Originator", "Release",
              "Organization", "Environment", "Description", "How-To-Repeat",
              "Fix", "Release-Note", "Audit-Trail", "Unformatted")

# Dictionary telling us which GNATS fields are multiline
multilinefields = {"Organization":1, "Environment":1, "Description":1,
                   "How-To-Repeat":1, "Fix":1, "Release-Note":1,
                   "Audit-Trail":1, "Unformatted":1}

# Mapping of GCC release to version. Our version string is updated every
# so we need to funnel all release's with 3.4 in the string to be version
# 3.4 for bug tracking purposes
# The key is a regex to match, the value is the version it corresponds
# with
releasetovermap = {r"3\.4":"3.4", r"3\.3":"3.3", r"3\.2\.2":"3.2.2",
                   r"3\.2\.1":"3.2.1", r"3\.2":"3.2", r"3\.1\.2":"3.1.2",
                   r"3\.1\.1":"3.1.1", r"3\.1":"3.1", r"3\.0\.4":"3.0.4",
                   r"3\.0\.3":"3.0.3", r"3\.0\.2":"3.0.2", r"3\.0\.1":"3.0.1",
                   r"3\.0":"3.0", r"2\.95\.4":"2.95.4", r"2\.95\.3":"2.95.3",
                   r"2\.95\.2":"2.95.2", r"2\.95\.1":"2.95.1",
                   r"2\.95":"2.95", r"2\.97":"2.97",
                   r"2\.96.*[rR][eE][dD].*[hH][aA][tT]":"2.96 (redhat)",
                   r"2\.96":"2.96"}

# These map the field name to the field id bugzilla assigns. We need
# the id when doing bug activity.
fieldids = {"State":8, "Responsible":15}

# These are the keywords we use in gcc bug tracking. They are transformed
# into bugzilla keywords.  The format here is <keyword>-><bugzilla keyword id>
keywordids = {"wrong-code":1, "ice-on-legal-code":2, "ice-on-illegal-code":3,
              "rejects-legal":4, "accepts-illegal":5, "pessimizes-code":6}

# Map from GNATS states to Bugzilla states.  Duplicates and reopened bugs
# are handled when parsing the audit trail, so no need for them here.
state_lookup = {"":"NEW", "open":"ASSIGNED", "analyzed":"ASSIGNED",
                "feedback":"WAITING", "closed":"CLOSED",
                "suspended":"SUSPENDED"}

# Table of versions that exist in the bugs, built up as we go along
versions_table = {}

# Delimiter gnatsweb uses for attachments
attachment_delimiter = "----gnatsweb-attachment----\n"

# Here starts the various regular expressions we use
# Matches an entire GNATS single line field
gnatfieldre = re.compile(r"""^([>\w\-]+)\s*:\s*(.*)\s*$""")

# Matches the name of a GNATS field
fieldnamere = re.compile(r"""^>(.*)$""")

# Matches the useless part of an envelope
uselessre = re.compile(r"""^(\S*?):\s*""", re.MULTILINE)

# Matches the filename in a content disposition
dispositionre = re.compile("(\\S+);\\s*filename=\"([^\"]+)\"")

# Matches the last changed date in the entire text of a bug
# If you have other editable fields that get audit trail entries, modify this
# The field names are explicitly listed in order to speed up matching
lastdatere = re.compile(r"""^(?:(?:State|Responsible|Priority|Severity)-Changed-When: )(.+?)$""", re.MULTILINE)

# Matches the From line of an email or the first line of an audit trail entry
# We use this re to find the begin lines of all the audit trail entries
# The field names are explicitly listed in order to speed up matching
fromtore=re.compile(r"""^(?:(?:State|Responsible|Priority|Severity)-Changed-From-To: |From: )""", re.MULTILINE)

# These re's match the various parts of an audit trail entry
changedfromtore=re.compile(r"""^(\w+?)-Changed-From-To: (.+?)$""", re.MULTILINE)
changedbyre=re.compile(r"""^\w+?-Changed-By: (.+?)$""", re.MULTILINE)
changedwhenre=re.compile(r"""^\w+?-Changed-When: (.+?)$""", re.MULTILINE)
changedwhyre=re.compile(r"""^\w+?-Changed-Why:\s*(.*?)$""", re.MULTILINE)

# This re matches audit trail text saying that the current bug is a duplicate of another
duplicatere=re.compile(r"""(?:")?Dup(?:licate)?(?:d)?(?:")? of .*?(\d+)""", re.IGNORECASE | re.MULTILINE)

# Get the text of a From: line
fromre=re.compile(r"""^From: (.*?)$""", re.MULTILINE)

# Get the text of a Date: Line
datere=re.compile(r"""^Date: (.*?)$""", re.MULTILINE)

#  Map of the responsible file to email addresses
responsible_map = {}
#  List of records in the responsible file
responsible_list = []
#  List of records in the categories file
categories_list = []
# List of pr's in the index
pr_list = []
# Map usernames to user ids
usermapping = {}
# Start with this user id
userid_base = 2

# Name of gnats user
gnats_username = "gnats@gcc.gnu.org"
# Name of unassigned user
unassigned_username = "unassigned@gcc.gnu.org"

gnats_db_dir = "."
product = "gcc"
productdesc = "GNU Compiler Connection"
milestoneurl = "http://gcc/gnu.org"
defaultmilestone = "3.4"

def write_non_bug_tables():
    """ Write out the non-bug related tables, such as products, profiles, etc."""
    # Set all non-unconfirmed bugs's everconfirmed flag
    print >>outfile, "update bugs set everconfirmed=1 where bug_status != 'UNCONFIRMED';"

    # Set all bugs assigned to the unassigned user to NEW
    print >>outfile, "update bugs set bug_status='NEW',assigned_to='NULL' where bug_status='ASSIGNED' AND assigned_to=3;"
    
    # Insert the products
    print >>outfile, "\ninsert into products ("
    print >>outfile, "  product, description, milestoneurl, isactive,"
    print >>outfile, "  defaultmilestone, votestoconfirm) values ("
    print >>outfile, "  '%s', '%s', '%s', 1, '%s', 1);" % (product,
                                                           productdesc,
                                                           milestoneurl,
                                                           defaultmilestone)

    # Insert the components    
    for category in categories_list:
        component = SqlQuote(category[0])
        productstr = SqlQuote(product)
        description = SqlQuote(category[1])
        initialowner = SqlQuote("3")
        print >>outfile, "\ninsert into components (";
        print >>outfile, "  value, program, initialowner, initialqacontact,"
        print >>outfile, "  description) values ("
        print >>outfile, "  %s, %s, %s, '', %s);" % (component, productstr,
                                                     initialowner, description)
        
    # Insert the versions
    for productstr, version_list in versions_table.items():
        productstr = SqlQuote(productstr)
        for version in version_list:
            version = SqlQuote(version)
            print >>outfile, "\ninsert into versions (value, program) "
            print >>outfile, "  values (%s, %s);" % (version, productstr)
            
    # Insert the users
    for username, userid in usermapping.items():
        realname = map_username_to_realname(username)
        username = SqlQuote(username)
        realname = SqlQuote(realname)
        print >>outfile, "\ninsert into profiles ("
        print >>outfile, "  userid, login_name, password, cryptpassword, realname, groupset"
        print >>outfile, ") values ("
        print >>outfile, "%s,%s,'password',encrypt('password'), %s, 0);" % (userid, username, realname)
    print >>outfile, "update profiles set groupset=1 << 32 where login_name like '%\@gcc.gnu.org';"
    
def unixdate2datetime(unixdate):
    """ Convert a unix date to a datetime value """
    year, month, day, hour, min, sec, x, x, x, x = email.Utils.parsedate_tz(unixdate)
    return "%d-%02d-%02d %02d:%02d:%02d" % (year,month,day,hour,min,sec)

def unixdate2timestamp(unixdate):
    """ Convert a unix date to a timestamp value """
    year, month, day, hour, min, sec, x, x, x, x = email.Utils.parsedate_tz(unixdate)
    return "%d%02d%02d%02d%02d%02d" % (year,month,day,hour,min,sec)

def SqlQuote(str):
    """ Perform SQL quoting on a string """
    return "'%s'" % str.replace("'", """''""").replace("\\", "\\\\").replace("\0","\\0")

def convert_gccver_to_ver(gccver):
    """ Given a gcc version, convert it to a Bugzilla version. """
    for k in releasetovermap.keys():
        if re.search(".*%s.*" % k, gccver) is not None:
            return releasetovermap[k]
    result = re.search(r""".*(\d\.\d) \d+ \(experimental\).*""", gccver)
    if result is not None:
        return result.group(1)
    return "unknown"

def load_index(fname):
    """ Load in the GNATS index file """
    global pr_list
    ifp = open(fname)
    for record in ifp.xreadlines():
        fields = record.split("|")
        pr_list.append(fields[0])
    ifp.close()
    
def load_categories(fname):
    """ Load in the GNATS categories file """
    global categories_list
    cfp = open(fname)
    for record in cfp.xreadlines():
        if re.search("^#", record) is not None:
            continue
        categories_list.append(record.split(":"))
    cfp.close()
    
def map_username_to_realname(username): 
    """ Given a username, find the real name """
    name = username
    name = re.sub("@.*", "", name)
    for responsible_record in responsible_list:
	if responsible_record[0] == name:
	    return responsible_record[1]
    if len(responsible_record) > 2:
        if responsible_record[2] == username:
	    return responsible_record[1]
    return ""


def get_userid(responsible):
    """ Given an email address, get the user id """
    global responsible_map
    global usermapping
    global userid_base
    if responsible is None:
        return -1
    responsible = responsible.lower()
    responsible = re.sub("sources.redhat.com", "gcc.gnu.org", responsible)
    if responsible_map.has_key(responsible):
        responsible = responsible_map[responsible]
    if usermapping.has_key(responsible):
        return usermapping[responsible]
    else:
        usermapping[responsible] = userid_base
        userid_base += 1
    return usermapping[responsible]

def load_responsible(fname):
    """ Load in the GNATS responsible file """
    global responsible_map
    global responsible_list
    rfp = open(fname)
    for record in rfp.xreadlines():
        if re.search("^#", record) is not None:
            continue
        split_record = record.split(":")
        responsible_map[split_record[0]] = split_record[2].rstrip()
        responsible_list.append(record.split(":"))
    rfp.close()

def split_csl(list):
    """ Split a comma separated list """
    newlist = re.split(r"""\s*,\s*""", list)
    return newlist

def fix_email_addrs(addrs):
    """ Perform various fixups and cleaning on an e-mail address """
    addrs = split_csl(addrs)
    trimmed_addrs = []
    for addr in addrs:
        addr = re.sub(r"""\(.*\)""","",addr)
        addr = re.sub(r""".*<(.*)>.*""","\\1",addr)
        addr = addr.rstrip()
        addr = addr.lstrip()
        trimmed_addrs.append(addr)
    addrs = ", ".join(trimmed_addrs)
    return addrs

class Bugzillabug(object):
    """ Class representing a bugzilla bug """
    def __init__(self, gbug):
        """ Initialize a bugzilla bug from a GNATS bug.  """
        self.bug_id = gbug.bug_id
        self.long_descs = []
        self.bug_ccs = [get_userid("gcc-bugs@gcc.gnu.org")]
        self.bug_activity = []
        self.attachments = gbug.attachments
        self.gnatsfields = gbug.fields
        self.need_unformatted = gbug.has_unformatted_attach == 0
        self.need_unformatted &= gbug.fields.has_key("Unformatted")
        self.translate_pr()
        self.update_versions()
        if self.fields.has_key("Audit-Trail"):
            self.parse_audit_trail()
            self.write_bug()
            
    def parse_fromto(type, string):
        """ Parses the from and to parts of a changed-from-to line """
        fromstr = ""
        tostr = ""

        # Some slightly messed up changed lines have unassigned-new,
        # instead of unassigned->new. So we make the > optional.        
        result = re.search(r"""(.*)-(?:>?)(.*)""", string)
        
        # Only know how to handle parsing of State and Responsible
        # changed-from-to right now
        if type == "State":
            fromstr = state_lookup[result.group(1)]
            tostr = state_lookup[result.group(2)]
        elif type == "Responsible":
            if result.group(1) != "":
                fromstr = result.group(1)
            if result.group(2) != "":
                tostr = result.group(2)
            if responsible_map.has_key(fromstr):
                fromstr = responsible_map[fromstr]
            if responsible_map.has_key(tostr):
                tostr = responsible_map[tostr]  
        return (fromstr, tostr)
    parse_fromto = staticmethod(parse_fromto)
    
    def parse_audit_trail(self):
        """ Parse a GNATS audit trail """
        trail = self.fields["Audit-Trail"]
        # Begin to split the audit trail into pieces
        result = fromtore.finditer(trail)
        starts = []
        ends = []
        pieces = []
        # Make a list of the pieces
        for x in result:
            pieces.append (x)
        # Find the start and end of each piece
        if len(pieces) > 0:
            for x in xrange(len(pieces)-1):
                starts.append(pieces[x].start())
                ends.append(pieces[x+1].start())
            starts.append(pieces[-1].start())
            ends.append(len(trail))
        pieces = []
        # Now make the list of actual text of the pieces
        for x in xrange(len(starts)):
            pieces.append(trail[starts[x]:ends[x]])
        # And parse the actual pieces
        for piece in pieces:
            result = changedfromtore.search(piece)
            # See what things we actually have inside this entry, and
            # handle them appropriately
            if result is not None:
                type = result.group(1)
                changedfromto = result.group(2)
                # If the bug was reopened, mark it as such
                if changedfromto.find("closed->analyzed") != -1:
                    if self.fields["bug_status"] == "'NEW'":
                        self.fields["bug_status"] = "'REOPENED'"
                if type == "State" or type == "Responsible":
                    oldstate, newstate = self.parse_fromto (type, changedfromto)
                result = changedbyre.search(piece)
                if result is not None:
                    changedby = result.group(1)
                result = changedwhenre.search(piece)
                if result is not None:
                    changedwhen = result.group(1)
                    changedwhen = unixdate2datetime(changedwhen)
                    changedwhen = SqlQuote(changedwhen)
                result = changedwhyre.search(piece)
                changedwhy = piece[result.start(1):]
                #changedwhy = changedwhy.lstrip()
                changedwhy = changedwhy.rstrip()
                changedby = get_userid(changedby)
		# Put us on the cc list if we aren't there already
                if changedby != self.fields["userid"] \
                       and changedby not in self.bug_ccs:
                    self.bug_ccs.append(changedby)
                # If it's a duplicate, mark it as such
                result = duplicatere.search(changedwhy)
                if result is not None:
                    newtext = "*** This bug has been marked as a duplicate of %s ***" % result.group(1)
                    newtext = SqlQuote(newtext)
                    self.long_descs.append((self.bug_id, changedby,
                                            changedwhen, newtext))
                    self.fields["bug_status"] = "'RESOLVED'"
                    self.fields["resolution"] = "'DUPLICATE'"
                    self.fields["userid"] = changedby
                else:
                    newtext = "%s-Changed-From-To: %s\n%s-Changed-Why: %s\n" % (type, changedfromto, type, changedwhy)
                    newtext = SqlQuote(newtext)
                    self.long_descs.append((self.bug_id, changedby,
                                            changedwhen, newtext))
                if type == "State" or type == "Responsible":
                    newstate = SqlQuote("%s" % newstate)
                    oldstate = SqlQuote("%s" % oldstate)
                    fieldid = fieldids[type]
                    self.bug_activity.append((newstate, oldstate, fieldid, changedby, changedwhen))
                    
            else:
		# It's an email
                result = fromre.search(piece)
                if result is None:
                    continue
                fromstr = result.group(1)
                fromstr = fix_email_addrs(fromstr)
                fromstr = get_userid(fromstr)
                result = datere.search(piece)
                if result is None:
                    continue
                datestr = result.group(1)
                datestr = SqlQuote(unixdate2timestamp(datestr))
                if fromstr != self.fields["userid"] \
                       and fromstr not in self.bug_ccs:
                    self.bug_ccs.append(fromstr)
                self.long_descs.append((self.bug_id, fromstr, datestr,
                                        SqlQuote(piece)))
                
                    

    def write_bug(self):
	""" Output a bug to the data file """
        fields = self.fields
        print >>outfile, "\ninsert into bugs("
        print >>outfile, "  bug_id, assigned_to, bug_severity, priority, bug_status, creation_ts, delta_ts,"
        print >>outfile, "  short_desc,"
        print >>outfile, "  reporter, version,"
        print >>outfile, "  product, component, resolution, target_milestone, qa_contact,"
        print >>outfile, "  gccbuild, gcctarget, gcchost, keywords"
        print >>outfile, "  ) values ("
        print >>outfile, "%s, %s, %s, %s, %s, %s, %s," % (self.bug_id, fields["userid"], fields["bug_severity"], fields["priority"], fields["bug_status"], fields["creation_ts"], fields["delta_ts"])
        print >>outfile, "%s," % (fields["short_desc"])
        print >>outfile, "%s, %s," % (fields["reporter"], fields["version"])
        print >>outfile, "%s, %s, %s, %s, 0," %(fields["product"], fields["component"], fields["resolution"], fields["target_milestone"])
        print >>outfile, "%s, %s, %s, %s" % (fields["gccbuild"], fields["gcctarget"], fields["gcchost"], fields["keywords"])
        print >>outfile, ");"
        if self.fields["keywords"] != 0:
            print >>outfile, "\ninsert into keywords (bug_id, keywordid) values ("
            print >>outfile, " %s, %s);" % (self.bug_id, fields["keywordid"])
        for id, who, when, text in self.long_descs:
            print >>outfile, "\ninsert into longdescs ("
            print >>outfile, "  bug_id, who, bug_when, thetext) values("
            print >>outfile, "  %s, %s, %s, %s);" % (id, who, when, text)
        for name, data, who in self.attachments:
            print >>outfile, "\ninsert into attachments ("
            print >>outfile, "  bug_id, filename, description, mimetype, ispatch, submitter_id) values ("
	    ftype = None
	    # It's *magic*!
	    if name.endswith(".ii") == 1:
		ftype = "text/x-c++"
 	    elif name.endswith(".i") == 1:
		ftype = "text/x-c"
	    else:
		ftype = magicf.detect(cStringIO.StringIO(data))
            if ftype is None:
                ftype = "application/octet-stream"
            
            print >>outfile, "%s,%s,%s, %s,0, %s,%s);" %(self.bug_id, SqlQuote(name), SqlQuote(name), SqlQuote (ftype), who)
            print >>outfile, "\ninsert into attach_data ("
            print >>outfile, "\n(id, thedata) values (last_insert_id(),"
            print >>outfile, "%s);" % (SqlQuote(zlib.compress(data)))
        for newstate, oldstate, fieldid, changedby, changedwhen in self.bug_activity:
            print >>outfile, "\ninsert into bugs_activity ("
            print >>outfile, "  bug_id, who, bug_when, fieldid, added, removed) values ("
            print >>outfile, "  %s, %s, %s, %s, %s, %s);" % (self.bug_id,
                                                             changedby,
                                                             changedwhen,
                                                             fieldid,
                                                             newstate,
                                                             oldstate)
        for cc in self.bug_ccs:
            print >>outfile, "\ninsert into cc(bug_id, who) values (%s, %s);" %(self.bug_id, cc)
    def update_versions(self):
	""" Update the versions table to account for the version on this bug """
        global versions_table
        if self.fields.has_key("Release") == 0 \
               or self.fields.has_key("Category") == 0:
            return
        curr_product = "gcc"
        curr_version = self.fields["Release"]
        if curr_version == "":
            return
        curr_version = convert_gccver_to_ver (curr_version)
        if versions_table.has_key(curr_product) == 0:
            versions_table[curr_product] = []
        for version in versions_table[curr_product]:
            if version == curr_version:
                return
        versions_table[curr_product].append(curr_version)
    def translate_pr(self):
	""" Transform a GNATS PR into a Bugzilla bug """
        self.fields = self.gnatsfields
        if (self.fields.has_key("Organization") == 0) \
           or self.fields["Organization"].find("GCC"):
            self.fields["Originator"] = ""
            self.fields["Organization"] = ""
        self.fields["Organization"].lstrip()
        if (self.fields.has_key("Release") == 0) \
               or self.fields["Release"] == "" \
               or self.fields["Release"].find("unknown-1.0") != -1:
            self.fields["Release"]="unknown"
        if self.fields.has_key("Responsible"):
            result = re.search(r"""\w+""", self.fields["Responsible"])
            self.fields["Responsible"] = "%s%s" % (result.group(0), "@gcc.gnu.org")
        self.fields["gcchost"] = ""
        self.fields["gcctarget"] = ""
        self.fields["gccbuild"] = ""
        if self.fields.has_key("Environment"):
            result = re.search("^host: (.+?)$", self.fields["Environment"],
                               re.MULTILINE)
            if result is not None:
                self.fields["gcchost"] = result.group(1)
            result = re.search("^target: (.+?)$", self.fields["Environment"],
                               re.MULTILINE)
            if result is not None:
                self.fields["gcctarget"] = result.group(1)
            result = re.search("^build: (.+?)$", self.fields["Environment"],
                               re.MULTILINE)
            if result is not None:
                self.fields["gccbuild"] = result.group(1)
        self.fields["userid"] = get_userid(self.fields["Responsible"])
        self.fields["bug_severity"] = "normal"
        if self.fields["Class"] == "change-request":
            self.fields["bug_severity"] = "enhancement"
        elif self.fields.has_key("Severity"):
            if self.fields["Severity"] == "critical":
                self.fields["bug_severity"] = "critical"
            elif self.fields["Severity"] == "serious":
                self.fields["bug_severity"] = "major"
        elif self.fields.has_key("Synopsis"):
            if re.search("crash|assert", self.fields["Synopsis"]):
                self.fields["bug_severity"] = "critical"
            elif re.search("wrong|error", self.fields["Synopsis"]):
                self.fields["bug_severity"] = "major"
        self.fields["bug_severity"] = SqlQuote(self.fields["bug_severity"])
        self.fields["keywords"] = 0
        if keywordids.has_key(self.fields["Class"]):
            self.fields["keywords"] = self.fields["Class"]
            self.fields["keywordid"] = keywordids[self.fields["Class"]]
            self.fields["keywords"] = SqlQuote(self.fields["keywords"])
        self.fields["priority"] = "P1"
        if self.fields.has_key("Severity") and self.fields.has_key("Priority"):
            severity = self.fields["Severity"]
            priority = self.fields["Priority"]
            if severity == "critical":
                if priority == "high":
                    self.fields["priority"] = "Highest"
                else:
                    self.fields["priority"] = "High"
            elif severity == "serious":
                if priority == "low":
                    self.fields["priority"] = "Low"
                else:
                    self.fields["priority"] = "Normal"
            else:
                if priority == "high":
                    self.fields["priority"] = "Low"
                else:
                    self.fields["priority"] = "Lowest"
        self.fields["priority"] = SqlQuote(self.fields["priority"])
        state = self.fields["State"]
        if (state == "open" or state == "analyzed") and self.fields["userid"] != 3:
            self.fields["bug_status"] = "ASSIGNED"
            self.fields["resolution"] = ""
        elif state == "feedback":
            self.fields["bug_status"] = "WAITING"
            self.fields["resolution"] = ""
        elif state == "closed":
            self.fields["bug_status"] = "CLOSED"
            if self.fields.has_key("Class"):
                theclass = self.fields["Class"]
                if theclass.find("duplicate") != -1:
                    self.fields["resolution"]="DUPLICATE"
                elif theclass.find("mistaken") != -1:
                    self.fields["resolution"]="INVALID"                    
                else:
                    self.fields["resolution"]="FIXED"
            else:
                self.fields["resolution"]="FIXED"
        elif state == "suspended":
            self.fields["bug_status"] = "SUSPENDED"
            self.fields["resolution"] = ""
        elif state == "analyzed" and self.fields["userid"] == 3:
            self.fields["bug_status"] = "NEW"
            self.fields["resolution"] = ""
        else:
            self.fields["bug_status"] = "UNCONFIRMED"
            self.fields["resolution"] = ""
        self.fields["bug_status"] = SqlQuote(self.fields["bug_status"])
        self.fields["resolution"] = SqlQuote(self.fields["resolution"])
        self.fields["creation_ts"] = ""
        if self.fields.has_key("Arrival-Date") and self.fields["Arrival-Date"] != "":
            self.fields["creation_ts"] = unixdate2datetime(self.fields["Arrival-Date"])
        self.fields["creation_ts"] = SqlQuote(self.fields["creation_ts"])
        self.fields["delta_ts"] = ""
        if self.fields.has_key("Audit-Trail"):
            result = lastdatere.findall(self.fields["Audit-Trail"])
            result.reverse()
            if len(result) > 0:
                self.fields["delta_ts"] = unixdate2timestamp(result[0])
        if self.fields["delta_ts"] == "":
            if self.fields.has_key("Arrival-Date") and self.fields["Arrival-Date"] != "":
                self.fields["delta_ts"] = unixdate2timestamp(self.fields["Arrival-Date"])
        self.fields["delta_ts"] = SqlQuote(self.fields["delta_ts"])
        self.fields["short_desc"] = SqlQuote(self.fields["Synopsis"])
        if self.fields.has_key("Reply-To") and self.fields["Reply-To"] != "":
            self.fields["reporter"] = get_userid(self.fields["Reply-To"])
        elif self.fields.has_key("Mail-Header"):
            result = re.search(r"""From .*?([\w.]+@[\w.]+)""", self.fields["Mail-Header"])
            if result:
                self.fields["reporter"] = get_userid(result.group(1))
            else:
                self.fields["reporter"] = get_userid(gnats_username)
        else:
            self.fields["reporter"] = get_userid(gnats_username)
        long_desc = self.fields["Description"]
        long_desc2 = ""
        for field in ["Release", "Environment", "How-To-Repeat"]:
            if self.fields.has_key(field) and self.fields[field] != "":
                long_desc += ("\n\n%s:\n" % field) + self.fields[field]
        if self.fields.has_key("Fix") and self.fields["Fix"] != "":
            long_desc2 = "Fix:\n" + self.fields["Fix"]
        if self.need_unformatted  == 1 and self.fields["Unformatted"] != "":
            long_desc += "\n\nUnformatted:\n" + self.fields["Unformatted"]
        if long_desc != "":
            self.long_descs.append((self.bug_id, self.fields["reporter"],
                                    self.fields["creation_ts"],
                                    SqlQuote(long_desc)))
        if long_desc2 != "":
            self.long_descs.append((self.bug_id, self.fields["reporter"],
                                    self.fields["creation_ts"],
                                    SqlQuote(long_desc2)))
        for field in ["gcchost", "gccbuild", "gcctarget"]:
            self.fields[field] = SqlQuote(self.fields[field])
        self.fields["version"] = ""
        if self.fields["Release"] != "":
            self.fields["version"] = convert_gccver_to_ver (self.fields["Release"])
        self.fields["version"] = SqlQuote(self.fields["version"])
        self.fields["product"] = SqlQuote("gcc")
        self.fields["component"] = "invalid"
        if self.fields.has_key("Category"):
            self.fields["component"] = self.fields["Category"]
        self.fields["component"] = SqlQuote(self.fields["component"])
        self.fields["target_milestone"] = "---"
        if self.fields["version"].find("3.4") != -1:
            self.fields["target_milestone"] = "3.4"
        self.fields["target_milestone"] = SqlQuote(self.fields["target_milestone"])
        if self.fields["userid"] == 2:
            self.fields["userid"] = "\'NULL\'"

class GNATSbug(object):
    """ Represents a single GNATS PR """
    def __init__(self, filename):
        self.attachments = []
        self.has_unformatted_attach = 0
        fp = open (filename)
        self.fields = self.parse_pr(fp.xreadlines())
        self.bug_id = int(self.fields["Number"])
        if self.fields.has_key("Unformatted"):
            self.find_gnatsweb_attachments()
        if self.fields.has_key("How-To-Repeat"):
            self.find_regular_attachments("How-To-Repeat")
        if self.fields.has_key("Fix"):
            self.find_regular_attachments("Fix")

    def get_attacher(fields):
        if fields.has_key("Reply-To") and fields["Reply-To"] != "":
            return get_userid(fields["Reply-To"])
        else:
            result = None
            if fields.has_key("Mail-Header"):
                result = re.search(r"""From .*?([\w.]+\@[\w.]+)""",
                                   fields["Mail-Header"])
            if result is not None:
                reporter = get_userid(result.group(1))
            else:
                reporter = get_userid(gnats_username)
    get_attacher = staticmethod(get_attacher)
    def find_regular_attachments(self, which):
        fields = self.fields
        while re.search("^begin [0-7]{3}", fields[which],
                        re.DOTALL | re.MULTILINE):
            outfp = cStringIO.StringIO()
            infp = cStringIO.StringIO(fields[which])
            filename, start, end = specialuu.decode(infp, outfp, quiet=0)
            fields[which]=fields[which].replace(fields[which][start:end],
                                                "See attachments for %s\n" % filename)
            self.attachments.append((filename, outfp.getvalue(),
                                     self.get_attacher(fields)))

    def decode_gnatsweb_attachment(self, attachment):
        result = re.split(r"""\n\n""", attachment, 1)
        if len(result) == 1:
            return -1
        envelope, body = result
        envelope = uselessre.split(envelope)
        envelope.pop(0)
        # Turn the list of key, value into a dict of key => value
        attachinfo = dict([(envelope[i], envelope[i+1]) for i in xrange(0,len(envelope),2)])
        for x in attachinfo.keys():
            attachinfo[x] = attachinfo[x].rstrip()
        if (attachinfo.has_key("Content-Type") == 0) or \
           (attachinfo.has_key("Content-Disposition") == 0):
            raise ValueError, "Unable to parse file attachment"
        result = dispositionre.search(attachinfo["Content-Disposition"])
        filename = result.group(2)
        filename = re.sub(".*/","", filename)
        filename = re.sub(".*\\\\","", filename)
        attachinfo["filename"]=filename
        result = re.search("""(\S+);.*""", attachinfo["Content-Type"])
        if result is not None:
            attachinfo["Content-Type"] = result.group(1)
        if attachinfo.has_key("Content-Transfer-Encoding"):
            if attachinfo["Content-Transfer-Encoding"] == "base64":
                attachinfo["data"] = base64.decodestring(body)
        else:
            attachinfo["data"]=body

        return (attachinfo["filename"], attachinfo["data"],
                self.get_attacher(self.fields))

    def find_gnatsweb_attachments(self):
        fields = self.fields
        attachments = re.split(attachment_delimiter, fields["Unformatted"])
        fields["Unformatted"] = attachments.pop(0)
        for attachment in attachments:
            result = self.decode_gnatsweb_attachment (attachment)
            if result != -1:
                self.attachments.append(result)
            self.has_unformatted_attach = 1
    def parse_pr(lines):
        #fields = {"envelope":[]}
        fields = {"envelope":array.array("c")}
        hdrmulti = "envelope"
        for line in lines:
            line = line.rstrip('\n')
            line += '\n'
            result = gnatfieldre.search(line)
            if result is None:
                if hdrmulti != "":
                    if fields.has_key(hdrmulti):
                        #fields[hdrmulti].append(line)
                        fields[hdrmulti].fromstring(line)
                    else:
                        #fields[hdrmulti] = [line]
                        fields[hdrmulti] = array.array("c", line)
                continue
            hdr, arg = result.groups()
            ghdr = "*not valid*"
            result = fieldnamere.search(hdr)
            if result != None:
                ghdr = result.groups()[0]
            if ghdr in fieldnames:
                if multilinefields.has_key(ghdr):
                    hdrmulti = ghdr
                    #fields[ghdr] = [""]
                    fields[ghdr] = array.array("c")
                else:
                    hdrmulti = ""
                    #fields[ghdr] = [arg]
                    fields[ghdr] = array.array("c", arg)
            elif hdrmulti != "":
                #fields[hdrmulti].append(line)
                fields[hdrmulti].fromstring(line)
            if hdrmulti == "envelope" and \
               (hdr == "Reply-To" or hdr == "From" \
                or hdr == "X-GNATS-Notify"):
                arg = fix_email_addrs(arg)
                #fields[hdr] = [arg]
                fields[hdr] = array.array("c", arg)
	if fields.has_key("Reply-To") and len(fields["Reply-To"]) > 0:
            fields["Reply-To"] = fields["Reply-To"]
        else:
            fields["Reply-To"] = fields["From"]
        if fields.has_key("From"):
            del fields["From"]
        if fields.has_key("X-GNATS-Notify") == 0:
            fields["X-GNATS-Notify"] = array.array("c")
            #fields["X-GNATS-Notify"] = ""
        for x in fields.keys():
            fields[x] = fields[x].tostring()
            #fields[x] = "".join(fields[x])            
        for x in fields.keys():
            if multilinefields.has_key(x):
                fields[x] = fields[x].rstrip()

        return fields
    parse_pr = staticmethod(parse_pr)
load_index("%s/gnats-adm/index" % gnats_db_dir)
load_categories("%s/gnats-adm/categories" % gnats_db_dir)
load_responsible("%s/gnats-adm/responsible" % gnats_db_dir)
get_userid(gnats_username)
get_userid(unassigned_username)
for x in pr_list:
    print "Processing %s..." % x
    a = GNATSbug ("%s/%s" % (gnats_db_dir, x))
    b = Bugzillabug(a)
write_non_bug_tables()
outfile.close()