Roundup Tracker

Many people have a body of issues in an existing tracker, BugZilla being a very common issue tracker. I was faced with the task of moving all our issues from a bugzilla instance into my shiny new roundup instance. Bugzilla helpfully provides an option to format query results and bugs in RDF (an XML dialect) so using the wonders of ElementTree it isn't too hard to import issues into a roundup instance, provided you can either figure out how to map the information onto the roundup schema, or modify your roundup schema to match. I took the later course, and then modified my schema over time to work better with roundup.

The following is a simple script to chug through the bugzilla issues and then import them into roundup. I cache the issue downloads so I can experiment without killing the bugzilla instance, so it should be easy enough to tweak the script to your heart's content.

I should add the caveat that I haven't used this script in a while, so it might have bitrotted slightly over time. I'm confident that it should work for most people with only minimal tweaking.

-- MichaelTwomey

bugzilla_to_roundup.py::

• #!/usr/bin/env python """Bugzilla to roundup Converts entries in bugzilla to roundup issues """ import os

  from optparse import OptionParser import urllib2 import sys import logging

  from elementtree import ElementTree from roundup import instance from roundup.date import Date logging.basicConfig() logger = logging.getLogger() logger.setLevel(logging.INFO) # Map bugzilla states to ours bug_status_mapping = {

  • "UNCONFIRMED": "open", "NEW": "open", "ASSIGNED": "accepted", "REOPENED": "open", # this our chatting equiv? "RESOLVED": "complete", "VERIFIED": "verified", "CLOSED": "cancelled", }
  # Map severity to ours bug_severity_mapping = {
  • "blocker": "critical", "critical": "critical", "major": "urgent", "normal": "bug", "minor": "bug", "trivial": "bug", "enhancement": "feature", }
  # Map properties, this doesn't cover severity and status bug_property_mapping = {
  • "product": "product", "version": "version", "assigned_to": "assignedto", "reporter": "creator", "creation_ts": "creation", "short_desc": "title" }
  usage="""%prog bugzilla_base_url roundup_instace

  e.g. %prog http://example.com/cgi-bin/bugzilla/ $HOME/Servers/roundup/issues """ def get_bugzilla_bug_list(url_prefix):

  • """Uses bugzilla to get a list of bug ids. Uses the RDF output of bugzilla. """ result = []

    url = "%sbuglist.cgi?short_desc_type=allwordssubstr&short_desc=&long_desc_type=allwordssubstr&long_desc=&bug_file_loc_type=allwordssubstr&bug_file_loc=&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&bug_status=RESOLVED&bug_status=VERIFIED&bug_status=CLOSED&emailtype1=substring&email1=&emailtype2=substring&email2=&bugidtype=include&bug_id=&votes=&changedin=&chfieldfrom=&chfieldto=Now&chfieldvalue=&cmdtype=doit&newqueryname=&order=Reuse+same+sort+as+last+time&field0-0-0=noop&type0-0-0=noop&value0-0-0=&format=rdf" % url_prefix f = urllib2.urlopen(url) rdf_text = f.read() fp = open("bugzilla.rdf", "w") fp.write(rdf_text) fp.close()

    tree = ElementTree.parse("bugzilla.rdf") root = tree.getroot()

    for bug in root.findall(".//{http://www.bugzilla.org/rdf#}id"):

    • result.append(bug.text)
    return result
  def download_bugs(url_prefix, bug_ids, output_dir):
  • """Downloads all the bugs given from bugzilla to XML files""" for bug_id in bug_ids:
    • f = urllib2.urlopen("%s/xml.cgi?id=%s" % (url_prefix, bug_id)) bug_text = f.read() fp = open(os.path.join(output_dir, "%s.xml" % bug_id), "w") fp.write(bug_text) fp.close()

  class BugzillaBug:

  • def init(self, tree):

    • self._tree = tree self._root = self._tree.getroot()

    def getattr(self, name):

    • # Special case for all the long_desc's there if name in ["long_desc",]:
      • return self._root.findall("bug/%s" % name)
      return self._root.find("bug/%s" % name).text

    def getitem(self, key):

    • return self.getattr(key)

    def keys(self):
    • return [x.tag for x in self._root.find("bug")]
  def parse_bug(bug_filename):
  • """Parses a bug""" logger.debug("Parsing %s" % bug_filename)

    tree = ElementTree.parse(bug_filename) return BugzillaBug(tree)

  def dump_bugs(bug_ids, cache_dir):
  • """Dumps out info on the given bugs""" for bug_id in bug_ids:
    • bug = parse_bug(os.path.join(cache_dir, "%s.xml" % bug_id)) for key in bug.keys():
      • if key == "long_desc":
        • continue
        print "%s: %s = %s" % (bug_id, key, bug[key])
      for long_desc in bug.long_desc:
      • print "%s:long_desc %s = %s" % (bug_id, "who", long_desc.find("who").text) print "%s:long_desc %s = %s" % (bug_id, "bug_when", long_desc.find("bug_when").text) thetext = long_desc.find("thetext").text thetext = thetext.replace("\n", "\\n") print "%s:long_desc %s = %s" % (bug_id, "thetext", thetext)
  def convert_bugs(bug_ids, cache_dir, roundup_instance):
  • """Converts the bugzilla dumped XML into roundup bugs""" tracker = instance.open(roundup_instance) # NOTE: Is it worth sorting the bugs so the ids increase in chronological order? for bug_id in bug_ids:
    • filename = os.path.join(cache_dir, "%s.xml" % bug_id) logger.info("Processing bug %s" % bug_id) bz_bug = parse_bug(filename) roundup_bug = {} for bz_prop, roundup_prop in bug_property_mapping.items():
      • roundup_bug[roundup_prop] = bz_bug[bz_prop]
      roundup_bug['status'] = bug_status_mapping[bz_bug.bug_status] roundup_bug['priority'] = bug_severity_mapping[bz_bug.bug_severity] roundup_bug['messages'] = [] for long_desc in bz_bug.long_desc:
      • message = {} message['creator'] = long_desc.find('who').text message['author'] = long_desc.find('who').text message['creation'] = long_desc.find('bug_when').text message['file'] = long_desc.find('thetext').text roundup_bug['messages'].append(message)
      bug_id = add_bug_to_roundup(roundup_bug, tracker) for message in roundup_bug['messages']:
      • add_message_to_roundup(bug_id, message, tracker)
      # Append the bug's XML as a file xml = open(filename, "r").read() add_file_to_roundup(bug_id, filename, xml, "text/xml", tracker)
  def add_bug_to_roundup(bug, tracker):
  • """Add a roundup bug (dict) to a roundup instance :param bug: A dictionary of bug data. Pretty much a plain set of

    property -> value mappings, with the only exception being messages which maps to a list of message dictionaries. :param tracker: The roundup tracker instance, this is obtained via

    roundup.instance.open. """ try:

    • # Open up the db and get the user, creating if necessary
    • db = tracker.open('admin') username = get_roundup_user(db, bug['creator']) db.commit() db.close() db = tracker.open(username) # Get the issue's properties product_id = get_roundup_property(db, 'product', bug['product']) version_id = get_roundup_property(db, 'version', bug['version']) status_id = get_roundup_property(db, 'status', bug['status']) priority_id = get_roundup_property(db, 'priority', bug['priority']) # Get the users relating to the issue # From bugzilla the username is the email, so can use that as a starting point creator_id = get_roundup_user(db, bug['creator']) assignedto_id = get_roundup_user(db, bug['assignedto']) bug_id = db.issue.create(
      • title=bug['title'], product=bug['product'], version=bug['version'], status=bug['status'], priority=bug['priority'], #creator=bug['creator'], assignedto=bug['assignedto'].split("@")[0], #creation=bug['creation'], )
      db.commit()
    finally:
    • #Ensure the db is always closed no matter what db.close() logger.info("Roundup db connection closed")
    logger.info("Added issue: %s" % bug_id) return bug_id
  def add_message_to_roundup(bug_id, message, tracker):
  • """Adds the given message to to roundup""" try:
    • # Open up the db and get the user, creating if necessary db = tracker.open('admin') username = get_roundup_user(db, message['creator']) db.commit() db.close() # Re-open as the user db = tracker.open(username) # Create the author if not there get_roundup_user(db, message['author']) # Create the message message_id = db.msg.create(
      • content=message['file'], author=message['author'].split("@")[0], date=Date(message['creation']), #creation=message['creation'] )
      # Add the message to the bug bug = db.issue.getnode(bug_id) messages = bug.messages messages.append(message_id) bug.messages = messages # Force a setattr db.commit()
    finally:
    • #Ensure the db is always closed no matter what db.close() logger.info("Roundup db connection closed")
    logger.info("Added message: %s to issue: %s" % (message_id, bug_id)) return message_id
  def add_file_to_roundup(bug_id, filename, content, mimetype, tracker):
  • filename = os.path.basename(filename) # ensure there are no directory parts try:
    • db = tracker.open("admin") # Create the file file_id = db.file.create(
      • content=content, name=filename, type=mimetype )
      #Append the file to the bug bug = db.issue.getnode(bug_id) files = bug.files files.append(file_id) bug.files = files db.commit()
    finally:
    • db.close() logger.debug("Closed db connection")
    logger.info("Added file %s to issue %s" % (file_id, bug_id)) return file_id
  def get_roundup_property(db, propname, value):
  • """Obtains the id of the given property. If it doesn't exist it is created. """ klass = db.getclass(propname) try:
    • result = klass.lookup(value)

    except KeyError:

    • result = klass.create(name=value)
    return result
  def get_roundup_user(db, email):
  • """Find (or create) a user based on their email address This assumes everyone has a username which is also their email address prefix. """ username = email.split("@")[0] try:
    • db.user.lookup(username)

    except KeyError:

    • db.user.create(username=username, address=email)
    return username
  def main():

  • parser = OptionParser(usage=usage) parser.add_option("", "--download", dest="download", default=False, action="store_true",

    • help="Download bugs only. Downloads the individual bug xml files to dir called 'cache'.")
    parser.add_option("", "--dump", dest="dump", default=False, action="store_true",
    • help="Dumps out details on the parse entries")
    options, args = parser.parse_args() if len(args) != 2:
    • parser.error("You need to give the bugzilla URL and the roundup instance home dir")
    bugzilla_url, roundup_instance = args bug_ids = get_bugzilla_bug_list(bugzilla_url) if options.download:
    • download_bugs(bugzilla_url, bug_ids, "cache") sys.exit(0)
    if options.dump:
    • dump_bugs(bug_ids, "cache") sys.exit(0)
    convert_bugs(bug_ids, "cache", roundup_instance)

  if name == "main":

  • main()

From wiki Sat Feb 9 00:38:56 +1100 2008 From: wiki Date: Sat, 09 Feb 2008 00:38:56 +1100 Subject: get the python code Message-ID: <20080209003856+1100@www.mechanicalcat.net>

the python code is available in a usable fashion via the diff page (http://www.mechanicalcat.net/tech/roundup/wiki/ImportingFromBugzilla/diff)

Tobias