Roundup Tracker

   1   #!/usr/bin/env python
   2 
   3   ##
   4   # File: roundup-feeder
   5   # Desc: RSS issue feeder for roundup. It is partially derived from the 
   6   #       `spycyroll' RSS aggregator, from http://spycyroll.sourceforge.net/.
   7   #
   8   #       I've made it as straightforward as possible.
   9   #
  10   #       Creates new roundup issues for each new item in its RSS feed, and keeps
  11   #       track of those already entered in the roundup DB in a pickled list.
  12   #
  13   # Auth: Cian Synnott <cian@dmz.ie>
  14   # $Id: roundup-feeder.py,v 1.2 2004/09/06 21:30:12 cian Exp $
  15   ##
  16 
  17   import sys
  18   import pickle
  19 
  20   # Mark Pilgrim's rather relaxed RSS parser (http://feedparser.sourceforge.net/)
  21   import feedparser
  22 
  23   # Ease of configuration
  24   from ConfigParser import ConfigParser
  25 
  26   # The all-important roundup stuff
  27   import roundup.instance
  28   from roundup import date
  29 
  30   DEFAULT_CONFIG_FILE = "config.ini"
  31 
  32   class Channel:
  33     """An RSS channel.
  34 
  35        Allows an RSS channel to be loaded and stored in memory
  36 
  37        rss :   url to the RSS or RDF file. From this url, it figures out the rest;
  38        title : Title as specified in RSS file UNLESS you specify it while creating
  39              the object
  40        link :  url for the site, as specified in the RSS file
  41        description : optional textual description.
  42        items[]: collection of type NewsItem for the items in the feed
  43     """
  44 
  45     def __init__(self, rss, title=None):
  46       self.rss         = rss
  47       self.title       = title
  48       self.link        = None
  49       self.description = None
  50       self.items       = []
  51 
  52     def load(self, rss=None):
  53       """Downloads and parses a channel
  54 
  55       sets the feed's title, link and description.
  56       sets and returns items[], for NewsItems defined
  57       """
  58       if rss is None:
  59         rss = self.rss
  60       prss = feedparser.parse(rss)
  61       channel = prss['channel']
  62       items = prss['items']
  63       if 'link' in channel.keys():
  64         self.link = channel['link']
  65       else:
  66         self.link = ''
  67       title = channel['title']
  68       self.description = channel['description']
  69       if self.title is None:
  70         self.title = title
  71       self.items = []
  72       for item in items:
  73         self.items.append(NewsItem(item))
  74       return self.items
  75 
  76   class NewsItem:
  77       """Each item in a channel"""
  78    
  79       def __init__(self, dict):
  80           self.link = dict.get('link', '')
  81           self.title = dict['title']
  82           self.description = dict['description']
  83           if 'date' in dict.keys():
  84             self.date = dict['date']
  85           else:
  86             self.date = None
  87 
  88   def loadChannels(config):
  89     """Loads all channels in a configuration
  90     """
  91     feeds = {}
  92 
  93     for f in config.sections():
  94       if config.has_option(f, 'name'):
  95         feeds[f] = config.get(f, 'name')
  96       else:
  97         feeds[f] = None
  98 
  99     channels = []
 100 
 101     for f in feeds.keys():
 102       c = Channel(f, feeds[f])
 103       try:
 104         c.load()
 105       except:
 106         continue
 107 
 108       channels.append(c)
 109 
 110     return channels
 111 
 112 
 113   class RoundupFeeder:
 114     """A roundup RSS feeder class
 115 
 116        A class for maintaining a `connection' to the roundup database and feeding 
 117        issues into it.
 118 
 119        instance : An instance of the roundup tracker we're dealing with
 120        db       : The roundup database belonging to that instance
 121        cl       : The roundup class we enter issues as 
 122        uid      : The user id we've opend the db as
 123     """
 124 
 125     def __init__(self, home, user, klass):
 126       """Initialise the RoundupFeeder
 127 
 128          home  : The location of the roundup tracker on the filesystem
 129          user  : The user to open the database as
 130          klass : The name of the `issue' class in the database
 131       """
 132 
 133       # Connect to our roundup database
 134       self.instance = roundup.instance.open(home)
 135       self.db       = self.instance.open('admin')
 136 
 137       # First lookup and reconnect as this user
 138       try:
 139         self.uid = self.db.user.lookup(user)
 140         username = self.db.user.get(self.uid, 'username')
 141 
 142         self.db.close()
 143         self.db = self.instance.open(username)
 144       except:
 145         print '''
 146   Cannot open the tracker "%s" with username "%s".
 147   Are you sure this user exists in the database?
 148   '''%(home, user)
 149         sys.exit(1)
 150 
 151       try:
 152         self.cl = self.db.getclass(klass)
 153       except:
 154         print '''
 155   It appears that the configured Roundup class "%s" does not exist in the
 156   database. Valid class names are: %s
 157   '''%(klass, ', '.join(self.db.getclasses()))
 158         sys.exit(1)
 159 
 160     def cleanup(self):
 161       """Cleans up the RoundupFeeder
 162   
 163          Closes the database we've been dealing with. This will need to be called
 164          once you have created a RoundupFeeder; perhaps use a try: finally:
 165          structure.
 166       """
 167       self.db.close()
 168 
 169     def enterNewsItem(self, channel, item):
 170       """Enter a news item as a roundup issue
 171 
 172          Each news item should be a new issue in roundup.
 173       
 174          channel : The channel this item is from
 175         item     : The item to enter as an issue
 176       """
 177   
 178       # Setup issue title and issue content
 179       title = item.title
 180 
 181       if item.description:
 182         content = '''
 183   From: %s (%s)
 184 
 185   %s
 186 
 187   Link: %s
 188   '''%(channel.title, channel.link, item.description, item.link)
 189 
 190       else:
 191         content = '''
 192   From: %s (%s)
 193 
 194   Link: %s
 195   '''%(channel.title, channel.link, item.link)
 196 
 197       # Set up the issue
 198       issue = {}
 199       issue['title']      = title
 200       issue['files']      = []
 201       issue['nosy']       = []
 202       issue['superseder'] = []
 203 
 204       # Set up the message
 205       msg = {}
 206       msg['author']     = self.uid
 207       msg['date']       = date.Date('.')
 208       msg['summary']    = title
 209       msg['content']    = content
 210       msg['files']      = []
 211       msg['recipients'] = []
 212       msg['messageid']  = ''
 213       msg['inreplyto']  = ''
 214     
 215       # My issues have a 'type' property for the issue type - I set that to
 216       # roundup
 217       if self.cl.getprops().has_key('type'):
 218         issue['type'] = 'rssfeed'
 219     
 220       # My issues have a 'status' property - I set that to unread
 221       if self.cl.getprops().has_key('type'):
 222         issue['status'] = 'unread'
 223 
 224       # Now create new message & issue for this item
 225       try:
 226         message_id = self.db.msg.create(**msg)
 227         issue['messages'] = [message_id]
 228         nodeid = self.cl.create(**issue)
 229       except Exception, inst:
 230         print '''
 231   Error creating issue for item '%s'.
 232   '''%(item.link)
 233         print inst
 234       
 235       self.db.commit()
 236 
 237   if __name__ == '__main__':
 238 
 239     if len(sys.argv) > 1:
 240       config_file = sys.argv[1]
 241     else:
 242       config_file = DEFAULT_CONFIG_FILE
 243 
 244     config = ConfigParser()
 245     config.readfp(open(config_file))
 246 
 247     statefile     = config.get('DEFAULT', 'statefile'    )
 248     roundup_home  = config.get('DEFAULT', 'roundup-home' )
 249     roundup_user  = config.get('DEFAULT', 'roundup-user' )
 250     roundup_class = config.get('DEFAULT', 'roundup-class')
 251 
 252     channels = loadChannels(config)
 253 
 254     try:
 255       fp = open(statefile, 'r')
 256       olditems = pickle.load(fp)
 257       fp.close()
 258     except:
 259       olditems = []
 260 
 261     newitems = []
 262 
 263     feeder = RoundupFeeder(roundup_home, roundup_user, roundup_class)
 264 
 265     # Now use try: finally: to make sure the database gets closed
 266     try:
 267       for c in channels:
 268         for i in c.items:
 269           if i.link not in newitems:
 270             newitems.append(i.link)
 271             if i.link not in olditems:
 272               feeder.enterNewsItem(c, i)
 273 
 274     finally:
 275       feeder.cleanup()
 276 
 277     # Now pickle our list of RSS items for the next run.
 278     try:
 279       fp = open(statefile, 'w')
 280       pickle.dump(newitems, fp)
 281       fp.close()
 282     except:
 283       print "Couldn't dump statefile!"