code.delx.au - offlineimap/blob - offlineimap/folder/IMAP.py

   1 # IMAP folder support
   2 # Copyright (C) 2002-2007 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18
  19 from Base import BaseFolder
  20 import imaplib
  21 from offlineimap import imaputil, imaplibutil
  22 from offlineimap.ui import UIBase
  23 from offlineimap.version import versionstr
  24 import rfc822, time, string, random, binascii, re
  25 from StringIO import StringIO
  26 from copy import copy
  27
  28
  29 class IMAPFolder(BaseFolder):
  30     def __init__(self, imapserver, name, visiblename, accountname, repository):
  31         self.config = imapserver.config
  32         self.expunge = repository.getexpunge()
  33         self.name = imaputil.dequote(name)
  34         self.root = None # imapserver.root
  35         self.sep = imapserver.delim
  36         self.imapserver = imapserver
  37         self.messagelist = None
  38         self.visiblename = visiblename
  39         self.accountname = accountname
  40         self.repository = repository
  41         self.randomgenerator = random.Random()
  42         BaseFolder.__init__(self)
  43
  44     def getaccountname(self):
  45         return self.accountname
  46
  47     def suggeststhreads(self):
  48         return 1
  49
  50     def waitforthread(self):
  51         self.imapserver.connectionwait()
  52
  53     def getcopyinstancelimit(self):
  54         return 'MSGCOPY_' + self.repository.getname()
  55
  56     def getvisiblename(self):
  57         return self.visiblename
  58
  59     def getuidvalidity(self):
  60         imapobj = self.imapserver.acquireconnection()
  61         try:
  62             # Primes untagged_responses
  63             imapobj.select(self.getfullname(), readonly = 1)
  64             return long(imapobj.untagged_responses['UIDVALIDITY'][0])
  65         finally:
  66             self.imapserver.releaseconnection(imapobj)
  67
  68     def cachemessagelist(self):
  69         imapobj = self.imapserver.acquireconnection()
  70         self.messagelist = {}
  71
  72         try:
  73             # Primes untagged_responses
  74             imapobj.select(self.getfullname(), readonly = 1, force = 1)
  75             try:
  76                 # Some mail servers do not return an EXISTS response if
  77                 # the folder is empty.
  78                 maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
  79             except KeyError:
  80                 return
  81             if maxmsgid < 1:
  82                 # No messages; return
  83                 return
  84
  85             # Now, get the flags and UIDs for these.
  86             # We could conceivably get rid of maxmsgid and just say
  87             # '1:*' here.
  88             response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID INTERNALDATE)')[1]
  89         finally:
  90             self.imapserver.releaseconnection(imapobj)
  91         for messagestr in response:
  92             # Discard the message number.
  93             messagestr = string.split(messagestr, maxsplit = 1)[1]
  94             options = imaputil.flags2hash(messagestr)
  95             if not options.has_key('UID'):
  96                 UIBase.getglobalui().warn('No UID in message with options %s' %\
  97                                           str(options),
  98                                           minor = 1)
  99             else:
 100                 uid = long(options['UID'])
 101                 flags = imaputil.flagsimap2maildir(options['FLAGS'])
 102                 rtime = imaplibutil.Internaldate2epoch(messagestr)
 103                 self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime}
 104
 105     def getmessagelist(self):
 106         return self.messagelist
 107
 108     def getmessage(self, uid):
 109         ui = UIBase.getglobalui()
 110         imapobj = self.imapserver.acquireconnection()
 111         try:
 112             imapobj.select(self.getfullname(), readonly = 1)
 113             initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])')
 114             ui.debug('imap', 'Returned object from fetching %d: %s' % \
 115                      (uid, str(initialresult)))
 116             return initialresult[1][0][1].replace("\r\n", "\n")
 117
 118         finally:
 119             self.imapserver.releaseconnection(imapobj)
 120
 121     def getmessagetime(self, uid):
 122         return self.messagelist[uid]['time']
 123
 124     def getmessageflags(self, uid):
 125         return self.messagelist[uid]['flags']
 126
 127     def savemessage_getnewheader(self, content):
 128         headername = 'X-OfflineIMAP-%s-' % str(binascii.crc32(content)).replace('-', 'x')
 129         headername += binascii.hexlify(self.repository.getname()) + '-'
 130         headername += binascii.hexlify(self.getname())
 131         headervalue= '%d-' % long(time.time())
 132         headervalue += str(self.randomgenerator.random()).replace('.', '')
 133         headervalue += '-v' + versionstr
 134         return (headername, headervalue)
 135
 136     def savemessage_addheader(self, content, headername, headervalue):
 137         ui = UIBase.getglobalui()
 138         ui.debug('imap',
 139                  'savemessage_addheader: called to add %s: %s' % (headername,
 140                                                                   headervalue))
 141         insertionpoint = content.find("\r\n")
 142         ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint)
 143         leader = content[0:insertionpoint]
 144         ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader))
 145         if insertionpoint == 0 or insertionpoint == -1:
 146             newline = ''
 147             insertionpoint = 0
 148         else:
 149             newline = "\r\n"
 150         newline += "%s: %s" % (headername, headervalue)
 151         ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline))
 152         trailer = content[insertionpoint:]
 153         ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer))
 154         return leader + newline + trailer
 155
 156     def savemessage_searchforheader(self, imapobj, headername, headervalue):
 157         if imapobj.untagged_responses.has_key('APPENDUID'):
 158             return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1])
 159
 160         ui = UIBase.getglobalui()
 161         ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \
 162                  (headername, headervalue))
 163         # Now find the UID it got.
 164         headervalue = imapobj._quote(headervalue)
 165         try:
 166             matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0]
 167         except imapobj.error, err:
 168             # IMAP server doesn't implement search or had a problem.
 169             ui.debug('imap', "savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername))
 170             return 0
 171         ui.debug('imap', 'savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids))
 172
 173         if matchinguids == '':
 174             ui.debug('imap', "savemessage_searchforheader: UID SEARCH for message with header %s yielded no results" % headername)
 175             return 0
 176
 177         matchinguids = matchinguids.split(' ')
 178         ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \
 179                  repr(matchinguids))
 180         if len(matchinguids) != 1 or matchinguids[0] == None:
 181             raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids))
 182         matchinguids.sort()
 183         return long(matchinguids[0])
 184
 185     def savemessage(self, uid, content, flags, rtime):
 186         imapobj = self.imapserver.acquireconnection()
 187         ui = UIBase.getglobalui()
 188         ui.debug('imap', 'savemessage: called')
 189         try:
 190             try:
 191                 imapobj.select(self.getfullname()) # Needed for search
 192             except imapobj.readonly:
 193                 ui.msgtoreadonly(self, uid, content, flags)
 194                 # Return indicating message taken, but no UID assigned.
 195                 # Fudge it.
 196                 return 0
 197
 198             # This backend always assigns a new uid, so the uid arg is ignored.
 199             # In order to get the new uid, we need to save off the message ID.
 200
 201             message = rfc822.Message(StringIO(content))
 202             datetuple_msg = rfc822.parsedate(message.getheader('Date'))
 203             # Will be None if missing or not in a valid format.
 204
 205             # If time isn't known
 206             if rtime == None and datetuple_msg == None:
 207                 datetuple = time.localtime()
 208             elif rtime == None:
 209                 datetuple = datetuple_msg
 210             else:
 211                 datetuple = time.localtime(rtime)
 212
 213             try:
 214                 if datetuple[0] < 1981:
 215                     raise ValueError
 216                 # This could raise a value error if it's not a valid format.
 217                 date = imaplib.Time2Internaldate(datetuple)
 218             except (ValueError, OverflowError):
 219                 # Argh, sometimes it's a valid format but year is 0102
 220                 # or something.  Argh.  It seems that Time2Internaldate
 221                 # will rause a ValueError if the year is 0102 but not 1902,
 222                 # but some IMAP servers nonetheless choke on 1902.
 223                 date = imaplib.Time2Internaldate(time.localtime())
 224
 225             ui.debug('imap', 'savemessage: using date ' + str(date))
 226             content = re.sub("(?<!\r)\n", "\r\n", content)
 227             ui.debug('imap', 'savemessage: initial content is: ' + repr(content))
 228
 229             (headername, headervalue) = self.savemessage_getnewheader(content)
 230             ui.debug('imap', 'savemessage: new headers are: %s: %s' % \
 231                      (headername, headervalue))
 232             content = self.savemessage_addheader(content, headername,
 233                                                  headervalue)
 234             ui.debug('imap', 'savemessage: new content is: ' + repr(content))
 235             ui.debug('imap', 'savemessage: new content length is ' + \
 236                      str(len(content)))
 237
 238             assert(imapobj.append(self.getfullname(),
 239                                        imaputil.flagsmaildir2imap(flags),
 240                                        date, content)[0] == 'OK')
 241
 242             # Checkpoint.  Let it write out the messages, etc.
 243             assert(imapobj.check()[0] == 'OK')
 244
 245             # Keep trying until we get the UID.
 246             ui.debug('imap', 'savemessage: first attempt to get new UID')
 247             uid = self.savemessage_searchforheader(imapobj, headername,
 248                                                    headervalue)
 249             # See docs for savemessage in Base.py for explanation of this and other return values
 250             if uid <= 0:
 251                 ui.debug('imap', 'savemessage: first attempt to get new UID failed.  Going to run a NOOP and try again.')
 252                 assert(imapobj.noop()[0] == 'OK')
 253                 uid = self.savemessage_searchforheader(imapobj, headername,
 254                                                        headervalue)
 255         finally:
 256             self.imapserver.releaseconnection(imapobj)
 257
 258         if uid: # avoid UID FETCH 0 crash happening later on
 259             self.messagelist[uid] = {'uid': uid, 'flags': flags}
 260
 261         ui.debug('imap', 'savemessage: returning %d' % uid)
 262         return uid
 263
 264     def savemessageflags(self, uid, flags):
 265         imapobj = self.imapserver.acquireconnection()
 266         try:
 267             try:
 268                 imapobj.select(self.getfullname())
 269             except imapobj.readonly:
 270                 UIBase.getglobalui().flagstoreadonly(self, [uid], flags)
 271                 return
 272             result = imapobj.uid('store', '%d' % uid, 'FLAGS',
 273                                  imaputil.flagsmaildir2imap(flags))
 274             assert result[0] == 'OK', 'Error with store: ' + r[1]
 275         finally:
 276             self.imapserver.releaseconnection(imapobj)
 277         result = result[1][0]
 278         if not result:
 279             self.messagelist[uid]['flags'] = flags
 280         else:
 281             flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS']
 282             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 283
 284     def addmessageflags(self, uid, flags):
 285         self.addmessagesflags([uid], flags)
 286
 287     def addmessagesflags_noconvert(self, uidlist, flags):
 288         self.processmessagesflags('+', uidlist, flags)
 289
 290     def addmessagesflags(self, uidlist, flags):
 291         """This is here for the sake of UIDMaps.py -- deletemessages must
 292         add flags and get a converted UID, and if we don't have noconvert,
 293         then UIDMaps will try to convert it twice."""
 294         self.addmessagesflags_noconvert(uidlist, flags)
 295
 296     def deletemessageflags(self, uid, flags):
 297         self.deletemessagesflags([uid], flags)
 298
 299     def deletemessagesflags(self, uidlist, flags):
 300         self.processmessagesflags('-', uidlist, flags)
 301
 302     def processmessagesflags(self, operation, uidlist, flags):
 303         if len(uidlist) > 101:
 304             # Hack for those IMAP ervers with a limited line length
 305             self.processmessagesflags(operation, uidlist[:100], flags)
 306             self.processmessagesflags(operation, uidlist[100:], flags)
 307             return
 308
 309         imapobj = self.imapserver.acquireconnection()
 310         try:
 311             try:
 312                 imapobj.select(self.getfullname())
 313             except imapobj.readonly:
 314                 UIBase.getglobalui().flagstoreadonly(self, uidlist, flags)
 315                 return
 316             r = imapobj.uid('store',
 317                             imaputil.listjoin(uidlist),
 318                             operation + 'FLAGS',
 319                             imaputil.flagsmaildir2imap(flags))
 320             assert r[0] == 'OK', 'Error with store: ' + r[1]
 321             r = r[1]
 322         finally:
 323             self.imapserver.releaseconnection(imapobj)
 324         # Some IMAP servers do not always return a result.  Therefore,
 325         # only update the ones that it talks about, and manually fix
 326         # the others.
 327         needupdate = copy(uidlist)
 328         for result in r:
 329             if result == None:
 330                 # Compensate for servers that don't return anything from
 331                 # STORE.
 332                 continue
 333             attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1])
 334             if not ('UID' in attributehash and 'FLAGS' in attributehash):
 335                 # Compensate for servers that don't return a UID attribute.
 336                 continue
 337             flags = attributehash['FLAGS']
 338             uid = long(attributehash['UID'])
 339             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 340             try:
 341                 needupdate.remove(uid)
 342             except ValueError:          # Let it slide if it's not in the list
 343                 pass
 344         for uid in needupdate:
 345             if operation == '+':
 346                 for flag in flags:
 347                     if not flag in self.messagelist[uid]['flags']:
 348                         self.messagelist[uid]['flags'].append(flag)
 349                     self.messagelist[uid]['flags'].sort()
 350             elif operation == '-':
 351                 for flag in flags:
 352                     if flag in self.messagelist[uid]['flags']:
 353                         self.messagelist[uid]['flags'].remove(flag)
 354
 355     def deletemessage(self, uid):
 356         self.deletemessages_noconvert([uid])
 357
 358     def deletemessages(self, uidlist):
 359         self.deletemessages_noconvert(uidlist)
 360
 361     def deletemessages_noconvert(self, uidlist):
 362         # Weed out ones not in self.messagelist
 363         uidlist = [uid for uid in uidlist if uid in self.messagelist]
 364         if not len(uidlist):
 365             return
 366
 367         self.addmessagesflags_noconvert(uidlist, ['T'])
 368         imapobj = self.imapserver.acquireconnection()
 369         try:
 370             try:
 371                 imapobj.select(self.getfullname())
 372             except imapobj.readonly:
 373                 UIBase.getglobalui().deletereadonly(self, uidlist)
 374                 return
 375             if self.expunge:
 376                 assert(imapobj.expunge()[0] == 'OK')
 377         finally:
 378             self.imapserver.releaseconnection(imapobj)
 379         for uid in uidlist:
 380             del self.messagelist[uid]
 381
 382