code.delx.au - offlineimap/blob - offlineimap/folder/IMAP.py

   1 # IMAP folder support
   2 # Copyright (C) 2002-2007 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18
  19 from Base import BaseFolder
  20 import imaplib
  21 from offlineimap import imaputil, imaplibutil
  22 from offlineimap.ui import UIBase
  23 from offlineimap.version import versionstr
  24 import rfc822, time, string, random, binascii, re
  25 from StringIO import StringIO
  26 from copy import copy
  27
  28
  29 class IMAPFolder(BaseFolder):
  30     def __init__(self, imapserver, name, visiblename, accountname, repository):
  31         self.config = imapserver.config
  32         self.expunge = repository.getexpunge()
  33         self.name = imaputil.dequote(name)
  34         self.root = None # imapserver.root
  35         self.sep = imapserver.delim
  36         self.imapserver = imapserver
  37         self.messagelist = None
  38         self.visiblename = visiblename
  39         self.accountname = accountname
  40         self.repository = repository
  41         self.randomgenerator = random.Random()
  42         BaseFolder.__init__(self)
  43
  44     def selectro(self, imapobj):
  45         """Select this folder when we do not need write access.
  46         Prefer SELECT to EXAMINE if we can, since some servers
  47         (Courier) do not stabilize UID validity until the folder is
  48         selected."""
  49         try:
  50             imapobj.select(self.getfullname())
  51         except imapobj.readonly:
  52             imapobj.select(self.getfullname(), readonly = 1)
  53
  54     def getaccountname(self):
  55         return self.accountname
  56
  57     def suggeststhreads(self):
  58         return 1
  59
  60     def waitforthread(self):
  61         self.imapserver.connectionwait()
  62
  63     def getcopyinstancelimit(self):
  64         return 'MSGCOPY_' + self.repository.getname()
  65
  66     def getvisiblename(self):
  67         return self.visiblename
  68
  69     def getuidvalidity(self):
  70         imapobj = self.imapserver.acquireconnection()
  71         try:
  72             # Primes untagged_responses
  73             self.selectro(imapobj)
  74             return long(imapobj.untagged_responses['UIDVALIDITY'][0])
  75         finally:
  76             self.imapserver.releaseconnection(imapobj)
  77
  78     def quickchanged(self, statusfolder):
  79         # An IMAP folder has definitely changed if the number of
  80         # messages or the UID of the last message have changed.  Otherwise
  81         # only flag changes could have occurred.
  82         imapobj = self.imapserver.acquireconnection()
  83         try:
  84             # Primes untagged_responses
  85             imapobj.select(self.getfullname(), readonly = 1, force = 1)
  86             try:
  87                 # Some mail servers do not return an EXISTS response if
  88                 # the folder is empty.
  89                 maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
  90             except KeyError:
  91                 return True
  92
  93             # Different number of messages than last time?
  94             if maxmsgid != len(statusfolder.getmessagelist()):
  95                 return True
  96
  97             if maxmsgid < 1:
  98                 # No messages; return
  99                 return False
 100
 101             # Now, get the UID for the last message.
 102             response = imapobj.fetch('%d' % maxmsgid, '(UID)')[1]
 103         finally:
 104             self.imapserver.releaseconnection(imapobj)
 105
 106         # Discard the message number.
 107         messagestr = string.split(response[0], maxsplit = 1)[1]
 108         options = imaputil.flags2hash(messagestr)
 109         if not options.has_key('UID'):
 110             return True
 111         uid = long(options['UID'])
 112         saveduids = statusfolder.getmessagelist().keys()
 113         saveduids.sort()
 114         if uid != saveduids[-1]:
 115             return True
 116
 117         return False
 118
 119     def cachemessagelist(self):
 120         imapobj = self.imapserver.acquireconnection()
 121         self.messagelist = {}
 122
 123         try:
 124             # Primes untagged_responses
 125             imapobj.select(self.getfullname(), readonly = 1, force = 1)
 126             try:
 127                 # Some mail servers do not return an EXISTS response if
 128                 # the folder is empty.
 129                 maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
 130             except KeyError:
 131                 return
 132             if maxmsgid < 1:
 133                 # No messages; return
 134                 return
 135
 136             # Now, get the flags and UIDs for these.
 137             # We could conceivably get rid of maxmsgid and just say
 138             # '1:*' here.
 139             response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID INTERNALDATE)')[1]
 140         finally:
 141             self.imapserver.releaseconnection(imapobj)
 142         for messagestr in response:
 143             # Discard the message number.
 144             messagestr = string.split(messagestr, maxsplit = 1)[1]
 145             options = imaputil.flags2hash(messagestr)
 146             if not options.has_key('UID'):
 147                 UIBase.getglobalui().warn('No UID in message with options %s' %\
 148                                           str(options),
 149                                           minor = 1)
 150             else:
 151                 uid = long(options['UID'])
 152                 flags = imaputil.flagsimap2maildir(options['FLAGS'])
 153                 rtime = imaplibutil.Internaldate2epoch(messagestr)
 154                 self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime}
 155
 156     def getmessagelist(self):
 157         return self.messagelist
 158
 159     def getmessage(self, uid):
 160         ui = UIBase.getglobalui()
 161         imapobj = self.imapserver.acquireconnection()
 162         try:
 163             imapobj.select(self.getfullname(), readonly = 1)
 164             initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])')
 165             ui.debug('imap', 'Returned object from fetching %d: %s' % \
 166                      (uid, str(initialresult)))
 167             return initialresult[1][0][1].replace("\r\n", "\n")
 168
 169         finally:
 170             self.imapserver.releaseconnection(imapobj)
 171
 172     def getmessagetime(self, uid):
 173         return self.messagelist[uid]['time']
 174
 175     def getmessageflags(self, uid):
 176         return self.messagelist[uid]['flags']
 177
 178     def savemessage_getnewheader(self, content):
 179         headername = 'X-OfflineIMAP-%s-' % str(binascii.crc32(content)).replace('-', 'x')
 180         headername += binascii.hexlify(self.repository.getname()) + '-'
 181         headername += binascii.hexlify(self.getname())
 182         headervalue= '%d-' % long(time.time())
 183         headervalue += str(self.randomgenerator.random()).replace('.', '')
 184         headervalue += '-v' + versionstr
 185         return (headername, headervalue)
 186
 187     def savemessage_addheader(self, content, headername, headervalue):
 188         ui = UIBase.getglobalui()
 189         ui.debug('imap',
 190                  'savemessage_addheader: called to add %s: %s' % (headername,
 191                                                                   headervalue))
 192         insertionpoint = content.find("\r\n")
 193         ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint)
 194         leader = content[0:insertionpoint]
 195         ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader))
 196         if insertionpoint == 0 or insertionpoint == -1:
 197             newline = ''
 198             insertionpoint = 0
 199         else:
 200             newline = "\r\n"
 201         newline += "%s: %s" % (headername, headervalue)
 202         ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline))
 203         trailer = content[insertionpoint:]
 204         ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer))
 205         return leader + newline + trailer
 206
 207     def savemessage_searchforheader(self, imapobj, headername, headervalue):
 208         if imapobj.untagged_responses.has_key('APPENDUID'):
 209             return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1])
 210
 211         ui = UIBase.getglobalui()
 212         ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \
 213                  (headername, headervalue))
 214         # Now find the UID it got.
 215         headervalue = imapobj._quote(headervalue)
 216         try:
 217             matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0]
 218         except imapobj.error, err:
 219             # IMAP server doesn't implement search or had a problem.
 220             ui.debug('imap', "savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername))
 221             return 0
 222         ui.debug('imap', 'savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids))
 223
 224         if matchinguids == '':
 225             ui.debug('imap', "savemessage_searchforheader: UID SEARCH for message with header %s yielded no results" % headername)
 226             return 0
 227
 228         matchinguids = matchinguids.split(' ')
 229         ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \
 230                  repr(matchinguids))
 231         if len(matchinguids) != 1 or matchinguids[0] == None:
 232             raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids))
 233         matchinguids.sort()
 234         return long(matchinguids[0])
 235
 236     def savemessage(self, uid, content, flags, rtime):
 237         imapobj = self.imapserver.acquireconnection()
 238         ui = UIBase.getglobalui()
 239         ui.debug('imap', 'savemessage: called')
 240         try:
 241             try:
 242                 imapobj.select(self.getfullname()) # Needed for search
 243             except imapobj.readonly:
 244                 ui.msgtoreadonly(self, uid, content, flags)
 245                 # Return indicating message taken, but no UID assigned.
 246                 # Fudge it.
 247                 return 0
 248
 249             # This backend always assigns a new uid, so the uid arg is ignored.
 250             # In order to get the new uid, we need to save off the message ID.
 251
 252             message = rfc822.Message(StringIO(content))
 253             datetuple_msg = rfc822.parsedate(message.getheader('Date'))
 254             # Will be None if missing or not in a valid format.
 255
 256             # If time isn't known
 257             if rtime == None and datetuple_msg == None:
 258                 datetuple = time.localtime()
 259             elif rtime == None:
 260                 datetuple = datetuple_msg
 261             else:
 262                 datetuple = time.localtime(rtime)
 263
 264             try:
 265                 if datetuple[0] < 1981:
 266                     raise ValueError
 267
 268                 # Check for invalid date
 269                 datetuple_check = time.localtime(time.mktime(datetuple))
 270                 if datetuple[:2] != datetuple_check[:2]:
 271                     raise ValueError
 272
 273                 # This could raise a value error if it's not a valid format.
 274                 date = imaplib.Time2Internaldate(datetuple)
 275             except (ValueError, OverflowError):
 276                 # Argh, sometimes it's a valid format but year is 0102
 277                 # or something.  Argh.  It seems that Time2Internaldate
 278                 # will rause a ValueError if the year is 0102 but not 1902,
 279                 # but some IMAP servers nonetheless choke on 1902.
 280                 date = imaplib.Time2Internaldate(time.localtime())
 281
 282             ui.debug('imap', 'savemessage: using date ' + str(date))
 283             content = re.sub("(?<!\r)\n", "\r\n", content)
 284             ui.debug('imap', 'savemessage: initial content is: ' + repr(content))
 285
 286             (headername, headervalue) = self.savemessage_getnewheader(content)
 287             ui.debug('imap', 'savemessage: new headers are: %s: %s' % \
 288                      (headername, headervalue))
 289             content = self.savemessage_addheader(content, headername,
 290                                                  headervalue)
 291             ui.debug('imap', 'savemessage: new content is: ' + repr(content))
 292             ui.debug('imap', 'savemessage: new content length is ' + \
 293                      str(len(content)))
 294
 295             assert(imapobj.append(self.getfullname(),
 296                                        imaputil.flagsmaildir2imap(flags),
 297                                        date, content)[0] == 'OK')
 298
 299             # Checkpoint.  Let it write out the messages, etc.
 300             assert(imapobj.check()[0] == 'OK')
 301
 302             # Keep trying until we get the UID.
 303             ui.debug('imap', 'savemessage: first attempt to get new UID')
 304             uid = self.savemessage_searchforheader(imapobj, headername,
 305                                                    headervalue)
 306             # See docs for savemessage in Base.py for explanation of this and other return values
 307             if uid <= 0:
 308                 ui.debug('imap', 'savemessage: first attempt to get new UID failed.  Going to run a NOOP and try again.')
 309                 assert(imapobj.noop()[0] == 'OK')
 310                 uid = self.savemessage_searchforheader(imapobj, headername,
 311                                                        headervalue)
 312         finally:
 313             self.imapserver.releaseconnection(imapobj)
 314
 315         if uid: # avoid UID FETCH 0 crash happening later on
 316             self.messagelist[uid] = {'uid': uid, 'flags': flags}
 317
 318         ui.debug('imap', 'savemessage: returning %d' % uid)
 319         return uid
 320
 321     def savemessageflags(self, uid, flags):
 322         imapobj = self.imapserver.acquireconnection()
 323         try:
 324             try:
 325                 imapobj.select(self.getfullname())
 326             except imapobj.readonly:
 327                 UIBase.getglobalui().flagstoreadonly(self, [uid], flags)
 328                 return
 329             result = imapobj.uid('store', '%d' % uid, 'FLAGS',
 330                                  imaputil.flagsmaildir2imap(flags))
 331             assert result[0] == 'OK', 'Error with store: ' + '. '.join(r[1])
 332         finally:
 333             self.imapserver.releaseconnection(imapobj)
 334         result = result[1][0]
 335         if not result:
 336             self.messagelist[uid]['flags'] = flags
 337         else:
 338             flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS']
 339             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 340
 341     def addmessageflags(self, uid, flags):
 342         self.addmessagesflags([uid], flags)
 343
 344     def addmessagesflags_noconvert(self, uidlist, flags):
 345         self.processmessagesflags('+', uidlist, flags)
 346
 347     def addmessagesflags(self, uidlist, flags):
 348         """This is here for the sake of UIDMaps.py -- deletemessages must
 349         add flags and get a converted UID, and if we don't have noconvert,
 350         then UIDMaps will try to convert it twice."""
 351         self.addmessagesflags_noconvert(uidlist, flags)
 352
 353     def deletemessageflags(self, uid, flags):
 354         self.deletemessagesflags([uid], flags)
 355
 356     def deletemessagesflags(self, uidlist, flags):
 357         self.processmessagesflags('-', uidlist, flags)
 358
 359     def processmessagesflags(self, operation, uidlist, flags):
 360         if len(uidlist) > 101:
 361             # Hack for those IMAP ervers with a limited line length
 362             self.processmessagesflags(operation, uidlist[:100], flags)
 363             self.processmessagesflags(operation, uidlist[100:], flags)
 364             return
 365
 366         imapobj = self.imapserver.acquireconnection()
 367         try:
 368             try:
 369                 imapobj.select(self.getfullname())
 370             except imapobj.readonly:
 371                 UIBase.getglobalui().flagstoreadonly(self, uidlist, flags)
 372                 return
 373             r = imapobj.uid('store',
 374                             imaputil.listjoin(uidlist),
 375                             operation + 'FLAGS',
 376                             imaputil.flagsmaildir2imap(flags))
 377             assert r[0] == 'OK', 'Error with store: ' + '. '.join(r[1])
 378             r = r[1]
 379         finally:
 380             self.imapserver.releaseconnection(imapobj)
 381         # Some IMAP servers do not always return a result.  Therefore,
 382         # only update the ones that it talks about, and manually fix
 383         # the others.
 384         needupdate = copy(uidlist)
 385         for result in r:
 386             if result == None:
 387                 # Compensate for servers that don't return anything from
 388                 # STORE.
 389                 continue
 390             attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1])
 391             if not ('UID' in attributehash and 'FLAGS' in attributehash):
 392                 # Compensate for servers that don't return a UID attribute.
 393                 continue
 394             lflags = attributehash['FLAGS']
 395             uid = long(attributehash['UID'])
 396             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(lflags)
 397             try:
 398                 needupdate.remove(uid)
 399             except ValueError:          # Let it slide if it's not in the list
 400                 pass
 401         for uid in needupdate:
 402             if operation == '+':
 403                 for flag in flags:
 404                     if not flag in self.messagelist[uid]['flags']:
 405                         self.messagelist[uid]['flags'].append(flag)
 406                     self.messagelist[uid]['flags'].sort()
 407             elif operation == '-':
 408                 for flag in flags:
 409                     if flag in self.messagelist[uid]['flags']:
 410                         self.messagelist[uid]['flags'].remove(flag)
 411
 412     def deletemessage(self, uid):
 413         self.deletemessages_noconvert([uid])
 414
 415     def deletemessages(self, uidlist):
 416         self.deletemessages_noconvert(uidlist)
 417
 418     def deletemessages_noconvert(self, uidlist):
 419         # Weed out ones not in self.messagelist
 420         uidlist = [uid for uid in uidlist if uid in self.messagelist]
 421         if not len(uidlist):
 422             return
 423
 424         self.addmessagesflags_noconvert(uidlist, ['T'])
 425         imapobj = self.imapserver.acquireconnection()
 426         try:
 427             try:
 428                 imapobj.select(self.getfullname())
 429             except imapobj.readonly:
 430                 UIBase.getglobalui().deletereadonly(self, uidlist)
 431                 return
 432             if self.expunge:
 433                 assert(imapobj.expunge()[0] == 'OK')
 434         finally:
 435             self.imapserver.releaseconnection(imapobj)
 436         for uid in uidlist:
 437             del self.messagelist[uid]
 438
 439