code.delx.au - offlineimap/blob - offlineimap/folder/IMAP.py

   1 # IMAP folder support
   2 # Copyright (C) 2002-2007 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18
  19 from Base import BaseFolder
  20 import imaplib
  21 from offlineimap import imaputil, imaplibutil
  22 from offlineimap.ui import UIBase
  23 from offlineimap.version import versionstr
  24 import rfc822, time, string, random, binascii, re
  25 from StringIO import StringIO
  26 from copy import copy
  27
  28
  29 class IMAPFolder(BaseFolder):
  30     def __init__(self, imapserver, name, visiblename, accountname, repository):
  31         self.config = imapserver.config
  32         self.expunge = repository.getexpunge()
  33         self.name = imaputil.dequote(name)
  34         self.root = None # imapserver.root
  35         self.sep = imapserver.delim
  36         self.imapserver = imapserver
  37         self.messagelist = None
  38         self.visiblename = visiblename
  39         self.accountname = accountname
  40         self.repository = repository
  41         self.randomgenerator = random.Random()
  42         BaseFolder.__init__(self)
  43
  44     def selectro(self, imapobj):
  45         """Select this folder when we do not need write access.
  46         Prefer SELECT to EXAMINE if we can, since some servers
  47         (Courier) do not stabilize UID validity until the folder is
  48         selected."""
  49         try:
  50             imapobj.select(self.getfullname())
  51         except imapobj.readonly:
  52             imapobj.select(self.getfullname(), readonly = 1)
  53
  54     def getaccountname(self):
  55         return self.accountname
  56
  57     def suggeststhreads(self):
  58         return 1
  59
  60     def waitforthread(self):
  61         self.imapserver.connectionwait()
  62
  63     def getcopyinstancelimit(self):
  64         return 'MSGCOPY_' + self.repository.getname()
  65
  66     def getvisiblename(self):
  67         return self.visiblename
  68
  69     def getuidvalidity(self):
  70         imapobj = self.imapserver.acquireconnection()
  71         try:
  72             # Primes untagged_responses
  73             self.selectro(imapobj)
  74             return long(imapobj.untagged_responses['UIDVALIDITY'][0])
  75         finally:
  76             self.imapserver.releaseconnection(imapobj)
  77
  78     def quickchanged(self, statusfolder):
  79         # An IMAP folder has definitely changed if the number of
  80         # messages or the UID of the last message have changed.  Otherwise
  81         # only flag changes could have occurred.
  82         imapobj = self.imapserver.acquireconnection()
  83         try:
  84             # Primes untagged_responses
  85             imapobj.select(self.getfullname(), readonly = 1, force = 1)
  86             try:
  87                 # Some mail servers do not return an EXISTS response if
  88                 # the folder is empty.
  89                 maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
  90             except KeyError:
  91                 return True
  92
  93             # Different number of messages than last time?
  94             if maxmsgid != len(statusfolder.getmessagelist()):
  95                 return True
  96
  97             if maxmsgid < 1:
  98                 # No messages; return
  99                 return False
 100
 101             # Now, get the UID for the last message.
 102             response = imapobj.fetch('%d' % maxmsgid, '(UID)')[1]
 103         finally:
 104             self.imapserver.releaseconnection(imapobj)
 105
 106         # Discard the message number.
 107         messagestr = string.split(response[0], maxsplit = 1)[1]
 108         options = imaputil.flags2hash(messagestr)
 109         if not options.has_key('UID'):
 110             return True
 111         uid = long(options['UID'])
 112         saveduids = statusfolder.getmessagelist().keys()
 113         saveduids.sort()
 114         if uid != saveduids[-1]:
 115             return True
 116
 117         return False
 118
 119     def cachemessagelist(self):
 120         imapobj = self.imapserver.acquireconnection()
 121         self.messagelist = {}
 122
 123         try:
 124             # Primes untagged_responses
 125             imapobj.select(self.getfullname(), readonly = 1, force = 1)
 126             try:
 127                 # Some mail servers do not return an EXISTS response if
 128                 # the folder is empty.
 129                 maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
 130             except KeyError:
 131                 return
 132             if maxmsgid < 1:
 133                 # No messages; return
 134                 return
 135
 136             # Now, get the flags and UIDs for these.
 137             # We could conceivably get rid of maxmsgid and just say
 138             # '1:*' here.
 139             response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID INTERNALDATE)')[1]
 140         finally:
 141             self.imapserver.releaseconnection(imapobj)
 142         for messagestr in response:
 143             # Discard the message number.
 144             messagestr = string.split(messagestr, maxsplit = 1)[1]
 145             options = imaputil.flags2hash(messagestr)
 146             if not options.has_key('UID'):
 147                 UIBase.getglobalui().warn('No UID in message with options %s' %\
 148                                           str(options),
 149                                           minor = 1)
 150             else:
 151                 uid = long(options['UID'])
 152                 flags = imaputil.flagsimap2maildir(options['FLAGS'])
 153                 rtime = imaplibutil.Internaldate2epoch(messagestr)
 154                 self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime}
 155
 156     def getmessagelist(self):
 157         return self.messagelist
 158
 159     def getmessage(self, uid):
 160         ui = UIBase.getglobalui()
 161         imapobj = self.imapserver.acquireconnection()
 162         try:
 163             imapobj.select(self.getfullname(), readonly = 1)
 164             initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])')
 165             ui.debug('imap', 'Returned object from fetching %d: %s' % \
 166                      (uid, str(initialresult)))
 167             return initialresult[1][0][1].replace("\r\n", "\n")
 168
 169         finally:
 170             self.imapserver.releaseconnection(imapobj)
 171
 172     def getmessagetime(self, uid):
 173         return self.messagelist[uid]['time']
 174
 175     def getmessageflags(self, uid):
 176         return self.messagelist[uid]['flags']
 177
 178     def savemessage_getnewheader(self, content):
 179         headername = 'X-OfflineIMAP-%s-' % str(binascii.crc32(content)).replace('-', 'x')
 180         headername += binascii.hexlify(self.repository.getname()) + '-'
 181         headername += binascii.hexlify(self.getname())
 182         headervalue= '%d-' % long(time.time())
 183         headervalue += str(self.randomgenerator.random()).replace('.', '')
 184         headervalue += '-v' + versionstr
 185         return (headername, headervalue)
 186
 187     def savemessage_addheader(self, content, headername, headervalue):
 188         ui = UIBase.getglobalui()
 189         ui.debug('imap',
 190                  'savemessage_addheader: called to add %s: %s' % (headername,
 191                                                                   headervalue))
 192         insertionpoint = content.find("\r\n")
 193         ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint)
 194         leader = content[0:insertionpoint]
 195         ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader))
 196         if insertionpoint == 0 or insertionpoint == -1:
 197             newline = ''
 198             insertionpoint = 0
 199         else:
 200             newline = "\r\n"
 201         newline += "%s: %s" % (headername, headervalue)
 202         ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline))
 203         trailer = content[insertionpoint:]
 204         ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer))
 205         return leader + newline + trailer
 206
 207     def savemessage_searchforheader(self, imapobj, headername, headervalue):
 208         if imapobj.untagged_responses.has_key('APPENDUID'):
 209             return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1])
 210
 211         ui = UIBase.getglobalui()
 212         ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \
 213                  (headername, headervalue))
 214         # Now find the UID it got.
 215         headervalue = imapobj._quote(headervalue)
 216         try:
 217             matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0]
 218         except imapobj.error, err:
 219             # IMAP server doesn't implement search or had a problem.
 220             ui.debug('imap', "savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername))
 221             return 0
 222         ui.debug('imap', 'savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids))
 223
 224         if matchinguids == '':
 225             ui.debug('imap', "savemessage_searchforheader: UID SEARCH for message with header %s yielded no results" % headername)
 226             return 0
 227
 228         matchinguids = matchinguids.split(' ')
 229         ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \
 230                  repr(matchinguids))
 231         if len(matchinguids) != 1 or matchinguids[0] == None:
 232             raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids))
 233         matchinguids.sort()
 234         return long(matchinguids[0])
 235
 236     def savemessage(self, uid, content, flags, rtime):
 237         imapobj = self.imapserver.acquireconnection()
 238         ui = UIBase.getglobalui()
 239         ui.debug('imap', 'savemessage: called')
 240         try:
 241             try:
 242                 imapobj.select(self.getfullname()) # Needed for search
 243             except imapobj.readonly:
 244                 ui.msgtoreadonly(self, uid, content, flags)
 245                 # Return indicating message taken, but no UID assigned.
 246                 # Fudge it.
 247                 return 0
 248
 249             # This backend always assigns a new uid, so the uid arg is ignored.
 250             # In order to get the new uid, we need to save off the message ID.
 251
 252             message = rfc822.Message(StringIO(content))
 253             datetuple_msg = rfc822.parsedate(message.getheader('Date'))
 254             # Will be None if missing or not in a valid format.
 255
 256             # If time isn't known
 257             if rtime == None and datetuple_msg == None:
 258                 datetuple = time.localtime()
 259             elif rtime == None:
 260                 datetuple = datetuple_msg
 261             else:
 262                 datetuple = time.localtime(rtime)
 263
 264             try:
 265                 if datetuple[0] < 1981:
 266                     raise ValueError
 267
 268                 # Check for invalid date
 269                 datetuple_check = time.localtime(time.mktime(datetuple))
 270                 if datetuple[:2] != datetuple_check[:2]:
 271                     raise ValueError
 272
 273                 # This could raise a value error if it's not a valid format.
 274                 date = imaplib.Time2Internaldate(datetuple)
 275             except (ValueError, OverflowError):
 276                 # Argh, sometimes it's a valid format but year is 0102
 277                 # or something.  Argh.  It seems that Time2Internaldate
 278                 # will rause a ValueError if the year is 0102 but not 1902,
 279                 # but some IMAP servers nonetheless choke on 1902.
 280                 date = imaplib.Time2Internaldate(time.localtime())
 281
 282             ui.debug('imap', 'savemessage: using date ' + str(date))
 283             content = re.sub("(?<!\r)\n", "\r\n", content)
 284             ui.debug('imap', 'savemessage: initial content is: ' + repr(content))
 285
 286             (headername, headervalue) = self.savemessage_getnewheader(content)
 287             ui.debug('imap', 'savemessage: new headers are: %s: %s' % \
 288                      (headername, headervalue))
 289             content = self.savemessage_addheader(content, headername,
 290                                                  headervalue)
 291             ui.debug('imap', 'savemessage: new content is: ' + repr(content))
 292             ui.debug('imap', 'savemessage: new content length is ' + \
 293                      str(len(content)))
 294
 295             assert(imapobj.append(self.getfullname(),
 296                                        imaputil.flagsmaildir2imap(flags),
 297                                        date, content)[0] == 'OK')
 298
 299             # Checkpoint.  Let it write out the messages, etc.
 300             assert(imapobj.check()[0] == 'OK')
 301
 302             # Keep trying until we get the UID.
 303             ui.debug('imap', 'savemessage: first attempt to get new UID')
 304             uid = self.savemessage_searchforheader(imapobj, headername,
 305                                                    headervalue)
 306             # See docs for savemessage in Base.py for explanation of this and other return values
 307             if uid <= 0:
 308                 ui.debug('imap', 'savemessage: first attempt to get new UID failed.  Going to run a NOOP and try again.')
 309                 assert(imapobj.noop()[0] == 'OK')
 310                 uid = self.savemessage_searchforheader(imapobj, headername,
 311                                                        headervalue)
 312         finally:
 313             self.imapserver.releaseconnection(imapobj)
 314
 315         if uid: # avoid UID FETCH 0 crash happening later on
 316             self.messagelist[uid] = {'uid': uid, 'flags': flags}
 317
 318         ui.debug('imap', 'savemessage: returning %d' % uid)
 319         return uid
 320
 321     def savemessageflags(self, uid, flags):
 322         imapobj = self.imapserver.acquireconnection()
 323         try:
 324             try:
 325                 imapobj.select(self.getfullname())
 326             except imapobj.readonly:
 327                 UIBase.getglobalui().flagstoreadonly(self, [uid], flags)
 328                 return
 329             result = imapobj.uid('store', '%d' % uid, 'FLAGS',
 330                                  imaputil.flagsmaildir2imap(flags))
 331             assert result[0] == 'OK', 'Error with store: ' + '. '.join(r[1])
 332         finally:
 333             self.imapserver.releaseconnection(imapobj)
 334         result = result[1][0]
 335         if not result:
 336             self.messagelist[uid]['flags'] = flags
 337         else:
 338             flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS']
 339             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 340
 341     def addmessageflags(self, uid, flags):
 342         self.addmessagesflags([uid], flags)
 343
 344     def addmessagesflags_noconvert(self, uidlist, flags):
 345         self.processmessagesflags('+', uidlist, flags)
 346
 347     def addmessagesflags(self, uidlist, flags):
 348         """This is here for the sake of UIDMaps.py -- deletemessages must
 349         add flags and get a converted UID, and if we don't have noconvert,
 350         then UIDMaps will try to convert it twice."""
 351         self.addmessagesflags_noconvert(uidlist, flags)
 352
 353     def deletemessageflags(self, uid, flags):
 354         self.deletemessagesflags([uid], flags)
 355
 356     def deletemessagesflags(self, uidlist, flags):
 357         self.processmessagesflags('-', uidlist, flags)
 358
 359     def processmessagesflags(self, operation, uidlist, flags):
 360         if len(uidlist) > 101:
 361             # Hack for those IMAP ervers with a limited line length
 362             self.processmessagesflags(operation, uidlist[:100], flags)
 363             self.processmessagesflags(operation, uidlist[100:], flags)
 364             return
 365
 366         imapobj = self.imapserver.acquireconnection()
 367         try:
 368             # Making sure, that we have the necessary rights
 369             # ensuring that we access readonly: python's braindead imaplib.py
 370             # otherwise might raise an exception during the myrights() call
 371             imapobj.select(self.getfullname(),readonly=1)
 372             myrights = imapobj.myrights(self.getfullname())[1][0].split()[1]
 373             if 'T' in flags and not 'd' in myrights or \
 374                     'S' in flags and not 's' in myrights or \
 375                     filter(lambda x: x not in 'TS', flags) and not 'w' in myrights:
 376                 # no delete/expunge right, but needed or
 377                 # no store seen right, but needed or
 378                 # no write right, but needed
 379                 UIBase.getglobalui().flagstoreadonly(self, uidlist, flags)
 380                 return
 381
 382             try:
 383                 imapobj.select(self.getfullname())
 384             except imapobj.readonly:
 385                 # Above we made sure, we have the necessary rights.
 386                 # Ugly hack, to prevent an unnecessary exception:
 387                 #  readonly: mailbox status changed to READ-ONLY
 388                 # imaplib should take care of that itself.
 389                 # The connection is anyway released below, so we dont need to
 390                 # undo the hack.
 391                 imapobj.is_readonly = True
 392             r = imapobj.uid('store',
 393                             imaputil.listjoin(uidlist),
 394                             operation + 'FLAGS',
 395                             imaputil.flagsmaildir2imap(flags))
 396             assert r[0] == 'OK', 'Error with store: ' + '. '.join(r[1])
 397             r = r[1]
 398         finally:
 399             self.imapserver.releaseconnection(imapobj)
 400         # Some IMAP servers do not always return a result.  Therefore,
 401         # only update the ones that it talks about, and manually fix
 402         # the others.
 403         needupdate = copy(uidlist)
 404         for result in r:
 405             if result == None:
 406                 # Compensate for servers that don't return anything from
 407                 # STORE.
 408                 continue
 409             attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1])
 410             if not ('UID' in attributehash and 'FLAGS' in attributehash):
 411                 # Compensate for servers that don't return a UID attribute.
 412                 continue
 413             flags = attributehash['FLAGS']
 414             uid = long(attributehash['UID'])
 415             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 416             try:
 417                 needupdate.remove(uid)
 418             except ValueError:          # Let it slide if it's not in the list
 419                 pass
 420         for uid in needupdate:
 421             if operation == '+':
 422                 for flag in flags:
 423                     if not flag in self.messagelist[uid]['flags']:
 424                         self.messagelist[uid]['flags'].append(flag)
 425                     self.messagelist[uid]['flags'].sort()
 426             elif operation == '-':
 427                 for flag in flags:
 428                     if flag in self.messagelist[uid]['flags']:
 429                         self.messagelist[uid]['flags'].remove(flag)
 430
 431     def deletemessage(self, uid):
 432         self.deletemessages_noconvert([uid])
 433
 434     def deletemessages(self, uidlist):
 435         self.deletemessages_noconvert(uidlist)
 436
 437     def deletemessages_noconvert(self, uidlist):
 438         # Weed out ones not in self.messagelist
 439         uidlist = [uid for uid in uidlist if uid in self.messagelist]
 440         if not len(uidlist):
 441             return
 442
 443         self.addmessagesflags_noconvert(uidlist, ['T'])
 444         imapobj = self.imapserver.acquireconnection()
 445         try:
 446             # Making sure, that we have the necessary rights
 447             # ensuring that we access readonly: python's braindead imaplib.py
 448             # otherwise might raise an exception during the myrights() call
 449             imapobj.select(self.getfullname(),readonly=1)
 450             if not 'd' in imapobj.myrights(self.getfullname())[1][0].split()[1]:
 451                 # no delete/expunge rights
 452                 UIBase.getglobalui().deletereadonly(self, uidlist)
 453                 return
 454
 455             if self.expunge:
 456                 imapobj.select(self.getfullname())
 457                 assert(imapobj.expunge()[0] == 'OK')
 458         finally:
 459             self.imapserver.releaseconnection(imapobj)
 460         for uid in uidlist:
 461             del self.messagelist[uid]
 462
 463