code.delx.au - offlineimap/blob - offlineimap/folder/IMAP.py

   1 # IMAP folder support
   2 # Copyright (C) 2002-2007 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18
  19 from Base import BaseFolder
  20 from offlineimap import imaplib2, imaputil, imaplibutil
  21 from offlineimap.ui import UIBase
  22 from offlineimap.version import versionstr
  23 import rfc822, time, string, random, binascii, re
  24 from StringIO import StringIO
  25 from copy import copy
  26
  27
  28 class IMAPFolder(BaseFolder):
  29     def __init__(self, imapserver, name, visiblename, accountname, repository):
  30         self.config = imapserver.config
  31         self.expunge = repository.getexpunge()
  32         self.name = imaputil.dequote(name)
  33         self.root = None # imapserver.root
  34         self.sep = imapserver.delim
  35         self.imapserver = imapserver
  36         self.messagelist = None
  37         self.visiblename = visiblename
  38         self.accountname = accountname
  39         self.repository = repository
  40         self.randomgenerator = random.Random()
  41         BaseFolder.__init__(self)
  42
  43     def selectro(self, imapobj):
  44         """Select this folder when we do not need write access.
  45         Prefer SELECT to EXAMINE if we can, since some servers
  46         (Courier) do not stabilize UID validity until the folder is
  47         selected."""
  48         try:
  49             imapobj.select(self.getfullname())
  50         except imapobj.readonly:
  51             imapobj.select(self.getfullname(), readonly = 1)
  52
  53     def getaccountname(self):
  54         return self.accountname
  55
  56     def suggeststhreads(self):
  57         return 1
  58
  59     def waitforthread(self):
  60         self.imapserver.connectionwait()
  61
  62     def getcopyinstancelimit(self):
  63         return 'MSGCOPY_' + self.repository.getname()
  64
  65     def getvisiblename(self):
  66         return self.visiblename
  67
  68     def getuidvalidity(self):
  69         imapobj = self.imapserver.acquireconnection()
  70         try:
  71             # Primes untagged_responses
  72             self.selectro(imapobj)
  73             return long(imapobj.untagged_responses['UIDVALIDITY'][0])
  74         finally:
  75             self.imapserver.releaseconnection(imapobj)
  76
  77     def quickchanged(self, statusfolder):
  78         # An IMAP folder has definitely changed if the number of
  79         # messages or the UID of the last message have changed.  Otherwise
  80         # only flag changes could have occurred.
  81         imapobj = self.imapserver.acquireconnection()
  82         try:
  83             # Primes untagged_responses
  84             imapobj.select(self.getfullname(), readonly = 1, force = 1)
  85             try:
  86                 # Some mail servers do not return an EXISTS response if
  87                 # the folder is empty.
  88                 maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
  89             except KeyError:
  90                 return True
  91
  92             # Different number of messages than last time?
  93             if maxmsgid != len(statusfolder.getmessagelist()):
  94                 return True
  95
  96             if maxmsgid < 1:
  97                 # No messages; return
  98                 return False
  99
 100             # Now, get the UID for the last message.
 101             response = imapobj.fetch('%d' % maxmsgid, '(UID)')[1]
 102         finally:
 103             self.imapserver.releaseconnection(imapobj)
 104
 105         # Discard the message number.
 106         messagestr = string.split(response[0], maxsplit = 1)[1]
 107         options = imaputil.flags2hash(messagestr)
 108         if not options.has_key('UID'):
 109             return True
 110         uid = long(options['UID'])
 111         saveduids = statusfolder.getmessagelist().keys()
 112         saveduids.sort()
 113         if uid != saveduids[-1]:
 114             return True
 115
 116         return False
 117
 118     def cachemessagelist(self):
 119         imapobj = self.imapserver.acquireconnection()
 120         self.messagelist = {}
 121
 122         try:
 123             # Primes untagged_responses
 124             imapobj.select(self.getfullname(), readonly = 1, force = 1)
 125             try:
 126                 # Some mail servers do not return an EXISTS response if
 127                 # the folder is empty.
 128                 maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
 129             except KeyError:
 130                 return
 131             if maxmsgid < 1:
 132                 # No messages; return
 133                 return
 134
 135             # Now, get the flags and UIDs for these.
 136             # We could conceivably get rid of maxmsgid and just say
 137             # '1:*' here.
 138             response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID INTERNALDATE)')[1]
 139         finally:
 140             self.imapserver.releaseconnection(imapobj)
 141         for messagestr in response:
 142             # Discard the message number.
 143             messagestr = string.split(messagestr, maxsplit = 1)[1]
 144             options = imaputil.flags2hash(messagestr)
 145             if not options.has_key('UID'):
 146                 UIBase.getglobalui().warn('No UID in message with options %s' %\
 147                                           str(options),
 148                                           minor = 1)
 149             else:
 150                 uid = long(options['UID'])
 151                 flags = imaputil.flagsimap2maildir(options['FLAGS'])
 152                 rtime = imaplibutil.Internaldate2epoch(messagestr)
 153                 self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime}
 154
 155     def getmessagelist(self):
 156         return self.messagelist
 157
 158     def getmessage(self, uid):
 159         ui = UIBase.getglobalui()
 160         imapobj = self.imapserver.acquireconnection()
 161         try:
 162             imapobj.select(self.getfullname(), readonly = 1)
 163             initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])')
 164             ui.debug('imap', 'Returned object from fetching %d: %s' % \
 165                      (uid, str(initialresult)))
 166             return initialresult[1][0][1].replace("\r\n", "\n")
 167
 168         finally:
 169             self.imapserver.releaseconnection(imapobj)
 170
 171     def getmessagetime(self, uid):
 172         return self.messagelist[uid]['time']
 173
 174     def getmessageflags(self, uid):
 175         return self.messagelist[uid]['flags']
 176
 177     def savemessage_getnewheader(self, content):
 178         headername = 'X-OfflineIMAP-%s-' % str(binascii.crc32(content)).replace('-', 'x')
 179         headername += binascii.hexlify(self.repository.getname()) + '-'
 180         headername += binascii.hexlify(self.getname())
 181         headervalue= '%d-' % long(time.time())
 182         headervalue += str(self.randomgenerator.random()).replace('.', '')
 183         headervalue += '-v' + versionstr
 184         return (headername, headervalue)
 185
 186     def savemessage_addheader(self, content, headername, headervalue):
 187         ui = UIBase.getglobalui()
 188         ui.debug('imap',
 189                  'savemessage_addheader: called to add %s: %s' % (headername,
 190                                                                   headervalue))
 191         insertionpoint = content.find("\r\n")
 192         ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint)
 193         leader = content[0:insertionpoint]
 194         ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader))
 195         if insertionpoint == 0 or insertionpoint == -1:
 196             newline = ''
 197             insertionpoint = 0
 198         else:
 199             newline = "\r\n"
 200         newline += "%s: %s" % (headername, headervalue)
 201         ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline))
 202         trailer = content[insertionpoint:]
 203         ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer))
 204         return leader + newline + trailer
 205
 206     def savemessage_searchforheader(self, imapobj, headername, headervalue):
 207         if imapobj.untagged_responses.has_key('APPENDUID'):
 208             return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1])
 209
 210         ui = UIBase.getglobalui()
 211         ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \
 212                  (headername, headervalue))
 213         # Now find the UID it got.
 214         headervalue = imapobj._quote(headervalue)
 215         try:
 216             matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0]
 217         except imapobj.error, err:
 218             # IMAP server doesn't implement search or had a problem.
 219             ui.debug('imap', "savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername))
 220             return 0
 221         ui.debug('imap', 'savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids))
 222
 223         if matchinguids == '':
 224             ui.debug('imap', "savemessage_searchforheader: UID SEARCH for message with header %s yielded no results" % headername)
 225             return 0
 226
 227         matchinguids = matchinguids.split(' ')
 228         ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \
 229                  repr(matchinguids))
 230         if len(matchinguids) != 1 or matchinguids[0] == None:
 231             raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids))
 232         matchinguids.sort()
 233         return long(matchinguids[0])
 234
 235     def savemessage(self, uid, content, flags, rtime):
 236         imapobj = self.imapserver.acquireconnection()
 237         ui = UIBase.getglobalui()
 238         ui.debug('imap', 'savemessage: called')
 239         try:
 240             try:
 241                 imapobj.select(self.getfullname()) # Needed for search
 242             except imapobj.readonly:
 243                 ui.msgtoreadonly(self, uid, content, flags)
 244                 # Return indicating message taken, but no UID assigned.
 245                 # Fudge it.
 246                 return 0
 247
 248             # This backend always assigns a new uid, so the uid arg is ignored.
 249             # In order to get the new uid, we need to save off the message ID.
 250
 251             message = rfc822.Message(StringIO(content))
 252             datetuple_msg = rfc822.parsedate(message.getheader('Date'))
 253             # Will be None if missing or not in a valid format.
 254
 255             # If time isn't known
 256             if rtime == None and datetuple_msg == None:
 257                 datetuple = time.localtime()
 258             elif rtime == None:
 259                 datetuple = datetuple_msg
 260             else:
 261                 datetuple = time.localtime(rtime)
 262
 263             try:
 264                 if datetuple[0] < 1981:
 265                     raise ValueError
 266
 267                 # Check for invalid date
 268                 datetuple_check = time.localtime(time.mktime(datetuple))
 269                 if datetuple[:2] != datetuple_check[:2]:
 270                     raise ValueError
 271
 272                 # This could raise a value error if it's not a valid format.
 273                 date = imaplib2.Time2Internaldate(datetuple)
 274             except (ValueError, OverflowError):
 275                 # Argh, sometimes it's a valid format but year is 0102
 276                 # or something.  Argh.  It seems that Time2Internaldate
 277                 # will rause a ValueError if the year is 0102 but not 1902,
 278                 # but some IMAP servers nonetheless choke on 1902.
 279                 date = imaplib2.Time2Internaldate(time.localtime())
 280
 281             ui.debug('imap', 'savemessage: using date ' + str(date))
 282             content = re.sub("(?<!\r)\n", "\r\n", content)
 283             ui.debug('imap', 'savemessage: initial content is: ' + repr(content))
 284
 285             (headername, headervalue) = self.savemessage_getnewheader(content)
 286             ui.debug('imap', 'savemessage: new headers are: %s: %s' % \
 287                      (headername, headervalue))
 288             content = self.savemessage_addheader(content, headername,
 289                                                  headervalue)
 290             ui.debug('imap', 'savemessage: new content is: ' + repr(content))
 291             ui.debug('imap', 'savemessage: new content length is ' + \
 292                      str(len(content)))
 293
 294             assert(imapobj.append(self.getfullname(),
 295                                        imaputil.flagsmaildir2imap(flags),
 296                                        date, content)[0] == 'OK')
 297
 298             # Checkpoint.  Let it write out the messages, etc.
 299             assert(imapobj.check()[0] == 'OK')
 300
 301             # Keep trying until we get the UID.
 302             ui.debug('imap', 'savemessage: first attempt to get new UID')
 303             uid = self.savemessage_searchforheader(imapobj, headername,
 304                                                    headervalue)
 305             # See docs for savemessage in Base.py for explanation of this and other return values
 306             if uid <= 0:
 307                 ui.debug('imap', 'savemessage: first attempt to get new UID failed.  Going to run a NOOP and try again.')
 308                 assert(imapobj.noop()[0] == 'OK')
 309                 uid = self.savemessage_searchforheader(imapobj, headername,
 310                                                        headervalue)
 311         finally:
 312             self.imapserver.releaseconnection(imapobj)
 313
 314         if uid: # avoid UID FETCH 0 crash happening later on
 315             self.messagelist[uid] = {'uid': uid, 'flags': flags}
 316
 317         ui.debug('imap', 'savemessage: returning %d' % uid)
 318         return uid
 319
 320     def savemessageflags(self, uid, flags):
 321         imapobj = self.imapserver.acquireconnection()
 322         try:
 323             try:
 324                 imapobj.select(self.getfullname())
 325             except imapobj.readonly:
 326                 UIBase.getglobalui().flagstoreadonly(self, [uid], flags)
 327                 return
 328             result = imapobj.uid('store', '%d' % uid, 'FLAGS',
 329                                  imaputil.flagsmaildir2imap(flags))
 330             assert result[0] == 'OK', 'Error with store: ' + '. '.join(r[1])
 331         finally:
 332             self.imapserver.releaseconnection(imapobj)
 333         result = result[1][0]
 334         if not result:
 335             self.messagelist[uid]['flags'] = flags
 336         else:
 337             flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS']
 338             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 339
 340     def addmessageflags(self, uid, flags):
 341         self.addmessagesflags([uid], flags)
 342
 343     def addmessagesflags_noconvert(self, uidlist, flags):
 344         self.processmessagesflags('+', uidlist, flags)
 345
 346     def addmessagesflags(self, uidlist, flags):
 347         """This is here for the sake of UIDMaps.py -- deletemessages must
 348         add flags and get a converted UID, and if we don't have noconvert,
 349         then UIDMaps will try to convert it twice."""
 350         self.addmessagesflags_noconvert(uidlist, flags)
 351
 352     def deletemessageflags(self, uid, flags):
 353         self.deletemessagesflags([uid], flags)
 354
 355     def deletemessagesflags(self, uidlist, flags):
 356         self.processmessagesflags('-', uidlist, flags)
 357
 358     def processmessagesflags(self, operation, uidlist, flags):
 359         if len(uidlist) > 101:
 360             # Hack for those IMAP ervers with a limited line length
 361             self.processmessagesflags(operation, uidlist[:100], flags)
 362             self.processmessagesflags(operation, uidlist[100:], flags)
 363             return
 364
 365         imapobj = self.imapserver.acquireconnection()
 366         try:
 367             try:
 368                 imapobj.select(self.getfullname())
 369             except imapobj.readonly:
 370                 UIBase.getglobalui().flagstoreadonly(self, uidlist, flags)
 371                 return
 372             r = imapobj.uid('store',
 373                             imaputil.listjoin(uidlist),
 374                             operation + 'FLAGS',
 375                             imaputil.flagsmaildir2imap(flags))
 376             assert r[0] == 'OK', 'Error with store: ' + '. '.join(r[1])
 377             r = r[1]
 378         finally:
 379             self.imapserver.releaseconnection(imapobj)
 380         # Some IMAP servers do not always return a result.  Therefore,
 381         # only update the ones that it talks about, and manually fix
 382         # the others.
 383         needupdate = copy(uidlist)
 384         for result in r:
 385             if result == None:
 386                 # Compensate for servers that don't return anything from
 387                 # STORE.
 388                 continue
 389             attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1])
 390             if not ('UID' in attributehash and 'FLAGS' in attributehash):
 391                 # Compensate for servers that don't return a UID attribute.
 392                 continue
 393             lflags = attributehash['FLAGS']
 394             uid = long(attributehash['UID'])
 395             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(lflags)
 396             try:
 397                 needupdate.remove(uid)
 398             except ValueError:          # Let it slide if it's not in the list
 399                 pass
 400         for uid in needupdate:
 401             if operation == '+':
 402                 for flag in flags:
 403                     if not flag in self.messagelist[uid]['flags']:
 404                         self.messagelist[uid]['flags'].append(flag)
 405                     self.messagelist[uid]['flags'].sort()
 406             elif operation == '-':
 407                 for flag in flags:
 408                     if flag in self.messagelist[uid]['flags']:
 409                         self.messagelist[uid]['flags'].remove(flag)
 410
 411     def deletemessage(self, uid):
 412         self.deletemessages_noconvert([uid])
 413
 414     def deletemessages(self, uidlist):
 415         self.deletemessages_noconvert(uidlist)
 416
 417     def deletemessages_noconvert(self, uidlist):
 418         # Weed out ones not in self.messagelist
 419         uidlist = [uid for uid in uidlist if uid in self.messagelist]
 420         if not len(uidlist):
 421             return
 422
 423         self.addmessagesflags_noconvert(uidlist, ['T'])
 424         imapobj = self.imapserver.acquireconnection()
 425         try:
 426             try:
 427                 imapobj.select(self.getfullname())
 428             except imapobj.readonly:
 429                 UIBase.getglobalui().deletereadonly(self, uidlist)
 430                 return
 431             if self.expunge:
 432                 assert(imapobj.expunge()[0] == 'OK')
 433         finally:
 434             self.imapserver.releaseconnection(imapobj)
 435         for uid in uidlist:
 436             del self.messagelist[uid]
 437
 438