code.delx.au - offlineimap/blob - offlineimap/folder/IMAP.py

   1 # IMAP folder support
   2 # Copyright (C) 2002-2004 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  18
  19 from Base import BaseFolder
  20 from offlineimap import imaputil, imaplib
  21 from offlineimap.ui import UIBase
  22 from offlineimap.version import versionstr
  23 import rfc822, time, string, random, binascii, re
  24 from StringIO import StringIO
  25 from copy import copy
  26
  27
  28 class IMAPFolder(BaseFolder):
  29     def __init__(self, imapserver, name, visiblename, accountname, repository):
  30         self.config = imapserver.config
  31         self.expunge = repository.getexpunge()
  32         self.name = imaputil.dequote(name)
  33         self.root = None # imapserver.root
  34         self.sep = imapserver.delim
  35         self.imapserver = imapserver
  36         self.messagelist = None
  37         self.visiblename = visiblename
  38         self.accountname = accountname
  39         self.repository = repository
  40         self.randomgenerator = random.Random()
  41         BaseFolder.__init__(self)
  42
  43     def getaccountname(self):
  44         return self.accountname
  45
  46     def suggeststhreads(self):
  47         return 1
  48
  49     def waitforthread(self):
  50         self.imapserver.connectionwait()
  51
  52     def getcopyinstancelimit(self):
  53         return 'MSGCOPY_' + self.repository.getname()
  54
  55     def getvisiblename(self):
  56         return self.visiblename
  57
  58     def getuidvalidity(self):
  59         imapobj = self.imapserver.acquireconnection()
  60         try:
  61             # Primes untagged_responses
  62             imapobj.select(self.getfullname(), readonly = 1)
  63             return long(imapobj.untagged_responses['UIDVALIDITY'][0])
  64         finally:
  65             self.imapserver.releaseconnection(imapobj)
  66
  67     def cachemessagelist(self):
  68         imapobj = self.imapserver.acquireconnection()
  69         self.messagelist = {}
  70
  71         try:
  72             # Primes untagged_responses
  73             imapobj.select(self.getfullname(), readonly = 1, force = 1)
  74             try:
  75                 # Some mail servers do not return an EXISTS response if
  76                 # the folder is empty.
  77                 maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
  78             except KeyError:
  79                 return
  80             if maxmsgid < 1:
  81                 # No messages; return
  82                 return
  83
  84             # Now, get the flags and UIDs for these.
  85             # We could conceivably get rid of maxmsgid and just say
  86             # '1:*' here.
  87             response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID)')[1]
  88         finally:
  89             self.imapserver.releaseconnection(imapobj)
  90         for messagestr in response:
  91             # Discard the message number.
  92             messagestr = string.split(messagestr, maxsplit = 1)[1]
  93             options = imaputil.flags2hash(messagestr)
  94             if not options.has_key('UID'):
  95                 UIBase.getglobalui().warn('No UID in message with options %s' %\
  96                                           str(options),
  97                                           minor = 1)
  98             else:
  99                 uid = long(options['UID'])
 100                 flags = imaputil.flagsimap2maildir(options['FLAGS'])
 101                 self.messagelist[uid] = {'uid': uid, 'flags': flags}
 102
 103     def getmessagelist(self):
 104         return self.messagelist
 105
 106     def getmessage(self, uid):
 107         ui = UIBase.getglobalui()
 108         imapobj = self.imapserver.acquireconnection()
 109         try:
 110             imapobj.select(self.getfullname(), readonly = 1)
 111             initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])')
 112             ui.debug('imap', 'Returned object from fetching %d: %s' % \
 113                      (uid, str(initialresult)))
 114             return initialresult[1][0][1].replace("\r\n", "\n")
 115
 116         finally:
 117             self.imapserver.releaseconnection(imapobj)
 118
 119     def getmessageflags(self, uid):
 120         return self.messagelist[uid]['flags']
 121
 122     def savemessage_getnewheader(self, content):
 123         headername = 'X-OfflineIMAP-%s-' % str(binascii.crc32(content)).replace('-', 'x')
 124         headername += binascii.hexlify(self.repository.getname()) + '-'
 125         headername += binascii.hexlify(self.getname())
 126         headervalue= '%d-' % long(time.time())
 127         headervalue += str(self.randomgenerator.random()).replace('.', '')
 128         headervalue += '-v' + versionstr
 129         return (headername, headervalue)
 130
 131     def savemessage_addheader(self, content, headername, headervalue):
 132         ui = UIBase.getglobalui()
 133         ui.debug('imap',
 134                  'savemessage_addheader: called to add %s: %s' % (headername,
 135                                                                   headervalue))
 136         insertionpoint = content.find("\r\n")
 137         ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint)
 138         leader = content[0:insertionpoint]
 139         ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader))
 140         if insertionpoint == 0 or insertionpoint == -1:
 141             newline = ''
 142             insertionpoint = 0
 143         else:
 144             newline = "\r\n"
 145         newline += "%s: %s" % (headername, headervalue)
 146         ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline))
 147         trailer = content[insertionpoint:]
 148         ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer))
 149         return leader + newline + trailer
 150
 151     def savemessage_searchforheader(self, imapobj, headername, headervalue):
 152         if imapobj.untagged_responses.has_key('APPENDUID'):
 153             return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1])
 154
 155         ui = UIBase.getglobalui()
 156         ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \
 157                  (headername, headervalue))
 158         # Now find the UID it got.
 159         headervalue = imapobj._quote(headervalue)
 160         try:
 161             matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0]
 162         except imapobj.error:
 163             # IMAP server doesn't implement search or had a problem.
 164             return 0
 165         ui.debug('imap', 'savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids))
 166
 167         matchinguids = matchinguids.split(' ')
 168         ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \
 169                  repr(matchinguids))
 170         if len(matchinguids) != 1 or matchinguids[0] == None:
 171             raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids))
 172         matchinguids.sort()
 173         return long(matchinguids[0])
 174
 175     def savemessage(self, uid, content, flags):
 176         imapobj = self.imapserver.acquireconnection()
 177         ui = UIBase.getglobalui()
 178         ui.debug('imap', 'savemessage: called')
 179         try:
 180             try:
 181                 imapobj.select(self.getfullname()) # Needed for search
 182             except imapobj.readonly:
 183                 ui.msgtoreadonly(self, uid, content, flags)
 184                 # Return indicating message taken, but no UID assigned.
 185                 # Fudge it.
 186                 return 0
 187
 188             # This backend always assigns a new uid, so the uid arg is ignored.
 189             # In order to get the new uid, we need to save off the message ID.
 190
 191             message = rfc822.Message(StringIO(content))
 192             datetuple = rfc822.parsedate(message.getheader('Date'))
 193             # Will be None if missing or not in a valid format.
 194             if datetuple == None:
 195                 datetuple = time.localtime()
 196             try:
 197                 if datetuple[0] < 1981:
 198                     raise ValueError
 199                 # This could raise a value error if it's not a valid format.
 200                 date = imaplib.Time2Internaldate(datetuple)
 201             except (ValueError, OverflowError):
 202                 # Argh, sometimes it's a valid format but year is 0102
 203                 # or something.  Argh.  It seems that Time2Internaldate
 204                 # will rause a ValueError if the year is 0102 but not 1902,
 205                 # but some IMAP servers nonetheless choke on 1902.
 206                 date = imaplib.Time2Internaldate(time.localtime())
 207
 208             ui.debug('imap', 'savemessage: using date ' + str(date))
 209             content = re.sub("(?<!\r)\n", "\r\n", content)
 210             ui.debug('imap', 'savemessage: initial content is: ' + repr(content))
 211
 212             (headername, headervalue) = self.savemessage_getnewheader(content)
 213             ui.debug('imap', 'savemessage: new headers are: %s: %s' % \
 214                      (headername, headervalue))
 215             content = self.savemessage_addheader(content, headername,
 216                                                  headervalue)
 217             ui.debug('imap', 'savemessage: new content is: ' + repr(content))
 218             ui.debug('imap', 'savemessage: new content length is ' + \
 219                      str(len(content)))
 220
 221             assert(imapobj.append(self.getfullname(),
 222                                        imaputil.flagsmaildir2imap(flags),
 223                                        date, content)[0] == 'OK')
 224
 225             # Checkpoint.  Let it write out the messages, etc.
 226             assert(imapobj.check()[0] == 'OK')
 227
 228             # Keep trying until we get the UID.
 229             try:
 230                 ui.debug('imap', 'savemessage: first attempt to get new UID')
 231                 uid = self.savemessage_searchforheader(imapobj, headername,
 232                                                        headervalue)
 233             except ValueError:
 234                 ui.debug('imap', 'savemessage: first attempt to get new UID failed.  Going to run a NOOP and try again.')
 235                 assert(imapobj.noop()[0] == 'OK')
 236                 uid = self.savemessage_searchforheader(imapobj, headername,
 237                                                        headervalue)
 238         finally:
 239             self.imapserver.releaseconnection(imapobj)
 240
 241         self.messagelist[uid] = {'uid': uid, 'flags': flags}
 242         ui.debug('imap', 'savemessage: returning %d' % uid)
 243         return uid
 244
 245     def savemessageflags(self, uid, flags):
 246         imapobj = self.imapserver.acquireconnection()
 247         try:
 248             try:
 249                 imapobj.select(self.getfullname())
 250             except imapobj.readonly:
 251                 UIBase.getglobalui().flagstoreadonly(self, [uid], flags)
 252                 return
 253             result = imapobj.uid('store', '%d' % uid, 'FLAGS',
 254                                  imaputil.flagsmaildir2imap(flags))
 255             assert result[0] == 'OK', 'Error with store: ' + r[1]
 256         finally:
 257             self.imapserver.releaseconnection(imapobj)
 258         result = result[1][0]
 259         if not result:
 260             self.messagelist[uid]['flags'] = flags
 261         else:
 262             flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS']
 263             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 264
 265     def addmessageflags(self, uid, flags):
 266         self.addmessagesflags([uid], flags)
 267
 268     def addmessagesflags_noconvert(self, uidlist, flags):
 269         self.processmessagesflags('+', uidlist, flags)
 270
 271     def addmessagesflags(self, uidlist, flags):
 272         """This is here for the sake of UIDMaps.py -- deletemessages must
 273         add flags and get a converted UID, and if we don't have noconvert,
 274         then UIDMaps will try to convert it twice."""
 275         self.addmessagesflags_noconvert(uidlist, flags)
 276
 277     def deletemessageflags(self, uid, flags):
 278         self.deletemessagesflags([uid], flags)
 279
 280     def deletemessagesflags(self, uidlist, flags):
 281         self.processmessagesflags('-', uidlist, flags)
 282
 283     def processmessagesflags(self, operation, uidlist, flags):
 284         if len(uidlist) > 101:
 285             # Hack for those IMAP ervers with a limited line length
 286             self.processmessagesflags(operation, uidlist[:100], flags)
 287             self.processmessagesflags(operation, uidlist[100:], flags)
 288             return
 289
 290         imapobj = self.imapserver.acquireconnection()
 291         try:
 292             try:
 293                 imapobj.select(self.getfullname())
 294             except imapobj.readonly:
 295                 UIBase.getglobalui().flagstoreadonly(self, uidlist, flags)
 296                 return
 297             r = imapobj.uid('store',
 298                             imaputil.listjoin(uidlist),
 299                             operation + 'FLAGS',
 300                             imaputil.flagsmaildir2imap(flags))
 301             assert r[0] == 'OK', 'Error with store: ' + r[1]
 302             r = r[1]
 303         finally:
 304             self.imapserver.releaseconnection(imapobj)
 305         # Some IMAP servers do not always return a result.  Therefore,
 306         # only update the ones that it talks about, and manually fix
 307         # the others.
 308         needupdate = copy(uidlist)
 309         for result in r:
 310             if result == None:
 311                 # Compensate for servers that don't return anything from
 312                 # STORE.
 313                 continue
 314             attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1])
 315             if not ('UID' in attributehash and 'FLAGS' in attributehash):
 316                 # Compensate for servers that don't return a UID attribute.
 317                 continue
 318             flags = attributehash['FLAGS']
 319             uid = long(attributehash['UID'])
 320             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 321             try:
 322                 needupdate.remove(uid)
 323             except ValueError:          # Let it slide if it's not in the list
 324                 pass
 325         for uid in needupdate:
 326             if operation == '+':
 327                 for flag in flags:
 328                     if not flag in self.messagelist[uid]['flags']:
 329                         self.messagelist[uid]['flags'].append(flag)
 330                     self.messagelist[uid]['flags'].sort()
 331             elif operation == '-':
 332                 for flag in flags:
 333                     if flag in self.messagelist[uid]['flags']:
 334                         self.messagelist[uid]['flags'].remove(flag)
 335
 336     def deletemessage(self, uid):
 337         self.deletemessages_noconvert([uid])
 338
 339     def deletemessages(self, uidlist):
 340         self.deletemessages_noconvert(uidlist)
 341
 342     def deletemessages_noconvert(self, uidlist):
 343         # Weed out ones not in self.messagelist
 344         uidlist = [uid for uid in uidlist if uid in self.messagelist]
 345         if not len(uidlist):
 346             return
 347
 348         self.addmessagesflags_noconvert(uidlist, ['T'])
 349         imapobj = self.imapserver.acquireconnection()
 350         try:
 351             try:
 352                 imapobj.select(self.getfullname())
 353             except imapobj.readonly:
 354                 UIBase.getglobalui().deletereadonly(self, uidlist)
 355                 return
 356             if self.expunge:
 357                 assert(imapobj.expunge()[0] == 'OK')
 358         finally:
 359             self.imapserver.releaseconnection(imapobj)
 360         for uid in uidlist:
 361             del self.messagelist[uid]
 362
 363