code.delx.au - offlineimap/blob - offlineimap/folder/IMAP.py

   1 # IMAP folder support
   2 # Copyright (C) 2002-2004 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18
  19 from Base import BaseFolder
  20 from offlineimap import imaputil, imaplib
  21 from offlineimap.ui import UIBase
  22 from offlineimap.version import versionstr
  23 import rfc822, time, string, random, binascii, re
  24 from StringIO import StringIO
  25 from copy import copy
  26
  27
  28 class IMAPFolder(BaseFolder):
  29     def __init__(self, imapserver, name, visiblename, accountname, repository):
  30         self.config = imapserver.config
  31         self.expunge = repository.getexpunge()
  32         self.name = imaputil.dequote(name)
  33         self.root = None # imapserver.root
  34         self.sep = imapserver.delim
  35         self.imapserver = imapserver
  36         self.messagelist = None
  37         self.visiblename = visiblename
  38         self.accountname = accountname
  39         self.repository = repository
  40         self.randomgenerator = random.Random()
  41         BaseFolder.__init__(self)
  42
  43     def getaccountname(self):
  44         return self.accountname
  45
  46     def suggeststhreads(self):
  47         return 1
  48
  49     def waitforthread(self):
  50         self.imapserver.connectionwait()
  51
  52     def getcopyinstancelimit(self):
  53         return 'MSGCOPY_' + self.repository.getname()
  54
  55     def getvisiblename(self):
  56         return self.visiblename
  57
  58     def getuidvalidity(self):
  59         imapobj = self.imapserver.acquireconnection()
  60         try:
  61             # Primes untagged_responses
  62             imapobj.select(self.getfullname(), readonly = 1)
  63             return long(imapobj.untagged_responses['UIDVALIDITY'][0])
  64         finally:
  65             self.imapserver.releaseconnection(imapobj)
  66
  67     def cachemessagelist(self):
  68         imapobj = self.imapserver.acquireconnection()
  69         self.messagelist = {}
  70
  71         try:
  72             # Primes untagged_responses
  73             imapobj.select(self.getfullname(), readonly = 1, force = 1)
  74             try:
  75                 # Some mail servers do not return an EXISTS response if
  76                 # the folder is empty.
  77                 maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
  78             except KeyError:
  79                 return
  80             if maxmsgid < 1:
  81                 # No messages; return
  82                 return
  83
  84             # Now, get the flags and UIDs for these.
  85             # We could conceivably get rid of maxmsgid and just say
  86             # '1:*' here.
  87             response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID)')[1]
  88         finally:
  89             self.imapserver.releaseconnection(imapobj)
  90         for messagestr in response:
  91             # Discard the message number.
  92             messagestr = string.split(messagestr, maxsplit = 1)[1]
  93             options = imaputil.flags2hash(messagestr)
  94             if not options.has_key('UID'):
  95                 UIBase.getglobalui().warn('No UID in message with options %s' %\
  96                                           str(options),
  97                                           minor = 1)
  98             else:
  99                 uid = long(options['UID'])
 100                 flags = imaputil.flagsimap2maildir(options['FLAGS'])
 101                 self.messagelist[uid] = {'uid': uid, 'flags': flags}
 102
 103     def getmessagelist(self):
 104         return self.messagelist
 105
 106     def getmessage(self, uid):
 107         ui = UIBase.getglobalui()
 108         imapobj = self.imapserver.acquireconnection()
 109         try:
 110             imapobj.select(self.getfullname(), readonly = 1)
 111             initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])')
 112             ui.debug('imap', 'Returned object from fetching %d: %s' % \
 113                      (uid, str(initialresult)))
 114             return initialresult[1][0][1].replace("\r\n", "\n")
 115
 116         finally:
 117             self.imapserver.releaseconnection(imapobj)
 118
 119     def getmessageflags(self, uid):
 120         return self.messagelist[uid]['flags']
 121
 122     def savemessage_getnewheader(self, content):
 123         headername = 'X-OfflineIMAP-%s-' % str(binascii.crc32(content)).replace('-', 'x')
 124         headername += binascii.hexlify(self.repository.getname()) + '-'
 125         headername += binascii.hexlify(self.getname())
 126         headervalue= '%d-' % long(time.time())
 127         headervalue += str(self.randomgenerator.random()).replace('.', '')
 128         headervalue += '-v' + versionstr
 129         return (headername, headervalue)
 130
 131     def savemessage_addheader(self, content, headername, headervalue):
 132         ui = UIBase.getglobalui()
 133         ui.debug('imap',
 134                  'savemessage_addheader: called to add %s: %s' % (headername,
 135                                                                   headervalue))
 136         insertionpoint = content.find("\r\n")
 137         ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint)
 138         leader = content[0:insertionpoint]
 139         ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader))
 140         if insertionpoint == 0 or insertionpoint == -1:
 141             newline = ''
 142             insertionpoint = 0
 143         else:
 144             newline = "\r\n"
 145         newline += "%s: %s" % (headername, headervalue)
 146         ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline))
 147         trailer = content[insertionpoint:]
 148         ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer))
 149         return leader + newline + trailer
 150
 151     def savemessage_searchforheader(self, imapobj, headername, headervalue):
 152         if imapobj.untagged_responses.has_key('APPENDUID'):
 153             return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1])
 154
 155         ui = UIBase.getglobalui()
 156         ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \
 157                  (headername, headervalue))
 158         # Now find the UID it got.
 159         headervalue = imapobj._quote(headervalue)
 160         try:
 161             matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0]
 162         except imapobj.error, err:
 163             # IMAP server doesn't implement search or had a problem.
 164             ui.debug('imap', "savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername))
 165             return 0
 166         ui.debug('imap', 'savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids))
 167
 168         if matchinguids == '':
 169             ui.debug('imap', "savemessage_searchforheader: UID SEARCH for message with header %s yielded no results" % headername)
 170             return 0
 171
 172         matchinguids = matchinguids.split(' ')
 173         ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \
 174                  repr(matchinguids))
 175         if len(matchinguids) != 1 or matchinguids[0] == None:
 176             raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids))
 177         matchinguids.sort()
 178         return long(matchinguids[0])
 179
 180     def savemessage(self, uid, content, flags):
 181         imapobj = self.imapserver.acquireconnection()
 182         ui = UIBase.getglobalui()
 183         ui.debug('imap', 'savemessage: called')
 184         try:
 185             try:
 186                 imapobj.select(self.getfullname()) # Needed for search
 187             except imapobj.readonly:
 188                 ui.msgtoreadonly(self, uid, content, flags)
 189                 # Return indicating message taken, but no UID assigned.
 190                 # Fudge it.
 191                 return 0
 192
 193             # This backend always assigns a new uid, so the uid arg is ignored.
 194             # In order to get the new uid, we need to save off the message ID.
 195
 196             message = rfc822.Message(StringIO(content))
 197             datetuple = rfc822.parsedate(message.getheader('Date'))
 198             # Will be None if missing or not in a valid format.
 199             if datetuple == None:
 200                 datetuple = time.localtime()
 201             try:
 202                 if datetuple[0] < 1981:
 203                     raise ValueError
 204                 # This could raise a value error if it's not a valid format.
 205                 date = imaplib.Time2Internaldate(datetuple)
 206             except (ValueError, OverflowError):
 207                 # Argh, sometimes it's a valid format but year is 0102
 208                 # or something.  Argh.  It seems that Time2Internaldate
 209                 # will rause a ValueError if the year is 0102 but not 1902,
 210                 # but some IMAP servers nonetheless choke on 1902.
 211                 date = imaplib.Time2Internaldate(time.localtime())
 212
 213             ui.debug('imap', 'savemessage: using date ' + str(date))
 214             content = re.sub("(?<!\r)\n", "\r\n", content)
 215             ui.debug('imap', 'savemessage: initial content is: ' + repr(content))
 216
 217             (headername, headervalue) = self.savemessage_getnewheader(content)
 218             ui.debug('imap', 'savemessage: new headers are: %s: %s' % \
 219                      (headername, headervalue))
 220             content = self.savemessage_addheader(content, headername,
 221                                                  headervalue)
 222             ui.debug('imap', 'savemessage: new content is: ' + repr(content))
 223             ui.debug('imap', 'savemessage: new content length is ' + \
 224                      str(len(content)))
 225
 226             assert(imapobj.append(self.getfullname(),
 227                                        imaputil.flagsmaildir2imap(flags),
 228                                        date, content)[0] == 'OK')
 229
 230             # Checkpoint.  Let it write out the messages, etc.
 231             assert(imapobj.check()[0] == 'OK')
 232
 233             # Keep trying until we get the UID.
 234             ui.debug('imap', 'savemessage: first attempt to get new UID')
 235             uid = self.savemessage_searchforheader(imapobj, headername,
 236                                                    headervalue)
 237             # See docs for savemessage in Base.py for explanation of this and other return values
 238             if uid <= 0:
 239                 ui.debug('imap', 'savemessage: first attempt to get new UID failed.  Going to run a NOOP and try again.')
 240                 assert(imapobj.noop()[0] == 'OK')
 241                 uid = self.savemessage_searchforheader(imapobj, headername,
 242                                                        headervalue)
 243         finally:
 244             self.imapserver.releaseconnection(imapobj)
 245
 246         if uid: # avoid UID FETCH 0 crash happening later on
 247             self.messagelist[uid] = {'uid': uid, 'flags': flags}
 248
 249         ui.debug('imap', 'savemessage: returning %d' % uid)
 250         return uid
 251
 252     def savemessageflags(self, uid, flags):
 253         imapobj = self.imapserver.acquireconnection()
 254         try:
 255             try:
 256                 imapobj.select(self.getfullname())
 257             except imapobj.readonly:
 258                 UIBase.getglobalui().flagstoreadonly(self, [uid], flags)
 259                 return
 260             result = imapobj.uid('store', '%d' % uid, 'FLAGS',
 261                                  imaputil.flagsmaildir2imap(flags))
 262             assert result[0] == 'OK', 'Error with store: ' + r[1]
 263         finally:
 264             self.imapserver.releaseconnection(imapobj)
 265         result = result[1][0]
 266         if not result:
 267             self.messagelist[uid]['flags'] = flags
 268         else:
 269             flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS']
 270             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 271
 272     def addmessageflags(self, uid, flags):
 273         self.addmessagesflags([uid], flags)
 274
 275     def addmessagesflags_noconvert(self, uidlist, flags):
 276         self.processmessagesflags('+', uidlist, flags)
 277
 278     def addmessagesflags(self, uidlist, flags):
 279         """This is here for the sake of UIDMaps.py -- deletemessages must
 280         add flags and get a converted UID, and if we don't have noconvert,
 281         then UIDMaps will try to convert it twice."""
 282         self.addmessagesflags_noconvert(uidlist, flags)
 283
 284     def deletemessageflags(self, uid, flags):
 285         self.deletemessagesflags([uid], flags)
 286
 287     def deletemessagesflags(self, uidlist, flags):
 288         self.processmessagesflags('-', uidlist, flags)
 289
 290     def processmessagesflags(self, operation, uidlist, flags):
 291         if len(uidlist) > 101:
 292             # Hack for those IMAP ervers with a limited line length
 293             self.processmessagesflags(operation, uidlist[:100], flags)
 294             self.processmessagesflags(operation, uidlist[100:], flags)
 295             return
 296
 297         imapobj = self.imapserver.acquireconnection()
 298         try:
 299             try:
 300                 imapobj.select(self.getfullname())
 301             except imapobj.readonly:
 302                 UIBase.getglobalui().flagstoreadonly(self, uidlist, flags)
 303                 return
 304             r = imapobj.uid('store',
 305                             imaputil.listjoin(uidlist),
 306                             operation + 'FLAGS',
 307                             imaputil.flagsmaildir2imap(flags))
 308             assert r[0] == 'OK', 'Error with store: ' + r[1]
 309             r = r[1]
 310         finally:
 311             self.imapserver.releaseconnection(imapobj)
 312         # Some IMAP servers do not always return a result.  Therefore,
 313         # only update the ones that it talks about, and manually fix
 314         # the others.
 315         needupdate = copy(uidlist)
 316         for result in r:
 317             if result == None:
 318                 # Compensate for servers that don't return anything from
 319                 # STORE.
 320                 continue
 321             attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1])
 322             if not ('UID' in attributehash and 'FLAGS' in attributehash):
 323                 # Compensate for servers that don't return a UID attribute.
 324                 continue
 325             flags = attributehash['FLAGS']
 326             uid = long(attributehash['UID'])
 327             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 328             try:
 329                 needupdate.remove(uid)
 330             except ValueError:          # Let it slide if it's not in the list
 331                 pass
 332         for uid in needupdate:
 333             if operation == '+':
 334                 for flag in flags:
 335                     if not flag in self.messagelist[uid]['flags']:
 336                         self.messagelist[uid]['flags'].append(flag)
 337                     self.messagelist[uid]['flags'].sort()
 338             elif operation == '-':
 339                 for flag in flags:
 340                     if flag in self.messagelist[uid]['flags']:
 341                         self.messagelist[uid]['flags'].remove(flag)
 342
 343     def deletemessage(self, uid):
 344         self.deletemessages_noconvert([uid])
 345
 346     def deletemessages(self, uidlist):
 347         self.deletemessages_noconvert(uidlist)
 348
 349     def deletemessages_noconvert(self, uidlist):
 350         # Weed out ones not in self.messagelist
 351         uidlist = [uid for uid in uidlist if uid in self.messagelist]
 352         if not len(uidlist):
 353             return
 354
 355         self.addmessagesflags_noconvert(uidlist, ['T'])
 356         imapobj = self.imapserver.acquireconnection()
 357         try:
 358             try:
 359                 imapobj.select(self.getfullname())
 360             except imapobj.readonly:
 361                 UIBase.getglobalui().deletereadonly(self, uidlist)
 362                 return
 363             if self.expunge:
 364                 assert(imapobj.expunge()[0] == 'OK')
 365         finally:
 366             self.imapserver.releaseconnection(imapobj)
 367         for uid in uidlist:
 368             del self.messagelist[uid]
 369
 370