code.delx.au - offlineimap/blob - offlineimap/folder/IMAP.py

   1 # IMAP folder support
   2 # Copyright (C) 2002-2004 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  18
  19 from Base import BaseFolder
  20 from offlineimap import imaputil, imaplib
  21 from offlineimap.ui import UIBase
  22 from offlineimap.version import versionstr
  23 import rfc822, time, string, random, binascii, re
  24 from StringIO import StringIO
  25 from copy import copy
  26
  27
  28 class IMAPFolder(BaseFolder):
  29     def __init__(self, imapserver, name, visiblename, accountname, repository):
  30         self.config = imapserver.config
  31         self.expunge = repository.getexpunge()
  32         self.name = imaputil.dequote(name)
  33         self.root = None # imapserver.root
  34         self.sep = imapserver.delim
  35         self.imapserver = imapserver
  36         self.messagelist = None
  37         self.visiblename = visiblename
  38         self.accountname = accountname
  39         self.repository = repository
  40         self.randomgenerator = random.Random()
  41         BaseFolder.__init__(self)
  42
  43     def getaccountname(self):
  44         return self.accountname
  45
  46     def suggeststhreads(self):
  47         return 1
  48
  49     def waitforthread(self):
  50         self.imapserver.connectionwait()
  51
  52     def getcopyinstancelimit(self):
  53         return 'MSGCOPY_' + self.repository.getname()
  54
  55     def getvisiblename(self):
  56         return self.visiblename
  57
  58     def getuidvalidity(self):
  59         imapobj = self.imapserver.acquireconnection()
  60         try:
  61             # Primes untagged_responses
  62             imapobj.select(self.getfullname(), readonly = 1)
  63             return long(imapobj.untagged_responses['UIDVALIDITY'][0])
  64         finally:
  65             self.imapserver.releaseconnection(imapobj)
  66
  67     def cachemessagelist(self):
  68         imapobj = self.imapserver.acquireconnection()
  69         self.messagelist = {}
  70
  71         try:
  72             # Primes untagged_responses
  73             imapobj.select(self.getfullname(), readonly = 1, force = 1)
  74             try:
  75                 # Some mail servers do not return an EXISTS response if
  76                 # the folder is empty.
  77                 maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
  78             except KeyError:
  79                 return
  80             if maxmsgid < 1:
  81                 # No messages; return
  82                 return
  83
  84             # Now, get the flags and UIDs for these.
  85             # We could conceivably get rid of maxmsgid and just say
  86             # '1:*' here.
  87             response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID)')[1]
  88         finally:
  89             self.imapserver.releaseconnection(imapobj)
  90         for messagestr in response:
  91             # Discard the message number.
  92             messagestr = string.split(messagestr, maxsplit = 1)[1]
  93             options = imaputil.flags2hash(messagestr)
  94             if not options.has_key('UID'):
  95                 UIBase.getglobalui().warn('No UID in message with options %s' %\
  96                                           str(options),
  97                                           minor = 1)
  98             else:
  99                 uid = long(options['UID'])
 100                 flags = imaputil.flagsimap2maildir(options['FLAGS'])
 101                 self.messagelist[uid] = {'uid': uid, 'flags': flags}
 102
 103     def getmessagelist(self):
 104         return self.messagelist
 105
 106     def getmessage(self, uid):
 107         ui = UIBase.getglobalui()
 108         imapobj = self.imapserver.acquireconnection()
 109         try:
 110             imapobj.select(self.getfullname(), readonly = 1)
 111             initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])')
 112             ui.debug('imap', 'Returned object from fetching %d: %s' % \
 113                      (uid, str(initialresult)))
 114             return initialresult[1][0][1].replace("\r\n", "\n")
 115
 116         finally:
 117             self.imapserver.releaseconnection(imapobj)
 118
 119     def getmessageflags(self, uid):
 120         return self.messagelist[uid]['flags']
 121
 122     def savemessage_getnewheader(self, content):
 123         headername = 'X-OfflineIMAP-%s-' % str(binascii.crc32(content)).replace('-', 'x')
 124         headername += binascii.hexlify(self.repository.getname()) + '-'
 125         headername += binascii.hexlify(self.getname())
 126         headervalue= '%d-' % long(time.time())
 127         headervalue += str(self.randomgenerator.random()).replace('.', '')
 128         headervalue += '-v' + versionstr
 129         return (headername, headervalue)
 130
 131     def savemessage_addheader(self, content, headername, headervalue):
 132         ui = UIBase.getglobalui()
 133         ui.debug('imap',
 134                  'savemessage_addheader: called to add %s: %s' % (headername,
 135                                                                   headervalue))
 136         insertionpoint = content.find("\r\n")
 137         ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint)
 138         leader = content[0:insertionpoint]
 139         ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader))
 140         if insertionpoint == 0 or insertionpoint == -1:
 141             newline = ''
 142             insertionpoint = 0
 143         else:
 144             newline = "\r\n"
 145         newline += "%s: %s" % (headername, headervalue)
 146         ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline))
 147         trailer = content[insertionpoint:]
 148         ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer))
 149         return leader + newline + trailer
 150
 151     def savemessage_searchforheader(self, imapobj, headername, headervalue):
 152         if imapobj.untagged_responses.has_key('APPENDUID'):
 153             return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1])
 154
 155         ui = UIBase.getglobalui()
 156         ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \
 157                  (headername, headervalue))
 158         # Now find the UID it got.
 159         headervalue = imapobj._quote(headervalue)
 160         try:
 161             matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0]
 162         except imapobj.error, err:
 163             # IMAP server doesn't implement search or had a problem.
 164             ui.debug('imap', "savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername)
 165             return 0
 166         ui.debug('imap', 'savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids))
 167
 168         if matchinguids == '':
 169             ui.debug('imap', "savemessage_searchforheader: UID SEARCH for message with header %s yielded no results" % headername)
 170             return 0
 171
 172         matchinguids = matchinguids.split(' ')
 173         ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \
 174                  repr(matchinguids))
 175         if len(matchinguids) != 1 or matchinguids[0] == None:
 176             raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids))
 177         matchinguids.sort()
 178         return long(matchinguids[0])
 179
 180     def savemessage(self, uid, content, flags):
 181         imapobj = self.imapserver.acquireconnection()
 182         ui = UIBase.getglobalui()
 183         ui.debug('imap', 'savemessage: called')
 184         try:
 185             try:
 186                 imapobj.select(self.getfullname()) # Needed for search
 187             except imapobj.readonly:
 188                 ui.msgtoreadonly(self, uid, content, flags)
 189                 # Return indicating message taken, but no UID assigned.
 190                 # Fudge it.
 191                 return 0
 192
 193             # This backend always assigns a new uid, so the uid arg is ignored.
 194             # In order to get the new uid, we need to save off the message ID.
 195
 196             message = rfc822.Message(StringIO(content))
 197             datetuple = rfc822.parsedate(message.getheader('Date'))
 198             # Will be None if missing or not in a valid format.
 199             if datetuple == None:
 200                 datetuple = time.localtime()
 201             try:
 202                 if datetuple[0] < 1981:
 203                     raise ValueError
 204                 # This could raise a value error if it's not a valid format.
 205                 date = imaplib.Time2Internaldate(datetuple)
 206             except (ValueError, OverflowError):
 207                 # Argh, sometimes it's a valid format but year is 0102
 208                 # or something.  Argh.  It seems that Time2Internaldate
 209                 # will rause a ValueError if the year is 0102 but not 1902,
 210                 # but some IMAP servers nonetheless choke on 1902.
 211                 date = imaplib.Time2Internaldate(time.localtime())
 212
 213             ui.debug('imap', 'savemessage: using date ' + str(date))
 214             content = re.sub("(?<!\r)\n", "\r\n", content)
 215             ui.debug('imap', 'savemessage: initial content is: ' + repr(content))
 216
 217             (headername, headervalue) = self.savemessage_getnewheader(content)
 218             ui.debug('imap', 'savemessage: new headers are: %s: %s' % \
 219                      (headername, headervalue))
 220             content = self.savemessage_addheader(content, headername,
 221                                                  headervalue)
 222             ui.debug('imap', 'savemessage: new content is: ' + repr(content))
 223             ui.debug('imap', 'savemessage: new content length is ' + \
 224                      str(len(content)))
 225
 226             assert(imapobj.append(self.getfullname(),
 227                                        imaputil.flagsmaildir2imap(flags),
 228                                        date, content)[0] == 'OK')
 229
 230             # Checkpoint.  Let it write out the messages, etc.
 231             assert(imapobj.check()[0] == 'OK')
 232
 233             # Keep trying until we get the UID.
 234             try:
 235                 ui.debug('imap', 'savemessage: first attempt to get new UID')
 236                 uid = self.savemessage_searchforheader(imapobj, headername,
 237                                                        headervalue)
 238             except ValueError:
 239                 ui.debug('imap', 'savemessage: first attempt to get new UID failed.  Going to run a NOOP and try again.')
 240                 assert(imapobj.noop()[0] == 'OK')
 241                 uid = self.savemessage_searchforheader(imapobj, headername,
 242                                                        headervalue)
 243         finally:
 244             self.imapserver.releaseconnection(imapobj)
 245
 246         self.messagelist[uid] = {'uid': uid, 'flags': flags}
 247         ui.debug('imap', 'savemessage: returning %d' % uid)
 248         return uid
 249
 250     def savemessageflags(self, uid, flags):
 251         imapobj = self.imapserver.acquireconnection()
 252         try:
 253             try:
 254                 imapobj.select(self.getfullname())
 255             except imapobj.readonly:
 256                 UIBase.getglobalui().flagstoreadonly(self, [uid], flags)
 257                 return
 258             result = imapobj.uid('store', '%d' % uid, 'FLAGS',
 259                                  imaputil.flagsmaildir2imap(flags))
 260             assert result[0] == 'OK', 'Error with store: ' + r[1]
 261         finally:
 262             self.imapserver.releaseconnection(imapobj)
 263         result = result[1][0]
 264         if not result:
 265             self.messagelist[uid]['flags'] = flags
 266         else:
 267             flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS']
 268             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 269
 270     def addmessageflags(self, uid, flags):
 271         self.addmessagesflags([uid], flags)
 272
 273     def addmessagesflags_noconvert(self, uidlist, flags):
 274         self.processmessagesflags('+', uidlist, flags)
 275
 276     def addmessagesflags(self, uidlist, flags):
 277         """This is here for the sake of UIDMaps.py -- deletemessages must
 278         add flags and get a converted UID, and if we don't have noconvert,
 279         then UIDMaps will try to convert it twice."""
 280         self.addmessagesflags_noconvert(uidlist, flags)
 281
 282     def deletemessageflags(self, uid, flags):
 283         self.deletemessagesflags([uid], flags)
 284
 285     def deletemessagesflags(self, uidlist, flags):
 286         self.processmessagesflags('-', uidlist, flags)
 287
 288     def processmessagesflags(self, operation, uidlist, flags):
 289         if len(uidlist) > 101:
 290             # Hack for those IMAP ervers with a limited line length
 291             self.processmessagesflags(operation, uidlist[:100], flags)
 292             self.processmessagesflags(operation, uidlist[100:], flags)
 293             return
 294
 295         imapobj = self.imapserver.acquireconnection()
 296         try:
 297             try:
 298                 imapobj.select(self.getfullname())
 299             except imapobj.readonly:
 300                 UIBase.getglobalui().flagstoreadonly(self, uidlist, flags)
 301                 return
 302             r = imapobj.uid('store',
 303                             imaputil.listjoin(uidlist),
 304                             operation + 'FLAGS',
 305                             imaputil.flagsmaildir2imap(flags))
 306             assert r[0] == 'OK', 'Error with store: ' + r[1]
 307             r = r[1]
 308         finally:
 309             self.imapserver.releaseconnection(imapobj)
 310         # Some IMAP servers do not always return a result.  Therefore,
 311         # only update the ones that it talks about, and manually fix
 312         # the others.
 313         needupdate = copy(uidlist)
 314         for result in r:
 315             if result == None:
 316                 # Compensate for servers that don't return anything from
 317                 # STORE.
 318                 continue
 319             attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1])
 320             if not ('UID' in attributehash and 'FLAGS' in attributehash):
 321                 # Compensate for servers that don't return a UID attribute.
 322                 continue
 323             flags = attributehash['FLAGS']
 324             uid = long(attributehash['UID'])
 325             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 326             try:
 327                 needupdate.remove(uid)
 328             except ValueError:          # Let it slide if it's not in the list
 329                 pass
 330         for uid in needupdate:
 331             if operation == '+':
 332                 for flag in flags:
 333                     if not flag in self.messagelist[uid]['flags']:
 334                         self.messagelist[uid]['flags'].append(flag)
 335                     self.messagelist[uid]['flags'].sort()
 336             elif operation == '-':
 337                 for flag in flags:
 338                     if flag in self.messagelist[uid]['flags']:
 339                         self.messagelist[uid]['flags'].remove(flag)
 340
 341     def deletemessage(self, uid):
 342         self.deletemessages_noconvert([uid])
 343
 344     def deletemessages(self, uidlist):
 345         self.deletemessages_noconvert(uidlist)
 346
 347     def deletemessages_noconvert(self, uidlist):
 348         # Weed out ones not in self.messagelist
 349         uidlist = [uid for uid in uidlist if uid in self.messagelist]
 350         if not len(uidlist):
 351             return
 352
 353         self.addmessagesflags_noconvert(uidlist, ['T'])
 354         imapobj = self.imapserver.acquireconnection()
 355         try:
 356             try:
 357                 imapobj.select(self.getfullname())
 358             except imapobj.readonly:
 359                 UIBase.getglobalui().deletereadonly(self, uidlist)
 360                 return
 361             if self.expunge:
 362                 assert(imapobj.expunge()[0] == 'OK')
 363         finally:
 364             self.imapserver.releaseconnection(imapobj)
 365         for uid in uidlist:
 366             del self.messagelist[uid]
 367
 368