code.delx.au - offlineimap/blob - offlineimap/folder/IMAP.py

   1 # IMAP folder support
   2 # Copyright (C) 2002-2004 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  18
  19 from Base import BaseFolder
  20 from offlineimap import imaputil, imaplib
  21 from offlineimap.ui import UIBase
  22 from offlineimap.version import versionstr
  23 import rfc822, time, string, random, binascii, re
  24 from StringIO import StringIO
  25 from copy import copy
  26
  27
  28 class IMAPFolder(BaseFolder):
  29     def __init__(self, imapserver, name, visiblename, accountname, repository):
  30         self.config = imapserver.config
  31         self.expunge = repository.getexpunge()
  32         self.name = imaputil.dequote(name)
  33         self.root = None # imapserver.root
  34         self.sep = imapserver.delim
  35         self.imapserver = imapserver
  36         self.messagelist = None
  37         self.visiblename = visiblename
  38         self.accountname = accountname
  39         self.repository = repository
  40         self.randomgenerator = random.Random()
  41         BaseFolder.__init__(self)
  42
  43     def getaccountname(self):
  44         return self.accountname
  45
  46     def suggeststhreads(self):
  47         return 1
  48
  49     def waitforthread(self):
  50         self.imapserver.connectionwait()
  51
  52     def getcopyinstancelimit(self):
  53         return 'MSGCOPY_' + self.repository.getname()
  54
  55     def getvisiblename(self):
  56         return self.visiblename
  57
  58     def getuidvalidity(self):
  59         imapobj = self.imapserver.acquireconnection()
  60         try:
  61             # Primes untagged_responses
  62             imapobj.select(self.getfullname(), readonly = 1)
  63             return long(imapobj.untagged_responses['UIDVALIDITY'][0])
  64         finally:
  65             self.imapserver.releaseconnection(imapobj)
  66
  67     def cachemessagelist(self):
  68         imapobj = self.imapserver.acquireconnection()
  69         self.messagelist = {}
  70
  71         try:
  72             # Primes untagged_responses
  73             imapobj.select(self.getfullname(), readonly = 1, force = 1)
  74             try:
  75                 # Some mail servers do not return an EXISTS response if
  76                 # the folder is empty.
  77                 maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
  78             except KeyError:
  79                 return
  80             if maxmsgid < 1:
  81                 # No messages; return
  82                 return
  83
  84             # Now, get the flags and UIDs for these.
  85             # We could conceivably get rid of maxmsgid and just say
  86             # '1:*' here.
  87             response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID)')[1]
  88         finally:
  89             self.imapserver.releaseconnection(imapobj)
  90         for messagestr in response:
  91             # Discard the message number.
  92             messagestr = string.split(messagestr, maxsplit = 1)[1]
  93             options = imaputil.flags2hash(messagestr)
  94             if not options.has_key('UID'):
  95                 UIBase.getglobalui().warn('No UID in message with options %s' %\
  96                                           str(options),
  97                                           minor = 1)
  98             else:
  99                 uid = long(options['UID'])
 100                 flags = imaputil.flagsimap2maildir(options['FLAGS'])
 101                 self.messagelist[uid] = {'uid': uid, 'flags': flags}
 102
 103     def getmessagelist(self):
 104         return self.messagelist
 105
 106     def getmessage(self, uid):
 107         ui = UIBase.getglobalui()
 108         imapobj = self.imapserver.acquireconnection()
 109         try:
 110             imapobj.select(self.getfullname(), readonly = 1)
 111             initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])')
 112             ui.debug('imap', 'Returned object from fetching %d: %s' % \
 113                      (uid, str(initialresult)))
 114             return initialresult[1][0][1].replace("\r\n", "\n")
 115
 116         finally:
 117             self.imapserver.releaseconnection(imapobj)
 118
 119     def getmessageflags(self, uid):
 120         return self.messagelist[uid]['flags']
 121
 122     def savemessage_getnewheader(self, content):
 123         headername = 'X-OfflineIMAP-%s-' % str(binascii.crc32(content)).replace('-', 'x')
 124         headername += binascii.hexlify(self.repository.getname()) + '-'
 125         headername += binascii.hexlify(self.getname())
 126         headervalue= '%d-' % long(time.time())
 127         headervalue += str(self.randomgenerator.random()).replace('.', '')
 128         headervalue += '-v' + versionstr
 129         return (headername, headervalue)
 130
 131     def savemessage_addheader(self, content, headername, headervalue):
 132         ui = UIBase.getglobalui()
 133         ui.debug('imap',
 134                  'savemessage_addheader: called to add %s: %s' % (headername,
 135                                                                   headervalue))
 136         insertionpoint = content.find("\r\n")
 137         ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint)
 138         leader = content[0:insertionpoint]
 139         ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader))
 140         if insertionpoint == 0 or insertionpoint == -1:
 141             newline = ''
 142             insertionpoint = 0
 143         else:
 144             newline = "\r\n"
 145         newline += "%s: %s" % (headername, headervalue)
 146         ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline))
 147         trailer = content[insertionpoint:]
 148         ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer))
 149         return leader + newline + trailer
 150
 151     def savemessage_searchforheader(self, imapobj, headername, headervalue):
 152         if imapobj.untagged_responses.has_key('APPENDUID'):
 153             return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1])
 154
 155         ui = UIBase.getglobalui()
 156         ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \
 157                  (headername, headervalue))
 158         # Now find the UID it got.
 159         headervalue = imapobj._quote(headervalue)
 160         try:
 161             matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0]
 162         except imapobj.error, err:
 163             # IMAP server doesn't implement search or had a problem.
 164             raise ValueError, "Got IMAP error while attempting to find UID for message with header %s: %s" % (headername, err)
 165         ui.debug('imap', 'savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids))
 166
 167         if matchinguids == '':
 168             raise ValueError, "Search for UID for message with header %s yielded no results" % headername
 169
 170         matchinguids = matchinguids.split(' ')
 171         ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \
 172                  repr(matchinguids))
 173         if len(matchinguids) != 1 or matchinguids[0] == None:
 174             raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids))
 175         matchinguids.sort()
 176         return long(matchinguids[0])
 177
 178     def savemessage(self, uid, content, flags):
 179         imapobj = self.imapserver.acquireconnection()
 180         ui = UIBase.getglobalui()
 181         ui.debug('imap', 'savemessage: called')
 182         try:
 183             try:
 184                 imapobj.select(self.getfullname()) # Needed for search
 185             except imapobj.readonly:
 186                 ui.msgtoreadonly(self, uid, content, flags)
 187                 # Return indicating message taken, but no UID assigned.
 188                 # Fudge it.
 189                 return 0
 190
 191             # This backend always assigns a new uid, so the uid arg is ignored.
 192             # In order to get the new uid, we need to save off the message ID.
 193
 194             message = rfc822.Message(StringIO(content))
 195             datetuple = rfc822.parsedate(message.getheader('Date'))
 196             # Will be None if missing or not in a valid format.
 197             if datetuple == None:
 198                 datetuple = time.localtime()
 199             try:
 200                 if datetuple[0] < 1981:
 201                     raise ValueError
 202                 # This could raise a value error if it's not a valid format.
 203                 date = imaplib.Time2Internaldate(datetuple)
 204             except (ValueError, OverflowError):
 205                 # Argh, sometimes it's a valid format but year is 0102
 206                 # or something.  Argh.  It seems that Time2Internaldate
 207                 # will rause a ValueError if the year is 0102 but not 1902,
 208                 # but some IMAP servers nonetheless choke on 1902.
 209                 date = imaplib.Time2Internaldate(time.localtime())
 210
 211             ui.debug('imap', 'savemessage: using date ' + str(date))
 212             content = re.sub("(?<!\r)\n", "\r\n", content)
 213             ui.debug('imap', 'savemessage: initial content is: ' + repr(content))
 214
 215             (headername, headervalue) = self.savemessage_getnewheader(content)
 216             ui.debug('imap', 'savemessage: new headers are: %s: %s' % \
 217                      (headername, headervalue))
 218             content = self.savemessage_addheader(content, headername,
 219                                                  headervalue)
 220             ui.debug('imap', 'savemessage: new content is: ' + repr(content))
 221             ui.debug('imap', 'savemessage: new content length is ' + \
 222                      str(len(content)))
 223
 224             assert(imapobj.append(self.getfullname(),
 225                                        imaputil.flagsmaildir2imap(flags),
 226                                        date, content)[0] == 'OK')
 227
 228             # Checkpoint.  Let it write out the messages, etc.
 229             assert(imapobj.check()[0] == 'OK')
 230
 231             # Keep trying until we get the UID.
 232             try:
 233                 ui.debug('imap', 'savemessage: first attempt to get new UID')
 234                 uid = self.savemessage_searchforheader(imapobj, headername,
 235                                                        headervalue)
 236             except ValueError:
 237                 ui.debug('imap', 'savemessage: first attempt to get new UID failed.  Going to run a NOOP and try again.')
 238                 assert(imapobj.noop()[0] == 'OK')
 239                 uid = self.savemessage_searchforheader(imapobj, headername,
 240                                                        headervalue)
 241         finally:
 242             self.imapserver.releaseconnection(imapobj)
 243
 244         self.messagelist[uid] = {'uid': uid, 'flags': flags}
 245         ui.debug('imap', 'savemessage: returning %d' % uid)
 246         return uid
 247
 248     def savemessageflags(self, uid, flags):
 249         imapobj = self.imapserver.acquireconnection()
 250         try:
 251             try:
 252                 imapobj.select(self.getfullname())
 253             except imapobj.readonly:
 254                 UIBase.getglobalui().flagstoreadonly(self, [uid], flags)
 255                 return
 256             result = imapobj.uid('store', '%d' % uid, 'FLAGS',
 257                                  imaputil.flagsmaildir2imap(flags))
 258             assert result[0] == 'OK', 'Error with store: ' + r[1]
 259         finally:
 260             self.imapserver.releaseconnection(imapobj)
 261         result = result[1][0]
 262         if not result:
 263             self.messagelist[uid]['flags'] = flags
 264         else:
 265             flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS']
 266             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 267
 268     def addmessageflags(self, uid, flags):
 269         self.addmessagesflags([uid], flags)
 270
 271     def addmessagesflags_noconvert(self, uidlist, flags):
 272         self.processmessagesflags('+', uidlist, flags)
 273
 274     def addmessagesflags(self, uidlist, flags):
 275         """This is here for the sake of UIDMaps.py -- deletemessages must
 276         add flags and get a converted UID, and if we don't have noconvert,
 277         then UIDMaps will try to convert it twice."""
 278         self.addmessagesflags_noconvert(uidlist, flags)
 279
 280     def deletemessageflags(self, uid, flags):
 281         self.deletemessagesflags([uid], flags)
 282
 283     def deletemessagesflags(self, uidlist, flags):
 284         self.processmessagesflags('-', uidlist, flags)
 285
 286     def processmessagesflags(self, operation, uidlist, flags):
 287         if len(uidlist) > 101:
 288             # Hack for those IMAP ervers with a limited line length
 289             self.processmessagesflags(operation, uidlist[:100], flags)
 290             self.processmessagesflags(operation, uidlist[100:], flags)
 291             return
 292
 293         imapobj = self.imapserver.acquireconnection()
 294         try:
 295             try:
 296                 imapobj.select(self.getfullname())
 297             except imapobj.readonly:
 298                 UIBase.getglobalui().flagstoreadonly(self, uidlist, flags)
 299                 return
 300             r = imapobj.uid('store',
 301                             imaputil.listjoin(uidlist),
 302                             operation + 'FLAGS',
 303                             imaputil.flagsmaildir2imap(flags))
 304             assert r[0] == 'OK', 'Error with store: ' + r[1]
 305             r = r[1]
 306         finally:
 307             self.imapserver.releaseconnection(imapobj)
 308         # Some IMAP servers do not always return a result.  Therefore,
 309         # only update the ones that it talks about, and manually fix
 310         # the others.
 311         needupdate = copy(uidlist)
 312         for result in r:
 313             if result == None:
 314                 # Compensate for servers that don't return anything from
 315                 # STORE.
 316                 continue
 317             attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1])
 318             if not ('UID' in attributehash and 'FLAGS' in attributehash):
 319                 # Compensate for servers that don't return a UID attribute.
 320                 continue
 321             flags = attributehash['FLAGS']
 322             uid = long(attributehash['UID'])
 323             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 324             try:
 325                 needupdate.remove(uid)
 326             except ValueError:          # Let it slide if it's not in the list
 327                 pass
 328         for uid in needupdate:
 329             if operation == '+':
 330                 for flag in flags:
 331                     if not flag in self.messagelist[uid]['flags']:
 332                         self.messagelist[uid]['flags'].append(flag)
 333                     self.messagelist[uid]['flags'].sort()
 334             elif operation == '-':
 335                 for flag in flags:
 336                     if flag in self.messagelist[uid]['flags']:
 337                         self.messagelist[uid]['flags'].remove(flag)
 338
 339     def deletemessage(self, uid):
 340         self.deletemessages_noconvert([uid])
 341
 342     def deletemessages(self, uidlist):
 343         self.deletemessages_noconvert(uidlist)
 344
 345     def deletemessages_noconvert(self, uidlist):
 346         # Weed out ones not in self.messagelist
 347         uidlist = [uid for uid in uidlist if uid in self.messagelist]
 348         if not len(uidlist):
 349             return
 350
 351         self.addmessagesflags_noconvert(uidlist, ['T'])
 352         imapobj = self.imapserver.acquireconnection()
 353         try:
 354             try:
 355                 imapobj.select(self.getfullname())
 356             except imapobj.readonly:
 357                 UIBase.getglobalui().deletereadonly(self, uidlist)
 358                 return
 359             if self.expunge:
 360                 assert(imapobj.expunge()[0] == 'OK')
 361         finally:
 362             self.imapserver.releaseconnection(imapobj)
 363         for uid in uidlist:
 364             del self.messagelist[uid]
 365
 366