code.delx.au - offlineimap/blob - offlineimap/folder/IMAP.py

   1 # IMAP folder support
   2 # Copyright (C) 2002-2004 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  18
  19 from Base import BaseFolder
  20 from offlineimap import imaputil, imaplib
  21 from offlineimap.ui import UIBase
  22 from offlineimap.version import versionstr
  23 import rfc822, time, string, random, binascii, re
  24 from StringIO import StringIO
  25 from copy import copy
  26
  27
  28 class IMAPFolder(BaseFolder):
  29     def __init__(self, imapserver, name, visiblename, accountname, repository):
  30         self.config = imapserver.config
  31         self.expunge = repository.getexpunge()
  32         self.name = imaputil.dequote(name)
  33         self.root = None # imapserver.root
  34         self.sep = imapserver.delim
  35         self.imapserver = imapserver
  36         self.messagelist = None
  37         self.visiblename = visiblename
  38         self.accountname = accountname
  39         self.repository = repository
  40         self.randomgenerator = random.Random()
  41         BaseFolder.__init__(self)
  42
  43     def getaccountname(self):
  44         return self.accountname
  45
  46     def suggeststhreads(self):
  47         return 1
  48
  49     def waitforthread(self):
  50         self.imapserver.connectionwait()
  51
  52     def getcopyinstancelimit(self):
  53         return 'MSGCOPY_' + self.repository.getname()
  54
  55     def getvisiblename(self):
  56         return self.visiblename
  57
  58     def getuidvalidity(self):
  59         imapobj = self.imapserver.acquireconnection()
  60         try:
  61             # Primes untagged_responses
  62             imapobj.select(self.getfullname(), readonly = 1)
  63             return long(imapobj.untagged_responses['UIDVALIDITY'][0])
  64         finally:
  65             self.imapserver.releaseconnection(imapobj)
  66
  67     def cachemessagelist(self):
  68         imapobj = self.imapserver.acquireconnection()
  69         self.messagelist = {}
  70
  71         try:
  72             # Primes untagged_responses
  73             imapobj.select(self.getfullname(), readonly = 1, force = 1)
  74             try:
  75                 # Some mail servers do not return an EXISTS response if
  76                 # the folder is empty.
  77                 maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
  78             except KeyError:
  79                 return
  80             if maxmsgid < 1:
  81                 # No messages; return
  82                 return
  83
  84             # Now, get the flags and UIDs for these.
  85             # We could conceivably get rid of maxmsgid and just say
  86             # '1:*' here.
  87             response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID)')[1]
  88         finally:
  89             self.imapserver.releaseconnection(imapobj)
  90         for messagestr in response:
  91             # Discard the message number.
  92             messagestr = string.split(messagestr, maxsplit = 1)[1]
  93             options = imaputil.flags2hash(messagestr)
  94             if not options.has_key('UID'):
  95                 UIBase.getglobalui().warn('No UID in message with options %s' %\
  96                                           str(options),
  97                                           minor = 1)
  98             else:
  99                 uid = long(options['UID'])
 100                 flags = imaputil.flagsimap2maildir(options['FLAGS'])
 101                 self.messagelist[uid] = {'uid': uid, 'flags': flags}
 102
 103     def getmessagelist(self):
 104         return self.messagelist
 105
 106     def getmessage(self, uid):
 107         ui = UIBase.getglobalui()
 108         imapobj = self.imapserver.acquireconnection()
 109         try:
 110             imapobj.select(self.getfullname(), readonly = 1)
 111             initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])')
 112             ui.debug('imap', 'Returned object from fetching %d: %s' % \
 113                      (uid, str(initialresult)))
 114             return initialresult[1][0][1].replace("\r\n", "\n")
 115
 116         finally:
 117             self.imapserver.releaseconnection(imapobj)
 118
 119     def getmessageflags(self, uid):
 120         return self.messagelist[uid]['flags']
 121
 122     def savemessage_getnewheader(self, content):
 123         headername = 'X-OfflineIMAP-%s-' % str(binascii.crc32(content)).replace('-', 'x')
 124         headername += binascii.hexlify(self.repository.getname()) + '-'
 125         headername += binascii.hexlify(self.getname())
 126         headervalue= '%d-' % long(time.time())
 127         headervalue += str(self.randomgenerator.random()).replace('.', '')
 128         headervalue += '-v' + versionstr
 129         return (headername, headervalue)
 130
 131     def savemessage_addheader(self, content, headername, headervalue):
 132         ui = UIBase.getglobalui()
 133         ui.debug('imap',
 134                  'savemessage_addheader: called to add %s: %s' % (headername,
 135                                                                   headervalue))
 136         insertionpoint = content.find("\r\n")
 137         ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint)
 138         leader = content[0:insertionpoint]
 139         ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader))
 140         if insertionpoint == 0 or insertionpoint == -1:
 141             newline = ''
 142             insertionpoint = 0
 143         else:
 144             newline = "\r\n"
 145         newline += "%s: %s" % (headername, headervalue)
 146         ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline))
 147         trailer = content[insertionpoint:]
 148         ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer))
 149         return leader + newline + trailer
 150
 151     def savemessage_searchforheader(self, imapobj, headername, headervalue):
 152         if imapobj.untagged_responses.has_key('APPENDUID'):
 153             return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1])
 154
 155         ui = UIBase.getglobalui()
 156         ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \
 157                  (headername, headervalue))
 158         # Now find the UID it got.
 159         headervalue = imapobj._quote(headervalue)
 160         try:
 161             matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0]
 162         except imapobj.error, err:
 163             # IMAP server doesn't implement search or had a problem.
 164             ui.debug('imap', "savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername))
 165             return 0
 166         ui.debug('imap', 'savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids))
 167
 168         if matchinguids == '':
 169             ui.debug('imap', "savemessage_searchforheader: UID SEARCH for message with header %s yielded no results" % headername)
 170             return 0
 171
 172         matchinguids = matchinguids.split(' ')
 173         ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \
 174                  repr(matchinguids))
 175         if len(matchinguids) != 1 or matchinguids[0] == None:
 176             raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids))
 177         matchinguids.sort()
 178         return long(matchinguids[0])
 179
 180     def savemessage(self, uid, content, flags):
 181         imapobj = self.imapserver.acquireconnection()
 182         ui = UIBase.getglobalui()
 183         ui.debug('imap', 'savemessage: called')
 184         try:
 185             try:
 186                 imapobj.select(self.getfullname()) # Needed for search
 187             except imapobj.readonly:
 188                 ui.msgtoreadonly(self, uid, content, flags)
 189                 # Return indicating message taken, but no UID assigned.
 190                 # Fudge it.
 191                 return 0
 192
 193             # This backend always assigns a new uid, so the uid arg is ignored.
 194             # In order to get the new uid, we need to save off the message ID.
 195
 196             message = rfc822.Message(StringIO(content))
 197             datetuple = rfc822.parsedate(message.getheader('Date'))
 198             # Will be None if missing or not in a valid format.
 199             if datetuple == None:
 200                 datetuple = time.localtime()
 201             try:
 202                 if datetuple[0] < 1981:
 203                     raise ValueError
 204                 # This could raise a value error if it's not a valid format.
 205                 date = imaplib.Time2Internaldate(datetuple)
 206             except (ValueError, OverflowError):
 207                 # Argh, sometimes it's a valid format but year is 0102
 208                 # or something.  Argh.  It seems that Time2Internaldate
 209                 # will rause a ValueError if the year is 0102 but not 1902,
 210                 # but some IMAP servers nonetheless choke on 1902.
 211                 date = imaplib.Time2Internaldate(time.localtime())
 212
 213             ui.debug('imap', 'savemessage: using date ' + str(date))
 214             content = re.sub("(?<!\r)\n", "\r\n", content)
 215             ui.debug('imap', 'savemessage: initial content is: ' + repr(content))
 216
 217             (headername, headervalue) = self.savemessage_getnewheader(content)
 218             ui.debug('imap', 'savemessage: new headers are: %s: %s' % \
 219                      (headername, headervalue))
 220             content = self.savemessage_addheader(content, headername,
 221                                                  headervalue)
 222             ui.debug('imap', 'savemessage: new content is: ' + repr(content))
 223             ui.debug('imap', 'savemessage: new content length is ' + \
 224                      str(len(content)))
 225
 226             assert(imapobj.append(self.getfullname(),
 227                                        imaputil.flagsmaildir2imap(flags),
 228                                        date, content)[0] == 'OK')
 229
 230             # Checkpoint.  Let it write out the messages, etc.
 231             assert(imapobj.check()[0] == 'OK')
 232
 233             # Keep trying until we get the UID.
 234             ui.debug('imap', 'savemessage: first attempt to get new UID')
 235             uid = self.savemessage_searchforheader(imapobj, headername,
 236                                                    headervalue)
 237             if uid <= 0:
 238                 ui.debug('imap', 'savemessage: first attempt to get new UID failed.  Going to run a NOOP and try again.')
 239                 assert(imapobj.noop()[0] == 'OK')
 240                 uid = self.savemessage_searchforheader(imapobj, headername,
 241                                                        headervalue)
 242         finally:
 243             self.imapserver.releaseconnection(imapobj)
 244
 245         self.messagelist[uid] = {'uid': uid, 'flags': flags}
 246         ui.debug('imap', 'savemessage: returning %d' % uid)
 247         return uid
 248
 249     def savemessageflags(self, uid, flags):
 250         imapobj = self.imapserver.acquireconnection()
 251         try:
 252             try:
 253                 imapobj.select(self.getfullname())
 254             except imapobj.readonly:
 255                 UIBase.getglobalui().flagstoreadonly(self, [uid], flags)
 256                 return
 257             result = imapobj.uid('store', '%d' % uid, 'FLAGS',
 258                                  imaputil.flagsmaildir2imap(flags))
 259             assert result[0] == 'OK', 'Error with store: ' + r[1]
 260         finally:
 261             self.imapserver.releaseconnection(imapobj)
 262         result = result[1][0]
 263         if not result:
 264             self.messagelist[uid]['flags'] = flags
 265         else:
 266             flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS']
 267             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 268
 269     def addmessageflags(self, uid, flags):
 270         self.addmessagesflags([uid], flags)
 271
 272     def addmessagesflags_noconvert(self, uidlist, flags):
 273         self.processmessagesflags('+', uidlist, flags)
 274
 275     def addmessagesflags(self, uidlist, flags):
 276         """This is here for the sake of UIDMaps.py -- deletemessages must
 277         add flags and get a converted UID, and if we don't have noconvert,
 278         then UIDMaps will try to convert it twice."""
 279         self.addmessagesflags_noconvert(uidlist, flags)
 280
 281     def deletemessageflags(self, uid, flags):
 282         self.deletemessagesflags([uid], flags)
 283
 284     def deletemessagesflags(self, uidlist, flags):
 285         self.processmessagesflags('-', uidlist, flags)
 286
 287     def processmessagesflags(self, operation, uidlist, flags):
 288         if len(uidlist) > 101:
 289             # Hack for those IMAP ervers with a limited line length
 290             self.processmessagesflags(operation, uidlist[:100], flags)
 291             self.processmessagesflags(operation, uidlist[100:], flags)
 292             return
 293
 294         imapobj = self.imapserver.acquireconnection()
 295         try:
 296             try:
 297                 imapobj.select(self.getfullname())
 298             except imapobj.readonly:
 299                 UIBase.getglobalui().flagstoreadonly(self, uidlist, flags)
 300                 return
 301             r = imapobj.uid('store',
 302                             imaputil.listjoin(uidlist),
 303                             operation + 'FLAGS',
 304                             imaputil.flagsmaildir2imap(flags))
 305             assert r[0] == 'OK', 'Error with store: ' + r[1]
 306             r = r[1]
 307         finally:
 308             self.imapserver.releaseconnection(imapobj)
 309         # Some IMAP servers do not always return a result.  Therefore,
 310         # only update the ones that it talks about, and manually fix
 311         # the others.
 312         needupdate = copy(uidlist)
 313         for result in r:
 314             if result == None:
 315                 # Compensate for servers that don't return anything from
 316                 # STORE.
 317                 continue
 318             attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1])
 319             if not ('UID' in attributehash and 'FLAGS' in attributehash):
 320                 # Compensate for servers that don't return a UID attribute.
 321                 continue
 322             flags = attributehash['FLAGS']
 323             uid = long(attributehash['UID'])
 324             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 325             try:
 326                 needupdate.remove(uid)
 327             except ValueError:          # Let it slide if it's not in the list
 328                 pass
 329         for uid in needupdate:
 330             if operation == '+':
 331                 for flag in flags:
 332                     if not flag in self.messagelist[uid]['flags']:
 333                         self.messagelist[uid]['flags'].append(flag)
 334                     self.messagelist[uid]['flags'].sort()
 335             elif operation == '-':
 336                 for flag in flags:
 337                     if flag in self.messagelist[uid]['flags']:
 338                         self.messagelist[uid]['flags'].remove(flag)
 339
 340     def deletemessage(self, uid):
 341         self.deletemessages_noconvert([uid])
 342
 343     def deletemessages(self, uidlist):
 344         self.deletemessages_noconvert(uidlist)
 345
 346     def deletemessages_noconvert(self, uidlist):
 347         # Weed out ones not in self.messagelist
 348         uidlist = [uid for uid in uidlist if uid in self.messagelist]
 349         if not len(uidlist):
 350             return
 351
 352         self.addmessagesflags_noconvert(uidlist, ['T'])
 353         imapobj = self.imapserver.acquireconnection()
 354         try:
 355             try:
 356                 imapobj.select(self.getfullname())
 357             except imapobj.readonly:
 358                 UIBase.getglobalui().deletereadonly(self, uidlist)
 359                 return
 360             if self.expunge:
 361                 assert(imapobj.expunge()[0] == 'OK')
 362         finally:
 363             self.imapserver.releaseconnection(imapobj)
 364         for uid in uidlist:
 365             del self.messagelist[uid]
 366
 367