code.delx.au - offlineimap/blob - offlineimap/folder/IMAP.py

   1 # IMAP folder support
   2 # Copyright (C) 2002-2004 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18
  19 from Base import BaseFolder
  20 from offlineimap import imaputil, imaplib
  21 from offlineimap.ui import UIBase
  22 from offlineimap.version import versionstr
  23 import rfc822, time, string, random, binascii, re
  24 from StringIO import StringIO
  25 from copy import copy
  26
  27
  28 class IMAPFolder(BaseFolder):
  29     def __init__(self, imapserver, name, visiblename, accountname, repository):
  30         self.config = imapserver.config
  31         self.expunge = repository.getexpunge()
  32         self.name = imaputil.dequote(name)
  33         self.root = None # imapserver.root
  34         self.sep = imapserver.delim
  35         self.imapserver = imapserver
  36         self.messagelist = None
  37         self.visiblename = visiblename
  38         self.accountname = accountname
  39         self.repository = repository
  40         self.randomgenerator = random.Random()
  41         BaseFolder.__init__(self)
  42
  43     def getaccountname(self):
  44         return self.accountname
  45
  46     def suggeststhreads(self):
  47         return 1
  48
  49     def waitforthread(self):
  50         self.imapserver.connectionwait()
  51
  52     def getcopyinstancelimit(self):
  53         return 'MSGCOPY_' + self.repository.getname()
  54
  55     def getvisiblename(self):
  56         return self.visiblename
  57
  58     def getuidvalidity(self):
  59         imapobj = self.imapserver.acquireconnection()
  60         try:
  61             # Primes untagged_responses
  62             imapobj.select(self.getfullname(), readonly = 1)
  63             return long(imapobj.untagged_responses['UIDVALIDITY'][0])
  64         finally:
  65             self.imapserver.releaseconnection(imapobj)
  66
  67     def cachemessagelist(self):
  68         imapobj = self.imapserver.acquireconnection()
  69         self.messagelist = {}
  70
  71         try:
  72             # Primes untagged_responses
  73             imapobj.select(self.getfullname(), readonly = 1, force = 1)
  74             try:
  75                 # Some mail servers do not return an EXISTS response if
  76                 # the folder is empty.
  77                 maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
  78             except KeyError:
  79                 return
  80             if maxmsgid < 1:
  81                 # No messages; return
  82                 return
  83
  84             # Now, get the flags and UIDs for these.
  85             # We could conceivably get rid of maxmsgid and just say
  86             # '1:*' here.
  87             response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID INTERNALDATE)')[1]
  88         finally:
  89             self.imapserver.releaseconnection(imapobj)
  90         for messagestr in response:
  91             # Discard the message number.
  92             messagestr = string.split(messagestr, maxsplit = 1)[1]
  93             options = imaputil.flags2hash(messagestr)
  94             if not options.has_key('UID'):
  95                 UIBase.getglobalui().warn('No UID in message with options %s' %\
  96                                           str(options),
  97                                           minor = 1)
  98             else:
  99                 uid = long(options['UID'])
 100                 flags = imaputil.flagsimap2maildir(options['FLAGS'])
 101                 rtime = imaplib.Internaldate2epoch(messagestr)
 102                 self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime}
 103
 104     def getmessagelist(self):
 105         return self.messagelist
 106
 107     def getmessage(self, uid):
 108         ui = UIBase.getglobalui()
 109         imapobj = self.imapserver.acquireconnection()
 110         try:
 111             imapobj.select(self.getfullname(), readonly = 1)
 112             initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])')
 113             ui.debug('imap', 'Returned object from fetching %d: %s' % \
 114                      (uid, str(initialresult)))
 115             return initialresult[1][0][1].replace("\r\n", "\n")
 116
 117         finally:
 118             self.imapserver.releaseconnection(imapobj)
 119
 120     def getmessagetime(self, uid):
 121         return self.messagelist[uid]['time']
 122
 123     def getmessageflags(self, uid):
 124         return self.messagelist[uid]['flags']
 125
 126     def savemessage_getnewheader(self, content):
 127         headername = 'X-OfflineIMAP-%s-' % str(binascii.crc32(content)).replace('-', 'x')
 128         headername += binascii.hexlify(self.repository.getname()) + '-'
 129         headername += binascii.hexlify(self.getname())
 130         headervalue= '%d-' % long(time.time())
 131         headervalue += str(self.randomgenerator.random()).replace('.', '')
 132         headervalue += '-v' + versionstr
 133         return (headername, headervalue)
 134
 135     def savemessage_addheader(self, content, headername, headervalue):
 136         ui = UIBase.getglobalui()
 137         ui.debug('imap',
 138                  'savemessage_addheader: called to add %s: %s' % (headername,
 139                                                                   headervalue))
 140         insertionpoint = content.find("\r\n")
 141         ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint)
 142         leader = content[0:insertionpoint]
 143         ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader))
 144         if insertionpoint == 0 or insertionpoint == -1:
 145             newline = ''
 146             insertionpoint = 0
 147         else:
 148             newline = "\r\n"
 149         newline += "%s: %s" % (headername, headervalue)
 150         ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline))
 151         trailer = content[insertionpoint:]
 152         ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer))
 153         return leader + newline + trailer
 154
 155     def savemessage_searchforheader(self, imapobj, headername, headervalue):
 156         if imapobj.untagged_responses.has_key('APPENDUID'):
 157             return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1])
 158
 159         ui = UIBase.getglobalui()
 160         ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \
 161                  (headername, headervalue))
 162         # Now find the UID it got.
 163         headervalue = imapobj._quote(headervalue)
 164         try:
 165             matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0]
 166         except imapobj.error, err:
 167             # IMAP server doesn't implement search or had a problem.
 168             ui.debug('imap', "savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername))
 169             return 0
 170         ui.debug('imap', 'savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids))
 171
 172         if matchinguids == '':
 173             ui.debug('imap', "savemessage_searchforheader: UID SEARCH for message with header %s yielded no results" % headername)
 174             return 0
 175
 176         matchinguids = matchinguids.split(' ')
 177         ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \
 178                  repr(matchinguids))
 179         if len(matchinguids) != 1 or matchinguids[0] == None:
 180             raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids))
 181         matchinguids.sort()
 182         return long(matchinguids[0])
 183
 184     def savemessage(self, uid, content, flags, rtime):
 185         imapobj = self.imapserver.acquireconnection()
 186         ui = UIBase.getglobalui()
 187         ui.debug('imap', 'savemessage: called')
 188         try:
 189             try:
 190                 imapobj.select(self.getfullname()) # Needed for search
 191             except imapobj.readonly:
 192                 ui.msgtoreadonly(self, uid, content, flags)
 193                 # Return indicating message taken, but no UID assigned.
 194                 # Fudge it.
 195                 return 0
 196
 197             # This backend always assigns a new uid, so the uid arg is ignored.
 198             # In order to get the new uid, we need to save off the message ID.
 199
 200             message = rfc822.Message(StringIO(content))
 201             datetuple_msg = rfc822.parsedate(message.getheader('Date'))
 202             # Will be None if missing or not in a valid format.
 203
 204             # If time isn't known
 205             if rtime == None and datetuple_msg == None:
 206                 datetuple = time.localtime()
 207             elif rtime == None:
 208                 datetuple = datetuple_msg
 209             else:
 210                 datetuple = time.localtime(rtime)
 211
 212             try:
 213                 if datetuple[0] < 1981:
 214                     raise ValueError
 215                 # This could raise a value error if it's not a valid format.
 216                 date = imaplib.Time2Internaldate(datetuple)
 217             except (ValueError, OverflowError):
 218                 # Argh, sometimes it's a valid format but year is 0102
 219                 # or something.  Argh.  It seems that Time2Internaldate
 220                 # will rause a ValueError if the year is 0102 but not 1902,
 221                 # but some IMAP servers nonetheless choke on 1902.
 222                 date = imaplib.Time2Internaldate(time.localtime())
 223
 224             ui.debug('imap', 'savemessage: using date ' + str(date))
 225             content = re.sub("(?<!\r)\n", "\r\n", content)
 226             ui.debug('imap', 'savemessage: initial content is: ' + repr(content))
 227
 228             (headername, headervalue) = self.savemessage_getnewheader(content)
 229             ui.debug('imap', 'savemessage: new headers are: %s: %s' % \
 230                      (headername, headervalue))
 231             content = self.savemessage_addheader(content, headername,
 232                                                  headervalue)
 233             ui.debug('imap', 'savemessage: new content is: ' + repr(content))
 234             ui.debug('imap', 'savemessage: new content length is ' + \
 235                      str(len(content)))
 236
 237             assert(imapobj.append(self.getfullname(),
 238                                        imaputil.flagsmaildir2imap(flags),
 239                                        date, content)[0] == 'OK')
 240
 241             # Checkpoint.  Let it write out the messages, etc.
 242             assert(imapobj.check()[0] == 'OK')
 243
 244             # Keep trying until we get the UID.
 245             ui.debug('imap', 'savemessage: first attempt to get new UID')
 246             uid = self.savemessage_searchforheader(imapobj, headername,
 247                                                    headervalue)
 248             # See docs for savemessage in Base.py for explanation of this and other return values
 249             if uid <= 0:
 250                 ui.debug('imap', 'savemessage: first attempt to get new UID failed.  Going to run a NOOP and try again.')
 251                 assert(imapobj.noop()[0] == 'OK')
 252                 uid = self.savemessage_searchforheader(imapobj, headername,
 253                                                        headervalue)
 254         finally:
 255             self.imapserver.releaseconnection(imapobj)
 256
 257         if uid: # avoid UID FETCH 0 crash happening later on
 258             self.messagelist[uid] = {'uid': uid, 'flags': flags}
 259
 260         ui.debug('imap', 'savemessage: returning %d' % uid)
 261         return uid
 262
 263     def savemessageflags(self, uid, flags):
 264         imapobj = self.imapserver.acquireconnection()
 265         try:
 266             try:
 267                 imapobj.select(self.getfullname())
 268             except imapobj.readonly:
 269                 UIBase.getglobalui().flagstoreadonly(self, [uid], flags)
 270                 return
 271             result = imapobj.uid('store', '%d' % uid, 'FLAGS',
 272                                  imaputil.flagsmaildir2imap(flags))
 273             assert result[0] == 'OK', 'Error with store: ' + r[1]
 274         finally:
 275             self.imapserver.releaseconnection(imapobj)
 276         result = result[1][0]
 277         if not result:
 278             self.messagelist[uid]['flags'] = flags
 279         else:
 280             flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS']
 281             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 282
 283     def addmessageflags(self, uid, flags):
 284         self.addmessagesflags([uid], flags)
 285
 286     def addmessagesflags_noconvert(self, uidlist, flags):
 287         self.processmessagesflags('+', uidlist, flags)
 288
 289     def addmessagesflags(self, uidlist, flags):
 290         """This is here for the sake of UIDMaps.py -- deletemessages must
 291         add flags and get a converted UID, and if we don't have noconvert,
 292         then UIDMaps will try to convert it twice."""
 293         self.addmessagesflags_noconvert(uidlist, flags)
 294
 295     def deletemessageflags(self, uid, flags):
 296         self.deletemessagesflags([uid], flags)
 297
 298     def deletemessagesflags(self, uidlist, flags):
 299         self.processmessagesflags('-', uidlist, flags)
 300
 301     def processmessagesflags(self, operation, uidlist, flags):
 302         if len(uidlist) > 101:
 303             # Hack for those IMAP ervers with a limited line length
 304             self.processmessagesflags(operation, uidlist[:100], flags)
 305             self.processmessagesflags(operation, uidlist[100:], flags)
 306             return
 307
 308         imapobj = self.imapserver.acquireconnection()
 309         try:
 310             try:
 311                 imapobj.select(self.getfullname())
 312             except imapobj.readonly:
 313                 UIBase.getglobalui().flagstoreadonly(self, uidlist, flags)
 314                 return
 315             r = imapobj.uid('store',
 316                             imaputil.listjoin(uidlist),
 317                             operation + 'FLAGS',
 318                             imaputil.flagsmaildir2imap(flags))
 319             assert r[0] == 'OK', 'Error with store: ' + r[1]
 320             r = r[1]
 321         finally:
 322             self.imapserver.releaseconnection(imapobj)
 323         # Some IMAP servers do not always return a result.  Therefore,
 324         # only update the ones that it talks about, and manually fix
 325         # the others.
 326         needupdate = copy(uidlist)
 327         for result in r:
 328             if result == None:
 329                 # Compensate for servers that don't return anything from
 330                 # STORE.
 331                 continue
 332             attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1])
 333             if not ('UID' in attributehash and 'FLAGS' in attributehash):
 334                 # Compensate for servers that don't return a UID attribute.
 335                 continue
 336             flags = attributehash['FLAGS']
 337             uid = long(attributehash['UID'])
 338             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 339             try:
 340                 needupdate.remove(uid)
 341             except ValueError:          # Let it slide if it's not in the list
 342                 pass
 343         for uid in needupdate:
 344             if operation == '+':
 345                 for flag in flags:
 346                     if not flag in self.messagelist[uid]['flags']:
 347                         self.messagelist[uid]['flags'].append(flag)
 348                     self.messagelist[uid]['flags'].sort()
 349             elif operation == '-':
 350                 for flag in flags:
 351                     if flag in self.messagelist[uid]['flags']:
 352                         self.messagelist[uid]['flags'].remove(flag)
 353
 354     def deletemessage(self, uid):
 355         self.deletemessages_noconvert([uid])
 356
 357     def deletemessages(self, uidlist):
 358         self.deletemessages_noconvert(uidlist)
 359
 360     def deletemessages_noconvert(self, uidlist):
 361         # Weed out ones not in self.messagelist
 362         uidlist = [uid for uid in uidlist if uid in self.messagelist]
 363         if not len(uidlist):
 364             return
 365
 366         self.addmessagesflags_noconvert(uidlist, ['T'])
 367         imapobj = self.imapserver.acquireconnection()
 368         try:
 369             try:
 370                 imapobj.select(self.getfullname())
 371             except imapobj.readonly:
 372                 UIBase.getglobalui().deletereadonly(self, uidlist)
 373                 return
 374             if self.expunge:
 375                 assert(imapobj.expunge()[0] == 'OK')
 376         finally:
 377             self.imapserver.releaseconnection(imapobj)
 378         for uid in uidlist:
 379             del self.messagelist[uid]
 380
 381