code.delx.au - offlineimap/blob - offlineimap/folder/IMAP.py

   1 # IMAP folder support
   2 # Copyright (C) 2002-2004 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18
  19 from Base import BaseFolder
  20 from offlineimap import imaputil, imaplib
  21 from offlineimap.ui import UIBase
  22 from offlineimap.version import versionstr
  23 import rfc822, time, string, random, binascii, re
  24 from StringIO import StringIO
  25 from copy import copy
  26
  27
  28 class IMAPFolder(BaseFolder):
  29     def __init__(self, imapserver, name, visiblename, accountname, repository):
  30         self.config = imapserver.config
  31         self.expunge = repository.getexpunge()
  32         self.name = imaputil.dequote(name)
  33         self.root = None # imapserver.root
  34         self.sep = imapserver.delim
  35         self.imapserver = imapserver
  36         self.messagelist = None
  37         self.visiblename = visiblename
  38         self.accountname = accountname
  39         self.repository = repository
  40         self.randomgenerator = random.Random()
  41         BaseFolder.__init__(self)
  42
  43     def getaccountname(self):
  44         return self.accountname
  45
  46     def suggeststhreads(self):
  47         return 1
  48
  49     def waitforthread(self):
  50         self.imapserver.connectionwait()
  51
  52     def getcopyinstancelimit(self):
  53         return 'MSGCOPY_' + self.repository.getname()
  54
  55     def getvisiblename(self):
  56         return self.visiblename
  57
  58     def getuidvalidity(self):
  59         imapobj = self.imapserver.acquireconnection()
  60         try:
  61             # Primes untagged_responses
  62             imapobj.select(self.getfullname(), readonly = 1)
  63             return long(imapobj.untagged_responses['UIDVALIDITY'][0])
  64         finally:
  65             self.imapserver.releaseconnection(imapobj)
  66
  67     def cachemessagelist(self):
  68         imapobj = self.imapserver.acquireconnection()
  69         self.messagelist = {}
  70
  71         try:
  72             # Primes untagged_responses
  73             imapobj.select(self.getfullname(), readonly = 1, force = 1)
  74             try:
  75                 # Some mail servers do not return an EXISTS response if
  76                 # the folder is empty.
  77                 maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
  78             except KeyError:
  79                 return
  80             if maxmsgid < 1:
  81                 # No messages; return
  82                 return
  83
  84             # Now, get the flags and UIDs for these.
  85             # We could conceivably get rid of maxmsgid and just say
  86             # '1:*' here.
  87             response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID INTERNALDATE)')[1]
  88         finally:
  89             self.imapserver.releaseconnection(imapobj)
  90         for messagestr in response:
  91             # Discard the message number.
  92             messagestr = string.split(messagestr, maxsplit = 1)[1]
  93             options = imaputil.flags2hash(messagestr)
  94             if not options.has_key('UID'):
  95                 UIBase.getglobalui().warn('No UID in message with options %s' %\
  96                                           str(options),
  97                                           minor = 1)
  98             else:
  99                 uid = long(options['UID'])
 100                 flags = imaputil.flagsimap2maildir(options['FLAGS'])
 101                 rtime = imaplib.Internaldate2epoch(messagestr)
 102                 self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime}
 103
 104     def getmessagelist(self):
 105         return self.messagelist
 106
 107     def getmessage(self, uid):
 108         ui = UIBase.getglobalui()
 109         imapobj = self.imapserver.acquireconnection()
 110         try:
 111             imapobj.select(self.getfullname(), readonly = 1)
 112             initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])')
 113             ui.debug('imap', 'Returned object from fetching %d: %s' % \
 114                      (uid, str(initialresult)))
 115             return initialresult[1][0][1].replace("\r\n", "\n")
 116
 117         finally:
 118             self.imapserver.releaseconnection(imapobj)
 119
 120     def getmessagetime(self, uid):
 121         return self.messagelist[uid]['time']
 122
 123     def getmessageflags(self, uid):
 124         return self.messagelist[uid]['flags']
 125
 126     def savemessage_getnewheader(self, content):
 127         headername = 'X-OfflineIMAP-%s-' % str(binascii.crc32(content)).replace('-', 'x')
 128         headername += binascii.hexlify(self.repository.getname()) + '-'
 129         headername += binascii.hexlify(self.getname())
 130         headervalue= '%d-' % long(time.time())
 131         headervalue += str(self.randomgenerator.random()).replace('.', '')
 132         headervalue += '-v' + versionstr
 133         return (headername, headervalue)
 134
 135     def savemessage_addheader(self, content, headername, headervalue):
 136         ui = UIBase.getglobalui()
 137         ui.debug('imap',
 138                  'savemessage_addheader: called to add %s: %s' % (headername,
 139                                                                   headervalue))
 140         insertionpoint = content.find("\r\n")
 141         ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint)
 142         leader = content[0:insertionpoint]
 143         ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader))
 144         if insertionpoint == 0 or insertionpoint == -1:
 145             newline = ''
 146             insertionpoint = 0
 147         else:
 148             newline = "\r\n"
 149         newline += "%s: %s" % (headername, headervalue)
 150         ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline))
 151         trailer = content[insertionpoint:]
 152         ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer))
 153         return leader + newline + trailer
 154
 155     def savemessage_searchforheader(self, imapobj, headername, headervalue):
 156         if imapobj.untagged_responses.has_key('APPENDUID'):
 157             return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1])
 158
 159         ui = UIBase.getglobalui()
 160         ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \
 161                  (headername, headervalue))
 162         # Now find the UID it got.
 163         headervalue = imapobj._quote(headervalue)
 164         try:
 165             matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0]
 166         except imapobj.error, err:
 167             # IMAP server doesn't implement search or had a problem.
 168             ui.debug('imap', "savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername))
 169             return 0
 170         ui.debug('imap', 'savemessage_searchforheader got initial matchinguids: ' + repr(matchinguids))
 171
 172         if matchinguids == '':
 173             ui.debug('imap', "savemessage_searchforheader: UID SEARCH for message with header %s yielded no results" % headername)
 174             return 0
 175
 176         matchinguids = matchinguids.split(' ')
 177         ui.debug('imap', 'savemessage_searchforheader: matchinguids now ' + \
 178                  repr(matchinguids))
 179         if len(matchinguids) != 1 or matchinguids[0] == None:
 180             raise ValueError, "While attempting to find UID for message with header %s, got wrong-sized matchinguids of %s" % (headername, str(matchinguids))
 181         matchinguids.sort()
 182         return long(matchinguids[0])
 183
 184     def savemessage(self, uid, content, flags, rtime):
 185         imapobj = self.imapserver.acquireconnection()
 186         ui = UIBase.getglobalui()
 187         ui.debug('imap', 'savemessage: called')
 188         try:
 189             try:
 190                 imapobj.select(self.getfullname()) # Needed for search
 191             except imapobj.readonly:
 192                 ui.msgtoreadonly(self, uid, content, flags)
 193                 # Return indicating message taken, but no UID assigned.
 194                 # Fudge it.
 195                 return 0
 196
 197             # This backend always assigns a new uid, so the uid arg is ignored.
 198             # In order to get the new uid, we need to save off the message ID.
 199
 200             # If time isn't known
 201             if rtime == None:
 202                 datetuple = time.localtime()
 203             else:
 204                 datetuple = time.localtime(rtime)
 205
 206             try:
 207                 if datetuple[0] < 1981:
 208                     raise ValueError
 209                 # This could raise a value error if it's not a valid format.
 210                 date = imaplib.Time2Internaldate(datetuple)
 211             except (ValueError, OverflowError):
 212                 # Argh, sometimes it's a valid format but year is 0102
 213                 # or something.  Argh.  It seems that Time2Internaldate
 214                 # will rause a ValueError if the year is 0102 but not 1902,
 215                 # but some IMAP servers nonetheless choke on 1902.
 216                 date = imaplib.Time2Internaldate(time.localtime())
 217
 218             ui.debug('imap', 'savemessage: using date ' + str(date))
 219             content = re.sub("(?<!\r)\n", "\r\n", content)
 220             ui.debug('imap', 'savemessage: initial content is: ' + repr(content))
 221
 222             (headername, headervalue) = self.savemessage_getnewheader(content)
 223             ui.debug('imap', 'savemessage: new headers are: %s: %s' % \
 224                      (headername, headervalue))
 225             content = self.savemessage_addheader(content, headername,
 226                                                  headervalue)
 227             ui.debug('imap', 'savemessage: new content is: ' + repr(content))
 228             ui.debug('imap', 'savemessage: new content length is ' + \
 229                      str(len(content)))
 230
 231             assert(imapobj.append(self.getfullname(),
 232                                        imaputil.flagsmaildir2imap(flags),
 233                                        date, content)[0] == 'OK')
 234
 235             # Checkpoint.  Let it write out the messages, etc.
 236             assert(imapobj.check()[0] == 'OK')
 237
 238             # Keep trying until we get the UID.
 239             ui.debug('imap', 'savemessage: first attempt to get new UID')
 240             uid = self.savemessage_searchforheader(imapobj, headername,
 241                                                    headervalue)
 242             # See docs for savemessage in Base.py for explanation of this and other return values
 243             if uid <= 0:
 244                 ui.debug('imap', 'savemessage: first attempt to get new UID failed.  Going to run a NOOP and try again.')
 245                 assert(imapobj.noop()[0] == 'OK')
 246                 uid = self.savemessage_searchforheader(imapobj, headername,
 247                                                        headervalue)
 248         finally:
 249             self.imapserver.releaseconnection(imapobj)
 250
 251         if uid: # avoid UID FETCH 0 crash happening later on
 252             self.messagelist[uid] = {'uid': uid, 'flags': flags}
 253
 254         ui.debug('imap', 'savemessage: returning %d' % uid)
 255         return uid
 256
 257     def savemessageflags(self, uid, flags):
 258         imapobj = self.imapserver.acquireconnection()
 259         try:
 260             try:
 261                 imapobj.select(self.getfullname())
 262             except imapobj.readonly:
 263                 UIBase.getglobalui().flagstoreadonly(self, [uid], flags)
 264                 return
 265             result = imapobj.uid('store', '%d' % uid, 'FLAGS',
 266                                  imaputil.flagsmaildir2imap(flags))
 267             assert result[0] == 'OK', 'Error with store: ' + r[1]
 268         finally:
 269             self.imapserver.releaseconnection(imapobj)
 270         result = result[1][0]
 271         if not result:
 272             self.messagelist[uid]['flags'] = flags
 273         else:
 274             flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS']
 275             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 276
 277     def addmessageflags(self, uid, flags):
 278         self.addmessagesflags([uid], flags)
 279
 280     def addmessagesflags_noconvert(self, uidlist, flags):
 281         self.processmessagesflags('+', uidlist, flags)
 282
 283     def addmessagesflags(self, uidlist, flags):
 284         """This is here for the sake of UIDMaps.py -- deletemessages must
 285         add flags and get a converted UID, and if we don't have noconvert,
 286         then UIDMaps will try to convert it twice."""
 287         self.addmessagesflags_noconvert(uidlist, flags)
 288
 289     def deletemessageflags(self, uid, flags):
 290         self.deletemessagesflags([uid], flags)
 291
 292     def deletemessagesflags(self, uidlist, flags):
 293         self.processmessagesflags('-', uidlist, flags)
 294
 295     def processmessagesflags(self, operation, uidlist, flags):
 296         if len(uidlist) > 101:
 297             # Hack for those IMAP ervers with a limited line length
 298             self.processmessagesflags(operation, uidlist[:100], flags)
 299             self.processmessagesflags(operation, uidlist[100:], flags)
 300             return
 301
 302         imapobj = self.imapserver.acquireconnection()
 303         try:
 304             try:
 305                 imapobj.select(self.getfullname())
 306             except imapobj.readonly:
 307                 UIBase.getglobalui().flagstoreadonly(self, uidlist, flags)
 308                 return
 309             r = imapobj.uid('store',
 310                             imaputil.listjoin(uidlist),
 311                             operation + 'FLAGS',
 312                             imaputil.flagsmaildir2imap(flags))
 313             assert r[0] == 'OK', 'Error with store: ' + r[1]
 314             r = r[1]
 315         finally:
 316             self.imapserver.releaseconnection(imapobj)
 317         # Some IMAP servers do not always return a result.  Therefore,
 318         # only update the ones that it talks about, and manually fix
 319         # the others.
 320         needupdate = copy(uidlist)
 321         for result in r:
 322             if result == None:
 323                 # Compensate for servers that don't return anything from
 324                 # STORE.
 325                 continue
 326             attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1])
 327             if not ('UID' in attributehash and 'FLAGS' in attributehash):
 328                 # Compensate for servers that don't return a UID attribute.
 329                 continue
 330             flags = attributehash['FLAGS']
 331             uid = long(attributehash['UID'])
 332             self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
 333             try:
 334                 needupdate.remove(uid)
 335             except ValueError:          # Let it slide if it's not in the list
 336                 pass
 337         for uid in needupdate:
 338             if operation == '+':
 339                 for flag in flags:
 340                     if not flag in self.messagelist[uid]['flags']:
 341                         self.messagelist[uid]['flags'].append(flag)
 342                     self.messagelist[uid]['flags'].sort()
 343             elif operation == '-':
 344                 for flag in flags:
 345                     if flag in self.messagelist[uid]['flags']:
 346                         self.messagelist[uid]['flags'].remove(flag)
 347
 348     def deletemessage(self, uid):
 349         self.deletemessages_noconvert([uid])
 350
 351     def deletemessages(self, uidlist):
 352         self.deletemessages_noconvert(uidlist)
 353
 354     def deletemessages_noconvert(self, uidlist):
 355         # Weed out ones not in self.messagelist
 356         uidlist = [uid for uid in uidlist if uid in self.messagelist]
 357         if not len(uidlist):
 358             return
 359
 360         self.addmessagesflags_noconvert(uidlist, ['T'])
 361         imapobj = self.imapserver.acquireconnection()
 362         try:
 363             try:
 364                 imapobj.select(self.getfullname())
 365             except imapobj.readonly:
 366                 UIBase.getglobalui().deletereadonly(self, uidlist)
 367                 return
 368             if self.expunge:
 369                 assert(imapobj.expunge()[0] == 'OK')
 370         finally:
 371             self.imapserver.releaseconnection(imapobj)
 372         for uid in uidlist:
 373             del self.messagelist[uid]
 374
 375