code.delx.au - offlineimap/blob - offlineimap/folder/Base.py

   1 # Base folder support
   2 # Copyright (C) 2002 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18
  19 from threading import *
  20 from offlineimap import threadutil
  21 from offlineimap.threadutil import InstanceLimitedThread
  22 from offlineimap.ui import UIBase
  23 import os.path, re
  24
  25 class BaseFolder:
  26     def __init__(self):
  27         self.uidlock = Lock()
  28
  29     def getname(self):
  30         """Returns name"""
  31         return self.name
  32
  33     def suggeststhreads(self):
  34         """Returns true if this folder suggests using threads for actions;
  35         false otherwise.  Probably only IMAP will return true."""
  36         return 0
  37
  38     def waitforthread(self):
  39         """For threading folders, waits until there is a resource available
  40         before firing off a thread.  For all others, returns immediately."""
  41         pass
  42
  43     def getcopyinstancelimit(self):
  44         """For threading folders, returns the instancelimitname for
  45         InstanceLimitedThreads."""
  46         raise NotImplementedException
  47
  48     def storesmessages(self):
  49         """Should be true for any backend that actually saves message bodies.
  50         (Almost all of them).  False for the LocalStatus backend.  Saves
  51         us from having to slurp up messages just for localstatus purposes."""
  52         return 1
  53
  54     def getvisiblename(self):
  55         return self.name
  56
  57     def getrepository(self):
  58         """Returns the repository object that this folder is within."""
  59         return self.repository
  60
  61     def getroot(self):
  62         """Returns the root of the folder, in a folder-specific fashion."""
  63         return self.root
  64
  65     def getsep(self):
  66         """Returns the separator for this folder type."""
  67         return self.sep
  68
  69     def getfullname(self):
  70         if self.getroot():
  71             return self.getroot() + self.getsep() + self.getname()
  72         else:
  73             return self.getname()
  74
  75     def getfolderbasename(self):
  76         foldername = self.getname()
  77         foldername = foldername.replace(self.repository.getsep(), '.')
  78         foldername = re.sub('/\.$', '/dot', foldername)
  79         foldername = re.sub('^\.$', 'dot', foldername)
  80         return foldername
  81
  82     def isuidvalidityok(self):
  83         if self.getsaveduidvalidity() != None:
  84             return self.getsaveduidvalidity() == self.getuidvalidity()
  85         else:
  86             self.saveuidvalidity()
  87             return 1
  88
  89     def _getuidfilename(self):
  90         return os.path.join(self.repository.getuiddir(),
  91                             self.getfolderbasename())
  92
  93     def getsaveduidvalidity(self):
  94         if hasattr(self, '_base_saved_uidvalidity'):
  95             return self._base_saved_uidvalidity
  96         uidfilename = self._getuidfilename()
  97         if not os.path.exists(uidfilename):
  98             self._base_saved_uidvalidity = None
  99         else:
 100             file = open(uidfilename, "rt")
 101             self._base_saved_uidvalidity = long(file.readline().strip())
 102             file.close()
 103         return self._base_saved_uidvalidity
 104
 105     def saveuidvalidity(self):
 106         newval = self.getuidvalidity()
 107         uidfilename = self._getuidfilename()
 108         self.uidlock.acquire()
 109         try:
 110             file = open(uidfilename + ".tmp", "wt")
 111             file.write("%d\n" % newval)
 112             file.close()
 113             os.rename(uidfilename + ".tmp", uidfilename)
 114             self._base_saved_uidvalidity = newval
 115         finally:
 116             self.uidlock.release()
 117
 118     def getuidvalidity(self):
 119         raise NotImplementedException
 120
 121     def cachemessagelist(self):
 122         """Reads the message list from disk or network and stores it in
 123         memory for later use.  This list will not be re-read from disk or
 124         memory unless this function is called again."""
 125         raise NotImplementedException
 126
 127     def getmessagelist(self):
 128         """Gets the current message list.
 129         You must call cachemessagelist() before calling this function!"""
 130         raise NotImplementedException
 131
 132     def getmessage(self, uid):
 133         """Returns the content of the specified message."""
 134         raise NotImplementedException
 135
 136     def savemessage(self, uid, content, flags, rtime):
 137         """Writes a new message, with the specified uid.
 138         If the uid is < 0, the backend should assign a new uid and return it.
 139
 140         If the backend cannot assign a new uid, it returns the uid passed in
 141         WITHOUT saving the message.
 142
 143         If the backend CAN assign a new uid, but cannot find out what this UID
 144         is (as is the case with many IMAP servers), it returns 0 but DOES save
 145         the message.
 146
 147         IMAP backend should be the only one that can assign a new uid.
 148
 149         If the uid is > 0, the backend should set the uid to this, if it can.
 150         If it cannot set the uid to that, it will save it anyway.
 151         It will return the uid assigned in any case.
 152         """
 153         raise NotImplementedException
 154
 155     def getmessagetime(self, uid):
 156         """Return the received time for the specified message."""
 157         raise NotImplementedException
 158
 159     def getmessageflags(self, uid):
 160         """Returns the flags for the specified message."""
 161         raise NotImplementedException
 162
 163     def savemessageflags(self, uid, flags):
 164         """Sets the specified message's flags to the given set."""
 165         raise NotImplementedException
 166
 167     def addmessageflags(self, uid, flags):
 168         """Adds the specified flags to the message's flag set.  If a given
 169         flag is already present, it will not be duplicated."""
 170         newflags = self.getmessageflags(uid)
 171         for flag in flags:
 172             if not flag in newflags:
 173                 newflags.append(flag)
 174         newflags.sort()
 175         self.savemessageflags(uid, newflags)
 176
 177     def addmessagesflags(self, uidlist, flags):
 178         for uid in uidlist:
 179             self.addmessageflags(uid, flags)
 180
 181     def deletemessageflags(self, uid, flags):
 182         """Removes each flag given from the message's flag set.  If a given
 183         flag is already removed, no action will be taken for that flag."""
 184         newflags = self.getmessageflags(uid)
 185         for flag in flags:
 186             if flag in newflags:
 187                 newflags.remove(flag)
 188         newflags.sort()
 189         self.savemessageflags(uid, newflags)
 190
 191     def deletemessagesflags(self, uidlist, flags):
 192         for uid in uidlist:
 193             self.deletemessageflags(uid, flags)
 194
 195     def deletemessage(self, uid):
 196         raise NotImplementedException
 197
 198     def deletemessages(self, uidlist):
 199         for uid in uidlist:
 200             self.deletemessage(uid)
 201
 202     def syncmessagesto_neguid_msg(self, uid, dest, applyto, register = 1):
 203         if register:
 204             UIBase.getglobalui().registerthread(self.getaccountname())
 205         UIBase.getglobalui().copyingmessage(uid, self, applyto)
 206         successobject = None
 207         successuid = None
 208         message = self.getmessage(uid)
 209         flags = self.getmessageflags(uid)
 210         rtime = self.getmessagetime(uid)
 211         for tryappend in applyto:
 212             successuid = tryappend.savemessage(uid, message, flags, rtime)
 213             if successuid >= 0:
 214                 successobject = tryappend
 215                 break
 216         # Did we succeed?
 217         if successobject != None:
 218             if successuid:       # Only if IMAP actually assigned a UID
 219                 # Copy the message to the other remote servers.
 220                 for appendserver in \
 221                         [x for x in applyto if x != successobject]:
 222                     appendserver.savemessage(successuid, message, flags, rtime)
 223                     # Copy to its new name on the local server and delete
 224                     # the one without a UID.
 225                     self.savemessage(successuid, message, flags, rtime)
 226             self.deletemessage(uid) # It'll be re-downloaded.
 227         else:
 228             # Did not find any server to take this message.  Ignore.
 229             pass
 230
 231
 232     def syncmessagesto_neguid(self, dest, applyto):
 233         """Pass 1 of folder synchronization.
 234
 235         Look for messages in self with a negative uid.  These are messages in
 236         Maildirs that were not added by us.  Try to add them to the dests,
 237         and once that succeeds, get the UID, add it to the others for real,
 238         add it to local for real, and delete the fake one."""
 239
 240         uidlist = [uid for uid in self.getmessagelist().keys() if uid < 0]
 241         threads = []
 242
 243         usethread = None
 244         if applyto != None:
 245             usethread = applyto[0]
 246
 247         for uid in uidlist:
 248             if usethread and usethread.suggeststhreads():
 249                 usethread.waitforthread()
 250                 thread = InstanceLimitedThread(\
 251                     usethread.getcopyinstancelimit(),
 252                     target = self.syncmessagesto_neguid_msg,
 253                     name = "New msg sync from %s" % self.getvisiblename(),
 254                     args = (uid, dest, applyto))
 255                 thread.setDaemon(1)
 256                 thread.start()
 257                 threads.append(thread)
 258             else:
 259                 self.syncmessagesto_neguid_msg(uid, dest, applyto, register = 0)
 260         for thread in threads:
 261             thread.join()
 262
 263     def copymessageto(self, uid, applyto, register = 1):
 264         # Sometimes, it could be the case that if a sync takes awhile,
 265         # a message might be deleted from the maildir before it can be
 266         # synced to the status cache.  This is only a problem with
 267         # self.getmessage().  So, don't call self.getmessage unless
 268         # really needed.
 269         if register:
 270             UIBase.getglobalui().registerthread(self.getaccountname())
 271         UIBase.getglobalui().copyingmessage(uid, self, applyto)
 272         message = ''
 273         # If any of the destinations actually stores the message body,
 274         # load it up.
 275         for object in applyto:
 276             if object.storesmessages():
 277                 message = self.getmessage(uid)
 278                 break
 279         flags = self.getmessageflags(uid)
 280         rtime = self.getmessagetime(uid)
 281         for object in applyto:
 282             newuid = object.savemessage(uid, message, flags, rtime)
 283             if newuid > 0 and newuid != uid:
 284                 # Change the local uid.
 285                 self.savemessage(newuid, message, flags, rtime)
 286                 self.deletemessage(uid)
 287                 uid = newuid
 288
 289
 290     def syncmessagesto_copy(self, dest, applyto):
 291         """Pass 2 of folder synchronization.
 292
 293         Look for messages present in self but not in dest.  If any, add
 294         them to dest."""
 295         threads = []
 296
 297         dest_messagelist = dest.getmessagelist()
 298         for uid in self.getmessagelist().keys():
 299             if uid < 0:                 # Ignore messages that pass 1 missed.
 300                 continue
 301             if not uid in dest_messagelist:
 302                 if self.suggeststhreads():
 303                     self.waitforthread()
 304                     thread = InstanceLimitedThread(\
 305                         self.getcopyinstancelimit(),
 306                         target = self.copymessageto,
 307                         name = "Copy message %d from %s" % (uid,
 308                                                             self.getvisiblename()),
 309                         args = (uid, applyto))
 310                     thread.setDaemon(1)
 311                     thread.start()
 312                     threads.append(thread)
 313                 else:
 314                     self.copymessageto(uid, applyto, register = 0)
 315         for thread in threads:
 316             thread.join()
 317
 318     def syncmessagesto_delete(self, dest, applyto):
 319         """Pass 3 of folder synchronization.
 320
 321         Look for message present in dest but not in self.
 322         If any, delete them."""
 323         deletelist = []
 324         self_messagelist = self.getmessagelist()
 325         for uid in dest.getmessagelist().keys():
 326             if uid < 0:
 327                 continue
 328             if not uid in self_messagelist:
 329                 deletelist.append(uid)
 330         if len(deletelist):
 331             UIBase.getglobalui().deletingmessages(deletelist, applyto)
 332             for object in applyto:
 333                 object.deletemessages(deletelist)
 334
 335     def syncmessagesto_flags(self, dest, applyto):
 336         """Pass 4 of folder synchronization.
 337
 338         Look for any flag matching issues -- set dest message to have the
 339         same flags that we have."""
 340
 341         # As an optimization over previous versions, we store up which flags
 342         # are being used for an add or a delete.  For each flag, we store
 343         # a list of uids to which it should be added.  Then, we can call
 344         # addmessagesflags() to apply them in bulk, rather than one
 345         # call per message as before.  This should result in some significant
 346         # performance improvements.
 347
 348         addflaglist = {}
 349         delflaglist = {}
 350
 351         for uid in self.getmessagelist().keys():
 352             if uid < 0:                 # Ignore messages missed by pass 1
 353                 continue
 354             selfflags = self.getmessageflags(uid)
 355             destflags = dest.getmessageflags(uid)
 356
 357             addflags = [x for x in selfflags if x not in destflags]
 358
 359             for flag in addflags:
 360                 if not flag in addflaglist:
 361                     addflaglist[flag] = []
 362                 addflaglist[flag].append(uid)
 363
 364             delflags = [x for x in destflags if x not in selfflags]
 365             for flag in delflags:
 366                 if not flag in delflaglist:
 367                     delflaglist[flag] = []
 368                 delflaglist[flag].append(uid)
 369
 370         for object in applyto:
 371             for flag in addflaglist.keys():
 372                 UIBase.getglobalui().addingflags(addflaglist[flag], flag, [object])
 373                 object.addmessagesflags(addflaglist[flag], [flag])
 374             for flag in delflaglist.keys():
 375                 UIBase.getglobalui().deletingflags(delflaglist[flag], flag, [object])
 376                 object.deletemessagesflags(delflaglist[flag], [flag])
 377
 378     def syncmessagesto(self, dest, applyto = None):
 379         """Syncs messages in this folder to the destination.
 380         If applyto is specified, it should be a list of folders (don't forget
 381         to include dest!) to which all write actions should be applied.
 382         It defaults to [dest] if not specified.  It is important that
 383         the UID generator be listed first in applyto; that is, the other
 384         applyto ones should be the ones that "copy" the main action."""
 385         if applyto == None:
 386             applyto = [dest]
 387
 388         self.syncmessagesto_neguid(dest, applyto)
 389         self.syncmessagesto_copy(dest, applyto)
 390         self.syncmessagesto_delete(dest, applyto)
 391
 392         # Now, the message lists should be identical wrt the uids present.
 393         # (except for potential negative uids that couldn't be placed
 394         # anywhere)
 395
 396         self.syncmessagesto_flags(dest, applyto)
 397
 398