code.delx.au - offlineimap/blob - offlineimap/folder/Base.py

   1 # Base folder support
   2 # Copyright (C) 2002 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18
  19 from threading import *
  20 from offlineimap import threadutil
  21 from offlineimap.threadutil import InstanceLimitedThread
  22 from offlineimap.ui import UIBase
  23 import os.path, re
  24
  25 class BaseFolder:
  26     def __init__(self):
  27         self.uidlock = Lock()
  28
  29     def getname(self):
  30         """Returns name"""
  31         return self.name
  32
  33     def suggeststhreads(self):
  34         """Returns true if this folder suggests using threads for actions;
  35         false otherwise.  Probably only IMAP will return true."""
  36         return 0
  37
  38     def waitforthread(self):
  39         """For threading folders, waits until there is a resource available
  40         before firing off a thread.  For all others, returns immediately."""
  41         pass
  42
  43     def getcopyinstancelimit(self):
  44         """For threading folders, returns the instancelimitname for
  45         InstanceLimitedThreads."""
  46         raise NotImplementedException
  47
  48     def storesmessages(self):
  49         """Should be true for any backend that actually saves message bodies.
  50         (Almost all of them).  False for the LocalStatus backend.  Saves
  51         us from having to slurp up messages just for localstatus purposes."""
  52         return 1
  53
  54     def getvisiblename(self):
  55         return self.name
  56
  57     def getrepository(self):
  58         """Returns the repository object that this folder is within."""
  59         return self.repository
  60
  61     def getroot(self):
  62         """Returns the root of the folder, in a folder-specific fashion."""
  63         return self.root
  64
  65     def getsep(self):
  66         """Returns the separator for this folder type."""
  67         return self.sep
  68
  69     def getfullname(self):
  70         if self.getroot():
  71             return self.getroot() + self.getsep() + self.getname()
  72         else:
  73             return self.getname()
  74
  75     def getfolderbasename(self):
  76         foldername = self.getname()
  77         foldername = foldername.replace(self.repository.getsep(), '.')
  78         foldername = re.sub('/\.$', '/dot', foldername)
  79         foldername = re.sub('^\.$', 'dot', foldername)
  80         return foldername
  81
  82     def isuidvalidityok(self):
  83         if self.getsaveduidvalidity() != None:
  84             return self.getsaveduidvalidity() == self.getuidvalidity()
  85         else:
  86             self.saveuidvalidity()
  87             return 1
  88
  89     def _getuidfilename(self):
  90         return os.path.join(self.repository.getuiddir(),
  91                             self.getfolderbasename())
  92
  93     def getsaveduidvalidity(self):
  94         if hasattr(self, '_base_saved_uidvalidity'):
  95             return self._base_saved_uidvalidity
  96         uidfilename = self._getuidfilename()
  97         if not os.path.exists(uidfilename):
  98             self._base_saved_uidvalidity = None
  99         else:
 100             file = open(uidfilename, "rt")
 101             self._base_saved_uidvalidity = long(file.readline().strip())
 102             file.close()
 103         return self._base_saved_uidvalidity
 104
 105     def saveuidvalidity(self):
 106         newval = self.getuidvalidity()
 107         uidfilename = self._getuidfilename()
 108         self.uidlock.acquire()
 109         try:
 110             file = open(uidfilename + ".tmp", "wt")
 111             file.write("%d\n" % newval)
 112             file.close()
 113             os.rename(uidfilename + ".tmp", uidfilename)
 114             self._base_saved_uidvalidity = newval
 115         finally:
 116             self.uidlock.release()
 117
 118     def getuidvalidity(self):
 119         raise NotImplementedException
 120
 121     def cachemessagelist(self):
 122         """Reads the message list from disk or network and stores it in
 123         memory for later use.  This list will not be re-read from disk or
 124         memory unless this function is called again."""
 125         raise NotImplementedException
 126
 127     def getmessagelist(self):
 128         """Gets the current message list.
 129         You must call cachemessagelist() before calling this function!"""
 130         raise NotImplementedException
 131
 132     def getmessage(self, uid):
 133         """Returns the content of the specified message."""
 134         raise NotImplementedException
 135
 136     def savemessage(self, uid, content, flags):
 137         """Writes a new message, with the specified uid.
 138         If the uid is < 0, the backend should assign a new uid and return it.
 139
 140         If the backend cannot assign a new uid, it returns the uid passed in
 141         WITHOUT saving the message.
 142
 143         If the backend CAN assign a new uid, but cannot find out what this UID
 144         is (as is the case with many IMAP servers), it returns 0 but DOES save
 145         the message.
 146
 147         IMAP backend should be the only one that can assign a new uid.
 148
 149         If the uid is > 0, the backend should set the uid to this, if it can.
 150         If it cannot set the uid to that, it will save it anyway.
 151         It will return the uid assigned in any case.
 152         """
 153         raise NotImplementedException
 154
 155     def getmessageflags(self, uid):
 156         """Returns the flags for the specified message."""
 157         raise NotImplementedException
 158
 159     def savemessageflags(self, uid, flags):
 160         """Sets the specified message's flags to the given set."""
 161         raise NotImplementedException
 162
 163     def addmessageflags(self, uid, flags):
 164         """Adds the specified flags to the message's flag set.  If a given
 165         flag is already present, it will not be duplicated."""
 166         newflags = self.getmessageflags(uid)
 167         for flag in flags:
 168             if not flag in newflags:
 169                 newflags.append(flag)
 170         newflags.sort()
 171         self.savemessageflags(uid, newflags)
 172
 173     def addmessagesflags(self, uidlist, flags):
 174         for uid in uidlist:
 175             self.addmessageflags(uid, flags)
 176
 177     def deletemessageflags(self, uid, flags):
 178         """Removes each flag given from the message's flag set.  If a given
 179         flag is already removed, no action will be taken for that flag."""
 180         newflags = self.getmessageflags(uid)
 181         for flag in flags:
 182             if flag in newflags:
 183                 newflags.remove(flag)
 184         newflags.sort()
 185         self.savemessageflags(uid, newflags)
 186
 187     def deletemessagesflags(self, uidlist, flags):
 188         for uid in uidlist:
 189             self.deletemessageflags(uid, flags)
 190
 191     def deletemessage(self, uid):
 192         raise NotImplementedException
 193
 194     def deletemessages(self, uidlist):
 195         for uid in uidlist:
 196             self.deletemessage(uid)
 197
 198     def syncmessagesto_neguid_msg(self, uid, dest, applyto, register = 1):
 199         if register:
 200             UIBase.getglobalui().registerthread(self.getaccountname())
 201         UIBase.getglobalui().copyingmessage(uid, self, applyto)
 202         successobject = None
 203         successuid = None
 204         message = self.getmessage(uid)
 205         flags = self.getmessageflags(uid)
 206         for tryappend in applyto:
 207             successuid = tryappend.savemessage(uid, message, flags)
 208             if successuid >= 0:
 209                 successobject = tryappend
 210                 break
 211         # Did we succeed?
 212         if successobject != None:
 213             if successuid:       # Only if IMAP actually assigned a UID
 214                 # Copy the message to the other remote servers.
 215                 for appendserver in \
 216                         [x for x in applyto if x != successobject]:
 217                     appendserver.savemessage(successuid, message, flags)
 218                     # Copy to its new name on the local server and delete
 219                     # the one without a UID.
 220                     self.savemessage(successuid, message, flags)
 221             self.deletemessage(uid) # It'll be re-downloaded.
 222         else:
 223             # Did not find any server to take this message.  Ignore.
 224             pass
 225
 226
 227     def syncmessagesto_neguid(self, dest, applyto):
 228         """Pass 1 of folder synchronization.
 229
 230         Look for messages in self with a negative uid.  These are messages in
 231         Maildirs that were not added by us.  Try to add them to the dests,
 232         and once that succeeds, get the UID, add it to the others for real,
 233         add it to local for real, and delete the fake one."""
 234
 235         uidlist = [uid for uid in self.getmessagelist().keys() if uid < 0]
 236         threads = []
 237
 238         usethread = None
 239         if applyto != None:
 240             usethread = applyto[0]
 241
 242         for uid in uidlist:
 243             if usethread and usethread.suggeststhreads():
 244                 usethread.waitforthread()
 245                 thread = InstanceLimitedThread(\
 246                     usethread.getcopyinstancelimit(),
 247                     target = self.syncmessagesto_neguid_msg,
 248                     name = "New msg sync from %s" % self.getvisiblename(),
 249                     args = (uid, dest, applyto))
 250                 thread.setDaemon(1)
 251                 thread.start()
 252                 threads.append(thread)
 253             else:
 254                 self.syncmessagesto_neguid_msg(uid, dest, applyto, register = 0)
 255         for thread in threads:
 256             thread.join()
 257
 258     def copymessageto(self, uid, applyto, register = 1):
 259         # Sometimes, it could be the case that if a sync takes awhile,
 260         # a message might be deleted from the maildir before it can be
 261         # synced to the status cache.  This is only a problem with
 262         # self.getmessage().  So, don't call self.getmessage unless
 263         # really needed.
 264         if register:
 265             UIBase.getglobalui().registerthread(self.getaccountname())
 266         UIBase.getglobalui().copyingmessage(uid, self, applyto)
 267         message = ''
 268         # If any of the destinations actually stores the message body,
 269         # load it up.
 270         for object in applyto:
 271             if object.storesmessages():
 272                 message = self.getmessage(uid)
 273                 break
 274         flags = self.getmessageflags(uid)
 275         for object in applyto:
 276             newuid = object.savemessage(uid, message, flags)
 277             if newuid > 0 and newuid != uid:
 278                 # Change the local uid.
 279                 self.savemessage(newuid, message, flags)
 280                 self.deletemessage(uid)
 281                 uid = newuid
 282
 283
 284     def syncmessagesto_copy(self, dest, applyto):
 285         """Pass 2 of folder synchronization.
 286
 287         Look for messages present in self but not in dest.  If any, add
 288         them to dest."""
 289         threads = []
 290
 291         dest_messagelist = dest.getmessagelist()
 292         for uid in self.getmessagelist().keys():
 293             if uid < 0:                 # Ignore messages that pass 1 missed.
 294                 continue
 295             if not uid in dest_messagelist:
 296                 if self.suggeststhreads():
 297                     self.waitforthread()
 298                     thread = InstanceLimitedThread(\
 299                         self.getcopyinstancelimit(),
 300                         target = self.copymessageto,
 301                         name = "Copy message %d from %s" % (uid,
 302                                                             self.getvisiblename()),
 303                         args = (uid, applyto))
 304                     thread.setDaemon(1)
 305                     thread.start()
 306                     threads.append(thread)
 307                 else:
 308                     self.copymessageto(uid, applyto, register = 0)
 309         for thread in threads:
 310             thread.join()
 311
 312     def syncmessagesto_delete(self, dest, applyto):
 313         """Pass 3 of folder synchronization.
 314
 315         Look for message present in dest but not in self.
 316         If any, delete them."""
 317         deletelist = []
 318         self_messagelist = self.getmessagelist()
 319         for uid in dest.getmessagelist().keys():
 320             if uid < 0:
 321                 continue
 322             if not uid in self_messagelist:
 323                 deletelist.append(uid)
 324         if len(deletelist):
 325             UIBase.getglobalui().deletingmessages(deletelist, applyto)
 326             for object in applyto:
 327                 object.deletemessages(deletelist)
 328
 329     def syncmessagesto_flags(self, dest, applyto):
 330         """Pass 4 of folder synchronization.
 331
 332         Look for any flag matching issues -- set dest message to have the
 333         same flags that we have."""
 334
 335         # As an optimization over previous versions, we store up which flags
 336         # are being used for an add or a delete.  For each flag, we store
 337         # a list of uids to which it should be added.  Then, we can call
 338         # addmessagesflags() to apply them in bulk, rather than one
 339         # call per message as before.  This should result in some significant
 340         # performance improvements.
 341
 342         addflaglist = {}
 343         delflaglist = {}
 344
 345         for uid in self.getmessagelist().keys():
 346             if uid < 0:                 # Ignore messages missed by pass 1
 347                 continue
 348             selfflags = self.getmessageflags(uid)
 349             destflags = dest.getmessageflags(uid)
 350
 351             addflags = [x for x in selfflags if x not in destflags]
 352
 353             for flag in addflags:
 354                 if not flag in addflaglist:
 355                     addflaglist[flag] = []
 356                 addflaglist[flag].append(uid)
 357
 358             delflags = [x for x in destflags if x not in selfflags]
 359             for flag in delflags:
 360                 if not flag in delflaglist:
 361                     delflaglist[flag] = []
 362                 delflaglist[flag].append(uid)
 363
 364         for object in applyto:
 365             for flag in addflaglist.keys():
 366                 UIBase.getglobalui().addingflags(addflaglist[flag], flag, [object])
 367                 object.addmessagesflags(addflaglist[flag], [flag])
 368             for flag in delflaglist.keys():
 369                 UIBase.getglobalui().deletingflags(delflaglist[flag], flag, [object])
 370                 object.deletemessagesflags(delflaglist[flag], [flag])
 371
 372     def syncmessagesto(self, dest, applyto = None):
 373         """Syncs messages in this folder to the destination.
 374         If applyto is specified, it should be a list of folders (don't forget
 375         to include dest!) to which all write actions should be applied.
 376         It defaults to [dest] if not specified.  It is important that
 377         the UID generator be listed first in applyto; that is, the other
 378         applyto ones should be the ones that "copy" the main action."""
 379         if applyto == None:
 380             applyto = [dest]
 381
 382         self.syncmessagesto_neguid(dest, applyto)
 383         self.syncmessagesto_copy(dest, applyto)
 384         self.syncmessagesto_delete(dest, applyto)
 385
 386         # Now, the message lists should be identical wrt the uids present.
 387         # (except for potential negative uids that couldn't be placed
 388         # anywhere)
 389
 390         self.syncmessagesto_flags(dest, applyto)
 391
 392