code.delx.au - offlineimap/blob - offlineimap/folder/Base.py

   1 # Base folder support
   2 # Copyright (C) 2002 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18
  19 from threading import *
  20 from offlineimap import threadutil
  21 from offlineimap.threadutil import InstanceLimitedThread
  22 from offlineimap.ui import UIBase
  23 import os.path, re
  24
  25 class BaseFolder:
  26     def __init__(self):
  27         self.uidlock = Lock()
  28
  29     def getname(self):
  30         """Returns name"""
  31         return self.name
  32
  33     def suggeststhreads(self):
  34         """Returns true if this folder suggests using threads for actions;
  35         false otherwise.  Probably only IMAP will return true."""
  36         return 0
  37
  38     def waitforthread(self):
  39         """For threading folders, waits until there is a resource available
  40         before firing off a thread.  For all others, returns immediately."""
  41         pass
  42
  43     def getcopyinstancelimit(self):
  44         """For threading folders, returns the instancelimitname for
  45         InstanceLimitedThreads."""
  46         raise NotImplementedException
  47
  48     def storesmessages(self):
  49         """Should be true for any backend that actually saves message bodies.
  50         (Almost all of them).  False for the LocalStatus backend.  Saves
  51         us from having to slurp up messages just for localstatus purposes."""
  52         return 1
  53
  54     def getvisiblename(self):
  55         return self.name
  56
  57     def getrepository(self):
  58         """Returns the repository object that this folder is within."""
  59         return self.repository
  60
  61     def getroot(self):
  62         """Returns the root of the folder, in a folder-specific fashion."""
  63         return self.root
  64
  65     def getsep(self):
  66         """Returns the separator for this folder type."""
  67         return self.sep
  68
  69     def getfullname(self):
  70         if self.getroot():
  71             return self.getroot() + self.getsep() + self.getname()
  72         else:
  73             return self.getname()
  74
  75     def getfolderbasename(self):
  76         foldername = self.getname()
  77         foldername = foldername.replace(self.repository.getsep(), '.')
  78         foldername = re.sub('/\.$', '/dot', foldername)
  79         foldername = re.sub('^\.$', 'dot', foldername)
  80         return foldername
  81
  82     def isuidvalidityok(self):
  83         if self.getsaveduidvalidity() != None:
  84             return self.getsaveduidvalidity() == self.getuidvalidity()
  85         else:
  86             self.saveuidvalidity()
  87             return 1
  88
  89     def _getuidfilename(self):
  90         return os.path.join(self.repository.getuiddir(),
  91                             self.getfolderbasename())
  92
  93     def getsaveduidvalidity(self):
  94         if hasattr(self, '_base_saved_uidvalidity'):
  95             return self._base_saved_uidvalidity
  96         uidfilename = self._getuidfilename()
  97         if not os.path.exists(uidfilename):
  98             self._base_saved_uidvalidity = None
  99         else:
 100             file = open(uidfilename, "rt")
 101             self._base_saved_uidvalidity = long(file.readline().strip())
 102             file.close()
 103         return self._base_saved_uidvalidity
 104
 105     def saveuidvalidity(self):
 106         newval = self.getuidvalidity()
 107         uidfilename = self._getuidfilename()
 108         self.uidlock.acquire()
 109         try:
 110             file = open(uidfilename + ".tmp", "wt")
 111             file.write("%d\n" % newval)
 112             file.close()
 113             os.rename(uidfilename + ".tmp", uidfilename)
 114             self._base_saved_uidvalidity = newval
 115         finally:
 116             self.uidlock.release()
 117
 118     def getuidvalidity(self):
 119         raise NotImplementedException
 120
 121     def cachemessagelist(self):
 122         """Reads the message list from disk or network and stores it in
 123         memory for later use.  This list will not be re-read from disk or
 124         memory unless this function is called again."""
 125         raise NotImplementedException
 126
 127     def getmessagelist(self):
 128         """Gets the current message list.
 129         You must call cachemessagelist() before calling this function!"""
 130         raise NotImplementedException
 131
 132     def uidexists(self,uid):
 133         """Returns true if uid exists"""
 134         mlist = self.getmessagelist()
 135         if uid in mlist:
 136                 return 1
 137         else:
 138                 return 0
 139         return 0
 140
 141     def getmessageuidlist(self):
 142         """Gets a list of UIDs.
 143         You may have to call cachemessagelist() before calling this function!"""
 144         return self.getmessagelist().keys()
 145
 146     def getmessagecount(self):
 147         """Gets the number of messages."""
 148         return len(self.getmessagelist().keys())
 149
 150     def getmessage(self, uid):
 151         """Returns the content of the specified message."""
 152         raise NotImplementedException
 153
 154     def savemessage(self, uid, content, flags, rtime):
 155         """Writes a new message, with the specified uid.
 156         If the uid is < 0, the backend should assign a new uid and return it.
 157
 158         If the backend cannot assign a new uid, it returns the uid passed in
 159         WITHOUT saving the message.
 160
 161         If the backend CAN assign a new uid, but cannot find out what this UID
 162         is (as is the case with many IMAP servers), it returns 0 but DOES save
 163         the message.
 164
 165         IMAP backend should be the only one that can assign a new uid.
 166
 167         If the uid is > 0, the backend should set the uid to this, if it can.
 168         If it cannot set the uid to that, it will save it anyway.
 169         It will return the uid assigned in any case.
 170         """
 171         raise NotImplementedException
 172
 173     def getmessagetime(self, uid):
 174         """Return the received time for the specified message."""
 175         raise NotImplementedException
 176
 177     def getmessageflags(self, uid):
 178         """Returns the flags for the specified message."""
 179         raise NotImplementedException
 180
 181     def savemessageflags(self, uid, flags):
 182         """Sets the specified message's flags to the given set."""
 183         raise NotImplementedException
 184
 185     def addmessageflags(self, uid, flags):
 186         """Adds the specified flags to the message's flag set.  If a given
 187         flag is already present, it will not be duplicated."""
 188         newflags = self.getmessageflags(uid)
 189         for flag in flags:
 190             if not flag in newflags:
 191                 newflags.append(flag)
 192         newflags.sort()
 193         self.savemessageflags(uid, newflags)
 194
 195     def addmessagesflags(self, uidlist, flags):
 196         for uid in uidlist:
 197             self.addmessageflags(uid, flags)
 198
 199     def deletemessageflags(self, uid, flags):
 200         """Removes each flag given from the message's flag set.  If a given
 201         flag is already removed, no action will be taken for that flag."""
 202         newflags = self.getmessageflags(uid)
 203         for flag in flags:
 204             if flag in newflags:
 205                 newflags.remove(flag)
 206         newflags.sort()
 207         self.savemessageflags(uid, newflags)
 208
 209     def deletemessagesflags(self, uidlist, flags):
 210         for uid in uidlist:
 211             self.deletemessageflags(uid, flags)
 212
 213     def deletemessage(self, uid):
 214         raise NotImplementedException
 215
 216     def deletemessages(self, uidlist):
 217         for uid in uidlist:
 218             self.deletemessage(uid)
 219
 220     def syncmessagesto_neguid_msg(self, uid, dest, applyto, register = 1):
 221         if register:
 222             UIBase.getglobalui().registerthread(self.getaccountname())
 223         UIBase.getglobalui().copyingmessage(uid, self, applyto)
 224         successobject = None
 225         successuid = None
 226         message = self.getmessage(uid)
 227         flags = self.getmessageflags(uid)
 228         rtime = self.getmessagetime(uid)
 229         for tryappend in applyto:
 230             successuid = tryappend.savemessage(uid, message, flags, rtime)
 231             if successuid >= 0:
 232                 successobject = tryappend
 233                 break
 234         # Did we succeed?
 235         if successobject != None:
 236             if successuid:       # Only if IMAP actually assigned a UID
 237                 # Copy the message to the other remote servers.
 238                 for appendserver in \
 239                         [x for x in applyto if x != successobject]:
 240                     appendserver.savemessage(successuid, message, flags, rtime)
 241                     # Copy to its new name on the local server and delete
 242                     # the one without a UID.
 243                     self.savemessage(successuid, message, flags, rtime)
 244             self.deletemessage(uid) # It'll be re-downloaded.
 245         else:
 246             # Did not find any server to take this message.  Ignore.
 247             pass
 248
 249
 250     def syncmessagesto_neguid(self, dest, applyto):
 251         """Pass 1 of folder synchronization.
 252
 253         Look for messages in self with a negative uid.  These are messages in
 254         Maildirs that were not added by us.  Try to add them to the dests,
 255         and once that succeeds, get the UID, add it to the others for real,
 256         add it to local for real, and delete the fake one."""
 257
 258         uidlist = [uid for uid in self.getmessageuidlist() if uid < 0]
 259         threads = []
 260
 261         usethread = None
 262         if applyto != None:
 263             usethread = applyto[0]
 264
 265         for uid in uidlist:
 266             if usethread and usethread.suggeststhreads():
 267                 usethread.waitforthread()
 268                 thread = InstanceLimitedThread(\
 269                     usethread.getcopyinstancelimit(),
 270                     target = self.syncmessagesto_neguid_msg,
 271                     name = "New msg sync from %s" % self.getvisiblename(),
 272                     args = (uid, dest, applyto))
 273                 thread.setDaemon(1)
 274                 thread.start()
 275                 threads.append(thread)
 276             else:
 277                 self.syncmessagesto_neguid_msg(uid, dest, applyto, register = 0)
 278         for thread in threads:
 279             thread.join()
 280
 281     def copymessageto(self, uid, applyto, register = 1):
 282         # Sometimes, it could be the case that if a sync takes awhile,
 283         # a message might be deleted from the maildir before it can be
 284         # synced to the status cache.  This is only a problem with
 285         # self.getmessage().  So, don't call self.getmessage unless
 286         # really needed.
 287         if register:
 288             UIBase.getglobalui().registerthread(self.getaccountname())
 289         UIBase.getglobalui().copyingmessage(uid, self, applyto)
 290         message = ''
 291         # If any of the destinations actually stores the message body,
 292         # load it up.
 293         for object in applyto:
 294             if object.storesmessages():
 295                 message = self.getmessage(uid)
 296                 break
 297         flags = self.getmessageflags(uid)
 298         rtime = self.getmessagetime(uid)
 299         for object in applyto:
 300             newuid = object.savemessage(uid, message, flags, rtime)
 301             if newuid > 0 and newuid != uid:
 302                 # Change the local uid.
 303                 self.savemessage(newuid, message, flags, rtime)
 304                 self.deletemessage(uid)
 305                 uid = newuid
 306
 307
 308     def syncmessagesto_copy(self, dest, applyto):
 309         """Pass 2 of folder synchronization.
 310
 311         Look for messages present in self but not in dest.  If any, add
 312         them to dest."""
 313         threads = []
 314
 315         for uid in self.getmessageuidlist():
 316             if uid < 0:                 # Ignore messages that pass 1 missed.
 317                 continue
 318             if not dest.uidexists(uid):
 319                 if self.suggeststhreads():
 320                     self.waitforthread()
 321                     thread = InstanceLimitedThread(\
 322                         self.getcopyinstancelimit(),
 323                         target = self.copymessageto,
 324                         name = "Copy message %d from %s" % (uid,
 325                                                             self.getvisiblename()),
 326                         args = (uid, applyto))
 327                     thread.setDaemon(1)
 328                     thread.start()
 329                     threads.append(thread)
 330                 else:
 331                     self.copymessageto(uid, applyto, register = 0)
 332         for thread in threads:
 333             thread.join()
 334
 335     def syncmessagesto_delete(self, dest, applyto):
 336         """Pass 3 of folder synchronization.
 337
 338         Look for message present in dest but not in self.
 339         If any, delete them."""
 340         deletelist = []
 341         for uid in dest.getmessageuidlist():
 342             if uid < 0:
 343                 continue
 344             if not self.uidexists(uid):
 345                 deletelist.append(uid)
 346         if len(deletelist):
 347             UIBase.getglobalui().deletingmessages(deletelist, applyto)
 348             for object in applyto:
 349                 object.deletemessages(deletelist)
 350
 351     def syncmessagesto_flags(self, dest, applyto):
 352         """Pass 4 of folder synchronization.
 353
 354         Look for any flag matching issues -- set dest message to have the
 355         same flags that we have."""
 356
 357         # As an optimization over previous versions, we store up which flags
 358         # are being used for an add or a delete.  For each flag, we store
 359         # a list of uids to which it should be added.  Then, we can call
 360         # addmessagesflags() to apply them in bulk, rather than one
 361         # call per message as before.  This should result in some significant
 362         # performance improvements.
 363
 364         addflaglist = {}
 365         delflaglist = {}
 366
 367         for uid in self.getmessageuidlist():
 368             if uid < 0:                 # Ignore messages missed by pass 1
 369                 continue
 370             selfflags = self.getmessageflags(uid)
 371             destflags = dest.getmessageflags(uid)
 372
 373             addflags = [x for x in selfflags if x not in destflags]
 374
 375             for flag in addflags:
 376                 if not flag in addflaglist:
 377                     addflaglist[flag] = []
 378                 addflaglist[flag].append(uid)
 379
 380             delflags = [x for x in destflags if x not in selfflags]
 381             for flag in delflags:
 382                 if not flag in delflaglist:
 383                     delflaglist[flag] = []
 384                 delflaglist[flag].append(uid)
 385
 386         for object in applyto:
 387             for flag in addflaglist.keys():
 388                 UIBase.getglobalui().addingflags(addflaglist[flag], flag, [object])
 389                 object.addmessagesflags(addflaglist[flag], [flag])
 390             for flag in delflaglist.keys():
 391                 UIBase.getglobalui().deletingflags(delflaglist[flag], flag, [object])
 392                 object.deletemessagesflags(delflaglist[flag], [flag])
 393
 394     def syncmessagesto(self, dest, applyto = None):
 395         """Syncs messages in this folder to the destination.
 396         If applyto is specified, it should be a list of folders (don't forget
 397         to include dest!) to which all write actions should be applied.
 398         It defaults to [dest] if not specified.  It is important that
 399         the UID generator be listed first in applyto; that is, the other
 400         applyto ones should be the ones that "copy" the main action."""
 401         if applyto == None:
 402             applyto = [dest]
 403
 404         self.syncmessagesto_neguid(dest, applyto)
 405         self.syncmessagesto_copy(dest, applyto)
 406         self.syncmessagesto_delete(dest, applyto)
 407
 408         # Now, the message lists should be identical wrt the uids present.
 409         # (except for potential negative uids that couldn't be placed
 410         # anywhere)
 411
 412         self.syncmessagesto_flags(dest, applyto)
 413
 414