Reduce memory usage when scanning Maildirs

[offlineimap] / offlineimap / folder / Maildir.py

diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py
index 56a63e4306a1217469ec8bc232432df04d50597f..811d759a48ab3ea28680317e7d5f13de623f6b9e 100644 (file)
--- a/offlineimap/folder/Maildir.py
+++ b/offlineimap/folder/Maildir.py
@@ -16,11 +16,16 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
+import os.path, os, re, time, socket
 from Base import BaseFolder
 from offlineimap import imaputil
 from offlineimap.ui import UIBase
 from threading import Lock
-import os.path, os, re, time, socket, md5
+
+try:
+    from hashlib import md5
+except ImportError:
+    from md5 import md5
 
 uidmatchre = re.compile(',U=(\d+)')
 flagmatchre = re.compile(':.*2,([A-Z]+)')
@@ -45,8 +50,10 @@ def gettimeseq():
         timelock.release()
 
 class MaildirFolder(BaseFolder):
-    def __init__(self, root, name, sep, repository, accountname):
+    def __init__(self, root, name, sep, repository, accountname, config):
         self.name = name
+        self.config = config
+        self.dofsync = config.getdefaultboolean("general", "fsync", True)
         self.root = root
         self.sep = sep
         self.messagelist = None
@@ -77,12 +84,12 @@ class MaildirFolder(BaseFolder):
         files = []
         nouidcounter = -1               # Messages without UIDs get
                                         # negative UID numbers.
-        foldermd5 = md5.new(self.getvisiblename()).hexdigest()
+        foldermd5 = md5(self.getvisiblename()).hexdigest()
         folderstr = ',FMD5=' + foldermd5
         for dirannex in ['new', 'cur']:
             fulldirname = os.path.join(self.getfullname(), dirannex)
-            files.extend([os.path.join(fulldirname, filename) for
-                          filename in os.listdir(fulldirname)])
+            files.extend(os.path.join(fulldirname, filename) for
+                         filename in os.listdir(fulldirname))
         for file in files:
             messagename = os.path.basename(file)
             foldermatch = messagename.find(folderstr) != -1
@@ -170,7 +177,7 @@ class MaildirFolder(BaseFolder):
                            os.getpid(),
                            socket.gethostname(),
                            uid,
-                           md5.new(self.getvisiblename()).hexdigest())
+                           md5(self.getvisiblename()).hexdigest())
             if os.path.exists(os.path.join(tmpdir, messagename)):
                 time.sleep(2)
                 attempts += 1
@@ -183,7 +190,8 @@ class MaildirFolder(BaseFolder):
 
         # Make sure the data hits the disk
         file.flush()
-        os.fsync(file.fileno())
+        if self.dofsync:
+            os.fsync(file.fileno())
 
         file.close()
         if rtime != None:
@@ -191,17 +199,17 @@ class MaildirFolder(BaseFolder):
         ui.debug('maildir', 'savemessage: moving from %s to %s' % \
                  (tmpmessagename, messagename))
         if tmpmessagename != messagename: # then rename it
-            os.link(os.path.join(tmpdir, tmpmessagename),
+            os.rename(os.path.join(tmpdir, tmpmessagename),
                     os.path.join(tmpdir, messagename))
-            os.unlink(os.path.join(tmpdir, tmpmessagename))
 
-        try:
-            # fsync the directory (safer semantics in Linux)
-            fd = os.open(tmpdir, os.O_RDONLY)
-            os.fsync(fd)
-            os.close(fd)
-        except:
-            pass
+        if self.dofsync:
+            try:
+                # fsync the directory (safer semantics in Linux)
+                fd = os.open(tmpdir, os.O_RDONLY)
+                os.fsync(fd)
+                os.close(fd)
+            except:
+                pass
 
         self.messagelist[uid] = {'uid': uid, 'flags': [],
                                  'filename': os.path.join(tmpdir, messagename)}