Reduce memory usage when scanning Maildirs

[offlineimap] / offlineimap / folder / Maildir.py

diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py
index c5d509282c18ece03fe560391245e0ababbebf83..811d759a48ab3ea28680317e7d5f13de623f6b9e 100644 (file)
--- a/offlineimap/folder/Maildir.py
+++ b/offlineimap/folder/Maildir.py
@@ -16,11 +16,16 @@
 #    along with this program; if not, write to the Free Software
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 
+import os.path, os, re, time, socket
 from Base import BaseFolder
 from offlineimap import imaputil
 from offlineimap.ui import UIBase
 from threading import Lock
-import os.path, os, re, time, socket, md5
+
+try:
+    from hashlib import md5
+except ImportError:
+    from md5 import md5
 
 uidmatchre = re.compile(',U=(\d+)')
 flagmatchre = re.compile(':.*2,([A-Z]+)')
@@ -45,8 +50,10 @@ def gettimeseq():
         timelock.release()
 
 class MaildirFolder(BaseFolder):
-    def __init__(self, root, name, sep, repository, accountname):
+    def __init__(self, root, name, sep, repository, accountname, config):
         self.name = name
+        self.config = config
+        self.dofsync = config.getdefaultboolean("general", "fsync", True)
         self.root = root
         self.sep = sep
         self.messagelist = None
@@ -77,12 +84,12 @@ class MaildirFolder(BaseFolder):
         files = []
         nouidcounter = -1               # Messages without UIDs get
                                         # negative UID numbers.
-        foldermd5 = md5.new(self.getvisiblename()).hexdigest()
+        foldermd5 = md5(self.getvisiblename()).hexdigest()
         folderstr = ',FMD5=' + foldermd5
         for dirannex in ['new', 'cur']:
             fulldirname = os.path.join(self.getfullname(), dirannex)
-            files.extend([os.path.join(fulldirname, filename) for
-                          filename in os.listdir(fulldirname)])
+            files.extend(os.path.join(fulldirname, filename) for
+                         filename in os.listdir(fulldirname))
         for file in files:
             messagename = os.path.basename(file)
             foldermatch = messagename.find(folderstr) != -1
@@ -158,8 +165,7 @@ class MaildirFolder(BaseFolder):
         # Otherwise, save the message in tmp/ and then call savemessageflags()
         # to give it a permanent home.
         tmpdir = os.path.join(self.getfullname(), 'tmp')
-        file = fd = None
-        messagename = tmpmessaename = None
+        messagename = None
         attempts = 0
         while 1:
             if attempts > 15:
@@ -171,25 +177,21 @@ class MaildirFolder(BaseFolder):
                            os.getpid(),
                            socket.gethostname(),
                            uid,
-                           md5.new(self.getvisiblename()).hexdigest())
-            tmpmessagename = messagename.split(',')[0]
-            try:
-                fd = os.open(os.path.join(tmpdir, tmpmessagename),
-                        os.O_WRONLY + os.O_CREAT + os.O_EXCL)
-                file = os.fdopen(fd, 'w')
-                ui.debug('maildir', 'savemessage: using temporary name %s' % tmpmessagename)
-            except OSError, e:
-                if e.errno == 17:
-                    time.sleep(2)
-                    attempts += 1
-                    continue
-                raise
-
+                           md5(self.getvisiblename()).hexdigest())
+            if os.path.exists(os.path.join(tmpdir, messagename)):
+                time.sleep(2)
+                attempts += 1
+            else:
+                break
+        tmpmessagename = messagename.split(',')[0]
+        ui.debug('maildir', 'savemessage: using temporary name %s' % tmpmessagename)
+        file = open(os.path.join(tmpdir, tmpmessagename), "wt")
         file.write(content)
 
         # Make sure the data hits the disk
         file.flush()
-        os.fsync(fd)
+        if self.dofsync:
+            os.fsync(file.fileno())
 
         file.close()
         if rtime != None:
@@ -197,17 +199,17 @@ class MaildirFolder(BaseFolder):
         ui.debug('maildir', 'savemessage: moving from %s to %s' % \
                  (tmpmessagename, messagename))
         if tmpmessagename != messagename: # then rename it
-            os.link(os.path.join(tmpdir, tmpmessagename),
+            os.rename(os.path.join(tmpdir, tmpmessagename),
                     os.path.join(tmpdir, messagename))
-            os.unlink(os.path.join(tmpdir, tmpmessagename))
 
-        try:
-            # fsync the directory (safer semantics in Linux)
-            fd = os.open(tmpdir, os.O_RDONLY)
-            os.fsync(fd)
-            os.close(fd)
-        except:
-            pass
+        if self.dofsync:
+            try:
+                # fsync the directory (safer semantics in Linux)
+                fd = os.open(tmpdir, os.O_RDONLY)
+                os.fsync(fd)
+                os.close(fd)
+            except:
+                pass
 
         self.messagelist[uid] = {'uid': uid, 'flags': [],
                                  'filename': os.path.join(tmpdir, messagename)}