code.delx.au - offlineimap/blob - offlineimap/imapserver.py

   1 # IMAP server support
   2 # Copyright (C) 2002 - 2007 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18
  19 from offlineimap import imaplib2, imaplibutil, imaputil, threadutil
  20 from offlineimap.ui import UIBase
  21 from offlineimap.accounts import syncfolder
  22 from threading import *
  23 import thread, hmac, os, time
  24 import base64
  25
  26 from StringIO import StringIO
  27 from platform import system
  28
  29 try:
  30     # do we have a recent pykerberos?
  31     have_gss = False
  32     import kerberos
  33     if 'authGSSClientWrap' in dir(kerberos):
  34         have_gss = True
  35 except ImportError:
  36     pass
  37
  38 class UsefulIMAPMixIn:
  39     def getstate(self):
  40         return self.state
  41     def getselectedfolder(self):
  42         if self.getstate() == 'SELECTED':
  43             return self.selectedfolder
  44         return None
  45
  46     def select(self, mailbox='INBOX', readonly=None, force = 0):
  47         if (not force) and self.getselectedfolder() == mailbox \
  48            and self.is_readonly == readonly:
  49             # No change; return.
  50             return
  51         result = self.__class__.__bases__[1].select(self, mailbox, readonly)
  52         if result[0] != 'OK':
  53             raise ValueError, "Error from select: %s" % str(result)
  54         if self.getstate() == 'SELECTED':
  55             self.selectedfolder = mailbox
  56         else:
  57             self.selectedfolder = None
  58
  59     def _mesg(self, s, tn=None, secs=None):
  60         imaplibutil.new_mesg(self, s, tn, secs)
  61
  62 class UsefulIMAP4(UsefulIMAPMixIn, imaplib2.IMAP4):
  63     # This is a hack around Darwin's implementation of realloc() (which
  64     # Python uses inside the socket code). On Darwin, we split the
  65     # message into 100k chunks, which should be small enough - smaller
  66     # might start seriously hurting performance ...
  67
  68     def read(self, size):
  69         if (system() == 'Darwin') and (size>0) :
  70             read = 0
  71             io = StringIO()
  72             while read < size:
  73                 sz = min(size-read, 8192)
  74                 data = imaplib2.IMAP4.read (self, sz)
  75                 read += len(data)
  76                 io.write(data)
  77                 if len(data) < sz:
  78                     break
  79             return io.getvalue()
  80         else:
  81             return imaplib2.IMAP4.read (self, size)
  82
  83 class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL):
  84     # This is the same hack as above, to be used in the case of an SSL
  85     # connexion.
  86
  87     def read(self, size):
  88         if (system() == 'Darwin') and (size>0) :
  89             read = 0
  90             io = StringIO()
  91             while read < size:
  92                 sz = min(size-read,8192)
  93                 data = imaplibutil.WrappedIMAP4_SSL.read (self, sz)
  94                 read += len(data)
  95                 io.write(data)
  96                 if len(data) < sz:
  97                     break
  98             return io.getvalue()
  99         else:
 100             return imaplibutil.WrappedIMAP4_SSL.read (self,size)
 101
 102 class UsefulIMAP4_Tunnel(UsefulIMAPMixIn, imaplibutil.IMAP4_Tunnel): pass
 103
 104 class IMAPServer:
 105     GSS_STATE_STEP = 0
 106     GSS_STATE_WRAP = 1
 107     def __init__(self, config, reposname,
 108                  username = None, password = None, hostname = None,
 109                  port = None, ssl = 1, maxconnections = 1, tunnel = None,
 110                  reference = '""', sslclientcert = None, sslclientkey = None,
 111                  idlefolders = []):
 112         self.reposname = reposname
 113         self.config = config
 114         self.username = username
 115         self.password = password
 116         self.passworderror = None
 117         self.goodpassword = None
 118         self.hostname = hostname
 119         self.tunnel = tunnel
 120         self.port = port
 121         self.usessl = ssl
 122         self.sslclientcert = sslclientcert
 123         self.sslclientkey = sslclientkey
 124         self.delim = None
 125         self.root = None
 126         if port == None:
 127             if ssl:
 128                 self.port = 993
 129             else:
 130                 self.port = 143
 131         self.maxconnections = maxconnections
 132         self.availableconnections = []
 133         self.assignedconnections = []
 134         self.lastowner = {}
 135         self.semaphore = BoundedSemaphore(self.maxconnections)
 136         self.connectionlock = Lock()
 137         self.reference = reference
 138         self.idlefolders = idlefolders
 139         self.gss_step = self.GSS_STATE_STEP
 140         self.gss_vc = None
 141         self.gssapi = False
 142
 143     def getpassword(self):
 144         if self.goodpassword != None:
 145             return self.goodpassword
 146
 147         if self.password != None and self.passworderror == None:
 148             return self.password
 149
 150         self.password = UIBase.getglobalui().getpass(self.reposname,
 151                                                      self.config,
 152                                                      self.passworderror)
 153         self.passworderror = None
 154
 155         return self.password
 156
 157     def getdelim(self):
 158         """Returns this server's folder delimiter.  Can only be called
 159         after one or more calls to acquireconnection."""
 160         return self.delim
 161
 162     def getroot(self):
 163         """Returns this server's folder root.  Can only be called after one
 164         or more calls to acquireconnection."""
 165         return self.root
 166
 167
 168     def releaseconnection(self, connection):
 169         """Releases a connection, returning it to the pool."""
 170         self.connectionlock.acquire()
 171         self.assignedconnections.remove(connection)
 172         self.availableconnections.append(connection)
 173         self.connectionlock.release()
 174         self.semaphore.release()
 175
 176     def md5handler(self, response):
 177         ui = UIBase.getglobalui()
 178         challenge = response.strip()
 179         ui.debug('imap', 'md5handler: got challenge %s' % challenge)
 180
 181         passwd = self.getpassword()
 182         retval = self.username + ' ' + hmac.new(passwd, challenge).hexdigest()
 183         ui.debug('imap', 'md5handler: returning %s' % retval)
 184         return retval
 185
 186     def plainauth(self, imapobj):
 187         UIBase.getglobalui().debug('imap',
 188                                    'Attempting plain authentication')
 189         imapobj.login(self.username, self.getpassword())
 190
 191     def gssauth(self, response):
 192         data = base64.b64encode(response)
 193         try:
 194             if self.gss_step == self.GSS_STATE_STEP:
 195                 if not self.gss_vc:
 196                     rc, self.gss_vc = kerberos.authGSSClientInit('imap@' +
 197                                                                  self.hostname)
 198                     response = kerberos.authGSSClientResponse(self.gss_vc)
 199                 rc = kerberos.authGSSClientStep(self.gss_vc, data)
 200                 if rc != kerberos.AUTH_GSS_CONTINUE:
 201                    self.gss_step = self.GSS_STATE_WRAP
 202             elif self.gss_step == self.GSS_STATE_WRAP:
 203                 rc = kerberos.authGSSClientUnwrap(self.gss_vc, data)
 204                 response = kerberos.authGSSClientResponse(self.gss_vc)
 205                 rc = kerberos.authGSSClientWrap(self.gss_vc, response,
 206                                                 self.username)
 207             response = kerberos.authGSSClientResponse(self.gss_vc)
 208         except kerberos.GSSError, err:
 209             # Kerberos errored out on us, respond with None to cancel the
 210             # authentication
 211             UIBase.getglobalui().debug('imap',
 212                                        '%s: %s' % (err[0][0], err[1][0]))
 213             return None
 214
 215         if not response:
 216             response = ''
 217         return base64.b64decode(response)
 218
 219     def acquireconnection(self):
 220         """Fetches a connection from the pool, making sure to create a new one
 221         if needed, to obey the maximum connection limits, etc.
 222         Opens a connection to the server and returns an appropriate
 223         object."""
 224
 225         self.semaphore.acquire()
 226         self.connectionlock.acquire()
 227         imapobj = None
 228
 229         if len(self.availableconnections): # One is available.
 230             # Try to find one that previously belonged to this thread
 231             # as an optimization.  Start from the back since that's where
 232             # they're popped on.
 233             threadid = thread.get_ident()
 234             imapobj = None
 235             for i in range(len(self.availableconnections) - 1, -1, -1):
 236                 tryobj = self.availableconnections[i]
 237                 if self.lastowner[tryobj] == threadid:
 238                     imapobj = tryobj
 239                     del(self.availableconnections[i])
 240                     break
 241             if not imapobj:
 242                 imapobj = self.availableconnections[0]
 243                 del(self.availableconnections[0])
 244             self.assignedconnections.append(imapobj)
 245             self.lastowner[imapobj] = thread.get_ident()
 246             self.connectionlock.release()
 247             return imapobj
 248
 249         self.connectionlock.release()   # Release until need to modify data
 250
 251         success = 0
 252         while not success:
 253             # Generate a new connection.
 254             if self.tunnel:
 255                 UIBase.getglobalui().connecting('tunnel', self.tunnel)
 256                 imapobj = UsefulIMAP4_Tunnel(self.tunnel)
 257                 success = 1
 258             elif self.usessl:
 259                 UIBase.getglobalui().connecting(self.hostname, self.port)
 260                 imapobj = UsefulIMAP4_SSL(self.hostname, self.port,
 261                                           self.sslclientkey, self.sslclientcert)
 262             else:
 263                 UIBase.getglobalui().connecting(self.hostname, self.port)
 264                 imapobj = UsefulIMAP4(self.hostname, self.port)
 265
 266             imapobj.mustquote = imaplibutil.mustquote
 267
 268             if not self.tunnel:
 269                 try:
 270                     # Try GSSAPI and continue if it fails
 271                     if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss:
 272                         UIBase.getglobalui().debug('imap',
 273                             'Attempting GSSAPI authentication')
 274                         try:
 275                             imapobj.authenticate('GSSAPI', self.gssauth)
 276                         except imapobj.error, val:
 277                             UIBase.getglobalui().debug('imap',
 278                                 'GSSAPI Authentication failed')
 279                         else:
 280                             self.gssapi = True
 281                             self.password = None
 282
 283                     if not self.gssapi:
 284                         if 'AUTH=CRAM-MD5' in imapobj.capabilities:
 285                             UIBase.getglobalui().debug('imap',
 286                                                    'Attempting CRAM-MD5 authentication')
 287                             try:
 288                                 imapobj.authenticate('CRAM-MD5', self.md5handler)
 289                             except imapobj.error, val:
 290                                 self.plainauth(imapobj)
 291                         else:
 292                             self.plainauth(imapobj)
 293                     # Would bail by here if there was a failure.
 294                     success = 1
 295                     self.goodpassword = self.password
 296                 except imapobj.error, val:
 297                     self.passworderror = str(val)
 298                     self.password = None
 299
 300         if self.delim == None:
 301             listres = imapobj.list(self.reference, '""')[1]
 302             if listres == [None] or listres == None:
 303                 # Some buggy IMAP servers do not respond well to LIST "" ""
 304                 # Work around them.
 305                 listres = imapobj.list(self.reference, '"*"')[1]
 306             self.delim, self.root = \
 307                         imaputil.imapsplit(listres[0])[1:]
 308             self.delim = imaputil.dequote(self.delim)
 309             self.root = imaputil.dequote(self.root)
 310
 311         self.connectionlock.acquire()
 312         self.assignedconnections.append(imapobj)
 313         self.lastowner[imapobj] = thread.get_ident()
 314         self.connectionlock.release()
 315         return imapobj
 316
 317     def connectionwait(self):
 318         """Waits until there is a connection available.  Note that between
 319         the time that a connection becomes available and the time it is
 320         requested, another thread may have grabbed it.  This function is
 321         mainly present as a way to avoid spawning thousands of threads
 322         to copy messages, then have them all wait for 3 available connections.
 323         It's OK if we have maxconnections + 1 or 2 threads, which is what
 324         this will help us do."""
 325         threadutil.semaphorewait(self.semaphore)
 326
 327     def close(self):
 328         # Make sure I own all the semaphores.  Let the threads finish
 329         # their stuff.  This is a blocking method.
 330         self.connectionlock.acquire()
 331         threadutil.semaphorereset(self.semaphore, self.maxconnections)
 332         for imapobj in self.assignedconnections + self.availableconnections:
 333             imapobj.logout()
 334         self.assignedconnections = []
 335         self.availableconnections = []
 336         self.lastowner = {}
 337         self.connectionlock.release()
 338
 339     def keepalive(self, timeout, event):
 340         """Sends a NOOP to each connection recorded.   It will wait a maximum
 341         of timeout seconds between doing this, and will continue to do so
 342         until the Event object as passed is true.  This method is expected
 343         to be invoked in a separate thread, which should be join()'d after
 344         the event is set."""
 345         ui = UIBase.getglobalui()
 346         ui.debug('imap', 'keepalive thread started')
 347         while 1:
 348             ui.debug('imap', 'keepalive: top of loop')
 349             if event.isSet():
 350                 ui.debug('imap', 'keepalive: event is set; exiting')
 351                 return
 352             ui.debug('imap', 'keepalive: acquiring connectionlock')
 353             self.connectionlock.acquire()
 354             numconnections = len(self.assignedconnections) + \
 355                              len(self.availableconnections)
 356             self.connectionlock.release()
 357             ui.debug('imap', 'keepalive: connectionlock released')
 358             threads = []
 359
 360             for i in range(numconnections):
 361                 ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections))
 362                 if len(self.idlefolders) > i:
 363                     idler = IdleThread(self, self.idlefolders[i])
 364                 else:
 365                     idler = IdleThread(self)
 366                 idler.start()
 367                 threads.append(idler)
 368                 ui.debug('imap', 'keepalive: thread started')
 369
 370             ui.debug('imap', 'keepalive: waiting for timeout')
 371             event.wait(timeout)
 372
 373             ui.debug('imap', 'keepalive: joining threads')
 374
 375             for idler in threads:
 376                 # Make sure all the commands have completed.
 377                 idler.stop()
 378                 idler.join()
 379
 380             ui.debug('imap', 'keepalive: bottom of loop')
 381
 382 class IdleThread(object):
 383     def __init__(self, parent, folder=None):
 384         self.parent = parent
 385         self.folder = folder
 386         self.event = Event()
 387         if folder is None:
 388             self.thread = Thread(target=self.noop)
 389         else:
 390             self.thread = Thread(target=self.idle)
 391         self.thread.setDaemon(1)
 392
 393     def start(self):
 394         self.thread.start()
 395
 396     def stop(self):
 397         self.event.set()
 398
 399     def join(self):
 400         self.thread.join()
 401
 402     def noop(self):
 403         imapobj = self.parent.acquireconnection()
 404         imapobj.noop()
 405         self.event.wait()
 406         self.parent.releaseconnection(imapobj)
 407
 408     def dosync(self):
 409         remoterepos = self.parent.repos
 410         account = remoterepos.account
 411         localrepos = account.localrepos
 412         remoterepos = account.remoterepos
 413         statusrepos = account.statusrepos
 414         remotefolder = remoterepos.getfolder(self.folder)
 415         syncfolder(account.name, remoterepos, remotefolder, localrepos, statusrepos, quick=False)
 416         ui = UIBase.getglobalui()
 417         ui.unregisterthread(currentThread())
 418
 419     def idle(self):
 420         imapobj = self.parent.acquireconnection()
 421         imapobj.select(self.folder)
 422         self.parent.releaseconnection(imapobj)
 423         while True:
 424             if self.event.isSet():
 425                 return
 426             self.needsync = False
 427             def callback(args):
 428                 if not self.event.isSet():
 429                     self.needsync = True
 430                     self.event.set()
 431             imapobj = self.parent.acquireconnection()
 432             if "IDLE" in imapobj.capabilities:
 433                 imapobj.idle(callback=callback)
 434             else:
 435                 imapobj.noop()
 436             self.event.wait()
 437             if self.event.isSet():
 438                 imapobj.noop()
 439             self.parent.releaseconnection(imapobj)
 440             if self.needsync:
 441                 self.event.clear()
 442                 self.dosync()
 443
 444 class ConfigedIMAPServer(IMAPServer):
 445     """This class is designed for easier initialization given a ConfigParser
 446     object and an account name.  The passwordhash is used if
 447     passwords for certain accounts are known.  If the password for this
 448     account is listed, it will be obtained from there."""
 449     def __init__(self, repository, passwordhash = {}):
 450         """Initialize the object.  If the account is not a tunnel,
 451         the password is required."""
 452         self.repos = repository
 453         self.config = self.repos.getconfig()
 454         usetunnel = self.repos.getpreauthtunnel()
 455         if not usetunnel:
 456             host = self.repos.gethost()
 457             user = self.repos.getuser()
 458             port = self.repos.getport()
 459             ssl = self.repos.getssl()
 460             sslclientcert = self.repos.getsslclientcert()
 461             sslclientkey = self.repos.getsslclientkey()
 462         reference = self.repos.getreference()
 463         idlefolders = self.repos.getidlefolders()
 464         server = None
 465         password = None
 466
 467         if repository.getname() in passwordhash:
 468             password = passwordhash[repository.getname()]
 469
 470         # Connect to the remote server.
 471         if usetunnel:
 472             IMAPServer.__init__(self, self.config, self.repos.getname(),
 473                                 tunnel = usetunnel,
 474                                 reference = reference,
 475                                 idlefolders = idlefolders,
 476                                 maxconnections = self.repos.getmaxconnections())
 477         else:
 478             if not password:
 479                 password = self.repos.getpassword()
 480             IMAPServer.__init__(self, self.config, self.repos.getname(),
 481                                 user, password, host, port, ssl,
 482                                 self.repos.getmaxconnections(),
 483                                 reference = reference,
 484                                 idlefolders = idlefolders,
 485                                 sslclientcert = sslclientcert,
 486                                 sslclientkey = sslclientkey)