code.delx.au - offlineimap/blob - offlineimap/imapserver.py

   1 # IMAP server support
   2 # Copyright (C) 2002 - 2007 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18
  19 from offlineimap import imaplib2, imaplibutil, imaputil, threadutil
  20 from offlineimap.ui import UIBase
  21 from offlineimap.accounts import syncfolder
  22 from threading import *
  23 import thread, hmac, os, time
  24 import base64
  25
  26 from StringIO import StringIO
  27 from platform import system
  28
  29 try:
  30     # do we have a recent pykerberos?
  31     have_gss = False
  32     import kerberos
  33     if 'authGSSClientWrap' in dir(kerberos):
  34         have_gss = True
  35 except ImportError:
  36     pass
  37
  38 class UsefulIMAPMixIn:
  39     def getstate(self):
  40         return self.state
  41     def getselectedfolder(self):
  42         if self.getstate() == 'SELECTED':
  43             return self.selectedfolder
  44         return None
  45
  46     def select(self, mailbox='INBOX', readonly=None, force = 0):
  47         if (not force) and self.getselectedfolder() == mailbox \
  48            and self.is_readonly == readonly:
  49             # No change; return.
  50             return
  51         result = self.__class__.__bases__[1].select(self, mailbox, readonly)
  52         if result[0] != 'OK':
  53             raise ValueError, "Error from select: %s" % str(result)
  54         if self.getstate() == 'SELECTED':
  55             self.selectedfolder = mailbox
  56         else:
  57             self.selectedfolder = None
  58
  59     def _mesg(self, s, tn=None, secs=None):
  60         imaplibutil.new_mesg(self, s, tn, secs)
  61
  62 class UsefulIMAP4(UsefulIMAPMixIn, imaplib2.IMAP4):
  63     # This is a hack around Darwin's implementation of realloc() (which
  64     # Python uses inside the socket code). On Darwin, we split the
  65     # message into 100k chunks, which should be small enough - smaller
  66     # might start seriously hurting performance ...
  67
  68     def read(self, size):
  69         if (system() == 'Darwin') and (size>0) :
  70             read = 0
  71             io = StringIO()
  72             while read < size:
  73                 sz = min(size-read, 8192)
  74                 data = imaplib2.IMAP4.read (self, sz)
  75                 read += len(data)
  76                 io.write(data)
  77                 if len(data) < sz:
  78                     break
  79             return io.getvalue()
  80         else:
  81             return imaplib2.IMAP4.read (self, size)
  82
  83 class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL):
  84     # This is the same hack as above, to be used in the case of an SSL
  85     # connexion.
  86
  87     def read(self, size):
  88         if (system() == 'Darwin') and (size>0) :
  89             read = 0
  90             io = StringIO()
  91             while read < size:
  92                 sz = min(size-read,8192)
  93                 data = imaplibutil.WrappedIMAP4_SSL.read (self, sz)
  94                 read += len(data)
  95                 io.write(data)
  96                 if len(data) < sz:
  97                     break
  98             return io.getvalue()
  99         else:
 100             return imaplibutil.WrappedIMAP4_SSL.read (self,size)
 101
 102 class UsefulIMAP4_Tunnel(UsefulIMAPMixIn, imaplibutil.IMAP4_Tunnel): pass
 103
 104 class IMAPServer:
 105     GSS_STATE_STEP = 0
 106     GSS_STATE_WRAP = 1
 107     def __init__(self, config, reposname,
 108                  username = None, password = None, hostname = None,
 109                  port = None, ssl = 1, maxconnections = 1, tunnel = None,
 110                  reference = '""', sslclientcert = None, sslclientkey = None,
 111                  idlefolders = []):
 112         self.reposname = reposname
 113         self.config = config
 114         self.username = username
 115         self.password = password
 116         self.passworderror = None
 117         self.goodpassword = None
 118         self.hostname = hostname
 119         self.tunnel = tunnel
 120         self.port = port
 121         self.usessl = ssl
 122         self.sslclientcert = sslclientcert
 123         self.sslclientkey = sslclientkey
 124         self.delim = None
 125         self.root = None
 126         if port == None:
 127             if ssl:
 128                 self.port = 993
 129             else:
 130                 self.port = 143
 131         self.maxconnections = maxconnections
 132         self.availableconnections = []
 133         self.assignedconnections = []
 134         self.lastowner = {}
 135         self.semaphore = BoundedSemaphore(self.maxconnections)
 136         self.connectionlock = Lock()
 137         self.reference = reference
 138         self.idlefolders = idlefolders
 139         self.gss_step = self.GSS_STATE_STEP
 140         self.gss_vc = None
 141         self.gssapi = False
 142
 143     def getpassword(self):
 144         if self.goodpassword != None:
 145             return self.goodpassword
 146
 147         if self.password != None and self.passworderror == None:
 148             return self.password
 149
 150         self.password = UIBase.getglobalui().getpass(self.reposname,
 151                                                      self.config,
 152                                                      self.passworderror)
 153         self.passworderror = None
 154
 155         return self.password
 156
 157     def getdelim(self):
 158         """Returns this server's folder delimiter.  Can only be called
 159         after one or more calls to acquireconnection."""
 160         return self.delim
 161
 162     def getroot(self):
 163         """Returns this server's folder root.  Can only be called after one
 164         or more calls to acquireconnection."""
 165         return self.root
 166
 167
 168     def releaseconnection(self, connection):
 169         """Releases a connection, returning it to the pool."""
 170         self.connectionlock.acquire()
 171         self.assignedconnections.remove(connection)
 172         self.availableconnections.append(connection)
 173         self.connectionlock.release()
 174         self.semaphore.release()
 175
 176     def md5handler(self, response):
 177         ui = UIBase.getglobalui()
 178         challenge = response.strip()
 179         ui.debug('imap', 'md5handler: got challenge %s' % challenge)
 180
 181         passwd = self.getpassword()
 182         retval = self.username + ' ' + hmac.new(passwd, challenge).hexdigest()
 183         ui.debug('imap', 'md5handler: returning %s' % retval)
 184         return retval
 185
 186     def plainauth(self, imapobj):
 187         UIBase.getglobalui().debug('imap',
 188                                    'Attempting plain authentication')
 189         imapobj.login(self.username, self.getpassword())
 190
 191     def gssauth(self, response):
 192         data = base64.b64encode(response)
 193         try:
 194             if self.gss_step == self.GSS_STATE_STEP:
 195                 if not self.gss_vc:
 196                     rc, self.gss_vc = kerberos.authGSSClientInit('imap@' +
 197                                                                  self.hostname)
 198                     response = kerberos.authGSSClientResponse(self.gss_vc)
 199                 rc = kerberos.authGSSClientStep(self.gss_vc, data)
 200                 if rc != kerberos.AUTH_GSS_CONTINUE:
 201                    self.gss_step = self.GSS_STATE_WRAP
 202             elif self.gss_step == self.GSS_STATE_WRAP:
 203                 rc = kerberos.authGSSClientUnwrap(self.gss_vc, data)
 204                 response = kerberos.authGSSClientResponse(self.gss_vc)
 205                 rc = kerberos.authGSSClientWrap(self.gss_vc, response,
 206                                                 self.username)
 207             response = kerberos.authGSSClientResponse(self.gss_vc)
 208         except kerberos.GSSError, err:
 209             # Kerberos errored out on us, respond with None to cancel the
 210             # authentication
 211             UIBase.getglobalui().debug('imap',
 212                                        '%s: %s' % (err[0][0], err[1][0]))
 213             return None
 214
 215         if not response:
 216             response = ''
 217         return base64.b64decode(response)
 218
 219     def acquireconnection(self):
 220         """Fetches a connection from the pool, making sure to create a new one
 221         if needed, to obey the maximum connection limits, etc.
 222         Opens a connection to the server and returns an appropriate
 223         object."""
 224
 225         self.semaphore.acquire()
 226         self.connectionlock.acquire()
 227         imapobj = None
 228
 229         if len(self.availableconnections): # One is available.
 230             # Try to find one that previously belonged to this thread
 231             # as an optimization.  Start from the back since that's where
 232             # they're popped on.
 233             threadid = thread.get_ident()
 234             imapobj = None
 235             for i in range(len(self.availableconnections) - 1, -1, -1):
 236                 tryobj = self.availableconnections[i]
 237                 if self.lastowner[tryobj] == threadid:
 238                     imapobj = tryobj
 239                     del(self.availableconnections[i])
 240                     break
 241             if not imapobj:
 242                 imapobj = self.availableconnections[0]
 243                 del(self.availableconnections[0])
 244             self.assignedconnections.append(imapobj)
 245             self.lastowner[imapobj] = thread.get_ident()
 246             self.connectionlock.release()
 247             return imapobj
 248
 249         self.connectionlock.release()   # Release until need to modify data
 250
 251         success = 0
 252         while not success:
 253             # Generate a new connection.
 254             if self.tunnel:
 255                 UIBase.getglobalui().connecting('tunnel', self.tunnel)
 256                 imapobj = UsefulIMAP4_Tunnel(self.tunnel)
 257                 success = 1
 258             elif self.usessl:
 259                 UIBase.getglobalui().connecting(self.hostname, self.port)
 260                 imapobj = UsefulIMAP4_SSL(self.hostname, self.port,
 261                                           self.sslclientkey, self.sslclientcert)
 262             else:
 263                 UIBase.getglobalui().connecting(self.hostname, self.port)
 264                 imapobj = UsefulIMAP4(self.hostname, self.port)
 265
 266             imapobj.mustquote = imaplibutil.mustquote
 267
 268             if not self.tunnel:
 269                 try:
 270                     # Try GSSAPI and continue if it fails
 271                     if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss:
 272                         UIBase.getglobalui().debug('imap',
 273                             'Attempting GSSAPI authentication')
 274                         try:
 275                             imapobj.authenticate('GSSAPI', self.gssauth)
 276                         except imapobj.error, val:
 277                             UIBase.getglobalui().debug('imap',
 278                                 'GSSAPI Authentication failed')
 279                         else:
 280                             self.gssapi = True
 281                             self.password = None
 282
 283                     if not self.gssapi:
 284                         if 'AUTH=CRAM-MD5' in imapobj.capabilities:
 285                             UIBase.getglobalui().debug('imap',
 286                                                    'Attempting CRAM-MD5 authentication')
 287                             try:
 288                                 imapobj.authenticate('CRAM-MD5', self.md5handler)
 289                             except imapobj.error, val:
 290                                 self.plainauth(imapobj)
 291                         else:
 292                             self.plainauth(imapobj)
 293                     # Would bail by here if there was a failure.
 294                     success = 1
 295                     self.goodpassword = self.password
 296                 except imapobj.error, val:
 297                     self.passworderror = str(val)
 298                     self.password = None
 299
 300         if self.delim == None:
 301             listres = imapobj.list(self.reference, '""')[1]
 302             if listres == [None] or listres == None:
 303                 # Some buggy IMAP servers do not respond well to LIST "" ""
 304                 # Work around them.
 305                 listres = imapobj.list(self.reference, '"*"')[1]
 306             self.delim, self.root = \
 307                         imaputil.imapsplit(listres[0])[1:]
 308             self.delim = imaputil.dequote(self.delim)
 309             self.root = imaputil.dequote(self.root)
 310
 311         self.connectionlock.acquire()
 312         self.assignedconnections.append(imapobj)
 313         self.lastowner[imapobj] = thread.get_ident()
 314         self.connectionlock.release()
 315         return imapobj
 316
 317     def connectionwait(self):
 318         """Waits until there is a connection available.  Note that between
 319         the time that a connection becomes available and the time it is
 320         requested, another thread may have grabbed it.  This function is
 321         mainly present as a way to avoid spawning thousands of threads
 322         to copy messages, then have them all wait for 3 available connections.
 323         It's OK if we have maxconnections + 1 or 2 threads, which is what
 324         this will help us do."""
 325         threadutil.semaphorewait(self.semaphore)
 326
 327     def close(self):
 328         # Make sure I own all the semaphores.  Let the threads finish
 329         # their stuff.  This is a blocking method.
 330         self.connectionlock.acquire()
 331         threadutil.semaphorereset(self.semaphore, self.maxconnections)
 332         for imapobj in self.assignedconnections + self.availableconnections:
 333             imapobj.logout()
 334         self.assignedconnections = []
 335         self.availableconnections = []
 336         self.lastowner = {}
 337         # reset kerberos state
 338         self.gss_step = self.GSS_STATE_STEP
 339         self.gss_vc = None
 340         self.gssapi = False
 341         self.connectionlock.release()
 342
 343     def keepalive(self, timeout, event):
 344         """Sends a NOOP to each connection recorded.   It will wait a maximum
 345         of timeout seconds between doing this, and will continue to do so
 346         until the Event object as passed is true.  This method is expected
 347         to be invoked in a separate thread, which should be join()'d after
 348         the event is set."""
 349         ui = UIBase.getglobalui()
 350         ui.debug('imap', 'keepalive thread started')
 351         while 1:
 352             ui.debug('imap', 'keepalive: top of loop')
 353             if event.isSet():
 354                 ui.debug('imap', 'keepalive: event is set; exiting')
 355                 return
 356             ui.debug('imap', 'keepalive: acquiring connectionlock')
 357             self.connectionlock.acquire()
 358             numconnections = len(self.assignedconnections) + \
 359                              len(self.availableconnections)
 360             self.connectionlock.release()
 361             ui.debug('imap', 'keepalive: connectionlock released')
 362             threads = []
 363
 364             for i in range(numconnections):
 365                 ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections))
 366                 if len(self.idlefolders) > i:
 367                     idler = IdleThread(self, self.idlefolders[i])
 368                 else:
 369                     idler = IdleThread(self)
 370                 idler.start()
 371                 threads.append(idler)
 372                 ui.debug('imap', 'keepalive: thread started')
 373
 374             ui.debug('imap', 'keepalive: waiting for timeout')
 375             event.wait(timeout)
 376
 377             ui.debug('imap', 'keepalive: joining threads')
 378
 379             for idler in threads:
 380                 # Make sure all the commands have completed.
 381                 idler.stop()
 382                 idler.join()
 383
 384             ui.debug('imap', 'keepalive: bottom of loop')
 385
 386 class IdleThread(object):
 387     def __init__(self, parent, folder=None):
 388         self.parent = parent
 389         self.folder = folder
 390         self.event = Event()
 391         if folder is None:
 392             self.thread = Thread(target=self.noop)
 393         else:
 394             self.thread = Thread(target=self.idle)
 395         self.thread.setDaemon(1)
 396
 397     def start(self):
 398         self.thread.start()
 399
 400     def stop(self):
 401         self.event.set()
 402
 403     def join(self):
 404         self.thread.join()
 405
 406     def noop(self):
 407         imapobj = self.parent.acquireconnection()
 408         imapobj.noop()
 409         self.event.wait()
 410         self.parent.releaseconnection(imapobj)
 411
 412     def dosync(self):
 413         remoterepos = self.parent.repos
 414         account = remoterepos.account
 415         localrepos = account.localrepos
 416         remoterepos = account.remoterepos
 417         statusrepos = account.statusrepos
 418         remotefolder = remoterepos.getfolder(self.folder)
 419         syncfolder(account.name, remoterepos, remotefolder, localrepos, statusrepos, quick=False)
 420         ui = UIBase.getglobalui()
 421         ui.unregisterthread(currentThread())
 422
 423     def idle(self):
 424         imapobj = self.parent.acquireconnection()
 425         imapobj.select(self.folder)
 426         self.parent.releaseconnection(imapobj)
 427         while True:
 428             if self.event.isSet():
 429                 return
 430             self.needsync = False
 431             def callback(args):
 432                 if not self.event.isSet():
 433                     self.needsync = True
 434                     self.event.set()
 435             imapobj = self.parent.acquireconnection()
 436             if "IDLE" in imapobj.capabilities:
 437                 imapobj.idle(callback=callback)
 438             else:
 439                 imapobj.noop()
 440             self.event.wait()
 441             if self.event.isSet():
 442                 imapobj.noop()
 443             self.parent.releaseconnection(imapobj)
 444             if self.needsync:
 445                 self.event.clear()
 446                 self.dosync()
 447
 448 class ConfigedIMAPServer(IMAPServer):
 449     """This class is designed for easier initialization given a ConfigParser
 450     object and an account name.  The passwordhash is used if
 451     passwords for certain accounts are known.  If the password for this
 452     account is listed, it will be obtained from there."""
 453     def __init__(self, repository, passwordhash = {}):
 454         """Initialize the object.  If the account is not a tunnel,
 455         the password is required."""
 456         self.repos = repository
 457         self.config = self.repos.getconfig()
 458         usetunnel = self.repos.getpreauthtunnel()
 459         if not usetunnel:
 460             host = self.repos.gethost()
 461             user = self.repos.getuser()
 462             port = self.repos.getport()
 463             ssl = self.repos.getssl()
 464             sslclientcert = self.repos.getsslclientcert()
 465             sslclientkey = self.repos.getsslclientkey()
 466         reference = self.repos.getreference()
 467         idlefolders = self.repos.getidlefolders()
 468         server = None
 469         password = None
 470
 471         if repository.getname() in passwordhash:
 472             password = passwordhash[repository.getname()]
 473
 474         # Connect to the remote server.
 475         if usetunnel:
 476             IMAPServer.__init__(self, self.config, self.repos.getname(),
 477                                 tunnel = usetunnel,
 478                                 reference = reference,
 479                                 idlefolders = idlefolders,
 480                                 maxconnections = self.repos.getmaxconnections())
 481         else:
 482             if not password:
 483                 password = self.repos.getpassword()
 484             IMAPServer.__init__(self, self.config, self.repos.getname(),
 485                                 user, password, host, port, ssl,
 486                                 self.repos.getmaxconnections(),
 487                                 reference = reference,
 488                                 idlefolders = idlefolders,
 489                                 sslclientcert = sslclientcert,
 490                                 sslclientkey = sslclientkey)