code.delx.au - offlineimap/blob - offlineimap/imapserver.py

   1 # IMAP server support
   2 # Copyright (C) 2002 - 2007 John Goerzen
   3 # <jgoerzen@complete.org>
   4 #
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, write to the Free Software
  17 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  18
  19 import imaplib
  20 from offlineimap import imaplibutil, imaputil, threadutil
  21 from offlineimap.ui import UIBase
  22 from threading import *
  23 import thread, hmac, os, time
  24 import base64
  25
  26 from StringIO import StringIO
  27 from platform import system
  28
  29 try:
  30     # do we have a recent pykerberos?
  31     have_gss = False
  32     import kerberos
  33     if 'authGSSClientWrap' in dir(kerberos):
  34         have_gss = True
  35 except ImportError:
  36     pass
  37
  38 class UsefulIMAPMixIn:
  39     def getstate(self):
  40         return self.state
  41     def getselectedfolder(self):
  42         if self.getstate() == 'SELECTED':
  43             return self.selectedfolder
  44         return None
  45
  46     def select(self, mailbox='INBOX', readonly=None, force = 0):
  47         if (not force) and self.getselectedfolder() == mailbox \
  48            and self.is_readonly == readonly:
  49             # No change; return.
  50             return
  51         result = self.__class__.__bases__[1].select(self, mailbox, readonly)
  52         if result[0] != 'OK':
  53             raise ValueError, "Error from select: %s" % str(result)
  54         if self.getstate() == 'SELECTED':
  55             self.selectedfolder = mailbox
  56         else:
  57             self.selectedfolder = None
  58
  59     def _mesg(self, s, secs=None):
  60         imaplibutil.new_mesg(self, s, secs)
  61
  62 class UsefulIMAP4(UsefulIMAPMixIn, imaplib.IMAP4):
  63     def open(self, host = '', port = imaplib.IMAP4_PORT):
  64         imaplibutil.new_open(self, host, port)
  65
  66     # This is a hack around Darwin's implementation of realloc() (which
  67     # Python uses inside the socket code). On Darwin, we split the
  68     # message into 100k chunks, which should be small enough - smaller
  69     # might start seriously hurting performance ...
  70
  71     def read(self, size):
  72         if (system() == 'Darwin') and (size>0) :
  73             read = 0
  74             io = StringIO()
  75             while read < size:
  76                 data = imaplib.IMAP4.read (self, min(size-read,8192))
  77                 read += len(data)
  78                 io.write(data)
  79             return io.getvalue()
  80         else:
  81             return imaplib.IMAP4.read (self, size)
  82
  83 class UsefulIMAP4_SSL(UsefulIMAPMixIn, imaplibutil.WrappedIMAP4_SSL):
  84     def open(self, host = '', port = imaplib.IMAP4_SSL_PORT):
  85         imaplibutil.new_open_ssl(self, host, port)
  86
  87     # This is the same hack as above, to be used in the case of an SSL
  88     # connexion.
  89
  90     def read(self, size):
  91         if (system() == 'Darwin') and (size>0) :
  92             read = 0
  93             io = StringIO()
  94             while read < size:
  95                 data = imaplibutil.WrappedIMAP4_SSL.read (self, min(size-read,8192))
  96                 read += len(data)
  97                 io.write(data)
  98             return io.getvalue()
  99         else:
 100             return imaplibutil.WrappedIMAP4_SSL.read (self,size)
 101
 102 class UsefulIMAP4_Tunnel(UsefulIMAPMixIn, imaplibutil.IMAP4_Tunnel): pass
 103
 104 class IMAPServer:
 105     GSS_STATE_STEP = 0
 106     GSS_STATE_WRAP = 1
 107     def __init__(self, config, reposname,
 108                  username = None, password = None, hostname = None,
 109                  port = None, ssl = 1, maxconnections = 1, tunnel = None,
 110                  reference = '""', sslclientcert = None, sslclientkey = None):
 111         self.reposname = reposname
 112         self.config = config
 113         self.username = username
 114         self.password = password
 115         self.passworderror = None
 116         self.goodpassword = None
 117         self.hostname = hostname
 118         self.tunnel = tunnel
 119         self.port = port
 120         self.usessl = ssl
 121         self.sslclientcert = sslclientcert
 122         self.sslclientkey = sslclientkey
 123         self.delim = None
 124         self.root = None
 125         if port == None:
 126             if ssl:
 127                 self.port = 993
 128             else:
 129                 self.port = 143
 130         self.maxconnections = maxconnections
 131         self.availableconnections = []
 132         self.assignedconnections = []
 133         self.lastowner = {}
 134         self.semaphore = BoundedSemaphore(self.maxconnections)
 135         self.connectionlock = Lock()
 136         self.reference = reference
 137         self.gss_step = self.GSS_STATE_STEP
 138         self.gss_vc = None
 139         self.gssapi = False
 140
 141     def getpassword(self):
 142         if self.goodpassword != None:
 143             return self.goodpassword
 144
 145         if self.password != None and self.passworderror == None:
 146             return self.password
 147
 148         self.password = UIBase.getglobalui().getpass(self.reposname,
 149                                                      self.config,
 150                                                      self.passworderror)
 151         self.passworderror = None
 152
 153         return self.password
 154
 155     def getdelim(self):
 156         """Returns this server's folder delimiter.  Can only be called
 157         after one or more calls to acquireconnection."""
 158         return self.delim
 159
 160     def getroot(self):
 161         """Returns this server's folder root.  Can only be called after one
 162         or more calls to acquireconnection."""
 163         return self.root
 164
 165
 166     def releaseconnection(self, connection):
 167         """Releases a connection, returning it to the pool."""
 168         self.connectionlock.acquire()
 169         self.assignedconnections.remove(connection)
 170         self.availableconnections.append(connection)
 171         self.connectionlock.release()
 172         self.semaphore.release()
 173
 174     def md5handler(self, response):
 175         ui = UIBase.getglobalui()
 176         challenge = response.strip()
 177         ui.debug('imap', 'md5handler: got challenge %s' % challenge)
 178
 179         passwd = self.getpassword()
 180         retval = self.username + ' ' + hmac.new(passwd, challenge).hexdigest()
 181         ui.debug('imap', 'md5handler: returning %s' % retval)
 182         return retval
 183
 184     def plainauth(self, imapobj):
 185         UIBase.getglobalui().debug('imap',
 186                                    'Attempting plain authentication')
 187         imapobj.login(self.username, self.getpassword())
 188
 189     def gssauth(self, response):
 190         data = base64.b64encode(response)
 191         try:
 192             if self.gss_step == self.GSS_STATE_STEP:
 193                 if not self.gss_vc:
 194                     rc, self.gss_vc = kerberos.authGSSClientInit('imap@' +
 195                                                                  self.hostname)
 196                     response = kerberos.authGSSClientResponse(self.gss_vc)
 197                 rc = kerberos.authGSSClientStep(self.gss_vc, data)
 198                 if rc != kerberos.AUTH_GSS_CONTINUE:
 199                    self.gss_step = self.GSS_STATE_WRAP
 200             elif self.gss_step == self.GSS_STATE_WRAP:
 201                 rc = kerberos.authGSSClientUnwrap(self.gss_vc, data)
 202                 response = kerberos.authGSSClientResponse(self.gss_vc)
 203                 rc = kerberos.authGSSClientWrap(self.gss_vc, response,
 204                                                 self.username)
 205             response = kerberos.authGSSClientResponse(self.gss_vc)
 206         except kerberos.GSSError, err:
 207             # Kerberos errored out on us, respond with None to cancel the
 208             # authentication
 209             UIBase.getglobalui().debug('imap',
 210                                        '%s: %s' % (err[0][0], err[1][0]))
 211             return None
 212
 213         if not response:
 214             response = ''
 215         return base64.b64decode(response)
 216
 217     def acquireconnection(self):
 218         """Fetches a connection from the pool, making sure to create a new one
 219         if needed, to obey the maximum connection limits, etc.
 220         Opens a connection to the server and returns an appropriate
 221         object."""
 222
 223         self.semaphore.acquire()
 224         self.connectionlock.acquire()
 225         imapobj = None
 226
 227         if len(self.availableconnections): # One is available.
 228             # Try to find one that previously belonged to this thread
 229             # as an optimization.  Start from the back since that's where
 230             # they're popped on.
 231             threadid = thread.get_ident()
 232             imapobj = None
 233             for i in range(len(self.availableconnections) - 1, -1, -1):
 234                 tryobj = self.availableconnections[i]
 235                 if self.lastowner[tryobj] == threadid:
 236                     imapobj = tryobj
 237                     del(self.availableconnections[i])
 238                     break
 239             if not imapobj:
 240                 imapobj = self.availableconnections[0]
 241                 del(self.availableconnections[0])
 242             self.assignedconnections.append(imapobj)
 243             self.lastowner[imapobj] = thread.get_ident()
 244             self.connectionlock.release()
 245             return imapobj
 246
 247         self.connectionlock.release()   # Release until need to modify data
 248
 249         success = 0
 250         while not success:
 251             # Generate a new connection.
 252             if self.tunnel:
 253                 UIBase.getglobalui().connecting('tunnel', self.tunnel)
 254                 imapobj = UsefulIMAP4_Tunnel(self.tunnel)
 255                 success = 1
 256             elif self.usessl:
 257                 UIBase.getglobalui().connecting(self.hostname, self.port)
 258                 imapobj = UsefulIMAP4_SSL(self.hostname, self.port,
 259                                           self.sslclientkey, self.sslclientcert)
 260             else:
 261                 UIBase.getglobalui().connecting(self.hostname, self.port)
 262                 imapobj = UsefulIMAP4(self.hostname, self.port)
 263
 264             imapobj.mustquote = imaplibutil.mustquote
 265
 266             if not self.tunnel:
 267                 try:
 268                     # Try GSSAPI and continue if it fails
 269                     if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss:
 270                         UIBase.getglobalui().debug('imap',
 271                             'Attempting GSSAPI authentication')
 272                         try:
 273                             imapobj.authenticate('GSSAPI', self.gssauth)
 274                         except imapobj.error, val:
 275                             UIBase.getglobalui().debug('imap',
 276                                 'GSSAPI Authentication failed')
 277                         else:
 278                             self.gssapi = True
 279                             self.password = None
 280
 281                     if not self.gssapi:
 282                         if 'AUTH=CRAM-MD5' in imapobj.capabilities:
 283                             UIBase.getglobalui().debug('imap',
 284                                                    'Attempting CRAM-MD5 authentication')
 285                             try:
 286                                 imapobj.authenticate('CRAM-MD5', self.md5handler)
 287                             except imapobj.error, val:
 288                                 self.plainauth(imapobj)
 289                         else:
 290                             self.plainauth(imapobj)
 291                     # Would bail by here if there was a failure.
 292                     success = 1
 293                     self.goodpassword = self.password
 294                 except imapobj.error, val:
 295                     self.passworderror = str(val)
 296                     self.password = None
 297
 298         if self.delim == None:
 299             listres = imapobj.list(self.reference, '""')[1]
 300             if listres == [None] or listres == None:
 301                 # Some buggy IMAP servers do not respond well to LIST "" ""
 302                 # Work around them.
 303                 listres = imapobj.list(self.reference, '"*"')[1]
 304             self.delim, self.root = \
 305                         imaputil.imapsplit(listres[0])[1:]
 306             self.delim = imaputil.dequote(self.delim)
 307             self.root = imaputil.dequote(self.root)
 308
 309         self.connectionlock.acquire()
 310         self.assignedconnections.append(imapobj)
 311         self.lastowner[imapobj] = thread.get_ident()
 312         self.connectionlock.release()
 313         return imapobj
 314
 315     def connectionwait(self):
 316         """Waits until there is a connection available.  Note that between
 317         the time that a connection becomes available and the time it is
 318         requested, another thread may have grabbed it.  This function is
 319         mainly present as a way to avoid spawning thousands of threads
 320         to copy messages, then have them all wait for 3 available connections.
 321         It's OK if we have maxconnections + 1 or 2 threads, which is what
 322         this will help us do."""
 323         threadutil.semaphorewait(self.semaphore)
 324
 325     def close(self):
 326         # Make sure I own all the semaphores.  Let the threads finish
 327         # their stuff.  This is a blocking method.
 328         self.connectionlock.acquire()
 329         threadutil.semaphorereset(self.semaphore, self.maxconnections)
 330         for imapobj in self.assignedconnections + self.availableconnections:
 331             imapobj.logout()
 332         self.assignedconnections = []
 333         self.availableconnections = []
 334         self.lastowner = {}
 335         self.connectionlock.release()
 336
 337     def keepalive(self, timeout, event):
 338         """Sends a NOOP to each connection recorded.   It will wait a maximum
 339         of timeout seconds between doing this, and will continue to do so
 340         until the Event object as passed is true.  This method is expected
 341         to be invoked in a separate thread, which should be join()'d after
 342         the event is set."""
 343         ui = UIBase.getglobalui()
 344         ui.debug('imap', 'keepalive thread started')
 345         while 1:
 346             ui.debug('imap', 'keepalive: top of loop')
 347             time.sleep(timeout)
 348             ui.debug('imap', 'keepalive: after wait')
 349             if event.isSet():
 350                 ui.debug('imap', 'keepalive: event is set; exiting')
 351                 return
 352             ui.debug('imap', 'keepalive: acquiring connectionlock')
 353             self.connectionlock.acquire()
 354             numconnections = len(self.assignedconnections) + \
 355                              len(self.availableconnections)
 356             self.connectionlock.release()
 357             ui.debug('imap', 'keepalive: connectionlock released')
 358             threads = []
 359             imapobjs = []
 360
 361             for i in range(numconnections):
 362                 ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections))
 363                 imapobj = self.acquireconnection()
 364                 ui.debug('imap', 'keepalive: connection %d acquired' % i)
 365                 imapobjs.append(imapobj)
 366                 thr = threadutil.ExitNotifyThread(target = imapobj.noop)
 367                 thr.setDaemon(1)
 368                 thr.start()
 369                 threads.append(thr)
 370                 ui.debug('imap', 'keepalive: thread started')
 371
 372             ui.debug('imap', 'keepalive: joining threads')
 373
 374             for thr in threads:
 375                 # Make sure all the commands have completed.
 376                 thr.join()
 377
 378             ui.debug('imap', 'keepalive: releasing connections')
 379
 380             for imapobj in imapobjs:
 381                 self.releaseconnection(imapobj)
 382
 383             ui.debug('imap', 'keepalive: bottom of loop')
 384
 385 class ConfigedIMAPServer(IMAPServer):
 386     """This class is designed for easier initialization given a ConfigParser
 387     object and an account name.  The passwordhash is used if
 388     passwords for certain accounts are known.  If the password for this
 389     account is listed, it will be obtained from there."""
 390     def __init__(self, repository, passwordhash = {}):
 391         """Initialize the object.  If the account is not a tunnel,
 392         the password is required."""
 393         self.repos = repository
 394         self.config = self.repos.getconfig()
 395         usetunnel = self.repos.getpreauthtunnel()
 396         if not usetunnel:
 397             host = self.repos.gethost()
 398             user = self.repos.getuser()
 399             port = self.repos.getport()
 400             ssl = self.repos.getssl()
 401             sslclientcert = self.repos.getsslclientcert()
 402             sslclientkey = self.repos.getsslclientkey()
 403         reference = self.repos.getreference()
 404         server = None
 405         password = None
 406
 407         if repository.getname() in passwordhash:
 408             password = passwordhash[repository.getname()]
 409
 410         # Connect to the remote server.
 411         if usetunnel:
 412             IMAPServer.__init__(self, self.config, self.repos.getname(),
 413                                 tunnel = usetunnel,
 414                                 reference = reference,
 415                                 maxconnections = self.repos.getmaxconnections())
 416         else:
 417             if not password:
 418                 password = self.repos.getpassword()
 419             IMAPServer.__init__(self, self.config, self.repos.getname(),
 420                                 user, password, host, port, ssl,
 421                                 self.repos.getmaxconnections(),
 422                                 reference = reference,
 423                                 sslclientcert = sslclientcert,
 424                                 sslclientkey = sslclientkey)