+++ /dev/null
-#!/usr/bin/env python2.5
-
-MINFILE_SIZE = 1024
-FILEBUFFER_SIZE = 1024**2
-
-import os, sys, bisect
-
-from copy import copy
-from base64 import standard_b64encode as b64encode
-from collections import defaultdict
-import cPickle
-try:
- import hashlib
- def _getSha1(filename):
- return hashlib.sha1()
-except ImportError:
- import sha
- def _getSha1(filename):
- return sha.new()
-def getSha1(filename):
- sha1 = _getSha1(filename)
- f = file(filename, 'r')
- data = f.read(FILEBUFFER_SIZE)
- while data:
- sha1.update(data)
- data = f.read(FILEBUFFER_SIZE)
- return b64encode(sha1.digest())
-
-try:
- import psyco
- psyco.full()
-except ImportError:
- print >>sys.stderr, "WARNING: Could not load psyco"
-
-class DiskObject(object):
- def __repr__(self):
- return repr(self.getFullPath())
- def __str__(self):
- return self.getFullPath()
- def __lt__(self, other):
- if not hasattr(other, 'getFullPath'):
- raise NotImplemented()
- return self.getFullPath() < other.getFullPath()
- def __eq__(self, other):
- if not hasattr(other, 'getFullPath'):
- raise NotImplemented()
- return self.getFullPath() == other.getFullPath()
- def __hash__(self):
- return hash(self.getFullPath())
-
-class Folder(DiskObject):
- def __init__(self, name, parent = None):
- if name.find(os.path.sep) >= 0 and name != os.path.sep:
- print name
- parent_name, name = os.path.split(name)
- parent = Folder(parent_name, parent)
-
- self.name = name
- self.parent = parent
- if parent:
- parent.addChild(self)
- self.children = {}
- def getFullPath(self):
- folderStack = []
- f = self
- while f:
- folderStack.append(f.name)
- f = f.parent
- return os.path.sep.join(reversed(folderStack))
- def addChild(self, child):
- self.children[child.name] = child
-
-def findDirectory(rootDir, dirName, createNonExistant = False):
- dir = dirName.split(os.path.sep)[1:]
- if dir == ['']:
- dir = []
-
- ret = rootDir
- for folderName in dir:
- try:
- ret = ret.children[folderName]
- except KeyError, e:
- if not createNonExistant:
- raise e
- ret = Folder(folderName, ret)
-
- return ret
-
-class FileObject(DiskObject):
- def __init__(self, name, folder):
- self.name = name
- self.folder = folder
- statinfo = os.stat(self.getFullPath())
- self.mtime_size = (statinfo.st_mtime, statinfo.st_size)
- def getDiskID(self):
- statinfo = os.stat(self.getFullPath())
- return (statinfo.st_dev, statinfo.st_ino) # Identify the file
- def get_mtime_size(self):
- return self.mtime_size
- def getFullPath(self):
- return '%(folder)s/%(file)s' % { 'folder': self.folder.getFullPath(), 'file': self.name }
-
-class GlobalFileInfo(object):
- def __init__(self):
- self.files = defaultdict(list)
- self.filelist = {}
- self.root = Folder('')
-
- def _scanDirUpdateFile(self, dirObject, dirPath, filename):
- def printPath(word):
- print '%s "%s"' % (word, filename[-80:])
- fullpath = os.path.join(dirPath, filename)
- if os.path.islink(fullpath) or not os.path.isfile(fullpath):
- printPath('Skipping')
- return
- try:
- file = FileObject(filename, dirObject)
- new_mtime_size = file.get_mtime_size()
-
- if file in self.filelist:
- if file.get_mtime_size() == self.filelist[file].get_mtime_size():
- printPath('Skipping')
- return
- old_sha1 = self.filelist[file].sha1
- del self.filelist[file]
- self.files[old_sha1].remove(file)
-
- if file.get_mtime_size()[1] < MINFILE_SIZE:
- printPath('Skipping')
- return
- printPath('Scanning')
-
- file.sha1 = getSha1(fullpath)
- self.files[file.sha1].append(file)
- self.filelist[file] = file
- except IOError:
- print >>sys.stderr, 'WARNING: Could not get sha1 of "%s"\n' % (fullpath)
-
- def scanDir(self, dirName):
- root = findDirectory(self.root, dirName, createNonExistant = True)
-
- for dirPath, dirs, files in os.walk(dirName):
- print 'Scanning directory "%s"\n' % dirPath
- folder = findDirectory(self.root, dirPath, createNonExistant = True)
- # Add the children Directories
- if '.svn' in dirs:
- dirs.remove('.svn')
- for d in dirs:
- Folder(d, folder) # As a side effect, this is added to the parent correctly
-
- for f in files:
- sys.stdout.write("\033[A\033[300D\033[2K")
- self._scanDirUpdateFile(folder, dirPath, f)
- sys.stdout.write("\033[A\033[100D\033[2K")
- def findDuplicates(self):
- return [(sha1, list(filenames)) for sha1, filenames in self.files.items() if len(filenames) > 1]
-
-def main():
- try:
- files = cPickle.load(open(sys.argv[1]))
- except IOError:
- files = GlobalFileInfo()
-
- for dir in sys.argv[2:]:
- if dir[-1] == '/':
- dir = dir[:-1]
- files.scanDir(dir)
-
- cPickle.dump(files, open(sys.argv[1], 'wb'), 2)
- print "Done"
-
-### print files.files
-
-if __name__ == "__main__":
- main()
+++ /dev/null
-#!/usr/bin/env python2.5
-
-MINFILE_SIZE = 1024
-FILEBUFFER_SIZE = 1024**2
-APPLICATION_VERSION = '0.2'
-
-import os, sys, bisect
-
-import python24_adapter
-from copy import copy
-from base64 import standard_b64encode as b64encode
-from collections import defaultdict
-import cPickle
-
-try:
- import hashlib
- def _getSha1(filename):
- return hashlib.sha1()
-except ImportError:
- import sha
- def _getSha1(filename):
- return sha.new()
-def getSha1(filename):
- if _sha1_cache.has_key(filename):
- return b64encode(_sha1_cache[filename])
-
- sha1 = _getSha1(filename)
- f = file(filename, 'r')
- data = f.read(FILEBUFFER_SIZE)
- while data:
- sha1.update(data)
- data = f.read(FILEBUFFER_SIZE)
-
- ret = sha1.digest()
- _sha1_cache[filename] = ret
- return b64encode(ret)
-
-try:
- import psyco
- psyco.full()
-except ImportError:
- print >>sys.stderr, "WARNING: Could not load psyco"
-
-def __versionUpgrade0_1(input):
- import base64
- return '0.2', dict((filename, base64.b64decode(sha1hash)) for filename, sha1hash in input)
-
-def loadCache(filename = os.path.expanduser('~/.sha1_cache'), version = APPLICATION_VERSION):
- global _sha1_cache
- try:
- cache_version, cache = cPickle.load(open(filename, 'rb'))
- if cache_version == '0.1':
- cache_version, cache = __versionUpgrade0_1(cache)
-
- if cache_version != version:
- raise Exception("Invalid Version")
- print 'WARNING: Using the cache file "%s", sha1 hash may be old' % filename
- except:
- cache = {}
- _sha1_cache = cache
-
-def storeCache(filename = os.path.expanduser('~/.sha1_cache'), version = APPLICATION_VERSION):
- fd = open(filename, 'wb')
- try:
- cPickle.dump((version, _sha1_cache), fd)
- finally:
- fd.close()
-
-class GlobalFileInfo(object):
- def __init__(self):
- self.files = defaultdict(lambda : defaultdict(list))
-
- def _scanDirUpdateFile(self, dirPath, filename):
- def printPath(word):
- print '%s "%s"' % (word, filename[-80:])
- fullpath = os.path.abspath(os.path.join(dirPath, filename))
- if os.path.islink(fullpath) or not os.path.isfile(fullpath):
- printPath('Skipping')
- return
- try:
- statInfo = os.stat(fullpath)
-
- if statInfo.st_size < MINFILE_SIZE:
- printPath('Skipping')
- return
- printPath('Scanning')
-
- fileHash = getSha1(fullpath)
- self.files[(fileHash, statInfo.st_size)][(statInfo.st_dev, statInfo.st_ino)].append(fullpath)
- except IOError:
- print >>sys.stderr, 'WARNING: Could not get sha1 of "%s"\n' % (fullpath)
-
- def scanDir(self, dirName):
- for dirPath, dirs, files in os.walk(dirName):
- print 'Scanning directory "%s"\n' % dirPath
- # Add the children Directories
- if '.svn' in dirs:
- dirs.remove('.svn')
-
- for f in files:
- sys.stdout.write("\033[A\033[300D\033[2K")
- self._scanDirUpdateFile(dirPath, f)
- sys.stdout.write("\033[A\033[100D\033[2K")
- def findDuplicates(self):
- return [(key, inodes) for key, inodes in self.files.items() if len(inodes) > 1]
-
-def prettyFormatDups(dups):
- return '\n'.join( \
- '%s\n\t%s' % (key, \
- '\n\t'.join('%s: %s' % (inode_key, ', '.join(files)) for inode_key, files in inodes.items()) \
- ) for key, inodes in dups \
- )
-
- ret = []
- for key, inodes in dups:
- section = []
- for inode_key, files in inodes.items():
- section.append('%s: %s' % (inode_key, ', '.join(files)))
- ret.append('%s\n\t%s' % (key, '\n\t'.join(section)))
-
- return '\n'.join(ret)
-
-
-def makeBashScript(dups, fd):
- spaceSaved = 0
- print >>fd, "#!/bin/bash"
- print >>fd, '# This script was created automatically by "%s"' % __file__
- # Print out a helper function
- print >>fd
- print >>fd, 'function doLink() {'
- print >>fd, '\tINPUT_FILE="${1}"'
- print >>fd, '\tshift'
- print >>fd, '\tfor file in "$@" ; do'
- print >>fd, '\t\tln "${INPUT_FILE}" "${file}"'
- print >>fd, '\tdone'
- print >>fd, '}'
-
- for dup_key, inodes in dups:
- print >>fd
- print >>fd, '# Handling %s' % str(dup_key)
- inodes_data = inodes.items()
- inodes_data.sort(key = lambda x: len(x[1]), reverse = True)
- for inode_key, files in inodes_data[1:]:
- print >>fd, '# Removing files connected to inode %d on device %d' % (inode_key[1], inode_key[0])
- print >>fd, 'rm -f "%s"' % '" "'.join(file for file in files)
- fileToLink = inodes_data[0][1][0] # Get the first filename of the largest group of (already) linked files
- print >>fd, '# Now link all the files together'
- print >>fd, 'doLink "%s" "%s"' % (fileToLink, '" "'.join('" "'.join(files) for inode_key, files in inodes_data[1:]))
- spaceSaved += sum(len(files) for inode_key, files in inodes_data[1:]) * dup_key[1]
-
- print >>fd
- print >>fd, '# Total space saved: %d B (%dK B) (%d MB)' % (spaceSaved, spaceSaved / 1024, spaceSaved / 1024**2)
-
-def main():
- loadCache()
- files = GlobalFileInfo()
-
- for dir in sys.argv[2:]:
- files.scanDir(dir)
-
- storeCache()
- print "Done."
- try:
- fd = open(sys.argv[1], 'wb')
- makeBashScript(files.findDuplicates(), fd)
- finally:
- fd.close()
-
-if __name__ == "__main__":
- main()
+++ /dev/null
-#!/usr/bin/env python
-
-import sys, os, re, itertools
-from wget_lib import *
-
-import twisted_wget
-from twisted_wget import reactor
-from Enum import enum
-
-DEBUG = True
-
-URL_TYPE = enum('ImageBoard', 'HtmlPage', 'Image', 'Other')
-
-def addtolist(list, *regexStrs):
- def decorator(func):
- for regexStr in regexStrs:
- regex = re.compile(regexStr)
- list.append( (regex, func) )
- return func
- return decorator
-
-class Downloader(object):
- htmlParsers = []
- class ParserException(Exception):
- pass
-
- def __init__(self, url):
- self.url = url
- self.deferred = None
-
- def downloadFiles(self):
- # XXX: This is a major hack and needs to be cleaned
- def commonCallback(downloadObject):
- self.workingUrls.remove(downloadObject)
- self.activeHosts.remove(downloadObject.host)
- self.__scheduleDownloadLater()
- def successCallback(downloadObject, data):
- print 'Downloaded %s' % downloadObject.url
- commonCallback(downloadObject)
- downloadObject.data = data
- downloadObject.callback(downloadObject)
- self.doneUrls.add(downloadObject)
- def errorCallback(downloadObject, data):
- commonCallback(downloadObject)
- print 'Error: %s' % data
- def doDownload(file):
- print 'About to download "%s"' % file.url
- twisted_wget.downloadURL(file.url,
- successBack = lambda data: successCallback(file, data),
- errorBack = lambda data: errorCallback(file, data)
- )
- self.waitingUrls.remove(file)
- self.workingUrls.add(file)
- self.activeHosts.add(file.host)
-
-
- self.deferred = None
- for file in list(self.waitingUrls):
- if file.host not in self.activeHosts:
- doDownload(file)
-
- # Notes:
- # - image_object.data is a string containing all of the data
- # - image_object.url is a string containing the url where the data was downloaded from
- def _parseImageBoard(self, image_object):
- assert(image_object.data != None)
- assert(image_object.url != None)
-
- for parser_regex, parser in self.htmlParsers:
- if parser_regex.search(image_object.url):
- return parser(image_object)
- raise DownloadManager.ParserException('Could not find the correct parser')
-
- @addtolist(htmlParsers, '\.4chan\.org')
- def _parseImageBoard_4chan(self, image_object):
- import htmldata
- def __extractImageUrlsFromList(urllist):
- for url_elem in urllist:
- if url_elem.tag_name.upper() == 'A' and isImageURL(url_elem.url):
- yield url_elem.url
-
- # TODO: Extract a better filename from the list
- urllist = __extractImageUrlsFromList( htmldata.urlextract(image_object.data, image_object.url) )
- urllist = xRemoveDups(urllist)
- urllist = itertools.ifilter(
- lambda elem: elem.find('/thumb/') == -1,
- itertools.ifilter(lambda elem: elem.find('/src.cgi/') == -1, urllist)
- )
-
- if DEBUG:
- urllist, urllist_dup = itertools.tee(urllist)
- print >>sys.stderr, 'Got the following urls: \n\t%s' % '\n\t'.join(urllist_dup)
-
- for url in urllist:
- self.downloadImage(url, referer = image_object.url)
-
-def main(output_directory):
- dm = DownloadManager()
- for url in sys.argv[1:]:
- dm.recursiveDownloadImages(url)
-
- reactor.run()
-
-if __name__ == "__main__":
- output_directory = os.environ.get('WGET_IMAGEBOARD_DIRECTORY',
- os.path.join(os.environ['HOME'], 'Images_old', 'wget'))
- main(output_directory)
+++ /dev/null
-#!/usr/bin/env python
-# Copyright 2007 James Bunton <jamesbunton@fastmail.fm>
-# Modified by Greg Darke <gdar9540@usyd.edu.au> (2007)
-# Licensed for distribution under the GPL version 2, check COPYING for details
-# Check to see if people are online...
-
-import commands_async, pwd, socket, sys
-
-def matchNames(names):
- def getFullName(gecos_entry):
- return gecos_entry[: entry.pw_gecos.find(',')]
- def parsePWDentry(entry):
- return (entry.pw_name.lower(), getFullName(entry.pw_gecos).lower())
-
- pwall = [parsePWDentry(entry) for entry in pwd.getpwall()]
-
- out = []
- for name in names:
- found = False
- name = name.lower()
- for entry in pwall:
- username, realname = entry
- if username.find(name) >= 0 or realname.find(name) >= 0:
- found = True
- out.append((username, realname))
- if not found:
- print "User '%s' not found in /etc/passwd, assuming you gave a username and you are not on the IT machines..." % name
- out.append((name, "[Not Found in /etc/passwd]"))
- return out
-
-def getSmbStatus():
- def halfparse(data):
- return data.split('\n')[4:]
-
- sshcmd = "ssh %s -q -o StrictHostKeyChecking=no -o BatchMode=true '/usr/samba/bin/smbstatus -b'"
-
- cmd_async = commands_async.CommandRunner()
- cmd_async.executeCommand(sshcmd % "ugitsmb.ug.it.usyd.edu.au")
- cmd_async.executeCommand(sshcmd % "itsmb.ug.it.usyd.edu.au")
- cmd_async.waitForCompletion()
-
- data = []
- for cmd, output in cmd_async.getOutputs().items():
- data += halfparse(output)
-
- out = []
- for line in data:
- line_split = line.strip().split()
- if not line_split or len(line_split) != 5:
- continue
-
- pid, username, group, _, ip = line_split
- host = socket.gethostbyaddr(ip[1:-1])[0]
- out.append((username, host))
- return out
-
-def getLastStatus():
- hosts = ["mono"]
- hosts += ['congo%d' % i for i in range(1,5)]
- hosts += ['nlp%d' % i for i in range(0,9)]
- #hosts += ['userf%d' % i for i in range(1,6)]
-
- sshcmd = "ssh %s -q -o StrictHostKeyChecking=no -o BatchMode=true 'last -a -t $(date +%%Y%%m%%d%%H%%M%%S)|grep \"still logged in\"'"
-### sshcmd = "rsh -n %s 'last -a -t $(date +%%Y%%m%%d%%H%%M%%S)|grep \"still logged in\"'"
-
- cmd_async = commands_async.CommandRunner()
- for host in hosts:
- cmd_async.executeCommand(sshcmd % host)
-
- cmd_async.waitForCompletion()
- data = "".join(output for cmd,output in cmd_async.getOutputs().items())
-
- out = []
- for line in data.split('\n'):
- if not line.strip():
- continue
- try:
- chunk = line.strip().split()
- username = chunk[0]
- ip = chunk[-1]
- except Exception, e:
- print "Error:", line, e
- return []
- if ip == 'in': # From 'still logged in'
- host = "unknown"
- else:
- try:
- host = socket.gethostbyaddr(ip)[0]
- except:
- host = "unknown"
- out.append((username, host))
- return out
-
-
-def printLocation((username, fullname), smbStatus):
- # Since we only want to know if they are at a location, and now how many times they are at
- # the location, we store it in a set
- locationList = set(ip for username2, ip in smbStatus if username == username2)
- if locationList:
- print "Username %s:\n Full name: '%s'\n %s\n" % \
- (username, fullname, '\n '.join('Location: %s' % ip for ip in locationList))
-
-def main():
- names = matchNames(sys.argv[1:])
- smbStatus = getSmbStatus()
- lastStatus = getLastStatus()
- status = smbStatus + lastStatus
-
- for name in names:
- printLocation(name, status)
-
-if __name__ == "__main__":
- main()
+++ /dev/null
-#!/usr/bin/env python
-# Copyright 2007 Greg Darke <gdar9540@usyd.edu.au>
-# Licensed for distribution under the GPL version 2, check COPYING for details
-# A async framework for sockets and fds (fds only supported under unix operating systems)
-# NOTE: Orig version submitted for NETS3603 assignment 1 (Semester 1 - 2007)
-
-
-from __future__ import division
-import os, sys, select, socket, bisect, fcntl
-from time import time
-
-class Callback(object):
- __slots__ = ['callback', 'callback_time']
- def __init__(self, callback_time, callback):
- self.callback_time = callback_time
- self.callback = callback
- def __call__(self):
- return self.callback()
- def __lt__(self, other):
- if hasattr(other, 'callback_time'):
- return self.callback_time < other.callback_time
- else:
- return NotImplemented
-
-class AsyncSocketOwner(object):
- """This is the object contains the 'main loop' of the application"""
- def __init__(self):
- self.sockets_input = []
- self.socket_callbacks = {}
- self.timer_callbacks = []
- self._exit = False
- self.state = {}
-
- def print_state(self):
-### sys.stdout.write('\033[H\033[2J')
- print "\n".join(['%s: %s' % v for v in self.state.items()])
- self.addCallback(1.0, self.print_state)
-
- def _check_timers_callbacks(self):
- now = time()
- i = bisect.bisect(self.timer_callbacks, Callback(now, None))
- self.state['Processing Callbacks'] = '%d of %d' % (i,
- len(self.timer_callbacks))
- needCall = self.timer_callbacks[0:i]
- self.timer_callbacks = self.timer_callbacks[i:]
-
- for callback in needCall:
- callback()
-
- def exit(self):
- self._exit = True
-
- def mainLoop(self):
- try:
- while not self._exit:
- if len(self.timer_callbacks) > 0:
- timeout = max(self.timer_callbacks[0].callback_time - time(), 0)
- # Wait until the next timer expires for input
- inputready, outputready, exceptready = \
- select.select(self.sockets_input, [], [], timeout)
- else:
- # Wait forever for input
- inputready, outputready, exceptready = \
- select.select(self.sockets_input, [], [])
-
- # Handle any data received
- self.state['Waiting sockets'] = len(inputready)
- self.state['Socket count'] = len(self.sockets_input)
- for s in inputready:
- self.socket_callbacks[s](s)
-
- # Handle timers:
- if len(self.timer_callbacks) > 0 and \
- self.timer_callbacks[0].callback_time < time():
- self._check_timers_callbacks()
- except KeyboardInterrupt:
- pass
-
- def _addFDCallback(self, fd, callback):
- """Add a callback for a file descriptor, also add it to the select call"""
- self.sockets_input.append(fd)
- self.socket_callbacks[fd] = callback
-
- def removeSocket(self, fd):
- """Removes the sepecified fd from the event loop"""
- self.sockets_input.remove(fd)
- del self.socket_callbacks[fd]
-
- def addFD(self, fd, callback):
- """Adds a file descriptor to the event loop"""
- # Turn blocking off
- flags = fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK
- fcntl.fcntl(fd, fcntl.F_SETFL, flags)
- self._addFDCallback(fd, callback)
-
- def addSocket(self, s, callback):
- """Adds a socket to the event loop"""
- # Disable blocking - So now we have an async socket
- s.setblocking(False)
- self._addFDCallback(s, callback)
-
- def addLineBufferedSocket(self, s, callback):
- sockWrapper = LineBufferedAsyncClientConnection(s, callback, self)
- s.setblocking(False)
- self._addFDCallback(s, sockWrapper._dataArrived)
-
- def addCallback(self, seconds, callback):
- """Add a timer callback"""
- # Keep the list of callbacks sorted to keep things more efficient (Note: This would be better with a heap)
- cb = Callback(time() + seconds, callback)
- bisect.insort(self.timer_callbacks, cb)
- return cb
-
- def removeCallback(self, callback_object):
- """Remove a callback from the list. NB: If the time has fired/is in the list to be
- fired, the outcome is undefined (currently it will be called - but this may change)"""
- if callback_object in self.timer_callbacks:
- self.timer_callbacks.remove(callback_object)
-
-class LineBufferedAsyncClientConnection(object):
- __slots__ = ['sock', 'callback', 'delim', 'eventLoop', 'linesBuffer', 'lineBuffer', 'closed']
- def __init__(self, sock, callback, eventLoop, delim = '\n'):
- self.sock = sock
- self.callback = callback
- self.delim = delim
- self.eventLoop = eventLoop
- self.linesBuffer = []
- self.lineBuffer = ''
-
- def _dataArrived(self, *args, **kwargs):
- data = self.sock.recv(65535)
- if not data:
- self.closed = True
- self.eventLoop.removeSocket(self.sock)
- return
-
- self.lineBuffer += data
- newLinePos = self.lineBuffer.rfind(self.delim)
- if newLinePos >= 0:
- self.linesBuffer += self.lineBuffer[:newLinePos].split(self.delim)
- self.lineBuffer = self.lineBuffer[newLinePos+1:]
- self.callback(self)
-
- def fileno(self):
- """Return the encapsulated socket's fileno (used for select.select)"""
- return self.sock.fileno()
-
- def readline(self):
- if not self.hasLine():
- raise Exception('No data in buffer')
- ret = self.linesBuffer[0]
- del self.linesBuffer[0]
- return ret
-
- def write(self, data):
- self.sock.write(data)
- send = write
-
- def hasLine(self):
- return len(self.linesBuffer) > 0
+++ /dev/null
-#! python
-# Copyright 2007 Greg Darke <gdar9540@usyd.edu.au>
-# Licensed for distribution under the GPL version 2, check COPYING for details
-# An enum like interface
-
-"""
-An enum like interface
-"""
-
-__all__ = ('enum', )
-
-import itertools
-
-def enum(*args):
- return EnumContainer(*args)
-
-class EnumElement(object):
- __slots__ = ('enumName', 'enumContainer')
- def __init__(self, enumName, enumContainer):
- self.enumName = enumName
- self.enumContainer = enumContainer
- def __repr__(self):
- return '%s(%s)' % (self.__class__.__name__, self.enumName)
- def __str__(self):
- return self.enumName
- def __eq__(self, other):
- if not isinstance(other, self.__class__): return NotImplemented
- return other is self
- def __hash__(self):
- return hash(self.enumName) ^ hash(self.enumContainer)
-
-class EnumContainer(object):
- def __init__(self, *enums):
- self.enumList = tuple( EnumElement(enumName, self) for enumName in enums)
- for enumName, enumElement in itertools.izip(enums, self.enumList):
- setattr(self, enumName, enumElement)
-
- def __contains__(self, enum):
- return enum in self.enumList
-
- def __repr__(self):
- return '%s(%s)' % (self.__class__.__name__, ', '.join(self.enumList))
-
- def explode(self):
- """Places contents of this enum into the callers global namespace"""
- import inspect
- frameObject, _, _, _, _ = inspect.stack[1] # The stackframe of who called us
- global_dict = frameObject.f_globals
- del frameObject
- for enum in self.enumList:
- if str(enum) in global_dict:
- raise ValueError, '%s is already in your global dict' % enum
- for enum in self.enumList:
- global_dict[str(enum)] = enum
+++ /dev/null
-#! python
-
-import sys
-import cgitb
-import inspect
-
-DEBUG_INCREMENT = 5
-DEBUG_LEVEL_DEBUG = DEBUG_INCREMENT * -2
-DEBUG_LEVEL_LOW = DEBUG_INCREMENT * -1
-DEBUG_LEVEL_MEDIUM = DEBUG_INCREMENT * 0
-DEBUG_LEVEL_HIGH = DEBUG_INCREMENT * 1
-DEBUG_LEVEL = DEBUG_LEVEL_MEDIUM
-
-__stackTraceEnabled = True
-
-def stackTraceEnabled(value):
- global __stackTraceEnabled
- __stackTraceEnabled = value
-
-def setDebugLevel(level):
- global DEBUG_LEVEL
- DEBUG_LEVEL = level
-
-def isBoundMethod(stackFrame):
- """Checks to see if the method that is running in the specified stackFrame is
-a bound method.
-Returns a 2-tuple containing if it is a bound method, and the object that it is
-bound to if it is bound."""
- def errout():
- return (False, None)
-
- if stackFrame.f_code.co_argcount < 1:
- return errout()
- firstVarName = stackFrame.f_code.co_varnames[0]
- firstVar = stackFrame.f_locals[firstVarName]
- if not hasattr(firstVar, stackFrame.f_code.co_name):
- return errout()
- if not hasattr(getattr(firstVar, stackFrame.f_code.co_name), 'func_code'):
- return errout()
- if getattr(getattr(firstVar, stackFrame.f_code.co_name), 'func_code') == stackFrame.f_code:
- return (True, firstVar)
- else:
- return errout()
-
-def createStackTrace(stackList):
- if not __stackTraceEnabled:
- return ''
- strStackList = []
- for stackItem in stackList:
- stackItemRepr = ""
- bm = isBoundMethod(stackItem[0]) # stackframe
- if bm[0]:
- stackItemRepr = '%s.' % bm[1].__class__.__name__
- stackItemRepr += stackItem[3] # Function Name
- del bm # Help remove circular dependencies (reduces memory useage)
- strStackList.append(stackItemRepr)
-
- return '=>'.join(strStackList)
-
-def debug(message, level=DEBUG_LEVEL_MEDIUM, indent_level = None):
- if level >= DEBUG_LEVEL:
- stack = inspect.stack()[1:-1] # Ignore this method
- stack.reverse()
- if indent_level == None:
- indent_level = len(stack)
- for line in message.split('\n'):
- print >>sys.stderr, '%s %s [%s]' %('>' * indent_level, line, createStackTrace(stack))
-
-def tracebackHook(etype, evalue, etb):
- print cgitb.text( (etype, evalue, etb), context = 5)
+++ /dev/null
-#! python
-
-from signal import signal, SIGHUP, SIGTERM
-
-class HUPInterrupt(Exception):
- pass
-class TERMInterrupt(Exception):
- pass
-
-def HUPHandler(signal, stackFrame):
- raise HUPInterrupt
-
-def TERMHandler(signal, stackFrame):
- raise TERMInterrupt
-
-# Install the handlers
-signal(SIGHUP, HUPHandler)
-signal(SIGTERM, TERMHandler)
+++ /dev/null
-#! python
-
-"""
-A small utility that provides similar functionality to the commands module, but
-allows you to get the output from multiple processes at the same time
-"""
-
-__author__ = "Greg Darke"
-
-import subprocess, fcntl, os
-from select import select
-try:
- import cStringIO as _StringIO
-except ImportError:
- import StringIO as _StringIO
-
-class CommandRunner(object):
- def __init__(self):
- self._outputs = {}
- self._processes = {}
- self._fds = []
-
- def _executeCommand(self, cmd):
- """Execute the command"""
- output = _StringIO.StringIO()
- isShell = isinstance(cmd, str)
- process = subprocess.Popen(args = cmd, shell = isShell,
- stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
-
- # Turn blocking off
- flags = fcntl.fcntl(process.stdout, fcntl.F_GETFL) | os.O_NONBLOCK
- fcntl.fcntl(process.stdout, fcntl.F_SETFL, flags)
-
- return (output, process)
-
- def executeCommand(self, cmd):
- """Executes a command, but does not return anything"""
- output, process = self._executeCommand(cmd)
- self._outputs[process.stdout] = output
- self._processes[cmd] = process
- self._fds.append(process.stdout)
-
- def _waitLoop(self):
- count = 0
- while self._fds:
- # While there are still some processes left
- inputReady, outputReady, exceptReady = \
- select(self._fds, [], self._fds)
-
- for fd in inputReady:
- data = fd.read()
- if not data:
- self._fds.remove(fd)
- continue
- self._outputs[fd].write(data)
-
- for fd in exceptReady:
- self._fds.remove(fd)
-
- def waitForCompletion(self):
- """Waits for all of the running processes to finish"""
- self._waitLoop()
-
- def getOutputs(self):
- """Returns a dictionay containing the command as the key, and a string (of the output) as the value"""
- outputs = dict((cmd, self._outputs[process.stdout].getvalue()) for cmd, process in self._processes.items())
- return outputs
+++ /dev/null
-#! /usr/bin/env python
-
-import os
-
-__all__ = ['load_options']
-
-def load_options():
- FILENAME = os.path.expanduser('~/priv/credentials.conf')
- options = {}
- try:
- options_fd = open(FILENAME, "r")
-
- for line in options_fd:
- if line.startswith('#'):
- continue
- line = line.strip().split('=')
- if len(line) < 2:
- continue
-
- key = line[0]
- value = '='.join(line[1:])
-
- options[key] = value
-
- options_fd.close()
- except:
- pass
- return options
+++ /dev/null
-#!/usr/bin/env python2.4
-
-import GregDebug, base64, os, sys, urlparse
-
-from twisted.internet import reactor, protocol
-from twisted.web.client import HTTPClientFactory
-from twisted.web.http import HTTPClient
-from twisted.web.client import _parse as parseURL
-
-__all__ = ('downloadURL', )
-
-def parseURL(url, defaultPort = None):
- """Based on twisted.web.client._parse"""
- parsed = urlparse.urlparse(url)
- scheme = parsed[0]
- path = urlparse.urlunparse(('','')+parsed[2:])
- if defaultPort is None:
- if scheme == 'https':
- defaultPort = 443
- else:
- defaultPort = 80
- host, port = parsed[1], defaultPort
-
- if '@' in host:
- authUser, host = host.split('@', 1)
- auth = (authUser, )
- if ':' in authUser:
- auth = tuple(authUser.split(':', 1))
- else:
- auth = None
-
- if ':' in host:
- host, port = host.rsplit(':', 1)
- port = int(port)
-
- return scheme, auth, host, port, path
-
-class HTTPProxyFactory(protocol.ClientFactory):
- def __init__(self, realFactory, proxyServer, proxyMethod = 'GET', proxyPassword = None):
- self.realFactory = realFactory
- self.proxyHost, self.proxyPort = proxyServer
- self.proxyMethod = proxyMethod
- self.proxyPassword = proxyPassword
-
- def buildProtocol(self, addr):
- protocol = HTTPProxyProtocol(self, self.realFactory.buildProtocol(addr) )
- return protocol
-
- def __getattr__(self, key):
- return getattr(self.realFactory, key)
-
-class HTTPProxyProtocol(protocol.Protocol):
- def __init__(self, factory, proxied):
- self.factory = factory
- self.proxied = proxied
- self.proxyPassword = factory.proxyPassword
- if self.proxyPassword is not None:
- self.proxyPassword = base64.standard_b64encode('%s:%s' % self.proxyPassword)
- if factory.proxyMethod == 'GET':
- self.__connectionMade = self.__connectionMade_GET
- else:
- raise NotImplementedError
-
- def __send(self, value):
- self.transport.write(value)
-
- def __getTransportWrites(self, function, *args, **kwargs):
- temp = self.transport.write
- request = []
- self.transport.write = lambda data: request.append(data)
- function(*args, **kwargs)
- self.proxied.connectionMade()
- self.transport.write = temp
- return request
-
- def __connectionMade_GET(self):
- self.factory.realFactory.path = self.factory.realFactory.url
- self.proxied.makeConnection(self.transport)
-
- self.__send('GET %s HTTP/1.0\r\n' % self.factory.realFactory.url)
- if self.proxyPassword is not None:
- self.__send('Proxy-Authorization: Basic %s\r\n' % self.proxyPassword)
-
- # Remove the real http client's get request
- for line in self.__getTransportWrites(self.proxied.connectionMade)[1:]:
- self.__send(line)
-
- def connectionMade(self):
- self.proxied.transport = self.transport
- self.__connectionMade()
-
- def dataReceived(self, data):
- self.proxied.dataReceived(data)
-
- def connectionLost(self, reason):
- self.proxied.connectionLost(reason)
-
-proxies = {}
-def downloadURL(url, method = 'GET', successBack = None, errorBack = None):
- factory = HTTPClientFactory(url, method = method)
- scheme, auth, host, port, path = parseURL(url)
- if successBack is not None:
- factory.deferred.addCallback(successBack)
- if errorBack is not None:
- factory.deferred.addErrback(errorBack)
- if scheme in proxies:
- (host, port), password, factory_type = proxies[scheme]
- # Change the factory to the proxies one
- factory = factory_type(realFactory = factory, proxyServer = (host, port), proxyMethod = method, proxyPassword = password)
-
- reactor.connectTCP(host, port, factory)
- return factory
-
-# Note: Does not currently honor the no-proxy variable
-def parseProxies():
- for k,v in ( (k,v) for k,v in os.environ.items() if v and k.endswith('_proxy')):
- proxy_type = k[:-len('_proxy')]
- if proxy_type == 'http':
- _, auth, host, port, _ = parseURL(v)
- proxies[proxy_type] = (host, port), auth, HTTPProxyFactory
-
-def main(urls):
- def summerise(string, summerisedLen = 100):
- if len(string) <= summerisedLen:
- return string
- else:
- summerisedLen -= 5
- start = summerisedLen // 2
- return '%s ... %s' % (string[:start], string[-(summerisedLen - start):])
-
- def s(data):
- print 'Success: "%r"' % summerise(data)
-### print 'factory: (\n\t%s\n)' % '\n\t'.join('%s:%s' % (attr, getattr(factory, attr)) for attr in dir(factory))
-
- def e(data):
- print data
-
- for url in urls:
- factory = downloadURL(url, successBack = s, errorBack = e)
- reactor.run()
-
-# Parse the environment variables for proxy servers
-parseProxies()
-if __name__ == "__main__":
- main(sys.argv[1:])
+++ /dev/null
-#! python
-
-__all__ = ('parse_url', 'isImageURL', 'unique', 'removeDups', 'xRemoveDups')
-
-IMAGE_EXTENSIONS = ('PNG', 'JPG', 'JPEG', 'BMP', 'GIF', 'SWF', 'TIF', 'TIFF')
-
-def parse_url(url):
- """Parses a url into a tuple of (hostname, directory, filename)."""
- return ('hostname', 'directory', 'filename')
-
-def isImageURL(url):
- """Checks if an filename is an image"""
- try:
- _, extension = url.rsplit('.', 1)
- except ValueError:
- # There was no '.' in the url
- return False
- else:
- return extension.upper() in IMAGE_EXTENSIONS
-
-def unique(l):
- list_iter = iter(l)
- last_item = list_iter.next()
- yield last_item
- for item in list_iter:
- if last_item != item:
- yield item
- last_item = item
-
-def removeDups(l):
- """Removes duplicates from the list (Note: The ordering of the list may change)"""
- return list(unique(sorted(l)))
-
-def xRemoveDups(l):
- """Removes duplicates from the list.
- Requires O(n) memory, objects must be hashable"""
- yielded = set()
- for elem in l:
- if elem in yielded:
- continue
- yielded.add(elem)
- yield elem
+++ /dev/null
-#! python
-# Copyright 2007 Greg Darke <starstuff@optusnet.com.au>
-# Licensed for distribution under the GPL version 2, check COPYING for details
-# Some little helper utils
-
-import subprocess, commands, itertools
-
-__all__ = ('getResolutions', )
-
-def _seperateGroups(lines):
- ret = []
- current_section = []
- for line in lines:
- if line.strip() == '':
- ret.append(current_section)
- current_section = []
- continue
- current_section.append(line)
- if current_section:
- ret.append(current_section)
- return ret
-
-def getResolutions():
- xdpyinfo_status, xdpyinfo_output = commands.getstatusoutput('xdpyinfo')
- lines = xdpyinfo_output.splitlines()
- groups = _seperateGroups(xdpyinfo_output.splitlines())
-
- screens = []
- for screen_data in itertools.islice(groups, 1, None, None):
- _, screen_number = screen_data[0].split()
- # remove the leading and trailing characters
- screen_number = screen_number[1:-1]
-
- _, screen_resolution_str, _, _, _ = screen_data[1].strip().split()
- screen_resolution = screen_resolution_str.split('x')
-
- screens.append( (screen_number, tuple(int(val) for val in screen_resolution)))
- return dict(screens)