From f667bb5991af1eb85cb9585b4dfbdf7ea80f6024 Mon Sep 17 00:00:00 2001 From: Greg Darke Date: Mon, 7 Jul 2008 12:33:39 +1000 Subject: [PATCH] Removing a lot of unused libraries --- bin/findsame_file.py | 175 ------------------------------------ bin/findsame_file_new.py | 170 ----------------------------------- bin/new_wget_image_board.py | 107 ---------------------- bin/pyfing | 113 ----------------------- lib/AsyncSocket.py | 160 --------------------------------- lib/Enum.py | 54 ----------- lib/GregDebug.py | 70 --------------- lib/SigHandler.py | 18 ---- lib/commands_async.py | 67 -------------- lib/priv_options.py | 28 ------ lib/twisted_wget.py | 145 ------------------------------ lib/wget_lib.py | 42 --------- lib/x11_helpers.py | 38 -------- 13 files changed, 1187 deletions(-) delete mode 100755 bin/findsame_file.py delete mode 100755 bin/findsame_file_new.py delete mode 100755 bin/new_wget_image_board.py delete mode 100755 bin/pyfing delete mode 100644 lib/AsyncSocket.py delete mode 100644 lib/Enum.py delete mode 100644 lib/GregDebug.py delete mode 100644 lib/SigHandler.py delete mode 100644 lib/commands_async.py delete mode 100644 lib/priv_options.py delete mode 100755 lib/twisted_wget.py delete mode 100644 lib/wget_lib.py delete mode 100644 lib/x11_helpers.py diff --git a/bin/findsame_file.py b/bin/findsame_file.py deleted file mode 100755 index 1a5ce8d..0000000 --- a/bin/findsame_file.py +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python2.5 - -MINFILE_SIZE = 1024 -FILEBUFFER_SIZE = 1024**2 - -import os, sys, bisect - -from copy import copy -from base64 import standard_b64encode as b64encode -from collections import defaultdict -import cPickle -try: - import hashlib - def _getSha1(filename): - return hashlib.sha1() -except ImportError: - import sha - def _getSha1(filename): - return sha.new() -def getSha1(filename): - sha1 = _getSha1(filename) - f = file(filename, 'r') - data = f.read(FILEBUFFER_SIZE) - while data: - sha1.update(data) - data = f.read(FILEBUFFER_SIZE) - return b64encode(sha1.digest()) - -try: - import psyco - psyco.full() -except ImportError: - print >>sys.stderr, "WARNING: Could not load psyco" - -class DiskObject(object): - def __repr__(self): - return repr(self.getFullPath()) - def __str__(self): - return self.getFullPath() - def __lt__(self, other): - if not hasattr(other, 'getFullPath'): - raise NotImplemented() - return self.getFullPath() < other.getFullPath() - def __eq__(self, other): - if not hasattr(other, 'getFullPath'): - raise NotImplemented() - return self.getFullPath() == other.getFullPath() - def __hash__(self): - return hash(self.getFullPath()) - -class Folder(DiskObject): - def __init__(self, name, parent = None): - if name.find(os.path.sep) >= 0 and name != os.path.sep: - print name - parent_name, name = os.path.split(name) - parent = Folder(parent_name, parent) - - self.name = name - self.parent = parent - if parent: - parent.addChild(self) - self.children = {} - def getFullPath(self): - folderStack = [] - f = self - while f: - folderStack.append(f.name) - f = f.parent - return os.path.sep.join(reversed(folderStack)) - def addChild(self, child): - self.children[child.name] = child - -def findDirectory(rootDir, dirName, createNonExistant = False): - dir = dirName.split(os.path.sep)[1:] - if dir == ['']: - dir = [] - - ret = rootDir - for folderName in dir: - try: - ret = ret.children[folderName] - except KeyError, e: - if not createNonExistant: - raise e - ret = Folder(folderName, ret) - - return ret - -class FileObject(DiskObject): - def __init__(self, name, folder): - self.name = name - self.folder = folder - statinfo = os.stat(self.getFullPath()) - self.mtime_size = (statinfo.st_mtime, statinfo.st_size) - def getDiskID(self): - statinfo = os.stat(self.getFullPath()) - return (statinfo.st_dev, statinfo.st_ino) # Identify the file - def get_mtime_size(self): - return self.mtime_size - def getFullPath(self): - return '%(folder)s/%(file)s' % { 'folder': self.folder.getFullPath(), 'file': self.name } - -class GlobalFileInfo(object): - def __init__(self): - self.files = defaultdict(list) - self.filelist = {} - self.root = Folder('') - - def _scanDirUpdateFile(self, dirObject, dirPath, filename): - def printPath(word): - print '%s "%s"' % (word, filename[-80:]) - fullpath = os.path.join(dirPath, filename) - if os.path.islink(fullpath) or not os.path.isfile(fullpath): - printPath('Skipping') - return - try: - file = FileObject(filename, dirObject) - new_mtime_size = file.get_mtime_size() - - if file in self.filelist: - if file.get_mtime_size() == self.filelist[file].get_mtime_size(): - printPath('Skipping') - return - old_sha1 = self.filelist[file].sha1 - del self.filelist[file] - self.files[old_sha1].remove(file) - - if file.get_mtime_size()[1] < MINFILE_SIZE: - printPath('Skipping') - return - printPath('Scanning') - - file.sha1 = getSha1(fullpath) - self.files[file.sha1].append(file) - self.filelist[file] = file - except IOError: - print >>sys.stderr, 'WARNING: Could not get sha1 of "%s"\n' % (fullpath) - - def scanDir(self, dirName): - root = findDirectory(self.root, dirName, createNonExistant = True) - - for dirPath, dirs, files in os.walk(dirName): - print 'Scanning directory "%s"\n' % dirPath - folder = findDirectory(self.root, dirPath, createNonExistant = True) - # Add the children Directories - if '.svn' in dirs: - dirs.remove('.svn') - for d in dirs: - Folder(d, folder) # As a side effect, this is added to the parent correctly - - for f in files: - sys.stdout.write("\033[A\033[300D\033[2K") - self._scanDirUpdateFile(folder, dirPath, f) - sys.stdout.write("\033[A\033[100D\033[2K") - def findDuplicates(self): - return [(sha1, list(filenames)) for sha1, filenames in self.files.items() if len(filenames) > 1] - -def main(): - try: - files = cPickle.load(open(sys.argv[1])) - except IOError: - files = GlobalFileInfo() - - for dir in sys.argv[2:]: - if dir[-1] == '/': - dir = dir[:-1] - files.scanDir(dir) - - cPickle.dump(files, open(sys.argv[1], 'wb'), 2) - print "Done" - -### print files.files - -if __name__ == "__main__": - main() diff --git a/bin/findsame_file_new.py b/bin/findsame_file_new.py deleted file mode 100755 index 293b9fb..0000000 --- a/bin/findsame_file_new.py +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env python2.5 - -MINFILE_SIZE = 1024 -FILEBUFFER_SIZE = 1024**2 -APPLICATION_VERSION = '0.2' - -import os, sys, bisect - -import python24_adapter -from copy import copy -from base64 import standard_b64encode as b64encode -from collections import defaultdict -import cPickle - -try: - import hashlib - def _getSha1(filename): - return hashlib.sha1() -except ImportError: - import sha - def _getSha1(filename): - return sha.new() -def getSha1(filename): - if _sha1_cache.has_key(filename): - return b64encode(_sha1_cache[filename]) - - sha1 = _getSha1(filename) - f = file(filename, 'r') - data = f.read(FILEBUFFER_SIZE) - while data: - sha1.update(data) - data = f.read(FILEBUFFER_SIZE) - - ret = sha1.digest() - _sha1_cache[filename] = ret - return b64encode(ret) - -try: - import psyco - psyco.full() -except ImportError: - print >>sys.stderr, "WARNING: Could not load psyco" - -def __versionUpgrade0_1(input): - import base64 - return '0.2', dict((filename, base64.b64decode(sha1hash)) for filename, sha1hash in input) - -def loadCache(filename = os.path.expanduser('~/.sha1_cache'), version = APPLICATION_VERSION): - global _sha1_cache - try: - cache_version, cache = cPickle.load(open(filename, 'rb')) - if cache_version == '0.1': - cache_version, cache = __versionUpgrade0_1(cache) - - if cache_version != version: - raise Exception("Invalid Version") - print 'WARNING: Using the cache file "%s", sha1 hash may be old' % filename - except: - cache = {} - _sha1_cache = cache - -def storeCache(filename = os.path.expanduser('~/.sha1_cache'), version = APPLICATION_VERSION): - fd = open(filename, 'wb') - try: - cPickle.dump((version, _sha1_cache), fd) - finally: - fd.close() - -class GlobalFileInfo(object): - def __init__(self): - self.files = defaultdict(lambda : defaultdict(list)) - - def _scanDirUpdateFile(self, dirPath, filename): - def printPath(word): - print '%s "%s"' % (word, filename[-80:]) - fullpath = os.path.abspath(os.path.join(dirPath, filename)) - if os.path.islink(fullpath) or not os.path.isfile(fullpath): - printPath('Skipping') - return - try: - statInfo = os.stat(fullpath) - - if statInfo.st_size < MINFILE_SIZE: - printPath('Skipping') - return - printPath('Scanning') - - fileHash = getSha1(fullpath) - self.files[(fileHash, statInfo.st_size)][(statInfo.st_dev, statInfo.st_ino)].append(fullpath) - except IOError: - print >>sys.stderr, 'WARNING: Could not get sha1 of "%s"\n' % (fullpath) - - def scanDir(self, dirName): - for dirPath, dirs, files in os.walk(dirName): - print 'Scanning directory "%s"\n' % dirPath - # Add the children Directories - if '.svn' in dirs: - dirs.remove('.svn') - - for f in files: - sys.stdout.write("\033[A\033[300D\033[2K") - self._scanDirUpdateFile(dirPath, f) - sys.stdout.write("\033[A\033[100D\033[2K") - def findDuplicates(self): - return [(key, inodes) for key, inodes in self.files.items() if len(inodes) > 1] - -def prettyFormatDups(dups): - return '\n'.join( \ - '%s\n\t%s' % (key, \ - '\n\t'.join('%s: %s' % (inode_key, ', '.join(files)) for inode_key, files in inodes.items()) \ - ) for key, inodes in dups \ - ) - - ret = [] - for key, inodes in dups: - section = [] - for inode_key, files in inodes.items(): - section.append('%s: %s' % (inode_key, ', '.join(files))) - ret.append('%s\n\t%s' % (key, '\n\t'.join(section))) - - return '\n'.join(ret) - - -def makeBashScript(dups, fd): - spaceSaved = 0 - print >>fd, "#!/bin/bash" - print >>fd, '# This script was created automatically by "%s"' % __file__ - # Print out a helper function - print >>fd - print >>fd, 'function doLink() {' - print >>fd, '\tINPUT_FILE="${1}"' - print >>fd, '\tshift' - print >>fd, '\tfor file in "$@" ; do' - print >>fd, '\t\tln "${INPUT_FILE}" "${file}"' - print >>fd, '\tdone' - print >>fd, '}' - - for dup_key, inodes in dups: - print >>fd - print >>fd, '# Handling %s' % str(dup_key) - inodes_data = inodes.items() - inodes_data.sort(key = lambda x: len(x[1]), reverse = True) - for inode_key, files in inodes_data[1:]: - print >>fd, '# Removing files connected to inode %d on device %d' % (inode_key[1], inode_key[0]) - print >>fd, 'rm -f "%s"' % '" "'.join(file for file in files) - fileToLink = inodes_data[0][1][0] # Get the first filename of the largest group of (already) linked files - print >>fd, '# Now link all the files together' - print >>fd, 'doLink "%s" "%s"' % (fileToLink, '" "'.join('" "'.join(files) for inode_key, files in inodes_data[1:])) - spaceSaved += sum(len(files) for inode_key, files in inodes_data[1:]) * dup_key[1] - - print >>fd - print >>fd, '# Total space saved: %d B (%dK B) (%d MB)' % (spaceSaved, spaceSaved / 1024, spaceSaved / 1024**2) - -def main(): - loadCache() - files = GlobalFileInfo() - - for dir in sys.argv[2:]: - files.scanDir(dir) - - storeCache() - print "Done." - try: - fd = open(sys.argv[1], 'wb') - makeBashScript(files.findDuplicates(), fd) - finally: - fd.close() - -if __name__ == "__main__": - main() diff --git a/bin/new_wget_image_board.py b/bin/new_wget_image_board.py deleted file mode 100755 index 3ee536b..0000000 --- a/bin/new_wget_image_board.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python - -import sys, os, re, itertools -from wget_lib import * - -import twisted_wget -from twisted_wget import reactor -from Enum import enum - -DEBUG = True - -URL_TYPE = enum('ImageBoard', 'HtmlPage', 'Image', 'Other') - -def addtolist(list, *regexStrs): - def decorator(func): - for regexStr in regexStrs: - regex = re.compile(regexStr) - list.append( (regex, func) ) - return func - return decorator - -class Downloader(object): - htmlParsers = [] - class ParserException(Exception): - pass - - def __init__(self, url): - self.url = url - self.deferred = None - - def downloadFiles(self): - # XXX: This is a major hack and needs to be cleaned - def commonCallback(downloadObject): - self.workingUrls.remove(downloadObject) - self.activeHosts.remove(downloadObject.host) - self.__scheduleDownloadLater() - def successCallback(downloadObject, data): - print 'Downloaded %s' % downloadObject.url - commonCallback(downloadObject) - downloadObject.data = data - downloadObject.callback(downloadObject) - self.doneUrls.add(downloadObject) - def errorCallback(downloadObject, data): - commonCallback(downloadObject) - print 'Error: %s' % data - def doDownload(file): - print 'About to download "%s"' % file.url - twisted_wget.downloadURL(file.url, - successBack = lambda data: successCallback(file, data), - errorBack = lambda data: errorCallback(file, data) - ) - self.waitingUrls.remove(file) - self.workingUrls.add(file) - self.activeHosts.add(file.host) - - - self.deferred = None - for file in list(self.waitingUrls): - if file.host not in self.activeHosts: - doDownload(file) - - # Notes: - # - image_object.data is a string containing all of the data - # - image_object.url is a string containing the url where the data was downloaded from - def _parseImageBoard(self, image_object): - assert(image_object.data != None) - assert(image_object.url != None) - - for parser_regex, parser in self.htmlParsers: - if parser_regex.search(image_object.url): - return parser(image_object) - raise DownloadManager.ParserException('Could not find the correct parser') - - @addtolist(htmlParsers, '\.4chan\.org') - def _parseImageBoard_4chan(self, image_object): - import htmldata - def __extractImageUrlsFromList(urllist): - for url_elem in urllist: - if url_elem.tag_name.upper() == 'A' and isImageURL(url_elem.url): - yield url_elem.url - - # TODO: Extract a better filename from the list - urllist = __extractImageUrlsFromList( htmldata.urlextract(image_object.data, image_object.url) ) - urllist = xRemoveDups(urllist) - urllist = itertools.ifilter( - lambda elem: elem.find('/thumb/') == -1, - itertools.ifilter(lambda elem: elem.find('/src.cgi/') == -1, urllist) - ) - - if DEBUG: - urllist, urllist_dup = itertools.tee(urllist) - print >>sys.stderr, 'Got the following urls: \n\t%s' % '\n\t'.join(urllist_dup) - - for url in urllist: - self.downloadImage(url, referer = image_object.url) - -def main(output_directory): - dm = DownloadManager() - for url in sys.argv[1:]: - dm.recursiveDownloadImages(url) - - reactor.run() - -if __name__ == "__main__": - output_directory = os.environ.get('WGET_IMAGEBOARD_DIRECTORY', - os.path.join(os.environ['HOME'], 'Images_old', 'wget')) - main(output_directory) diff --git a/bin/pyfing b/bin/pyfing deleted file mode 100755 index 664a1df..0000000 --- a/bin/pyfing +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python -# Copyright 2007 James Bunton -# Modified by Greg Darke (2007) -# Licensed for distribution under the GPL version 2, check COPYING for details -# Check to see if people are online... - -import commands_async, pwd, socket, sys - -def matchNames(names): - def getFullName(gecos_entry): - return gecos_entry[: entry.pw_gecos.find(',')] - def parsePWDentry(entry): - return (entry.pw_name.lower(), getFullName(entry.pw_gecos).lower()) - - pwall = [parsePWDentry(entry) for entry in pwd.getpwall()] - - out = [] - for name in names: - found = False - name = name.lower() - for entry in pwall: - username, realname = entry - if username.find(name) >= 0 or realname.find(name) >= 0: - found = True - out.append((username, realname)) - if not found: - print "User '%s' not found in /etc/passwd, assuming you gave a username and you are not on the IT machines..." % name - out.append((name, "[Not Found in /etc/passwd]")) - return out - -def getSmbStatus(): - def halfparse(data): - return data.split('\n')[4:] - - sshcmd = "ssh %s -q -o StrictHostKeyChecking=no -o BatchMode=true '/usr/samba/bin/smbstatus -b'" - - cmd_async = commands_async.CommandRunner() - cmd_async.executeCommand(sshcmd % "ugitsmb.ug.it.usyd.edu.au") - cmd_async.executeCommand(sshcmd % "itsmb.ug.it.usyd.edu.au") - cmd_async.waitForCompletion() - - data = [] - for cmd, output in cmd_async.getOutputs().items(): - data += halfparse(output) - - out = [] - for line in data: - line_split = line.strip().split() - if not line_split or len(line_split) != 5: - continue - - pid, username, group, _, ip = line_split - host = socket.gethostbyaddr(ip[1:-1])[0] - out.append((username, host)) - return out - -def getLastStatus(): - hosts = ["mono"] - hosts += ['congo%d' % i for i in range(1,5)] - hosts += ['nlp%d' % i for i in range(0,9)] - #hosts += ['userf%d' % i for i in range(1,6)] - - sshcmd = "ssh %s -q -o StrictHostKeyChecking=no -o BatchMode=true 'last -a -t $(date +%%Y%%m%%d%%H%%M%%S)|grep \"still logged in\"'" -### sshcmd = "rsh -n %s 'last -a -t $(date +%%Y%%m%%d%%H%%M%%S)|grep \"still logged in\"'" - - cmd_async = commands_async.CommandRunner() - for host in hosts: - cmd_async.executeCommand(sshcmd % host) - - cmd_async.waitForCompletion() - data = "".join(output for cmd,output in cmd_async.getOutputs().items()) - - out = [] - for line in data.split('\n'): - if not line.strip(): - continue - try: - chunk = line.strip().split() - username = chunk[0] - ip = chunk[-1] - except Exception, e: - print "Error:", line, e - return [] - if ip == 'in': # From 'still logged in' - host = "unknown" - else: - try: - host = socket.gethostbyaddr(ip)[0] - except: - host = "unknown" - out.append((username, host)) - return out - - -def printLocation((username, fullname), smbStatus): - # Since we only want to know if they are at a location, and now how many times they are at - # the location, we store it in a set - locationList = set(ip for username2, ip in smbStatus if username == username2) - if locationList: - print "Username %s:\n Full name: '%s'\n %s\n" % \ - (username, fullname, '\n '.join('Location: %s' % ip for ip in locationList)) - -def main(): - names = matchNames(sys.argv[1:]) - smbStatus = getSmbStatus() - lastStatus = getLastStatus() - status = smbStatus + lastStatus - - for name in names: - printLocation(name, status) - -if __name__ == "__main__": - main() diff --git a/lib/AsyncSocket.py b/lib/AsyncSocket.py deleted file mode 100644 index 76f10f1..0000000 --- a/lib/AsyncSocket.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python -# Copyright 2007 Greg Darke -# Licensed for distribution under the GPL version 2, check COPYING for details -# A async framework for sockets and fds (fds only supported under unix operating systems) -# NOTE: Orig version submitted for NETS3603 assignment 1 (Semester 1 - 2007) - - -from __future__ import division -import os, sys, select, socket, bisect, fcntl -from time import time - -class Callback(object): - __slots__ = ['callback', 'callback_time'] - def __init__(self, callback_time, callback): - self.callback_time = callback_time - self.callback = callback - def __call__(self): - return self.callback() - def __lt__(self, other): - if hasattr(other, 'callback_time'): - return self.callback_time < other.callback_time - else: - return NotImplemented - -class AsyncSocketOwner(object): - """This is the object contains the 'main loop' of the application""" - def __init__(self): - self.sockets_input = [] - self.socket_callbacks = {} - self.timer_callbacks = [] - self._exit = False - self.state = {} - - def print_state(self): -### sys.stdout.write('\033[H\033[2J') - print "\n".join(['%s: %s' % v for v in self.state.items()]) - self.addCallback(1.0, self.print_state) - - def _check_timers_callbacks(self): - now = time() - i = bisect.bisect(self.timer_callbacks, Callback(now, None)) - self.state['Processing Callbacks'] = '%d of %d' % (i, - len(self.timer_callbacks)) - needCall = self.timer_callbacks[0:i] - self.timer_callbacks = self.timer_callbacks[i:] - - for callback in needCall: - callback() - - def exit(self): - self._exit = True - - def mainLoop(self): - try: - while not self._exit: - if len(self.timer_callbacks) > 0: - timeout = max(self.timer_callbacks[0].callback_time - time(), 0) - # Wait until the next timer expires for input - inputready, outputready, exceptready = \ - select.select(self.sockets_input, [], [], timeout) - else: - # Wait forever for input - inputready, outputready, exceptready = \ - select.select(self.sockets_input, [], []) - - # Handle any data received - self.state['Waiting sockets'] = len(inputready) - self.state['Socket count'] = len(self.sockets_input) - for s in inputready: - self.socket_callbacks[s](s) - - # Handle timers: - if len(self.timer_callbacks) > 0 and \ - self.timer_callbacks[0].callback_time < time(): - self._check_timers_callbacks() - except KeyboardInterrupt: - pass - - def _addFDCallback(self, fd, callback): - """Add a callback for a file descriptor, also add it to the select call""" - self.sockets_input.append(fd) - self.socket_callbacks[fd] = callback - - def removeSocket(self, fd): - """Removes the sepecified fd from the event loop""" - self.sockets_input.remove(fd) - del self.socket_callbacks[fd] - - def addFD(self, fd, callback): - """Adds a file descriptor to the event loop""" - # Turn blocking off - flags = fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK - fcntl.fcntl(fd, fcntl.F_SETFL, flags) - self._addFDCallback(fd, callback) - - def addSocket(self, s, callback): - """Adds a socket to the event loop""" - # Disable blocking - So now we have an async socket - s.setblocking(False) - self._addFDCallback(s, callback) - - def addLineBufferedSocket(self, s, callback): - sockWrapper = LineBufferedAsyncClientConnection(s, callback, self) - s.setblocking(False) - self._addFDCallback(s, sockWrapper._dataArrived) - - def addCallback(self, seconds, callback): - """Add a timer callback""" - # Keep the list of callbacks sorted to keep things more efficient (Note: This would be better with a heap) - cb = Callback(time() + seconds, callback) - bisect.insort(self.timer_callbacks, cb) - return cb - - def removeCallback(self, callback_object): - """Remove a callback from the list. NB: If the time has fired/is in the list to be - fired, the outcome is undefined (currently it will be called - but this may change)""" - if callback_object in self.timer_callbacks: - self.timer_callbacks.remove(callback_object) - -class LineBufferedAsyncClientConnection(object): - __slots__ = ['sock', 'callback', 'delim', 'eventLoop', 'linesBuffer', 'lineBuffer', 'closed'] - def __init__(self, sock, callback, eventLoop, delim = '\n'): - self.sock = sock - self.callback = callback - self.delim = delim - self.eventLoop = eventLoop - self.linesBuffer = [] - self.lineBuffer = '' - - def _dataArrived(self, *args, **kwargs): - data = self.sock.recv(65535) - if not data: - self.closed = True - self.eventLoop.removeSocket(self.sock) - return - - self.lineBuffer += data - newLinePos = self.lineBuffer.rfind(self.delim) - if newLinePos >= 0: - self.linesBuffer += self.lineBuffer[:newLinePos].split(self.delim) - self.lineBuffer = self.lineBuffer[newLinePos+1:] - self.callback(self) - - def fileno(self): - """Return the encapsulated socket's fileno (used for select.select)""" - return self.sock.fileno() - - def readline(self): - if not self.hasLine(): - raise Exception('No data in buffer') - ret = self.linesBuffer[0] - del self.linesBuffer[0] - return ret - - def write(self, data): - self.sock.write(data) - send = write - - def hasLine(self): - return len(self.linesBuffer) > 0 diff --git a/lib/Enum.py b/lib/Enum.py deleted file mode 100644 index 6bb2d3d..0000000 --- a/lib/Enum.py +++ /dev/null @@ -1,54 +0,0 @@ -#! python -# Copyright 2007 Greg Darke -# Licensed for distribution under the GPL version 2, check COPYING for details -# An enum like interface - -""" -An enum like interface -""" - -__all__ = ('enum', ) - -import itertools - -def enum(*args): - return EnumContainer(*args) - -class EnumElement(object): - __slots__ = ('enumName', 'enumContainer') - def __init__(self, enumName, enumContainer): - self.enumName = enumName - self.enumContainer = enumContainer - def __repr__(self): - return '%s(%s)' % (self.__class__.__name__, self.enumName) - def __str__(self): - return self.enumName - def __eq__(self, other): - if not isinstance(other, self.__class__): return NotImplemented - return other is self - def __hash__(self): - return hash(self.enumName) ^ hash(self.enumContainer) - -class EnumContainer(object): - def __init__(self, *enums): - self.enumList = tuple( EnumElement(enumName, self) for enumName in enums) - for enumName, enumElement in itertools.izip(enums, self.enumList): - setattr(self, enumName, enumElement) - - def __contains__(self, enum): - return enum in self.enumList - - def __repr__(self): - return '%s(%s)' % (self.__class__.__name__, ', '.join(self.enumList)) - - def explode(self): - """Places contents of this enum into the callers global namespace""" - import inspect - frameObject, _, _, _, _ = inspect.stack[1] # The stackframe of who called us - global_dict = frameObject.f_globals - del frameObject - for enum in self.enumList: - if str(enum) in global_dict: - raise ValueError, '%s is already in your global dict' % enum - for enum in self.enumList: - global_dict[str(enum)] = enum diff --git a/lib/GregDebug.py b/lib/GregDebug.py deleted file mode 100644 index c431317..0000000 --- a/lib/GregDebug.py +++ /dev/null @@ -1,70 +0,0 @@ -#! python - -import sys -import cgitb -import inspect - -DEBUG_INCREMENT = 5 -DEBUG_LEVEL_DEBUG = DEBUG_INCREMENT * -2 -DEBUG_LEVEL_LOW = DEBUG_INCREMENT * -1 -DEBUG_LEVEL_MEDIUM = DEBUG_INCREMENT * 0 -DEBUG_LEVEL_HIGH = DEBUG_INCREMENT * 1 -DEBUG_LEVEL = DEBUG_LEVEL_MEDIUM - -__stackTraceEnabled = True - -def stackTraceEnabled(value): - global __stackTraceEnabled - __stackTraceEnabled = value - -def setDebugLevel(level): - global DEBUG_LEVEL - DEBUG_LEVEL = level - -def isBoundMethod(stackFrame): - """Checks to see if the method that is running in the specified stackFrame is -a bound method. -Returns a 2-tuple containing if it is a bound method, and the object that it is -bound to if it is bound.""" - def errout(): - return (False, None) - - if stackFrame.f_code.co_argcount < 1: - return errout() - firstVarName = stackFrame.f_code.co_varnames[0] - firstVar = stackFrame.f_locals[firstVarName] - if not hasattr(firstVar, stackFrame.f_code.co_name): - return errout() - if not hasattr(getattr(firstVar, stackFrame.f_code.co_name), 'func_code'): - return errout() - if getattr(getattr(firstVar, stackFrame.f_code.co_name), 'func_code') == stackFrame.f_code: - return (True, firstVar) - else: - return errout() - -def createStackTrace(stackList): - if not __stackTraceEnabled: - return '' - strStackList = [] - for stackItem in stackList: - stackItemRepr = "" - bm = isBoundMethod(stackItem[0]) # stackframe - if bm[0]: - stackItemRepr = '%s.' % bm[1].__class__.__name__ - stackItemRepr += stackItem[3] # Function Name - del bm # Help remove circular dependencies (reduces memory useage) - strStackList.append(stackItemRepr) - - return '=>'.join(strStackList) - -def debug(message, level=DEBUG_LEVEL_MEDIUM, indent_level = None): - if level >= DEBUG_LEVEL: - stack = inspect.stack()[1:-1] # Ignore this method - stack.reverse() - if indent_level == None: - indent_level = len(stack) - for line in message.split('\n'): - print >>sys.stderr, '%s %s [%s]' %('>' * indent_level, line, createStackTrace(stack)) - -def tracebackHook(etype, evalue, etb): - print cgitb.text( (etype, evalue, etb), context = 5) diff --git a/lib/SigHandler.py b/lib/SigHandler.py deleted file mode 100644 index 9b494d1..0000000 --- a/lib/SigHandler.py +++ /dev/null @@ -1,18 +0,0 @@ -#! python - -from signal import signal, SIGHUP, SIGTERM - -class HUPInterrupt(Exception): - pass -class TERMInterrupt(Exception): - pass - -def HUPHandler(signal, stackFrame): - raise HUPInterrupt - -def TERMHandler(signal, stackFrame): - raise TERMInterrupt - -# Install the handlers -signal(SIGHUP, HUPHandler) -signal(SIGTERM, TERMHandler) diff --git a/lib/commands_async.py b/lib/commands_async.py deleted file mode 100644 index f90a2a5..0000000 --- a/lib/commands_async.py +++ /dev/null @@ -1,67 +0,0 @@ -#! python - -""" -A small utility that provides similar functionality to the commands module, but -allows you to get the output from multiple processes at the same time -""" - -__author__ = "Greg Darke" - -import subprocess, fcntl, os -from select import select -try: - import cStringIO as _StringIO -except ImportError: - import StringIO as _StringIO - -class CommandRunner(object): - def __init__(self): - self._outputs = {} - self._processes = {} - self._fds = [] - - def _executeCommand(self, cmd): - """Execute the command""" - output = _StringIO.StringIO() - isShell = isinstance(cmd, str) - process = subprocess.Popen(args = cmd, shell = isShell, - stdout = subprocess.PIPE, stderr = subprocess.STDOUT) - - # Turn blocking off - flags = fcntl.fcntl(process.stdout, fcntl.F_GETFL) | os.O_NONBLOCK - fcntl.fcntl(process.stdout, fcntl.F_SETFL, flags) - - return (output, process) - - def executeCommand(self, cmd): - """Executes a command, but does not return anything""" - output, process = self._executeCommand(cmd) - self._outputs[process.stdout] = output - self._processes[cmd] = process - self._fds.append(process.stdout) - - def _waitLoop(self): - count = 0 - while self._fds: - # While there are still some processes left - inputReady, outputReady, exceptReady = \ - select(self._fds, [], self._fds) - - for fd in inputReady: - data = fd.read() - if not data: - self._fds.remove(fd) - continue - self._outputs[fd].write(data) - - for fd in exceptReady: - self._fds.remove(fd) - - def waitForCompletion(self): - """Waits for all of the running processes to finish""" - self._waitLoop() - - def getOutputs(self): - """Returns a dictionay containing the command as the key, and a string (of the output) as the value""" - outputs = dict((cmd, self._outputs[process.stdout].getvalue()) for cmd, process in self._processes.items()) - return outputs diff --git a/lib/priv_options.py b/lib/priv_options.py deleted file mode 100644 index f996c1b..0000000 --- a/lib/priv_options.py +++ /dev/null @@ -1,28 +0,0 @@ -#! /usr/bin/env python - -import os - -__all__ = ['load_options'] - -def load_options(): - FILENAME = os.path.expanduser('~/priv/credentials.conf') - options = {} - try: - options_fd = open(FILENAME, "r") - - for line in options_fd: - if line.startswith('#'): - continue - line = line.strip().split('=') - if len(line) < 2: - continue - - key = line[0] - value = '='.join(line[1:]) - - options[key] = value - - options_fd.close() - except: - pass - return options diff --git a/lib/twisted_wget.py b/lib/twisted_wget.py deleted file mode 100755 index aca0d55..0000000 --- a/lib/twisted_wget.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python2.4 - -import GregDebug, base64, os, sys, urlparse - -from twisted.internet import reactor, protocol -from twisted.web.client import HTTPClientFactory -from twisted.web.http import HTTPClient -from twisted.web.client import _parse as parseURL - -__all__ = ('downloadURL', ) - -def parseURL(url, defaultPort = None): - """Based on twisted.web.client._parse""" - parsed = urlparse.urlparse(url) - scheme = parsed[0] - path = urlparse.urlunparse(('','')+parsed[2:]) - if defaultPort is None: - if scheme == 'https': - defaultPort = 443 - else: - defaultPort = 80 - host, port = parsed[1], defaultPort - - if '@' in host: - authUser, host = host.split('@', 1) - auth = (authUser, ) - if ':' in authUser: - auth = tuple(authUser.split(':', 1)) - else: - auth = None - - if ':' in host: - host, port = host.rsplit(':', 1) - port = int(port) - - return scheme, auth, host, port, path - -class HTTPProxyFactory(protocol.ClientFactory): - def __init__(self, realFactory, proxyServer, proxyMethod = 'GET', proxyPassword = None): - self.realFactory = realFactory - self.proxyHost, self.proxyPort = proxyServer - self.proxyMethod = proxyMethod - self.proxyPassword = proxyPassword - - def buildProtocol(self, addr): - protocol = HTTPProxyProtocol(self, self.realFactory.buildProtocol(addr) ) - return protocol - - def __getattr__(self, key): - return getattr(self.realFactory, key) - -class HTTPProxyProtocol(protocol.Protocol): - def __init__(self, factory, proxied): - self.factory = factory - self.proxied = proxied - self.proxyPassword = factory.proxyPassword - if self.proxyPassword is not None: - self.proxyPassword = base64.standard_b64encode('%s:%s' % self.proxyPassword) - if factory.proxyMethod == 'GET': - self.__connectionMade = self.__connectionMade_GET - else: - raise NotImplementedError - - def __send(self, value): - self.transport.write(value) - - def __getTransportWrites(self, function, *args, **kwargs): - temp = self.transport.write - request = [] - self.transport.write = lambda data: request.append(data) - function(*args, **kwargs) - self.proxied.connectionMade() - self.transport.write = temp - return request - - def __connectionMade_GET(self): - self.factory.realFactory.path = self.factory.realFactory.url - self.proxied.makeConnection(self.transport) - - self.__send('GET %s HTTP/1.0\r\n' % self.factory.realFactory.url) - if self.proxyPassword is not None: - self.__send('Proxy-Authorization: Basic %s\r\n' % self.proxyPassword) - - # Remove the real http client's get request - for line in self.__getTransportWrites(self.proxied.connectionMade)[1:]: - self.__send(line) - - def connectionMade(self): - self.proxied.transport = self.transport - self.__connectionMade() - - def dataReceived(self, data): - self.proxied.dataReceived(data) - - def connectionLost(self, reason): - self.proxied.connectionLost(reason) - -proxies = {} -def downloadURL(url, method = 'GET', successBack = None, errorBack = None): - factory = HTTPClientFactory(url, method = method) - scheme, auth, host, port, path = parseURL(url) - if successBack is not None: - factory.deferred.addCallback(successBack) - if errorBack is not None: - factory.deferred.addErrback(errorBack) - if scheme in proxies: - (host, port), password, factory_type = proxies[scheme] - # Change the factory to the proxies one - factory = factory_type(realFactory = factory, proxyServer = (host, port), proxyMethod = method, proxyPassword = password) - - reactor.connectTCP(host, port, factory) - return factory - -# Note: Does not currently honor the no-proxy variable -def parseProxies(): - for k,v in ( (k,v) for k,v in os.environ.items() if v and k.endswith('_proxy')): - proxy_type = k[:-len('_proxy')] - if proxy_type == 'http': - _, auth, host, port, _ = parseURL(v) - proxies[proxy_type] = (host, port), auth, HTTPProxyFactory - -def main(urls): - def summerise(string, summerisedLen = 100): - if len(string) <= summerisedLen: - return string - else: - summerisedLen -= 5 - start = summerisedLen // 2 - return '%s ... %s' % (string[:start], string[-(summerisedLen - start):]) - - def s(data): - print 'Success: "%r"' % summerise(data) -### print 'factory: (\n\t%s\n)' % '\n\t'.join('%s:%s' % (attr, getattr(factory, attr)) for attr in dir(factory)) - - def e(data): - print data - - for url in urls: - factory = downloadURL(url, successBack = s, errorBack = e) - reactor.run() - -# Parse the environment variables for proxy servers -parseProxies() -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/lib/wget_lib.py b/lib/wget_lib.py deleted file mode 100644 index ce79d86..0000000 --- a/lib/wget_lib.py +++ /dev/null @@ -1,42 +0,0 @@ -#! python - -__all__ = ('parse_url', 'isImageURL', 'unique', 'removeDups', 'xRemoveDups') - -IMAGE_EXTENSIONS = ('PNG', 'JPG', 'JPEG', 'BMP', 'GIF', 'SWF', 'TIF', 'TIFF') - -def parse_url(url): - """Parses a url into a tuple of (hostname, directory, filename).""" - return ('hostname', 'directory', 'filename') - -def isImageURL(url): - """Checks if an filename is an image""" - try: - _, extension = url.rsplit('.', 1) - except ValueError: - # There was no '.' in the url - return False - else: - return extension.upper() in IMAGE_EXTENSIONS - -def unique(l): - list_iter = iter(l) - last_item = list_iter.next() - yield last_item - for item in list_iter: - if last_item != item: - yield item - last_item = item - -def removeDups(l): - """Removes duplicates from the list (Note: The ordering of the list may change)""" - return list(unique(sorted(l))) - -def xRemoveDups(l): - """Removes duplicates from the list. - Requires O(n) memory, objects must be hashable""" - yielded = set() - for elem in l: - if elem in yielded: - continue - yielded.add(elem) - yield elem diff --git a/lib/x11_helpers.py b/lib/x11_helpers.py deleted file mode 100644 index b479132..0000000 --- a/lib/x11_helpers.py +++ /dev/null @@ -1,38 +0,0 @@ -#! python -# Copyright 2007 Greg Darke -# Licensed for distribution under the GPL version 2, check COPYING for details -# Some little helper utils - -import subprocess, commands, itertools - -__all__ = ('getResolutions', ) - -def _seperateGroups(lines): - ret = [] - current_section = [] - for line in lines: - if line.strip() == '': - ret.append(current_section) - current_section = [] - continue - current_section.append(line) - if current_section: - ret.append(current_section) - return ret - -def getResolutions(): - xdpyinfo_status, xdpyinfo_output = commands.getstatusoutput('xdpyinfo') - lines = xdpyinfo_output.splitlines() - groups = _seperateGroups(xdpyinfo_output.splitlines()) - - screens = [] - for screen_data in itertools.islice(groups, 1, None, None): - _, screen_number = screen_data[0].split() - # remove the leading and trailing characters - screen_number = screen_number[1:-1] - - _, screen_resolution_str, _, _, _ = screen_data[1].strip().split() - screen_resolution = screen_resolution_str.split('x') - - screens.append( (screen_number, tuple(int(val) for val in screen_resolution))) - return dict(screens) -- 2.39.2