code.delx.au - webdl/blob - autograbber.py

   1 #!/usr/bin/env python
   2 # vim:ts=4:sts=4:sw=4:noet
   3
   4 from common import load_root_node
   5 import fnmatch
   6 import os
   7 import sys
   8
   9 class DownloadList(object):
  10         def __init__(self, filename):
  11                 self.seen_list = set()
  12                 try:
  13                         self.f = open(filename, "r")
  14                         for line in self.f:
  15                                 self.seen_list.add(line.decode("utf-8").strip())
  16                         self.f.close()
  17                 except Exception, e:
  18                         print >>sys.stderr, "Could not open:", filename, e
  19                 self.f = open(filename, "a")
  20
  21         def has_seen(self, node):
  22                 return node.title in self.seen_list
  23
  24         def mark_seen(self, node):
  25                 self.seen_list.add(node.title)
  26                 self.f.write(node.title.encode("utf-8") + "\n")
  27                 self.f.flush()
  28
  29
  30 def match(download_list, node, pattern, count=0):
  31         if node.can_download:
  32                 if not download_list.has_seen(node):
  33                         if node.download():
  34                                 download_list.mark_seen(node)
  35                         else:
  36                                 print >>sys.stderr, "Failed to download!", node.title
  37                 return
  38
  39         if count >= len(pattern):
  40                 print "No match found for pattern:", "/".join(pattern)
  41                 return
  42         p = pattern[count]
  43         for child in node.get_children():
  44                 if fnmatch.fnmatch(child.title, p):
  45                         match(download_list, child, pattern, count+1)
  46
  47
  48 def main(destdir, patternfile):
  49         os.chdir(destdir)
  50         node = load_root_node()
  51         download_list = DownloadList("downloaded_auto.txt")
  52
  53         for line in open(patternfile):
  54                 search = line.strip().split("/")
  55                 match(download_list, node, search)
  56
  57 if __name__ == "__main__":
  58         try:
  59                 destdir = os.path.abspath(sys.argv[1])
  60                 patternfile = os.path.abspath(sys.argv[2])
  61         except IndexError:
  62                 print >>sys.stderr, "Usage: %s destdir patternfile" % sys.argv[0]
  63                 sys.exit(1)
  64         try:
  65                 main(destdir, patternfile)
  66         except (KeyboardInterrupt, EOFError):
  67                 print "\nExiting..."
  68