code.delx.au - webdl/blob - autograbber.py

   1 #!/usr/bin/env python
   2
   3 from common import load_root_node
   4 import fnmatch
   5 import logging
   6 import os
   7 import sys
   8
   9 DOWNLOAD_HISTORY_FILES = [
  10     ".downloaded_auto.txt",
  11     "downloaded_auto.txt",
  12 ]
  13
  14 class DownloadList(object):
  15     def __init__(self):
  16         self.seen_list = set()
  17         for filename in DOWNLOAD_HISTORY_FILES:
  18             if os.path.isfile(filename):
  19                 break
  20         else:
  21             filename = DOWNLOAD_HISTORY_FILES[0]
  22         try:
  23             self.f = open(filename, "r")
  24             for line in self.f:
  25                 self.seen_list.add(line.strip())
  26             self.f.close()
  27         except Exception as e:
  28             logging.error("Could not open: %s -- %s", filename, e)
  29         self.f = open(filename, "a")
  30
  31     def has_seen(self, node):
  32         return node.title in self.seen_list
  33
  34     def mark_seen(self, node):
  35         self.seen_list.add(node.title)
  36         self.f.write(node.title + "\n")
  37         self.f.flush()
  38
  39
  40 def match(download_list, node, pattern, count=0):
  41     if node.can_download:
  42         if not download_list.has_seen(node):
  43             if node.download():
  44                 download_list.mark_seen(node)
  45             else:
  46                 logging.error("Failed to download! %s", node.title)
  47         return
  48
  49     if count >= len(pattern):
  50         logging.error("No match found for pattern:", "/".join(pattern))
  51         return
  52     p = pattern[count]
  53     for child in node.get_children():
  54         if fnmatch.fnmatch(child.title, p):
  55             match(download_list, child, pattern, count+1)
  56
  57
  58 def main(destdir, patternfile):
  59     os.chdir(destdir)
  60     node = load_root_node()
  61     download_list = DownloadList()
  62
  63     for line in open(patternfile):
  64         search = line.strip().split("/")
  65         match(download_list, node, search)
  66
  67 if __name__ == "__main__":
  68     try:
  69         destdir = os.path.abspath(sys.argv[1])
  70         patternfile = os.path.abspath(sys.argv[2])
  71     except IndexError:
  72         print("Usage: %s destdir patternfile" % sys.argv[0])
  73         sys.exit(1)
  74     try:
  75         main(destdir, patternfile)
  76     except (KeyboardInterrupt, EOFError):
  77         print("\nExiting...")
  78