Small cleanup

[webdl] / common.py

diff --git a/common.py b/common.py
index 4b89660d03941c4a32caa437f385dbe8401068e6..70f9726162cae78c8f890d64163f181211f2f283 100644 (file)
--- a/common.py
+++ b/common.py
@@ -1,6 +1,6 @@
 # vim:ts=4:sts=4:sw=4:noet
 
-from lxml import etree
+from lxml import etree, html
 import json
 try:
        import hashlib
@@ -31,8 +31,13 @@ class Node(object):
                self.can_download = False
 
        def get_children(self):
+               if not self.children:
+                       self.fill_children()
                return self.children
 
+       def fill_children(self):
+               pass
+
        def download(self):
                raise NotImplemented
 
@@ -85,6 +90,12 @@ def urlopen(url, max_age):
 
        return open(filename)
 
+def grab_html(url, max_age):
+       f = urlopen(url, max_age)
+       doc = html.parse(f)
+       f.close()
+       return doc
+
 def grab_xml(url, max_age):
        f = urlopen(url, max_age)
        doc = etree.parse(f)