Small cleanup

[webdl] / common.py

diff --git a/common.py b/common.py
index 26f8346c6c3dc35ed3267ecbb5f6064886f8e3ac..70f9726162cae78c8f890d64163f181211f2f283 100644 (file)
--- a/common.py
+++ b/common.py
@@ -1,6 +1,6 @@
 # vim:ts=4:sts=4:sw=4:noet
 
-from lxml import etree
+from lxml import etree, html
 import json
 try:
        import hashlib
@@ -31,8 +31,13 @@ class Node(object):
                self.can_download = False
 
        def get_children(self):
+               if not self.children:
+                       self.fill_children()
                return self.children
 
+       def fill_children(self):
+               pass
+
        def download(self):
                raise NotImplemented
 
@@ -85,6 +90,12 @@ def urlopen(url, max_age):
 
        return open(filename)
 
+def grab_html(url, max_age):
+       f = urlopen(url, max_age)
+       doc = html.parse(f)
+       f.close()
+       return doc
+
 def grab_xml(url, max_age):
        f = urlopen(url, max_age)
        doc = etree.parse(f)
@@ -97,30 +108,17 @@ def grab_json(url, max_age):
        f.close()
        return doc
 
-def download_rtmp(filename, vbase, vpath, hash_url=None):
-       filename = sanify_filename(filename)
-       print "Downloading: %s" % filename
-       if vpath.endswith(".flv"):
-               vpath = vpath[:-4]
-       cmd = [
-               "rtmpdump",
-               "-o", filename,
-               "-r", vbase,
-               "-y", vpath,
-       ]
-       if hash_url is not None:
-               cmd += ["--swfVfy", hash_url]
+def exec_subprocess(cmd):
        try:
                p = subprocess.Popen(cmd)
                ret = p.wait()
                if ret != 0:
-                       print >>sys.stderr, "rtmpdump exited with error code:", ret
+                       print >>sys.stderr, cmd[0], "exited with error code:", ret
                        return False
                else:
                        return True
        except OSError, e:
-               print >>sys.stderr, "Failed to run rtmpdump!", e
-               return False
+               print >>sys.stderr, "Failed to run", cmd[0], e
        except KeyboardInterrupt:
                print "Cancelled", cmd
                try:
@@ -129,6 +127,58 @@ def download_rtmp(filename, vbase, vpath, hash_url=None):
                except KeyboardInterrupt:
                        p.send_signal(signal.SIGKILL)
                        p.wait()
+       return False
+
+
+def convert_flv_mp4(orig_filename):
+       basename = os.path.splitext(orig_filename)[0]
+       flv_filename = basename + ".flv"
+       mp4_filename = basename + ".mp4"
+       os.rename(orig_filename, flv_filename)
+       print "Converting %s to mp4" % flv_filename
+       cmd = [
+               "ffmpeg",
+               "-i", flv_filename,
+               "-acodec", "copy",
+               "-vcodec", "copy",
+               mp4_filename,
+       ]
+       if not exec_subprocess(cmd):
+               return
+       try:
+               flv_size = os.stat(flv_filename).st_size
+               mp4_size = os.stat(mp4_filename).st_size
+               if abs(flv_size - mp4_size) < 0.05 * flv_size:
+                       os.unlink(flv_filename)
+               else:
+                       print >>sys.stderr, "The size of", mp4_filename, "is suspicious, did ffmpeg fail?"
+       except Exception, e:
+               print "Conversion failed", e
+
+def convert_filename(filename):
+       if filename.lower().endswith(".mp4"):
+               f = open(filename)
+               fourcc = f.read(4)
+               f.close()
+               if fourcc == "FLV\x01":
+                       convert_flv_mp4(filename)
+
+def download_rtmp(filename, vbase, vpath, hash_url=None):
+       filename = sanify_filename(filename)
+       print "Downloading: %s" % filename
+       if vpath.endswith(".flv"):
+               vpath = vpath[:-4]
+       cmd = [
+               "rtmpdump",
+               "-o", filename,
+               "-r", vbase,
+               "-y", vpath,
+       ]
+       if hash_url is not None:
+               cmd += ["--swfVfy", hash_url]
+       success = exec_subprocess(cmd)
+       convert_filename(filename)
+       return success
 
 def download_urllib(filename, url):
        filename = sanify_filename(filename)
@@ -143,6 +193,7 @@ def download_urllib(filename, url):
                        dst.write(buf)
                        sys.stdout.write(".")
                        sys.stdout.flush()
+               convert_filename(filename)
                return True
        except KeyboardInterrupt:
                print "\nCancelled", url