# vim:ts=4:sts=4:sw=4:noet
-from lxml import etree
+from lxml import etree, html
import json
try:
import hashlib
self.children = []
self.can_download = False
+ def get_children(self):
+ if not self.children:
+ self.fill_children()
+ return self.children
+
+ def fill_children(self):
+ pass
+
def download(self):
raise NotImplemented
def load_root_node():
root_node = Node("Root")
- print "Loading iView episode data...",
- sys.stdout.flush()
import iview
- iview_node = Node("ABC iView", root_node)
- iview.fill_nodes(iview_node)
- print "done"
+ iview.fill_nodes(root_node)
- print "Loading SBS episode data...",
- sys.stdout.flush()
import sbs
- sbs_node = Node("SBS", root_node)
- sbs.fill_nodes(sbs_node)
- print "done"
+ sbs.fill_nodes(root_node)
return root_node
+valid_chars = frozenset("-_.() abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
+def sanify_filename(filename):
+ filename = filename.encode("ascii", "ignore")
+ filename = "".join(c for c in filename if c in valid_chars)
+ return filename
+
def urlopen(url, max_age):
+### print url
if not os.path.isdir(CACHE_DIR):
os.makedirs(CACHE_DIR)
src = urllib.urlopen(url)
dst = open(filename, "w")
- shutil.copyfileobj(src, dst)
+ try:
+ shutil.copyfileobj(src, dst)
+ except Exception, e:
+ try:
+ os.unlink(filename)
+ except OSError:
+ pass
+ raise e
src.close()
dst.close()
return open(filename)
+def grab_html(url, max_age):
+ f = urlopen(url, max_age)
+ doc = html.parse(f)
+ f.close()
+ return doc
+
def grab_xml(url, max_age):
f = urlopen(url, max_age)
doc = etree.parse(f)
f.close()
return doc
-def download_rtmp(filename, vbase, vpath, hash_url=None):
- if vpath.endswith(".flv"):
- vpath = vpath[:-4]
- cmd = [
- "rtmpdump",
- "-o", filename,
- "-r", vbase,
- "-y", vpath,
- ]
- if hash_url is not None:
- cmd += ["--swfVfy", hash_url]
+def exec_subprocess(cmd):
try:
p = subprocess.Popen(cmd)
ret = p.wait()
if ret != 0:
- print >>sys.stderr, "rtmpdump exited with error code:", ret
+ print >>sys.stderr, cmd[0], "exited with error code:", ret
return False
else:
return True
except OSError, e:
- print >>sys.stderr, "Failed to run rtmpdump!", e
- return False
+ print >>sys.stderr, "Failed to run", cmd[0], e
except KeyboardInterrupt:
print "Cancelled", cmd
try:
except KeyboardInterrupt:
p.send_signal(signal.SIGKILL)
p.wait()
+ return False
+
+
+def convert_flv_mp4(orig_filename):
+ basename = os.path.splitext(orig_filename)[0]
+ flv_filename = basename + ".flv"
+ mp4_filename = basename + ".mp4"
+ os.rename(orig_filename, flv_filename)
+ print "Converting %s to mp4" % flv_filename
+ cmd = [
+ "ffmpeg",
+ "-i", flv_filename,
+ "-acodec", "copy",
+ "-vcodec", "copy",
+ mp4_filename,
+ ]
+ if not exec_subprocess(cmd):
+ return
+ try:
+ flv_size = os.stat(flv_filename).st_size
+ mp4_size = os.stat(mp4_filename).st_size
+ if abs(flv_size - mp4_size) < 0.05 * flv_size:
+ os.unlink(flv_filename)
+ else:
+ print >>sys.stderr, "The size of", mp4_filename, "is suspicious, did ffmpeg fail?"
+ except Exception, e:
+ print "Conversion failed", e
+
+def convert_filename(filename):
+ if filename.lower().endswith(".mp4"):
+ f = open(filename)
+ fourcc = f.read(4)
+ f.close()
+ if fourcc == "FLV\x01":
+ convert_flv_mp4(filename)
+
+def download_rtmp(filename, vbase, vpath, hash_url=None):
+ filename = sanify_filename(filename)
+ print "Downloading: %s" % filename
+ if vpath.endswith(".flv"):
+ vpath = vpath[:-4]
+ cmd = [
+ "rtmpdump",
+ "-o", filename,
+ "-r", vbase,
+ "-y", vpath,
+ ]
+ if hash_url is not None:
+ cmd += ["--swfVfy", hash_url]
+ success = exec_subprocess(cmd)
+ convert_filename(filename)
+ return success
def download_urllib(filename, url):
- print "Downloading: %s -> %s" % (url, filename)
+ filename = sanify_filename(filename)
+ print "Downloading: %s" % filename
try:
src = urllib.urlopen(url)
dst = open(filename, "w")
- shutil.copyfileobj(src, dst)
+ while True:
+ buf = src.read(1024*1024)
+ if not buf:
+ break
+ dst.write(buf)
+ sys.stdout.write(".")
+ sys.stdout.flush()
+ convert_filename(filename)
return True
except KeyboardInterrupt:
print "\nCancelled", url
finally:
- src.close()
- dst.close()
+ try:
+ src.close()
+ except:
+ pass
+ try:
+ dst.close()
+ except:
+ pass
return False