except ImportError:
pass
-CACHE_DIR = os.path.expanduser("~/.cache/webdl")
+CACHE_DIR = os.path.join(os.environ.get("XDG_CACHE_HOME", os.path.expanduser("~/.cache")), "webdl")
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:21.0) Gecko/20100101 Firefox/21.0"
class Node(object):
import sbs
sbs.fill_nodes(root_node)
- import plus7
- plus7.fill_nodes(root_node)
+### import plus7
+### plus7.fill_nodes(root_node)
import brightcove
brightcove.fill_nodes(root_node)
return open(filename)
src = _urlopen(url)
- dst = open(filename, "w")
+ dst = open(filename, "wb")
try:
shutil.copyfileobj(src, dst)
except Exception, e:
return False
-def convert_flv_mp4(orig_filename):
- basename = os.path.splitext(orig_filename)[0]
- flv_filename = basename + ".flv"
- mp4_filename = basename + ".mp4"
- if orig_filename != flv_filename:
- os.rename(orig_filename, flv_filename)
- print "Converting %s to mp4" % flv_filename
- if not avconv_remux(flv_filename, mp4_filename):
+def check_command_exists(cmd):
+ try:
+ subprocess.check_output(cmd)
+ return True
+ except Exception:
+ return False
+
+def generate_remux_cmd(infile, outfile):
+ if check_command_exists(["avconv", "--help"]):
+ return [
+ "avconv",
+ "-i", infile,
+ "-bsf:a", "aac_adtstoasc",
+ "-acodec", "copy",
+ "-vcodec", "copy",
+ outfile,
+ ]
+
+ if check_command_exists(["ffmpeg", "--help"]):
+ return [
+ "ffmpeg",
+ "-i", infile,
+ "-bsf:a", "aac_adtstoasc",
+ "-acodec", "copy",
+ "-vcodec", "copy",
+ outfile,
+ ]
+
+ raise Exception("You must install ffmpeg or libav-tools")
+
+def remux(infile, outfile):
+ print "Converting %s to mp4" % infile
+ cmd = generate_remux_cmd(infile, outfile)
+ if not exec_subprocess(cmd):
# failed, error has already been logged
- return
+ return False
try:
- flv_size = os.stat(flv_filename).st_size
- mp4_size = os.stat(mp4_filename).st_size
- if abs(flv_size - mp4_size) < 0.05 * flv_size:
- os.unlink(flv_filename)
+ flv_size = os.stat(infile).st_size
+ mp4_size = os.stat(outfile).st_size
+ if abs(flv_size - mp4_size) < 0.1 * flv_size:
+ os.unlink(infile)
+ return True
else:
- print >>sys.stderr, "The size of", mp4_filename, "is suspicious, did avconv fail?"
+ print >>sys.stderr, "The size of", outfile, "is suspicious, did avconv fail?"
+ return False
except Exception, e:
- print "Conversion failed", e
+ print >>sys.stderr, "Conversion failed", e
+ return False
-def avconv_remux(infile, outfile):
+def convert_to_mp4(filename):
+ with open(filename) as f:
+ fourcc = f.read(4)
+ basename, ext = os.path.splitext(filename)
+
+ if ext == ".mp4" and fourcc == "FLV\x01":
+ os.rename(filename, basename + ".flv")
+ ext = ".flv"
+ filename = basename + ext
+
+ if ext in (".flv", ".ts"):
+ filename_mp4 = basename + ".mp4"
+ return remux(filename, filename_mp4)
+
+ return ext == ".mp4"
+
+
+def download_hds(filename, video_url, pvswf=None):
+ filename = sanify_filename(filename)
+ video_url = video_url.replace("http://", "hds://")
+ print "Downloading: %s" % filename
cmd = [
- "avconv",
- "-i", infile,
- "-acodec", "copy",
- "-vcodec", "copy",
- outfile,
+ "livestreamer",
+ "-o", filename,
+ "%s pvswf=%s" % (video_url, pvswf),
+ "best",
]
- return exec_subprocess(cmd)
+ if exec_subprocess(cmd):
+ return convert_to_mp4(filename)
+ else:
+ return False
-def convert_filename(filename):
- if os.path.splitext(filename.lower())[1] in (".mp4", ".flv"):
- f = open(filename)
- fourcc = f.read(4)
- f.close()
- if fourcc == "FLV\x01":
- convert_flv_mp4(filename)
+def download_hls(filename, video_url):
+ filename = sanify_filename(filename)
+ video_url = video_url.replace("http://", "hlsvariant://")
+ print "Downloading: %s" % filename
+ cmd = [
+ "livestreamer",
+ "-o", filename,
+ video_url,
+ "best",
+ ]
+ if exec_subprocess(cmd):
+ return convert_to_mp4(filename)
+ else:
+ return False
def download_rtmp(filename, vbase, vpath, hash_url=None):
filename = sanify_filename(filename)
if hash_url is not None:
cmd += ["--swfVfy", hash_url]
if exec_subprocess(cmd):
- convert_filename(filename)
- return True
+ return convert_to_mp4(filename)
else:
return False
-def download_urllib(filename, url, referrer=None):
- filename = sanify_filename(filename)
- print "Downloading: %s" % filename
- try:
- src = _urlopen(url, referrer)
- dst = open(filename, "w")
- while True:
- buf = src.read(1024*1024)
- if not buf:
- break
- dst.write(buf)
- sys.stdout.write(".")
- sys.stdout.flush()
- print
- except KeyboardInterrupt:
- print "\nCancelled", url
- return False
- finally:
- try:
- src.close()
- except:
- pass
- try:
- dst.close()
- except:
- pass
-
- convert_filename(filename)
- return True
-
-def download_hls_get_stream(url, hack_url_func):
- url = hack_url_func(url)
-
- def parse_bandwidth(line):
- params = line.split(":", 1)[1].split(",")
- for kv in params:
- k, v = kv.split("=", 1)
- if k == "BANDWIDTH":
- return int(v)
- return 0
-
- m3u8 = grab_text(url, 0)
- best_bandwidth = None
- best_url = None
- for line in m3u8.split("\n"):
- if line.startswith("#EXT-X-STREAM-INF:"):
- bandwidth = parse_bandwidth(line)
- if best_bandwidth is None or bandwidth > best_bandwidth:
- best_bandwidth = bandwidth
- best_url = None
- elif not line.startswith("#"):
- if best_url is None:
- best_url = line.strip()
-
- if not best_url:
- raise Exception("Failed to find best stream for HLS: " + url)
-
- return best_url
-
-def download_hls_segments(tmpdir, url, hack_url_func):
- m3u8 = grab_text(url, 0)
- result = []
-
- local_m3u8_filename = tmpdir + "/index.m3u8"
- local_m3u8 = open(local_m3u8_filename, "w")
-
- i = 1
- fail_if_not_last_segment = None
- for line in m3u8.split("\n"):
- if not line.strip():
- continue
- if line.startswith("#"):
- local_m3u8.write(line + "\n")
- continue
-
- if fail_if_not_last_segment:
- raise e
-
- outfile = "%s/segment_%d.ts" % (tmpdir, i)
- i += 1
- try:
- download_hls_fetch_segment(hack_url_func(line), outfile)
- except urllib2.HTTPError, e:
- fail_if_not_last_segment = e
- continue
- local_m3u8.write(outfile + "\n")
- sys.stdout.write(".")
- sys.stdout.flush()
-
- sys.stdout.write("\n")
-
- local_m3u8.close()
- return local_m3u8_filename
-
-def download_hls_fetch_segment(segment, outfile):
- try:
- src = _urlopen(segment)
- dst = open(outfile, "w")
- shutil.copyfileobj(src, dst)
- except:
- print >>sys.stderr, "Failed to fetch", segment
- raise
- finally:
- try:
- src.close()
- except:
- pass
- try:
- dst.close()
- except:
- pass
-
-def download_hls(filename, m3u8_master_url, hack_url_func=None):
- if hack_url_func is None:
- hack_url_func = lambda url: url
-
- tmpdir = tempfile.mkdtemp(prefix="webdl-hls")
- filename = sanify_filename(filename)
-
- print "Downloading: %s" % filename
-
- try:
- best_stream_url = download_hls_get_stream(m3u8_master_url, hack_url_func)
- local_m3u8 = download_hls_segments(tmpdir, best_stream_url, hack_url_func)
- avconv_remux(local_m3u8, filename)
- return True
- except KeyboardInterrupt:
- print "\nCancelled", m3u8_master_url
- return False
- finally:
- shutil.rmtree(tmpdir)
-
def natural_sort(l, key=None):
ignore_list = ["a", "the"]
def key_func(k):