X-Git-Url: https://code.delx.au/webdl/blobdiff_plain/8c1040ccbc3e3dd49a0b38f7517181052bcf9f8e..4b32578817efa86e67d31f870c33b645c74934d3:/common.py diff --git a/common.py b/common.py index 94c0e2c..bfc49fc 100644 --- a/common.py +++ b/common.py @@ -24,7 +24,7 @@ try: except ImportError: pass -CACHE_DIR = os.path.expanduser("~/.cache/webdl") +CACHE_DIR = os.path.join(os.environ.get("XDG_CACHE_HOME", os.path.expanduser("~/.cache")), "webdl") USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:21.0) Gecko/20100101 Firefox/21.0" class Node(object): @@ -57,8 +57,8 @@ def load_root_node(): import sbs sbs.fill_nodes(root_node) - import plus7 - plus7.fill_nodes(root_node) +### import plus7 +### plus7.fill_nodes(root_node) import brightcove brightcove.fill_nodes(root_node) @@ -96,7 +96,7 @@ def urlopen(url, max_age): return open(filename) src = _urlopen(url) - dst = open(filename, "w") + dst = open(filename, "wb") try: shutil.copyfileobj(src, dst) except Exception, e: @@ -166,43 +166,86 @@ def exec_subprocess(cmd): return False -def convert_flv_mp4(orig_filename): - basename = os.path.splitext(orig_filename)[0] - flv_filename = basename + ".flv" - mp4_filename = basename + ".mp4" - if orig_filename != flv_filename: - os.rename(orig_filename, flv_filename) - print "Converting %s to mp4" % flv_filename - if not avconv_remux(flv_filename, mp4_filename): +def check_command_exists(cmd): + try: + subprocess.check_output(cmd) + return True + except Exception: + return False + +def generate_remux_cmd(infile, outfile): + if check_command_exists(["avconv", "--help"]): + return [ + "avconv", + "-i", infile, + "-bsf:a", "aac_adtstoasc", + "-acodec", "copy", + "-vcodec", "copy", + outfile, + ] + + if check_command_exists(["ffmpeg", "--help"]): + return [ + "ffmpeg", + "-i", infile, + "-bsf:a", "aac_adtstoasc", + "-acodec", "copy", + "-vcodec", "copy", + outfile, + ] + + raise Exception("You must install ffmpeg or libav-tools") + +def remux(infile, outfile): + print "Converting %s to mp4" % infile + cmd = generate_remux_cmd(infile, outfile) + if not exec_subprocess(cmd): # failed, error has already been logged - return + return False try: - flv_size = os.stat(flv_filename).st_size - mp4_size = os.stat(mp4_filename).st_size - if abs(flv_size - mp4_size) < 0.05 * flv_size: - os.unlink(flv_filename) + flv_size = os.stat(infile).st_size + mp4_size = os.stat(outfile).st_size + if abs(flv_size - mp4_size) < 0.1 * flv_size: + os.unlink(infile) + return True else: - print >>sys.stderr, "The size of", mp4_filename, "is suspicious, did avconv fail?" + print >>sys.stderr, "The size of", outfile, "is suspicious, did avconv fail?" + return False except Exception, e: - print "Conversion failed", e + print >>sys.stderr, "Conversion failed", e + return False -def avconv_remux(infile, outfile): +def convert_to_mp4(filename): + with open(filename) as f: + fourcc = f.read(4) + basename, ext = os.path.splitext(filename) + + if ext == ".mp4" and fourcc == "FLV\x01": + os.rename(filename, basename + ".flv") + ext = ".flv" + filename = basename + ext + + if ext in (".flv", ".ts"): + filename_mp4 = basename + ".mp4" + return remux(filename, filename_mp4) + + return ext == ".mp4" + + +def download_hds(filename, video_url, pvswf=None): + filename = sanify_filename(filename) + video_url = video_url.replace("http://", "hds://") + print "Downloading: %s" % filename cmd = [ - "avconv", - "-i", infile, - "-acodec", "copy", - "-vcodec", "copy", - outfile, + "livestreamer", + "-o", filename, + "%s pvswf=%s" % (video_url, pvswf), + "best", ] - return exec_subprocess(cmd) - -def convert_filename(filename): - if os.path.splitext(filename.lower())[1] in (".mp4", ".flv"): - f = open(filename) - fourcc = f.read(4) - f.close() - if fourcc == "FLV\x01": - convert_flv_mp4(filename) + if exec_subprocess(cmd): + return convert_to_mp4(filename) + else: + return False def download_rtmp(filename, vbase, vpath, hash_url=None): filename = sanify_filename(filename) @@ -218,44 +261,11 @@ def download_rtmp(filename, vbase, vpath, hash_url=None): if hash_url is not None: cmd += ["--swfVfy", hash_url] if exec_subprocess(cmd): - convert_filename(filename) - return True + return convert_to_mp4(filename) else: return False -def download_urllib(filename, url, referrer=None): - filename = sanify_filename(filename) - print "Downloading: %s" % filename - try: - src = _urlopen(url, referrer) - dst = open(filename, "w") - while True: - buf = src.read(1024*1024) - if not buf: - break - dst.write(buf) - sys.stdout.write(".") - sys.stdout.flush() - print - except KeyboardInterrupt: - print "\nCancelled", url - return False - finally: - try: - src.close() - except: - pass - try: - dst.close() - except: - pass - - convert_filename(filename) - return True - -def download_hls_get_stream(url, hack_url_func): - url = hack_url_func(url) - +def download_hls_get_stream(url): def parse_bandwidth(line): params = line.split(":", 1)[1].split(",") for kv in params: @@ -282,69 +292,63 @@ def download_hls_get_stream(url, hack_url_func): return best_url -def download_hls_segments(tmpdir, url, hack_url_func): +def download_hls_segments(outf, url): m3u8 = grab_text(url, 0) - result = [] - - local_m3u8_filename = tmpdir + "/index.m3u8" - local_m3u8 = open(local_m3u8_filename, "w") - i = 1 + fail_if_not_last_segment = None for line in m3u8.split("\n"): - if not line.strip(): - continue - if line.startswith("#"): - local_m3u8.write(line + "\n") + if not line.strip() or line.startswith("#"): continue - outfile = "%s/segment_%d.ts" % (tmpdir, i) - i += 1 - local_m3u8.write(outfile + "\n") - download_hls_fetch_segment(line, outfile) + if fail_if_not_last_segment: + raise e + + try: + download_hls_fetch_segment(outf, line) + except urllib2.HTTPError, e: + fail_if_not_last_segment = e + continue sys.stdout.write(".") sys.stdout.flush() sys.stdout.write("\n") - local_m3u8.close() - return local_m3u8_filename - -def download_hls_fetch_segment(segment, outfile): +def download_hls_fetch_segment(outf, segment_url): try: - src = _urlopen(segment) - dst = open(outfile, "w") - shutil.copyfileobj(src, dst) + src = _urlopen(segment_url) + shutil.copyfileobj(src, outf) + except: + raise finally: try: src.close() except: pass - try: - dst.close() - except: - pass def download_hls(filename, m3u8_master_url, hack_url_func=None): + filename = sanify_filename(filename) + print "Downloading: %s" % filename + if hack_url_func is None: hack_url_func = lambda url: url tmpdir = tempfile.mkdtemp(prefix="webdl-hls") - filename = sanify_filename(filename) - - print "Downloading: %s" % filename try: - best_stream_url = download_hls_get_stream(m3u8_master_url, hack_url_func) - local_m3u8 = download_hls_segments(tmpdir, best_stream_url, hack_url_func) - avconv_remux(local_m3u8, filename) - return False + best_stream_url = download_hls_get_stream(hack_url_func(m3u8_master_url)) + ts_file = open(filename, "wb") + download_hls_segments(ts_file, hack_url_func(best_stream_url)) except KeyboardInterrupt: print "\nCancelled", m3u8_master_url return False finally: shutil.rmtree(tmpdir) + try: + ts_file.close() + except: + pass - return True + return convert_to_mp4(filename) def natural_sort(l, key=None): ignore_list = ["a", "the"]