X-Git-Url: https://code.delx.au/webdl/blobdiff_plain/4ea7cd459da04dbde5f5cc1a26b3b343ab35cdce..0abf553c080a19c491103e3198f8b7c1fa0ce09c:/common.py diff --git a/common.py b/common.py index f51ec53..fb1cf16 100644 --- a/common.py +++ b/common.py @@ -1,21 +1,19 @@ -from lxml import etree, html -import cookielib +import python2_compat + +import hashlib +import http.cookiejar import json -try: - import hashlib -except ImportError: - import md5 as hashlib +import logging +import lxml.etree +import lxml.html import os import re import shutil import signal import subprocess -import sys -import tempfile import time -import urllib -import urllib2 -import urlparse +import urllib.parse +import urllib.request try: @@ -24,9 +22,20 @@ try: except ImportError: pass -CACHE_DIR = os.path.expanduser("~/.cache/webdl") + +logging.basicConfig( + format = "%(levelname)s %(message)s", + level = logging.INFO if os.environ.get("DEBUG", None) is None else logging.DEBUG, +) + +CACHE_DIR = os.path.join( + os.environ.get("XDG_CACHE_HOME", os.path.expanduser("~/.cache")), + "webdl" +) + USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:21.0) Gecko/20100101 Firefox/21.0" + class Node(object): def __init__(self, title, parent=None): self.title = title @@ -57,9 +66,6 @@ def load_root_node(): import sbs sbs.fill_nodes(root_node) - import plus7 - plus7.fill_nodes(root_node) - import brightcove brightcove.fill_nodes(root_node) @@ -67,39 +73,51 @@ def load_root_node(): valid_chars = frozenset("-_.()!@#%^ abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") def sanify_filename(filename): - filename = filename.encode("ascii", "ignore") filename = "".join(c for c in filename if c in valid_chars) + assert len(filename) > 0 return filename -cookiejar = cookielib.CookieJar() -urlopener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) +def ensure_scheme(url): + parts = urllib.parse.urlparse(url) + if parts.scheme: + return url + parts = list(parts) + parts[0] = "http" + return urllib.parse.urlunparse(parts) + +cookiejar = http.cookiejar.CookieJar() +urlopener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookiejar)) def _urlopen(url, referrer=None): - req = urllib2.Request(url) + url = ensure_scheme(url) + req = urllib.request.Request(url) req.add_header("User-Agent", USER_AGENT) if referrer: req.add_header("Referer", referrer) return urlopener.open(req) def urlopen(url, max_age): -### print url + logging.debug("urlopen(%r, %r)", url, max_age) + if not os.path.isdir(CACHE_DIR): os.makedirs(CACHE_DIR) if max_age <= 0: return _urlopen(url) - filename = hashlib.md5(url).hexdigest() + filename = hashlib.md5(url.encode("utf-8")).hexdigest() filename = os.path.join(CACHE_DIR, filename) if os.path.exists(filename): file_age = int(time.time()) - os.path.getmtime(filename) if file_age < max_age: - return open(filename) + logging.debug("loading from cache: %s", filename) + return open(filename, "rb") + logging.debug("downloading: %s -> %s", url, filename) src = _urlopen(url) - dst = open(filename, "w") + dst = open(filename, "wb") try: shutil.copyfileobj(src, dst) - except Exception, e: + except Exception as e: try: os.unlink(filename) except OSError: @@ -108,7 +126,7 @@ def urlopen(url, max_age): src.close() dst.close() - return open(filename) + return open(filename, "rb") def grab_text(url, max_age): f = urlopen(url, max_age) @@ -118,45 +136,47 @@ def grab_text(url, max_age): def grab_html(url, max_age): f = urlopen(url, max_age) - doc = html.parse(f, html.HTMLParser(encoding="utf-8", recover=True)) + doc = lxml.html.parse(f, lxml.html.HTMLParser(encoding="utf-8", recover=True)) f.close() return doc def grab_xml(url, max_age): f = urlopen(url, max_age) - doc = etree.parse(f, etree.XMLParser(encoding="utf-8", recover=True)) + doc = lxml.etree.parse(f, lxml.etree.XMLParser(encoding="utf-8", recover=True)) f.close() return doc def grab_json(url, max_age, skip_assignment=False, skip_function=False): f = urlopen(url, max_age) + text = f.read().decode("utf-8") + if skip_assignment: - text = f.read() pos = text.find("=") - doc = json.loads(text[pos+1:]) + text = text[pos+1:] + elif skip_function: - text = f.read() pos = text.find("(") rpos = text.rfind(")") - doc = json.loads(text[pos+1:rpos]) - else: - doc = json.load(f) + text = text[pos+1:rpos] + + doc = json.loads(text) f.close() return doc def exec_subprocess(cmd): + logging.debug("Executing: %s", cmd) try: p = subprocess.Popen(cmd) ret = p.wait() if ret != 0: - print >>sys.stderr, cmd[0], "exited with error code:", ret + logging.error("%s exited with error code: %s", cmd[0], ret) return False else: return True - except OSError, e: - print >>sys.stderr, "Failed to run", cmd[0], e + except OSError as e: + logging.error("Failed to run: %s -- %s", cmd[0], e) except KeyboardInterrupt: - print "Cancelled", cmd + logging.info("Cancelled: %s", cmd) try: p.terminate() p.wait() @@ -166,88 +186,123 @@ def exec_subprocess(cmd): return False -def convert_flv_mp4(orig_filename): - basename = os.path.splitext(orig_filename)[0] - flv_filename = basename + ".flv" - mp4_filename = basename + ".mp4" - if orig_filename != flv_filename: - os.rename(orig_filename, flv_filename) - print "Converting %s to mp4" % flv_filename - cmd = [ - "avconv", - "-i", flv_filename, - "-acodec", "copy", - "-vcodec", "copy", - mp4_filename, - ] +def check_command_exists(cmd): + try: + subprocess.check_output(cmd) + return True + except Exception: + return False + +def generate_remux_cmd(infile, outfile): + if check_command_exists(["avconv", "--help"]): + return [ + "avconv", + "-i", infile, + "-bsf:a", "aac_adtstoasc", + "-acodec", "copy", + "-vcodec", "copy", + outfile, + ] + + if check_command_exists(["ffmpeg", "--help"]): + return [ + "ffmpeg", + "-i", infile, + "-bsf:a", "aac_adtstoasc", + "-acodec", "copy", + "-vcodec", "copy", + outfile, + ] + + raise Exception("You must install ffmpeg or libav-tools") + +def remux(infile, outfile): + logging.info("Converting %s to mp4", infile) + cmd = generate_remux_cmd(infile, outfile) if not exec_subprocess(cmd): - return + # failed, error has already been logged + return False try: - flv_size = os.stat(flv_filename).st_size - mp4_size = os.stat(mp4_filename).st_size - if abs(flv_size - mp4_size) < 0.05 * flv_size: - os.unlink(flv_filename) + flv_size = os.stat(infile).st_size + mp4_size = os.stat(outfile).st_size + if abs(flv_size - mp4_size) < 0.1 * flv_size: + os.unlink(infile) + return True else: - print >>sys.stderr, "The size of", mp4_filename, "is suspicious, did avconv fail?" - except Exception, e: - print "Conversion failed", e + logging.error("The size of %s is suspicious, did the remux fail?", outfile) + return False + except Exception as e: + logging.error("Conversion failed! %s", e) + return False -def convert_filename(filename): - if os.path.splitext(filename.lower())[1] in (".mp4", ".flv"): - f = open(filename) +def convert_to_mp4(filename): + with open(filename, "rb") as f: fourcc = f.read(4) - f.close() - if fourcc == "FLV\x01": - convert_flv_mp4(filename) + basename, ext = os.path.splitext(filename) + + if ext == ".mp4" and fourcc == b"FLV\x01": + os.rename(filename, basename + ".flv") + ext = ".flv" + filename = basename + ext + + if ext in (".flv", ".ts"): + filename_mp4 = basename + ".mp4" + return remux(filename, filename_mp4) -def download_rtmp(filename, vbase, vpath, hash_url=None): + return ext == ".mp4" + + +def download_hds(filename, video_url, pvswf=None): filename = sanify_filename(filename) - print "Downloading: %s" % filename - if vpath.endswith(".flv"): - vpath = vpath[:-4] + logging.info("Downloading: %s", filename) + + video_url = "hds://" + video_url + if pvswf: + param = "%s pvswf=%s" % (video_url, pvswf) + else: + param = video_url + cmd = [ - "rtmpdump", + "livestreamer", "-o", filename, - "-r", vbase, - "-y", vpath, + param, + "best", ] - if hash_url is not None: - cmd += ["--swfVfy", hash_url] if exec_subprocess(cmd): - convert_filename(filename) - return True + return convert_to_mp4(filename) else: return False -def download_urllib(filename, url, referrer=None): +def download_hls(filename, video_url): filename = sanify_filename(filename) - print "Downloading: %s" % filename - try: - src = _urlopen(url, referrer) - dst = open(filename, "w") - while True: - buf = src.read(1024*1024) - if not buf: - break - dst.write(buf) - sys.stdout.write(".") - sys.stdout.flush() - print - except KeyboardInterrupt: - print "\nCancelled", url + video_url = "hlsvariant://" + video_url + logging.info("Downloading: %s", filename) + + cmd = [ + "livestreamer", + "-o", filename, + video_url, + "best", + ] + if exec_subprocess(cmd): + return convert_to_mp4(filename) + else: return False - finally: - try: - src.close() - except: - pass - try: - dst.close() - except: - pass - convert_filename(filename) - return True +def download_http(filename, video_url): + filename = sanify_filename(filename) + logging.info("Downloading: %s", filename) + + cmd = [ + "curl", + "--fail", "--retry", "3", + "-o", filename, + video_url, + ] + if exec_subprocess(cmd): + return convert_to_mp4(filename) + else: + return False def natural_sort(l, key=None): ignore_list = ["a", "the"] @@ -259,7 +314,7 @@ def natural_sort(l, key=None): for c in re.split("([0-9]+)", k): c = c.strip() if c.isdigit(): - newk.append(int(c)) + newk.append(c.zfill(5)) else: for subc in c.split(): if subc not in ignore_list: @@ -269,14 +324,14 @@ def natural_sort(l, key=None): return sorted(l, key=key_func) def append_to_qs(url, params): - r = list(urlparse.urlsplit(url)) - qs = urlparse.parse_qs(r[3]) - for k, v in params.iteritems(): + r = list(urllib.parse.urlsplit(url)) + qs = urllib.parse.parse_qs(r[3]) + for k, v in params.items(): if v is not None: qs[k] = v - elif qs.has_key(k): + elif k in qs: del qs[k] - r[3] = urllib.urlencode(qs, True) - url = urlparse.urlunsplit(r) + r[3] = urllib.parse.urlencode(sorted(qs.items()), True) + url = urllib.parse.urlunsplit(r) return url