-#!/usr/bin/env python
+#!/usr/bin/python2
from __future__ import division
import cookielib
import cgi
-import itertools
import json
from lxml import html
import os
referrer = ""
def urlopen(url, offset=None):
+ if url.startswith("//"):
+ url = "http:" + url
+
global referrer
req = urllib2.Request(url)
if referrer:
url = urlparse.urlunsplit(r)
return url
-def convert_from_old_itag(player_config):
- url_data = urlparse.parse_qs(player_config["args"]["url_encoded_fmt_stream_map"])
- url_data["url"] = []
- for itag_url in url_data["itag"]:
- pos = itag_url.find("url=")
- url_data["url"].append(itag_url[pos+4:])
- player_config["args"]["url_encoded_fmt_stream_map"] = urllib.urlencode(url_data, True)
-
def get_player_config(doc):
player_config = None
for script in doc.xpath("//script"):
if not script.text:
continue
for line in script.text.split("\n"):
- if "yt.playerConfig =" in line:
- p1 = line.find("=")
- p2 = line.rfind(";")
+ s = "ytplayer.config = {"
+ if s in line:
+ p1 = line.find(s) + len(s) - 1
+ p2 = line.find("};", p1) + 1
if p1 >= 0 and p2 > 0:
- return json.loads(line[p1+1:p2])
- if "'PLAYER_CONFIG': " in line:
- p1 = line.find(":")
- if p1 >= 0:
- player_config = json.loads(line[p1+1:])
- convert_from_old_itag(player_config)
- return player_config
+ return json.loads(line[p1:p2])
+
+def extract_function(output, script, func_name):
+ p1 = script.find("function " + func_name + "(")
+ p2 = script.find("}", p1)
+ code = script[p1:p2+1]
+ output.append(code)
+ deps = re.findall(R"[^\.][= ]([\$0-9a-zA-Z]+)\(", code)
+ deps = set(deps)
+ deps.remove(func_name)
+ for dep in deps:
+ extract_function(output, script, dep)
+
+def decode_signature(js_url, s):
+ script = urlopen(js_url).read()
+ func_name = re.search(R"\b([a-zA-Z]+)\([a-zA-Z]+\.s\);", script).groups()[0]
+
+ codes = []
+ extract_function(codes, script, func_name)
+
+ p = subprocess.Popen(
+ "js",
+ shell=True,
+ close_fds=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE
+ )
+ for code in codes:
+ p.stdin.write(code + "\n")
+ p.stdin.write("console.log(%s('%s'));\n" % (func_name, s))
+ p.stdin.close()
+
+ signature = p.stdout.read().strip()
+ if p.wait() != 0:
+ raise Exception("js failed to execute: %d" % p.returncode)
+
+ return signature
def get_best_video(player_config):
- url_data = urlparse.parse_qs(player_config["args"]["url_encoded_fmt_stream_map"])
- url_data = itertools.izip_longest(
- url_data["url"],
- url_data["type"],
- url_data["quality"],
- url_data.get("sig", []),
- )
+ url_data_list = player_config["args"]["url_encoded_fmt_stream_map"].split(",")
+ js_url = player_config["assets"]["js"]
+
best_url = None
best_quality = None
best_extension = None
- for video_url, mimetype, quality, signature in url_data:
- mimetype = mimetype.split(";")[0]
+ for url_data in url_data_list:
+ url_data = urlparse.parse_qs(url_data)
+ mimetype = url_data["type"][0].split(";")[0]
+ quality = url_data["quality"][0]
+
+ if url_data.has_key("stereo3d"):
+ continue
+ if quality not in QUALITIES:
+ continue
if mimetype not in MIMETYPES:
continue
+
extension = MIMETYPES[mimetype]
- quality = QUALITIES.get(quality.split(",")[0], -1)
- if best_quality is None or quality > best_quality:
- if signature:
- video_url = append_to_qs(video_url, {"signature": signature})
- best_url = video_url
- best_quality = quality
- best_extension = extension
+ quality = QUALITIES.get(quality, -1)
+
+ if best_quality is not None and quality < best_quality:
+ continue
+
+ video_url = url_data["url"][0]
+ if "sig" in url_data:
+ signature = url_data["sig"][0]
+ elif "s" in url_data:
+ signature = decode_signature(js_url, url_data["s"][0])
+ else:
+ signature = None
+
+ if signature:
+ video_url = append_to_qs(video_url, {"signature": signature})
+
+ best_url = video_url
+ best_quality = quality
+ best_extension = extension
return best_url, best_extension
def copy_with_progress(content_length, infile, outfile):
def print_status():
+ rate = 0
+ if now != last_ts:
+ rate = last_bytes_read / (now - last_ts)
sys.stdout.write("\33[2K\r")
sys.stdout.write("%s / %s (%s/sec)" % (
pp_size(bytes_read),
pp_size(content_length),
- pp_size(bytes_read / (now - start_ts)),
+ pp_size(rate),
))
sys.stdout.flush()
- start_ts = time.time()
last_ts = 0
+ last_bytes_read = 0
bytes_read = 0
while True:
now = time.time()
if now - last_ts > 0.5:
- last_ts = now
print_status()
+ last_ts = now
+ last_bytes_read = 0
buf = infile.read(32768)
if not buf:
break
outfile.write(buf)
+ last_bytes_read += len(buf)
bytes_read += len(buf)
# Newline at the end
if __name__ == "__main__":
- resource.setrlimit(resource.RLIMIT_AS, (MAX_MEMORY_BYTES, MAX_MEMORY_BYTES))
+### resource.setrlimit(resource.RLIMIT_AS, (MAX_MEMORY_BYTES, MAX_MEMORY_BYTES))
if os.environ.has_key("SCRIPT_NAME"):
cgimain()
else: