code.delx.au - webdl/blob - plus7.py

   1 import json
   2 import random
   3 import string
   4 import urllib
   5 from lxml.cssselect import CSSSelector
   6
   7 from common import grab_html, grab_json, download_rtmp, Node
   8
   9 BASE = "http://au.tv.yahoo.com"
  10 BROWSE = BASE + "/plus7/browse/"
  11
  12 METADATA_BASE = "http://video.query.yahoo.com/v1/public/yql?"
  13 METADATA_QUERY = {
  14     'q': 'SELECT streams,status FROM yahoo.media.video.streams WHERE id="%s" AND format="mp4,flv" AND protocol="rtmp,http" AND plrs="%s" AND offnetwork="false" AND site="autv_plus7" AND lang="en-AU" AND region="AU" AND override="none";',
  15     'callback': 'jsonp_callback',
  16     'env': 'prod',
  17     'format': 'json'
  18 }
  19
  20 HASH_URL = "http://d.yimg.com/m/up/ypp/au/player.swf"
  21
  22
  23 def extract_and_remove(tokens, key):
  24     lowertokens = [x.lower() for x in tokens]
  25     pos = lowertokens.index(key)
  26
  27     value = int(tokens[pos+1])
  28     tokens = tokens[:pos] + tokens[pos+2:]
  29
  30     return value, tokens
  31
  32
  33 def demangle_title(title, subtitle):
  34     tokens = title.split()
  35     insert_pos = len(tokens)
  36     if subtitle:
  37         insert_pos += 1
  38         tokens += ["-"] + subtitle.split()
  39
  40     try:
  41         season, tokens = extract_and_remove(tokens, "series")
  42         episode, tokens = extract_and_remove(tokens, "episode")
  43         if insert_pos < len(tokens):
  44             tokens.insert(insert_pos, "-")
  45         tokens.insert(insert_pos, "%sx%s" % (season, str(episode).zfill(2)))
  46     except ValueError:
  47         pass
  48
  49     return " ".join(tokens)
  50
  51 class Plus7Node(Node):
  52     def __init__(self, title, parent, url):
  53         Node.__init__(self, title, parent)
  54         self.url = url
  55         self.can_download = True
  56
  57     def get_video_id(self):
  58         doc = grab_html(self.url, 3600)
  59         for script in doc.xpath("//script"):
  60             if not script.text:
  61                 continue
  62             for line in script.text.split(";"):
  63                 line = line.strip()
  64                 if line.find("new Y.VideoPlatform.VideoPlayer") <= 0:
  65                     continue
  66
  67 ###             vidparams = line[line.find("(")+1 : line.rfind(")")]
  68 ###             vidparams = json.loads(vidparams)
  69 ###             return vidparams["playlist"]["mediaItems"][0]["id"]
  70
  71                 # Cannot parse it as JSON :(
  72                 pos1 = line.find('"mediaItems":')
  73                 if pos1 < 0:
  74                     continue
  75                 pos2 = line.find('"id":', pos1)
  76                 if pos2 < 0:
  77                     continue
  78                 pos3 = line.find('"', pos2+5)
  79                 pos4 = line.find('"', pos2+6)
  80                 if pos3 < 0 or pos4 < 0:
  81                     continue
  82                 return line[pos3+1:pos4]
  83
  84         raise Exception("Could not find video id on page " + self.url)
  85
  86     def generate_session(self):
  87         return "".join([random.choice(string.ascii_letters) for x in xrange(22)])
  88
  89     def download(self):
  90         vid_id = self.get_video_id()
  91         qs = dict(METADATA_QUERY.items()) # copy..
  92         qs["q"] = qs["q"] % (vid_id, self.generate_session())
  93         url = METADATA_BASE + urllib.urlencode(qs)
  94         doc = grab_json(url, 0, skip_function=True)
  95         stream_data = doc["query"]["results"]["mediaObj"][0]["streams"][0]
  96         vbase = stream_data["host"]
  97         vpath = stream_data["path"]
  98         filename = self.title + ".flv"
  99         return download_rtmp(filename, vbase, vpath, HASH_URL)
 100
 101
 102 class Plus7Series(Node):
 103     def __init__(self, title, parent, url):
 104         Node.__init__(self, title, parent)
 105         self.url = url
 106
 107     def fill_children(self):
 108         doc = grab_html(self.url, 3600)
 109         for item in CSSSelector("#related-episodes div.itemdetails")(doc):
 110             title = CSSSelector("span.title")(item)[0].text
 111             subtitle = CSSSelector("span.subtitle")(item)[0].xpath("string()")
 112             title = demangle_title(title, subtitle)
 113             url = CSSSelector("a")(item)[0].attrib["href"]
 114             Plus7Node(title, self, BASE + url)
 115
 116 class Plus7Root(Node):
 117     def __init__(self, parent):
 118         Node.__init__(self, "Yahoo Plus7 (broken!)", parent)
 119
 120     def fill_children(self):
 121         doc = grab_html(BROWSE, 3600)
 122         shows = []
 123         for script in doc.xpath("//script"):
 124             if not script.text or not script.text.startswith("var shows = "):
 125                 continue
 126             shows = script.text[12:]
 127             shows = shows.rstrip("; \n")
 128             shows = json.loads(shows)
 129         for show in shows:
 130             Plus7Series(show["title"], self, show["url"])
 131
 132 def fill_nodes(root_node):
 133     Plus7Root(root_node)
 134