X-Git-Url: https://code.delx.au/webdl/blobdiff_plain/5cf33ad1ef4fa0066c9ade22e3e6f151dd218a99..9832f6d2d387ba6e2375fc11824840cfdcb7e056:/sbs.py diff --git a/sbs.py b/sbs.py index ff20efc..a263168 100644 --- a/sbs.py +++ b/sbs.py @@ -1,62 +1,121 @@ -#!/usr/bin/env python -# vim:ts=4:sts=4:sw=4:noet +import requests_cache +from common import grab_html, grab_json, grab_xml, download_hls, Node, append_to_qs -from common import grab_xml as _grab_xml, download_rtmp, download_urllib, Node +import json +BASE = "http://www.sbs.com.au" +FULL_VIDEO_LIST = BASE + "/api/video_search/v2/?m=1&filters={section}{Programs}" +VIDEO_URL = BASE + "/ondemand/video/single/%s" -BASE_URL = "http://player.sbs.com.au" +NS = { + "smil": "http://www.w3.org/2005/SMIL21/Language", +} -def grab_xml(path, max_age): - return _grab_xml(BASE_URL + path, max_age) -class SbsNode(Node): - def __init__(self, title, parent, video_desc_url): - Node.__init__(self, title, parent) - self.video_desc_url = video_desc_url - self.can_download = True +class SbsVideoNode(Node): + def __init__(self, title, parent, url): + Node.__init__(self, title, parent) + self.video_id = url.split("/")[-1] + self.can_download = True - def download(self): - video = grab_xml(self.video_desc_url, 0) - vbase = video.xpath("//meta/@base")[0] - bestrate = 0 - bestvpath = None - for vpath in video.xpath("//switch/video"): - rate = float(vpath.xpath("@system-bitrate")[0]) - if rate > bestrate: - bestrate = rate - bestvpath = vpath.xpath("@src")[0] - filename = self.title + "." + bestvpath.rsplit(".", 1)[1] - if vbase.startswith("rtmp://"): - return download_rtmp(filename, vbase, bestvpath) - else: - return download_urllib(filename, vbase + bestvpath) + def download(self): + with requests_cache.disabled(): + doc = grab_html(VIDEO_URL % self.video_id) + player_params = self.get_player_params(doc) + release_url = player_params["releaseUrls"]["html"] + video_url = self.get_video_url(release_url) + if not video_url: + raise Exception("Unsupported video %s: %s" % (self.video_id, self.title)) + filename = self.title + ".ts" + return download_hls(filename, video_url) + + def get_player_params(self, doc): + for script in doc.xpath("//script"): + if not script.text: + continue + for line in script.text.split("\n"): + s = "var playerParams = {" + if s in line: + p1 = line.find(s) + len(s) - 1 + p2 = line.find("};", p1) + 1 + if p1 >= 0 and p2 > 0: + return json.loads(line[p1:p2]) + raise Exception("Unable to find player params for %s: %s" % (self.video_id, self.title)) + + def get_video_url(self, release_url): + with requests_cache.disabled(): + doc = grab_xml("http:" + release_url.replace("http:", "").replace("https:", "")) + video = doc.xpath("//smil:video", namespaces=NS) + if not video: + return + video_url = video[0].attrib["src"] + return video_url + +class SbsNavNode(Node): + def create_video_node(self, entry_data): + SbsVideoNode(entry_data["title"], self, entry_data["id"]) + + def find_existing_child(self, path): + for child in self.children: + if child.title == path: + return child + +class SbsRootNode(SbsNavNode): + def __init__(self, parent): + Node.__init__(self, "SBS", parent) + + def fill_children(self): + all_video_entries = self.load_all_video_entries() + category_and_entry_data = self.explode_videos_to_unique_categories(all_video_entries) + for category_path, entry_data in category_and_entry_data: + nav_node = self.create_nav_node(self, category_path) + nav_node.create_video_node(entry_data) + + def load_all_video_entries(self): + offset = 1 + amount = 500 + while True: + url = append_to_qs(FULL_VIDEO_LIST, {"range": "%s-%s" % (offset, offset+amount)}) + data = grab_json(url) + if "entries" not in data: + raise Exception("Missing data in SBS response", data) + entries = data["entries"] + if len(entries) == 0: + break + for entry in entries: + yield entry + offset += amount + + def explode_videos_to_unique_categories(self, all_video_entries): + for entry_data in all_video_entries: + for category_data in entry_data["media$categories"]: + category_path = self.calculate_category_path( + category_data["media$scheme"], + category_data["media$name"], + ) + if category_path: + yield category_path, entry_data + + def calculate_category_path(self, scheme, name): + if not scheme: + return + if scheme == name: + return + name = name.split("/") + if name[0] != scheme: + name.insert(0, scheme) + return name + + def create_nav_node(self, parent, category_path): + if not category_path: + return parent + + current_path = category_path[0] + current_node = parent.find_existing_child(current_path) + if not current_node: + current_node = SbsNavNode(current_path, parent) + return self.create_nav_node(current_node, category_path[1:]) def fill_nodes(root_node): - settings = grab_xml("/playerassets/programs/config/standalone_settings.xml", 24*3600) - menu_url = settings.xpath("/settings/setting[@name='menuURL']/@value")[0] - - root_menu = grab_xml(menu_url, 3600) - seen_category_titles = set() - for menu in root_menu.xpath("//menu"): - try: - category_title = menu.xpath("title/text()")[0] - playlist_url = menu.xpath("playlist/@xmlSrc")[0] - if category_title in seen_category_titles: - # append a number to the name - i = 2 - while True: - if (category_title+str(i)) not in seen_category_titles: - category_title += str(i) - break - i += 1 - seen_category_titles.add(category_title) - category_node = Node(category_title, root_node) - playlist = grab_xml(playlist_url, 3600) - for video_desc in playlist.xpath("//video"): - video_desc_url = video_desc.xpath("@src")[0] - video_title = video_desc.xpath("title/text()")[0].strip() - SbsNode(video_title, category_node, video_desc_url) - except IndexError: - continue - + SbsRootNode(root_node)