From 98cf0fdc0033f712bad0138621ff8a66504d293d Mon Sep 17 00:00:00 2001 From: James Bunton Date: Mon, 14 Dec 2015 21:18:51 +1100 Subject: [PATCH] sbs fetches list differently, now slower but will pull all videos --- sbs.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/sbs.py b/sbs.py index 8014767..a7c1858 100644 --- a/sbs.py +++ b/sbs.py @@ -1,11 +1,11 @@ #!/usr/bin/env python -from common import grab_html, grab_json, grab_xml, download_hls, Node +from common import grab_html, grab_json, grab_xml, download_hls, Node, append_to_qs import json BASE = "http://www.sbs.com.au" -FULL_VIDEO_LIST = BASE + "/api/video_feed/f/Bgtm9B/sbs-section-programs/?form=json" +FULL_VIDEO_LIST = BASE + "/api/video_search/v2/?m=1&filters={section}{Programs}" VIDEO_URL = BASE + "/ondemand/video/single/%s" NS = { @@ -60,14 +60,32 @@ class SbsRootNode(SbsNavNode): Node.__init__(self, "SBS", parent) def fill_children(self): - full_video_list = grab_json(FULL_VIDEO_LIST, 3600) - category_and_entry_data = self.explode_videos_to_unique_categories(full_video_list) + all_video_entries = self.load_all_video_entries() + category_and_entry_data = self.explode_videos_to_unique_categories(all_video_entries) for category_path, entry_data in category_and_entry_data: nav_node = self.create_nav_node(self, category_path) nav_node.create_video_node(entry_data) - def explode_videos_to_unique_categories(self, full_video_list): - for entry_data in full_video_list["entries"]: + def load_all_video_entries(self): + offset = 1 + amount = 500 + while True: + url = append_to_qs(FULL_VIDEO_LIST, {"range": "%s-%s" % (offset, offset+amount)}) + data = grab_json(url, 3600) + entries = data["entries"] + if len(entries) == 0: + break + for entry in entries: + yield entry + offset += amount + import sys + sys.stdout.write(".") + sys.stdout.flush() + sys.stdout.write("\n") + sys.stdout.flush() + + def explode_videos_to_unique_categories(self, all_video_entries): + for entry_data in all_video_entries: for category_data in entry_data["media$categories"]: category_path = self.calculate_category_path( category_data["media$scheme"], -- 2.39.2