From: James Bunton <jamesbunton@delx.net.au>
Date: Sun, 24 Feb 2019 00:04:32 +0000 (+1100)
Subject: sbs: filter duplicates
X-Git-Url: https://code.delx.au/webdl/commitdiff_plain/d2fbda5b1719e9a2463971f0f1ffd94e01d6e151

sbs: filter duplicates
---

diff --git a/sbs.py b/sbs.py
index dc353ca..1d6324f 100644
--- a/sbs.py
+++ b/sbs.py
@@ -77,17 +77,20 @@ class SbsRootNode(SbsNavNode):
 
     def load_all_video_entries(self):
         offset = 1
-        amount = 49
+        amount = 50
+        uniq = set()
         while True:
-            url = append_to_qs(FULL_VIDEO_LIST, {"range": "%s-%s" % (offset, offset+amount)})
+            url = append_to_qs(FULL_VIDEO_LIST, {"range": "%s-%s" % (offset, offset+amount-1)})
             data = grab_json(url)
             if "entries" not in data:
                 raise Exception("Missing data in SBS response", data)
             entries = data["entries"]
             if len(entries) == 0:
                 break
-            for entry in entries:
-                yield entry
+            for i, entry in enumerate(entries):
+                if entry["guid"] not in uniq:
+                    uniq.add(entry["guid"])
+                    yield entry
             offset += amount
             sys.stdout.write(".")
             sys.stdout.flush()