]>
code.delx.au - webdl/blob - common.py
f0886c8f808fa734e40c41ffcab896d6578e9fb2
19 autosocks
.try_autosocks()
25 format
= "%(levelname)s %(message)s",
26 level
= logging
.INFO
if os
.environ
.get("DEBUG", None) is None else logging
.DEBUG
,
29 CACHE_FILE
= os
.path
.join(
30 os
.environ
.get("XDG_CACHE_HOME", os
.path
.expanduser("~/.cache")),
34 if not os
.path
.isdir(os
.path
.dirname(CACHE_FILE
)):
35 os
.makedirs(os
.path
.dirname(CACHE_FILE
))
37 requests_cache
.install_cache(CACHE_FILE
, backend
='sqlite', expire_after
=3600)
41 def __init__(self
, title
, parent
=None):
44 parent
.children
.append(self
)
47 self
.can_download
= False
49 def get_children(self
):
54 def fill_children(self
):
62 root_node
= Node("Root")
65 iview
.fill_nodes(root_node
)
68 sbs
.fill_nodes(root_node
)
71 brightcove
.fill_nodes(root_node
)
75 valid_chars
= frozenset("-_.()!@#%^ abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
76 def sanify_filename(filename
):
77 filename
= "".join(c
for c
in filename
if c
in valid_chars
)
78 assert len(filename
) > 0
81 def ensure_scheme(url
):
82 parts
= urllib
.parse
.urlparse(url
)
87 return urllib
.parse
.urlunparse(parts
)
89 http_session
= requests
.Session()
90 http_session
.headers
["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:21.0) Gecko/20100101 Firefox/21.0"
93 logging
.debug("grab_text(%r)", url
)
94 request
= http_session
.prepare_request(requests
.Request("GET", url
))
95 response
= http_session
.send(request
)
99 logging
.debug("grab_html(%r)", url
)
100 request
= http_session
.prepare_request(requests
.Request("GET", url
))
101 response
= http_session
.send(request
, stream
=True)
102 doc
= lxml
.html
.parse(response
.raw
, lxml
.html
.HTMLParser(encoding
="utf-8", recover
=True))
107 logging
.debug("grab_xml(%r)", url
)
108 request
= http_session
.prepare_request(requests
.Request("GET", url
))
109 response
= http_session
.send(request
, stream
=True)
110 doc
= lxml
.etree
.parse(response
.raw
, lxml
.etree
.XMLParser(encoding
="utf-8", recover
=True))
115 logging
.debug("grab_json(%r)", url
)
116 request
= http_session
.prepare_request(requests
.Request("GET", url
))
117 response
= http_session
.send(request
)
118 return response
.json()
120 def exec_subprocess(cmd
):
121 logging
.debug("Executing: %s", cmd
)
123 p
= subprocess
.Popen(cmd
)
126 logging
.error("%s exited with error code: %s", cmd
[0], ret
)
131 logging
.error("Failed to run: %s -- %s", cmd
[0], e
)
132 except KeyboardInterrupt:
133 logging
.info("Cancelled: %s", cmd
)
137 except KeyboardInterrupt:
138 p
.send_signal(signal
.SIGKILL
)
143 def check_command_exists(cmd
):
145 subprocess
.check_output(cmd
, stderr
=subprocess
.STDOUT
)
151 for ffmpeg
in ["avconv", "ffmpeg"]:
152 if check_command_exists([ffmpeg
, "--help"]):
155 raise Exception("You must install ffmpeg or libav-tools")
158 for ffprobe
in ["avprobe", "ffprobe"]:
159 if check_command_exists([ffprobe
, "--help"]):
162 raise Exception("You must install ffmpeg or libav-tools")
164 def get_duration(filename
):
165 ffprobe
= find_ffprobe()
170 "-show_format_entry", "duration",
173 output
= subprocess
.check_output(cmd
).decode("utf-8")
174 for line
in output
.split("\n"):
175 if line
.startswith("duration="):
176 return float(line
.split("=")[1]) # ffprobe
177 if re
.match(R
'^[0-9.]*$', line
):
178 return float(line
) # avprobe
180 raise Exception("Unable to determine video duration of " + filename
)
182 def check_video_durations(flv_filename
, mp4_filename
):
183 flv_duration
= get_duration(flv_filename
)
184 mp4_duration
= get_duration(mp4_filename
)
186 if abs(flv_duration
- mp4_duration
) > 1:
188 "The duration of %s is suspicious, did the remux fail? Expected %s == %s",
189 mp4_filename
, flv_duration
, mp4_duration
195 def remux(infile
, outfile
):
196 logging
.info("Converting %s to mp4", infile
)
198 ffmpeg
= find_ffmpeg()
202 "-bsf:a", "aac_adtstoasc",
208 if not exec_subprocess(cmd
):
211 if not check_video_durations(infile
, outfile
):
217 def convert_to_mp4(filename
):
218 with
open(filename
, "rb") as f
:
220 basename
, ext
= os
.path
.splitext(filename
)
222 if ext
== ".mp4" and fourcc
== b
"FLV\x01":
223 os
.rename(filename
, basename
+ ".flv")
225 filename
= basename
+ ext
227 if ext
in (".flv", ".ts"):
228 filename_mp4
= basename
+ ".mp4"
229 return remux(filename
, filename_mp4
)
234 def download_hds(filename
, video_url
, pvswf
=None):
235 filename
= sanify_filename(filename
)
236 logging
.info("Downloading: %s", filename
)
238 video_url
= "hds://" + video_url
240 param
= "%s pvswf=%s" % (video_url
, pvswf
)
251 if exec_subprocess(cmd
):
252 return convert_to_mp4(filename
)
256 def download_hls(filename
, video_url
):
257 filename
= sanify_filename(filename
)
258 video_url
= "hlsvariant://" + video_url
259 logging
.info("Downloading: %s", filename
)
268 if exec_subprocess(cmd
):
269 return convert_to_mp4(filename
)
273 def download_http(filename
, video_url
):
274 filename
= sanify_filename(filename
)
275 logging
.info("Downloading: %s", filename
)
279 "--fail", "--retry", "3",
283 if exec_subprocess(cmd
):
284 return convert_to_mp4(filename
)
288 def natural_sort(l
, key
=None):
289 ignore_list
= ["a", "the"]
295 for c
in re
.split("([0-9]+)", k
):
298 newk
.append(c
.zfill(5))
300 for subc
in c
.split():
301 if subc
not in ignore_list
:
305 return sorted(l
, key
=key_func
)
307 def append_to_qs(url
, params
):
308 r
= list(urllib
.parse
.urlsplit(url
))
309 qs
= urllib
.parse
.parse_qs(r
[3])
310 for k
, v
in params
.items():
315 r
[3] = urllib
.parse
.urlencode(sorted(qs
.items()), True)
316 url
= urllib
.parse
.urlunsplit(r
)