Skip to content

Commit 5d993e0

Browse files
Billy2011Billy2011
committed
[PATCH] plugin.api.http_session: add prepare_new_request (streamlink#4521)
plugin.api.http_session: add prepare_new_request - Move stream.http.valid_args to HTTPSession.valid_request_args - Add HTTPSession.prepare_new_request - Update HTTPStream - Use newly added HTTPSession methods - Fix HLSStream - Prepare master URL appropriately - Fix DASHStream - Return correct JSON data, depending whether manifest URL exists - Filter custom args in DASHStream.parse_manifest - Use HTTPSession.prepare_new_request in plugins.twitch.UsherService - Rewrite Stream JSON tests and move test module
1 parent 3e43700 commit 5d993e0

File tree

8 files changed

+193
-50
lines changed

8 files changed

+193
-50
lines changed

‎src/streamlink/plugin/api/http_session.py

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
import ssl
22
import time
33
try:
4-
from typing import Any, Callable, List, Pattern, Tuple
4+
from typing import Any, Callable, Dict, List, Pattern, Tuple
55
except ImportError:
66
pass
77

88
import requests
99
# noinspection PyPackageRequirements
1010
import urllib3
11-
from requests import Session
11+
from requests import PreparedRequest, Request, Session
1212

1313
from streamlink.compat import is_py3
1414
from streamlink.exceptions import PluginError
@@ -72,6 +72,10 @@ def _parse_keyvalue_list(val):
7272
continue
7373

7474

75+
# requests.Request.__init__ keywords, except for "hooks"
76+
_VALID_REQUEST_ARGS = "method", "url", "headers", "files", "data", "params", "auth", "cookies", "json"
77+
78+
7579
class HTTPSession(Session):
7680
def __init__(self, *args, **kwargs):
7781
Session.__init__(self, *args, **kwargs)
@@ -146,6 +150,20 @@ def resolve_url(self, url):
146150
"""Resolves any redirects and returns the final URL."""
147151
return self.get(url, stream=True).url
148152

153+
@staticmethod
154+
def valid_request_args(**req_keywords):
155+
# type: () -> Dict
156+
return {k: v for k, v in req_keywords.items() if k in _VALID_REQUEST_ARGS}
157+
158+
def prepare_new_request(self, **req_keywords):
159+
# type: () -> PreparedRequest
160+
valid_args = self.valid_request_args(**req_keywords)
161+
valid_args.setdefault("method", "GET")
162+
request = Request(**valid_args)
163+
164+
# prepare request with the session context, which might add params, headers, cookies, etc.
165+
return self.prepare_request(request)
166+
149167
def request(self, method, url, *args, **kwargs):
150168
acceptable_status = kwargs.pop("acceptable_status", [])
151169
exception = kwargs.pop("exception", PluginError)

‎src/streamlink/plugins/twitch.py

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@
1212
from collections import namedtuple
1313
from random import random
1414

15-
import requests
16-
1715
from streamlink.compat import str, urlparse
1816
from streamlink.exceptions import NoStreamsError, PluginError
1917
from streamlink.plugin import Plugin, PluginArgument, PluginArguments, pluginmatcher
@@ -179,8 +177,7 @@ def _create_url(self, endpoint, **extra_params):
179177
}
180178
params.update(extra_params)
181179

182-
req = requests.Request("GET", url, params=params)
183-
req = self.session.http.prepare_request(req)
180+
req = self.session.http.prepare_new_request(url=url, params=params)
184181

185182
return req.url
186183

‎src/streamlink/stream/dash.py

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,10 @@
55
import os.path
66
from collections import defaultdict
77

8-
import requests
9-
108
from streamlink import PluginError, StreamError
119
from streamlink.compat import range, urlparse, urlunparse
1210
from streamlink.stream.dash_manifest import MPD, freeze_timeline, sleep_until, sleeper, utc
1311
from streamlink.stream.ffmpegmux import FFMPEGMuxer
14-
from streamlink.stream.http import normalize_key, valid_args
1512
from streamlink.stream.segmented import SegmentedStreamReader, SegmentedStreamWorker, SegmentedStreamWriter
1613
from streamlink.stream.stream import Stream
1714
from streamlink.utils.l10n import Language
@@ -157,11 +154,19 @@ def __init__(self,
157154
self.args = args
158155

159156
def __json__(self):
160-
req = requests.Request(method="GET", url=self.mpd.url, **valid_args(self.args))
161-
req = req.prepare()
157+
json = dict(type=self.shortname())
158+
159+
if self.mpd.url:
160+
args = self.args.copy()
161+
args.update(url=self.mpd.url)
162+
req = self.session.http.prepare_new_request(**args)
163+
json.update(
164+
# the MPD URL has already been prepared by the initial request in `parse_manifest`
165+
url=self.mpd.url,
166+
headers=dict(req.headers),
167+
)
162168

163-
headers = dict(map(normalize_key, req.headers.items()))
164-
return dict(type=type(self).shortname(), url=req.url, headers=headers)
169+
return json
165170

166171
@classmethod
167172
def parse_manifest(cls, session, url_or_manifest, **args):
@@ -176,7 +181,7 @@ def parse_manifest(cls, session, url_or_manifest, **args):
176181
if url_or_manifest.startswith('<?xml'):
177182
mpd = MPD(parse_xml(url_or_manifest, ignore_ns=True))
178183
else:
179-
res = session.http.get(url_or_manifest, **args)
184+
res = session.http.get(url_or_manifest, **session.http.valid_request_args(**args))
180185
url = res.url
181186

182187
urlp = list(urlparse(url))

‎src/streamlink/stream/file.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,13 @@ def __init__(self, session, path=None, fileobj=None):
1414
if not self.path and not self.fileobj:
1515
raise ValueError("path or fileobj must be set")
1616

17+
def __json__(self):
18+
json = super(FileStream, self).__json__()
19+
20+
if self.path:
21+
json["path"] = self.path
22+
23+
return json
24+
1725
def open(self):
1826
return self.fileobj or open(self.path)

‎src/streamlink/stream/hls.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -372,7 +372,7 @@ def __json__(self):
372372
json = HTTPStream.__json__(self)
373373

374374
if self.url_master:
375-
json["master"] = self.url_master
375+
json["master"] = self.to_manifest_url()
376376

377377
# Pretty sure HLS is GET only.
378378
del json["method"]
@@ -381,7 +381,13 @@ def __json__(self):
381381
return json
382382

383383
def to_manifest_url(self):
384-
return self.url_master
384+
if self.url_master is None:
385+
return None
386+
387+
args = self.args.copy()
388+
args.update(url=self.url_master)
389+
390+
return self.session.http.prepare_new_request(**args).url
385391

386392
def open(self):
387393
reader = self.__reader__(self)

‎src/streamlink/stream/http.py

Lines changed: 17 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,8 @@
1-
import requests
2-
31
from streamlink.exceptions import StreamError
42
from streamlink.stream.stream import Stream
53
from streamlink.stream.wrappers import StreamIOIterWrapper, StreamIOThreadWrapper
64

75

8-
VALID_ARGS = ("method", "url", "params", "headers", "cookies", "auth", "data", "json", "files")
9-
10-
11-
def normalize_key(keyval):
12-
key, val = keyval
13-
key = hasattr(key, "decode") and key.decode("utf8", "ignore") or key
14-
15-
return key, val
16-
17-
18-
def valid_args(args):
19-
return {k: v for k, v in args.items() if k in VALID_ARGS}
20-
21-
226
class HTTPStream(Stream):
237
"""A HTTP stream using the requests library.
248
@@ -42,33 +26,33 @@ def __repr__(self):
4226
return "<HTTPStream({0!r})>".format(self.url)
4327

4428
def __json__(self):
45-
args = self.args.copy()
46-
method = args.pop("method", "GET")
47-
req = requests.Request(method=method, **valid_args(args))
48-
req = self.session.http.prepare_request(req)
49-
50-
headers = dict(map(normalize_key, req.headers.items()))
51-
52-
return dict(type=type(self).shortname(), url=req.url,
53-
method=req.method, headers=headers,
54-
body=req.body)
29+
req = self.session.http.prepare_new_request(**self.args)
30+
31+
return dict(
32+
type=self.shortname(),
33+
method=req.method,
34+
url=req.url,
35+
headers=dict(req.headers),
36+
body=req.body,
37+
)
5538

5639
@property
5740
def url(self):
58-
args = self.args.copy()
59-
method = args.pop("method", "GET")
60-
return requests.Request(method=method, **valid_args(args)).prepare().url
41+
"""
42+
The URL to the stream, prepared by :mod:`requests` with parameters read from :attr:`args`.
43+
"""
44+
45+
return self.session.http.prepare_new_request(**self.args).url
6146

6247
def open(self):
63-
args = self.args.copy()
64-
method = args.pop("method", "GET")
48+
reqargs = self.session.http.valid_request_args(**self.args)
49+
reqargs.setdefault("method", "GET")
6550
timeout = self.session.options.get("stream-timeout")
6651
res = self.session.http.request(
67-
method=method,
6852
stream=True,
6953
exception=StreamError,
7054
timeout=timeout,
71-
**valid_args(args)
55+
**reqargs
7256
)
7357

7458
fd = StreamIOIterWrapper(res.iter_content(8192))

‎src/streamlink/stream/stream.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ def __repr__(self):
2020
return "<Stream()>"
2121

2222
def __json__(self):
23-
return dict(type=type(self).shortname())
23+
return dict(type=self.shortname())
2424

2525
def open(self):
2626
"""

‎tests/stream/test_stream_json.py

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
import pytest
2+
# noinspection PyUnresolvedReferences
3+
from requests.utils import DEFAULT_ACCEPT_ENCODING
4+
5+
from streamlink import Streamlink
6+
from streamlink.stream.dash import DASHStream
7+
from streamlink.stream.file import FileStream
8+
from streamlink.stream.hls import HLSStream
9+
from streamlink.stream.http import HTTPStream
10+
from streamlink.stream.stream import Stream
11+
from tests.mock import Mock
12+
13+
14+
@pytest.fixture(scope="module")
15+
def session():
16+
session = Streamlink()
17+
session.set_option("http-cookies", {"sessioncookiekey": "sessioncookieval"})
18+
session.set_option("http-headers", {"sessionheaderkey": "sessionheaderval"})
19+
session.set_option("http-query-params", {"sessionqueryparamkey": "sessionqueryparamval"})
20+
21+
return session
22+
23+
24+
@pytest.fixture(scope="module")
25+
def common_args():
26+
return dict(
27+
params={"queryparamkey": "queryparamval"},
28+
headers={
29+
"User-Agent": "Test",
30+
"headerkey": "headerval",
31+
},
32+
cookies={"cookiekey": "cookieval"},
33+
unknown="invalid",
34+
)
35+
36+
37+
@pytest.fixture(scope="module")
38+
def expected_headers():
39+
return {
40+
"User-Agent": "Test",
41+
"Accept": "*/*",
42+
"Accept-Encoding": DEFAULT_ACCEPT_ENCODING,
43+
"Connection": "keep-alive",
44+
"Cookie": "sessioncookiekey=sessioncookieval; cookiekey=cookieval",
45+
"headerkey": "headerval",
46+
"sessionheaderkey": "sessionheaderval",
47+
}
48+
49+
50+
def test_base_stream(session):
51+
stream = Stream(session)
52+
assert stream.__json__() == {
53+
"type": "stream",
54+
}
55+
assert stream.json == """{"type": "stream"}"""
56+
57+
58+
def test_file_stream_path(session):
59+
stream = FileStream(session, "/path/to/file")
60+
assert stream.__json__() == {
61+
"type": "file",
62+
"path": "/path/to/file",
63+
}
64+
65+
66+
def test_file_stream_handle(session):
67+
stream = FileStream(session, None, Mock())
68+
assert stream.__json__() == {
69+
"type": "file",
70+
}
71+
72+
73+
def test_http_stream(session, common_args, expected_headers):
74+
stream = HTTPStream(session, "http://host/path?foo=bar", **common_args)
75+
assert stream.__json__() == {
76+
"type": "http",
77+
"url": "http://host/path?foo=bar&sessionqueryparamkey=sessionqueryparamval&queryparamkey=queryparamval",
78+
"method": "GET",
79+
"body": None,
80+
"headers": expected_headers,
81+
}
82+
83+
84+
def test_hls_stream(session, common_args, expected_headers):
85+
stream = HLSStream(session, "http://host/stream.m3u8?foo=bar", **common_args)
86+
assert stream.__json__() == {
87+
"type": "hls",
88+
"url": "http://host/stream.m3u8?foo=bar&sessionqueryparamkey=sessionqueryparamval&queryparamkey=queryparamval",
89+
"headers": expected_headers,
90+
}
91+
92+
93+
def test_hls_stream_master(session, common_args, expected_headers):
94+
stream = HLSStream(session, "http://host/stream.m3u8?foo=bar", "http://host/master.m3u8?foo=bar", **common_args)
95+
assert stream.__json__() == {
96+
"type": "hls",
97+
"url": "http://host/stream.m3u8?foo=bar&sessionqueryparamkey=sessionqueryparamval&queryparamkey=queryparamval",
98+
"master": "http://host/master.m3u8?foo=bar&sessionqueryparamkey=sessionqueryparamval&queryparamkey=queryparamval",
99+
"headers": expected_headers,
100+
}
101+
102+
103+
def test_dash_stream(session, common_args):
104+
mpd = Mock(url=None)
105+
stream = DASHStream(session, mpd, **common_args)
106+
assert stream.__json__() == {
107+
"type": "dash",
108+
}
109+
110+
111+
def test_dash_stream_url(session, common_args, expected_headers):
112+
# DASHStream requires an MPD instance as input:
113+
# The URL of the MPD instance was already prepared by DASHStream.parse_manifest, so copy this behavior here.
114+
# This test verifies that session params, headers, etc. are added to the JSON data, without duplicates.
115+
args = common_args.copy()
116+
args.update(url="http://host/stream.mpd?foo=bar")
117+
url = session.http.prepare_new_request(**args).url
118+
119+
mpd = Mock(url=url)
120+
stream = DASHStream(session, mpd, **common_args)
121+
assert stream.__json__() == {
122+
"type": "dash",
123+
"url": "http://host/stream.mpd?foo=bar&sessionqueryparamkey=sessionqueryparamval&queryparamkey=queryparamval",
124+
"headers": expected_headers,
125+
}

0 commit comments

Comments
 (0)