plugin.api.http_session: add prepare_new_request

- Move stream.http.valid_args to HTTPSession.valid_request_args - Add HTTPSession.prepare_new_request - Update HTTPStream - Use newly added HTTPSession methods - Fix HLSStream - Prepare master URL appropriately - Fix DASHStream - Return correct JSON data, depending whether manifest URL exists - Filter custom args in DASHStream.parse_manifest - Use HTTPSession.prepare_new_request in plugins.twitch.UsherService - Rewrite Stream JSON tests and move test module
streamlink · back-to · May 13, 2022 · May 9, 2022 · May 9, 2022 · May 10, 2022
commit 05633e755df0906e0af09de86ef5bba807d5ffdc
diff --git a/src/streamlink/plugin/api/http_session.py b/src/streamlink/plugin/api/http_session.py
@@ -1,10 +1,10 @@
 import ssl
 import time
-from typing import Any, Callable, List, Pattern, Tuple
+from typing import Any, Callable, Dict, List, Pattern, Tuple
 
 import requests.adapters
 import urllib3
-from requests import Session
+from requests import PreparedRequest, Request, Session
 
 from streamlink.exceptions import PluginError
 from streamlink.packages.requests_file import FileAdapter
@@ -96,6 +96,10 @@ def _parse_keyvalue_list(val):
             continue
 
 
+# requests.Request.__init__ keywords, except for "hooks"
+_VALID_REQUEST_ARGS = "method", "url", "headers", "files", "data", "params", "auth", "cookies", "json"
+
+
 class HTTPSession(Session):
     def __init__(self):
         super().__init__()
@@ -168,6 +172,18 @@ def resolve_url(self, url):
         """Resolves any redirects and returns the final URL."""
         return self.get(url, stream=True).url
 
+    @staticmethod
+    def valid_request_args(**req_keywords) -> Dict:
+        return {k: v for k, v in req_keywords.items() if k in _VALID_REQUEST_ARGS}
+
+    def prepare_new_request(self, **req_keywords) -> PreparedRequest:
+        valid_args = self.valid_request_args(**req_keywords)
+        valid_args.setdefault("method", "GET")
+        request = Request(**valid_args)
+
+        # prepare request with the session context, which might add params, headers, cookies, etc.
+        return self.prepare_request(request)
+
     def request(self, method, url, *args, **kwargs):
         acceptable_status = kwargs.pop("acceptable_status", [])
         exception = kwargs.pop("exception", PluginError)

diff --git a/src/streamlink/plugins/twitch.py b/src/streamlink/plugins/twitch.py
@@ -13,8 +13,6 @@
 from typing import List, NamedTuple, Optional
 from urllib.parse import urlparse
 
-import requests
-
 from streamlink.exceptions import NoStreamsError, PluginError
 from streamlink.plugin import Plugin, PluginArgument, PluginArguments, pluginmatcher
 from streamlink.plugin.api import validate
@@ -190,8 +188,7 @@ def _create_url(self, endpoint, **extra_params):
         }
         params.update(extra_params)
 
-        req = requests.Request("GET", url, params=params)
-        req = self.session.http.prepare_request(req)
+        req = self.session.http.prepare_new_request(url=url, params=params)
 
         return req.url
 

diff --git a/src/streamlink/stream/dash.py b/src/streamlink/stream/dash.py
@@ -7,12 +7,9 @@
 from typing import Dict, Optional
 from urllib.parse import urlparse, urlunparse
 
-import requests
-
 from streamlink import PluginError, StreamError
 from streamlink.stream.dash_manifest import MPD, Representation, freeze_timeline, sleep_until, sleeper, utc
 from streamlink.stream.ffmpegmux import FFMPEGMuxer
-from streamlink.stream.http import normalize_key, valid_args
 from streamlink.stream.segmented import SegmentedStreamReader, SegmentedStreamWorker, SegmentedStreamWriter
 from streamlink.stream.stream import Stream
 from streamlink.utils.l10n import Language
@@ -173,11 +170,19 @@ def __init__(
         self.args = args
 
     def __json__(self):
-        req = requests.Request(method="GET", url=self.mpd.url, **valid_args(self.args))
-        req = req.prepare()
+        json = dict(type=self.shortname())
+
+        if self.mpd.url:
+            args = self.args.copy()
+            args.update(url=self.mpd.url)
+            req = self.session.http.prepare_new_request(**args)
+            json.update(
+                # the MPD URL has already been prepared by the initial request in `parse_manifest`
+                url=self.mpd.url,
+                headers=dict(req.headers),
+            )
 
-        headers = dict(map(normalize_key, req.headers.items()))
-        return dict(type=type(self).shortname(), url=req.url, headers=headers)
+        return json
 
     @classmethod
     def parse_manifest(
@@ -197,7 +202,7 @@ def parse_manifest(
         if url_or_manifest.startswith('<?xml'):
             mpd = MPD(parse_xml(url_or_manifest, ignore_ns=True))
         else:
-            res = session.http.get(url_or_manifest, **args)
+            res = session.http.get(url_or_manifest, **session.http.valid_request_args(**args))
             url = res.url
 
             urlp = list(urlparse(url))

diff --git a/src/streamlink/stream/file.py b/src/streamlink/stream/file.py
@@ -14,5 +14,13 @@ def __init__(self, session, path=None, fileobj=None):
         if not self.path and not self.fileobj:
             raise ValueError("path or fileobj must be set")
 
+    def __json__(self):
+        json = super().__json__()
+
+        if self.path:
+            json["path"] = self.path
+
+        return json
+
     def open(self):
         return self.fileobj or open(self.path)
diff --git a/src/streamlink/stream/hls.py b/src/streamlink/stream/hls.py
@@ -534,7 +534,7 @@ def __json__(self):
         json = super().__json__()
 
         if self.url_master:
-            json["master"] = self.url_master
+            json["master"] = self.to_manifest_url()
 
         # Pretty sure HLS is GET only.
         del json["method"]
@@ -543,7 +543,13 @@ def __json__(self):
         return json
 
     def to_manifest_url(self):
-        return self.url_master
+        if self.url_master is None:
+            return None
+
+        args = self.args.copy()
+        args.update(url=self.url_master)
+
+        return self.session.http.prepare_new_request(**args).url
 
     def open(self):
         reader = self.__reader__(self)

diff --git a/src/streamlink/stream/http.py b/src/streamlink/stream/http.py
@@ -1,26 +1,10 @@
 from typing import Dict
 
-import requests
-
 from streamlink.exceptions import StreamError
 from streamlink.stream.stream import Stream
 from streamlink.stream.wrappers import StreamIOIterWrapper, StreamIOThreadWrapper
 
 
-VALID_ARGS = ("method", "url", "params", "headers", "cookies", "auth", "data", "json", "files")
-
-
-def normalize_key(keyval):
-    key, val = keyval
-    key = hasattr(key, "decode") and key.decode("utf8", "ignore") or key
-
-    return key, val
-
-
-def valid_args(args):
-    return {k: v for k, v in args.items() if k in VALID_ARGS}
-
-
 class HTTPStream(Stream):
     """
     An HTTP stream using the :mod:`requests` library.
@@ -53,37 +37,33 @@ def __repr__(self):
         return "<HTTPStream({0!r})>".format(self.url)
 
     def __json__(self):
-        args = self.args.copy()
-        method = args.pop("method", "GET")
-        req = requests.Request(method=method, **valid_args(args))
-        req = self.session.http.prepare_request(req)
-
-        headers = dict(map(normalize_key, req.headers.items()))
-
-        return dict(type=type(self).shortname(), url=req.url,
-                    method=req.method, headers=headers,
-                    body=req.body)
+        req = self.session.http.prepare_new_request(**self.args)
+
+        return dict(
+            type=self.shortname(),
+            method=req.method,
+            url=req.url,
+            headers=dict(req.headers),
+            body=req.body,
+        )
 
     @property
     def url(self) -> str:
         """
         The URL to the stream, prepared by :mod:`requests` with parameters read from :attr:`args`.
         """
 
-        args = self.args.copy()
-        method = args.pop("method", "GET")
-        return requests.Request(method=method, **valid_args(args)).prepare().url
+        return self.session.http.prepare_new_request(**self.args).url
 
     def open(self):
-        args = self.args.copy()
-        method = args.pop("method", "GET")
+        reqargs = self.session.http.valid_request_args(**self.args)
+        reqargs.setdefault("method", "GET")
         timeout = self.session.options.get("stream-timeout")
         res = self.session.http.request(
-            method=method,
             stream=True,
             exception=StreamError,
             timeout=timeout,
-            **valid_args(args)
+            **reqargs,
         )
 
         fd = StreamIOIterWrapper(res.iter_content(8192))

diff --git a/src/streamlink/stream/stream.py b/src/streamlink/stream/stream.py
@@ -24,7 +24,7 @@ def __repr__(self):
         return "<Stream()>"
 
     def __json__(self):
-        return dict(type=type(self).shortname())
+        return dict(type=self.shortname())
 
     def open(self) -> "StreamIO":
         """

diff --git a/tests/stream/test_stream_json.py b/tests/stream/test_stream_json.py
@@ -0,0 +1,126 @@
+from unittest.mock import Mock
+
+import pytest
+# noinspection PyUnresolvedReferences
+from requests.utils import DEFAULT_ACCEPT_ENCODING
+
+from streamlink import Streamlink
+from streamlink.stream.dash import DASHStream
+from streamlink.stream.file import FileStream
+from streamlink.stream.hls import HLSStream
+from streamlink.stream.http import HTTPStream
+from streamlink.stream.stream import Stream
+
+
+@pytest.fixture(scope="module")
+def session():
+    session = Streamlink()
+    session.set_option("http-cookies", {"sessioncookiekey": "sessioncookieval"})
+    session.set_option("http-headers", {"sessionheaderkey": "sessionheaderval"})
+    session.set_option("http-query-params", {"sessionqueryparamkey": "sessionqueryparamval"})
+
+    return session
+
+
+@pytest.fixture(scope="module")
+def common_args():
+    return dict(
+        params={"queryparamkey": "queryparamval"},
+        headers={
+            "User-Agent": "Test",
+            "headerkey": "headerval",
+        },
+        cookies={"cookiekey": "cookieval"},
+        unknown="invalid",
+    )
+
+
+@pytest.fixture(scope="module")
+def expected_headers():
+    return {
+        "User-Agent": "Test",
+        "Accept": "*/*",
+        "Accept-Encoding": DEFAULT_ACCEPT_ENCODING,
+        "Connection": "keep-alive",
+        "Cookie": "sessioncookiekey=sessioncookieval; cookiekey=cookieval",
+        "headerkey": "headerval",
+        "sessionheaderkey": "sessionheaderval",
+    }
+
+
+def test_base_stream(session):
+    stream = Stream(session)
+    assert stream.__json__() == {
+        "type": "stream",
+    }
+    assert stream.json == """{"type": "stream"}"""
+
+
+def test_file_stream_path(session):
+    stream = FileStream(session, "/path/to/file")
+    assert stream.__json__() == {
+        "type": "file",
+        "path": "/path/to/file",
+    }
+
+
+def test_file_stream_handle(session):
+    stream = FileStream(session, None, Mock())
+    assert stream.__json__() == {
+        "type": "file",
+    }
+
+
+def test_http_stream(session, common_args, expected_headers):
+    stream = HTTPStream(session, "http://host/path?foo=bar", **common_args)
+    assert stream.__json__() == {
+        "type": "http",
+        "url": "http://host/path?foo=bar&sessionqueryparamkey=sessionqueryparamval&queryparamkey=queryparamval",
+        "method": "GET",
+        "body": None,
+        "headers": expected_headers,
+    }
+
+
+def test_hls_stream(session, common_args, expected_headers):
+    stream = HLSStream(session, "http://host/stream.m3u8?foo=bar", **common_args)
+    assert stream.__json__() == {
+        "type": "hls",
+        "url": "http://host/stream.m3u8?foo=bar&sessionqueryparamkey=sessionqueryparamval&queryparamkey=queryparamval",
+        "headers": expected_headers,
+    }
+
+
+def test_hls_stream_master(session, common_args, expected_headers):
+    stream = HLSStream(session, "http://host/stream.m3u8?foo=bar", "http://host/master.m3u8?foo=bar", **common_args)
+    assert stream.__json__() == {
+        "type": "hls",
+        "url": "http://host/stream.m3u8?foo=bar&sessionqueryparamkey=sessionqueryparamval&queryparamkey=queryparamval",
+        "master": "http://host/master.m3u8?foo=bar&sessionqueryparamkey=sessionqueryparamval&queryparamkey=queryparamval",
+        "headers": expected_headers,
+    }
+
+
+def test_dash_stream(session, common_args):
+    mpd = Mock(url=None)
+    stream = DASHStream(session, mpd, **common_args)
+    assert stream.__json__() == {
+        "type": "dash",
+    }
+
+
+def test_dash_stream_url(session, common_args, expected_headers):
+    # DASHStream requires an MPD instance as input:
+    # The URL of the MPD instance was already prepared by DASHStream.parse_manifest, so copy this behavior here.
+    # This test verifies that session params, headers, etc. are added to the JSON data, without duplicates.
+    args = common_args.copy()
+    args.update(url="http://host/stream.mpd?foo=bar")
+    url = session.http.prepare_new_request(**args).url
+
+    mpd = Mock(url=url)
+    stream = DASHStream(session, mpd, **common_args)
+    assert stream.__json__() == {
+        "type": "dash",
+        "url": "http://host/stream.mpd?foo=bar&sessionqueryparamkey=sessionqueryparamval&queryparamkey=queryparamval",
+        "headers": expected_headers,
+    }