URLLib3 传输实现（从 HTTPX 中提取）

urllib3-transport

使用 urllib3作为 HTTP 网络后端的  HTTPCore传输。（这最初是随 HTTPX一起提供的）。

当与 HTTPX 一起使用时，此传输通过保持相同的基础 HTTP 网络层，可以更轻松地从请求过渡到 HTTPX。

兼容：HTTPX 0.15.x，0.16.x（即HTTPCore 0.11.x和HTTPCore 0.12.x）。

注意：此处不支持所有urllib3 池管理器选项 - 请随时根据特定需求调整此要点。

用法

使用HTTPX:

import httpx
from urllib3_transport import URLLib3Transport

with httpx.Client(transport=URLLib3Transport()) as client:
    response = client.get("https://example.org")
    print(response)

如果要使用与 HTTPX(verify, cert, trust_env) 相同的选项传递自定义​ssl_context​, 请使用httpx.create_ssl_context() 帮助程序:

import httpx
from urllib3_transport import URLLib3Transport

ssl_context = httpx.create_ssl_context(verify="/tmp/client.pem")

with httpx.Client(transport=URLLib3Transport(ssl_context=ssl_context)) as client:
    response = client.get("https://example.org")
    print(response)

另请参阅更改 HTTPX 文档中的验证默认值。

urllib3_transport.py：

import socket
import ssl
from typing import Dict, Iterator, List, Optional, Tuple

import httpcore
import urllib3


class URLLib3ByteStream(httpcore.SyncByteStream):
    def __init__(self, response: urllib3.HTTPResponse) -> None:
        self._response = response

    def __iter__(self) -> Iterator[bytes]:
        try:
            for chunk in self._response.stream(4096, decode_content=False):
                yield chunk
        except socket.error as exc:
            raise httpcore.NetworkError(exc)

    def close(self) -> None:
        self._response.release_conn()


class URLLib3Transport(httpcore.SyncHTTPTransport):
    def __init__(
        self,
        *,
        ssl_context: ssl.SSLContext = None,
        pool_connections: int = 10,
        pool_maxsize: int = 10,
        pool_block: bool = False,
    ) -> None:
        self._pool = urllib3.PoolManager(
            ssl_context=ssl_context,
            num_pools=pool_connections,
            maxsize=pool_maxsize,
            block=pool_block,
        )

    def request(
        self,
        method: bytes,
        url: Tuple[bytes, bytes, Optional[int], bytes],
        headers: List[Tuple[bytes, bytes]] = None,
        stream: httpcore.SyncByteStream = None,
        ext: dict = None,
    ) -> Tuple[int, List[Tuple[bytes, bytes]], httpcore.SyncByteStream, dict]:
        headers = [] if headers is None else headers
        stream = httpcore.PlainByteStream(b"") if stream is None else stream
        ext = {} if ext is None else ext
        timeout: Dict[str, float] = ext["timeout"]

        urllib3_timeout = urllib3.util.Timeout(
            connect=timeout.get("connect"), read=timeout.get("read")
        )

        chunked = False
        content_length = 0
        for header_key, header_value in headers:
            header_key = header_key.lower()
            if header_key == b"transfer-encoding":
                chunked = header_value == b"chunked"
            if header_key == b"content-length":
                content_length = int(header_value.decode("ascii"))
        body = stream if chunked or content_length else None

        scheme, host, port, path = url
        default_port = {b"http": 80, "https": 443}.get(scheme)
        if port is None or port == default_port:
            url_str = "%s://%s%s" % (
                scheme.decode("ascii"),
                host.decode("ascii"),
                path.decode("ascii"),
            )
        else:
            url_str = "%s://%s:%d%s" % (
                scheme.decode("ascii"),
                host.decode("ascii"),
                port,
                path.decode("ascii"),
            )

        try:
            response = self._pool.urlopen(
                method=method.decode(),
                url=url_str,
                headers={
                    key.decode("ascii"): value.decode("ascii") for key, value in headers
                },
                body=body,
                redirect=False,
                assert_same_host=False,
                retries=0,
                preload_content=False,
                chunked=chunked,
                timeout=urllib3_timeout,
                pool_timeout=timeout.get("pool"),
            )
        except (urllib3.exceptions.SSLError, socket.error) as exc:
            raise httpcore.NetworkError(exc)

        status_code = response.status
        reason_phrase = response.reason
        headers = list(response.headers.items())
        stream = URLLib3ByteStream(response)
        ext = {"reason": reason_phrase, "http_version": "HTTP/1.1"}

        return (status_code, headers, stream, ext)

    def close(self) -> None:
        self._pool.clear()


class URLLib3ProxyTransport(URLLib3Transport):
    def __init__(
        self,
        *,
        proxy_url: str,
        proxy_headers: dict = None,
        ssl_context: ssl.SSLContext = None,
        pool_connections: int = 10,
        pool_maxsize: int = 10,
        pool_block: bool = False,
    ) -> None:
        self._pool = urllib3.ProxyManager(
            proxy_url=proxy_url,
            proxy_headers=proxy_headers,
            ssl_context=ssl_context,
            num_pools=pool_connections,
            maxsize=pool_maxsize,
            block=pool_block,
        )

许可证

MIT 许可

Copyright (c) 2020 Florimond Manca

特此免费授予获得本软件和相关文档文件（“软件”）副本的任何人不受限制地处理本软件的许可，包括但不限于使用、复制、修改、合并、发布、分发、再许可和/或出售本软件副本的权利，并允许向其提供本软件的人员这样做， 须符合以下条件：

上述版权声明和本许可声明应包含在本软件的所有副本或大部分内容中。

本软件按“原样”提供，不作任何明示或暗示的保证，包括但不限于适销性、特定用途适用性和无侵权的保证。在任何情况下，作者或版权所有者均不对任何索赔、损害或其他责任负责，无论是在合同诉讼、侵权行为或其他诉讼中，由本软件或本软件的使用或其他交易引起、由本软件引起或与之相关。

本文地址：https://gist.github.com/florimondmanca/d56764d78d748eb9f73165da388e546e