sqlmapproject/sqlmap
1
0
Fork 0
mirror of https://github.com/sqlmapproject/sqlmap.git synced 2026-10-12 02:22:55 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
776643e3bd3fa41cc0a9e8927bb7cb5a1c850acb
sqlmap/lib/request/http2.py
T

1611 lines
62 KiB
Python
Raw Blame History

#!/usr/bin/env python
"""
Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org)
See the file 'LICENSE' for copying permission
"""
# Native, dependency-free HTTP/2 client (RFC 9113) with HPACK (RFC 7541). Runtime code uses only
# the standard library. The accompanying tests optionally use python-hyper/hpack and hyper-h2 for
# differential and local peer validation. The implementation keeps Python 2.7-compatible syntax,
# but this rewrite was executed and tested on Python 3.13 only.
import base64
import socket
import ssl
import struct
import threading
from collections import OrderedDict
try:
from http.client import responses as _HTTP_RESPONSES
except ImportError:
from httplib import responses as _HTTP_RESPONSES
try:
from urllib.parse import quote, urljoin, urlsplit
except ImportError:
from urllib import quote
from urlparse import urljoin, urlsplit
from email.message import Message as _Message
REDIRECT_CODES = (301, 302, 303, 307, 308)
HUFFMAN_CODES = [
0x1ff8, 0x7fffd8, 0xfffffe2, 0xfffffe3, 0xfffffe4, 0xfffffe5, 0xfffffe6, 0xfffffe7, 0xfffffe8, 0xffffea,
0x3ffffffc, 0xfffffe9, 0xfffffea, 0x3ffffffd, 0xfffffeb, 0xfffffec, 0xfffffed, 0xfffffee, 0xfffffef,
0xffffff0, 0xffffff1, 0xffffff2, 0x3ffffffe, 0xffffff3, 0xffffff4, 0xffffff5, 0xffffff6, 0xffffff7, 0xffffff8,
0xffffff9, 0xffffffa, 0xffffffb, 0x14, 0x3f8, 0x3f9, 0xffa, 0x1ff9, 0x15, 0xf8, 0x7fa, 0x3fa, 0x3fb, 0xf9,
0x7fb, 0xfa, 0x16, 0x17, 0x18, 0x0, 0x1, 0x2, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x5c, 0xfb, 0x7ffc,
0x20, 0xffb, 0x3fc, 0x1ffa, 0x21, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68,
0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0xfc, 0x73, 0xfd, 0x1ffb, 0x7fff0, 0x1ffc, 0x3ffc,
0x22, 0x7ffd, 0x3, 0x23, 0x4, 0x24, 0x5, 0x25, 0x26, 0x27, 0x6, 0x74, 0x75, 0x28, 0x29, 0x2a, 0x7, 0x2b, 0x76,
0x2c, 0x8, 0x9, 0x2d, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7ffe, 0x7fc, 0x3ffd, 0x1ffd, 0xffffffc, 0xfffe6,
0x3fffd2, 0xfffe7, 0xfffe8, 0x3fffd3, 0x3fffd4, 0x3fffd5, 0x7fffd9, 0x3fffd6, 0x7fffda, 0x7fffdb, 0x7fffdc,
0x7fffdd, 0x7fffde, 0xffffeb, 0x7fffdf, 0xffffec, 0xffffed, 0x3fffd7, 0x7fffe0, 0xffffee, 0x7fffe1, 0x7fffe2,
0x7fffe3, 0x7fffe4, 0x1fffdc, 0x3fffd8, 0x7fffe5, 0x3fffd9, 0x7fffe6, 0x7fffe7, 0xffffef, 0x3fffda, 0x1fffdd,
0xfffe9, 0x3fffdb, 0x3fffdc, 0x7fffe8, 0x7fffe9, 0x1fffde, 0x7fffea, 0x3fffdd, 0x3fffde, 0xfffff0, 0x1fffdf,
0x3fffdf, 0x7fffeb, 0x7fffec, 0x1fffe0, 0x1fffe1, 0x3fffe0, 0x1fffe2, 0x7fffed, 0x3fffe1, 0x7fffee, 0x7fffef,
0xfffea, 0x3fffe2, 0x3fffe3, 0x3fffe4, 0x7ffff0, 0x3fffe5, 0x3fffe6, 0x7ffff1, 0x3ffffe0, 0x3ffffe1, 0xfffeb,
0x7fff1, 0x3fffe7, 0x7ffff2, 0x3fffe8, 0x1ffffec, 0x3ffffe2, 0x3ffffe3, 0x3ffffe4, 0x7ffffde, 0x7ffffdf,
0x3ffffe5, 0xfffff1, 0x1ffffed, 0x7fff2, 0x1fffe3, 0x3ffffe6, 0x7ffffe0, 0x7ffffe1, 0x3ffffe7, 0x7ffffe2,
0xfffff2, 0x1fffe4, 0x1fffe5, 0x3ffffe8, 0x3ffffe9, 0xffffffd, 0x7ffffe3, 0x7ffffe4, 0x7ffffe5, 0xfffec,
0xfffff3, 0xfffed, 0x1fffe6, 0x3fffe9, 0x1fffe7, 0x1fffe8, 0x7ffff3, 0x3fffea, 0x3fffeb, 0x1ffffee, 0x1ffffef,
0xfffff4, 0xfffff5, 0x3ffffea, 0x7ffff4, 0x3ffffeb, 0x7ffffe6, 0x3ffffec, 0x3ffffed, 0x7ffffe7, 0x7ffffe8,
0x7ffffe9, 0x7ffffea, 0x7ffffeb, 0xffffffe, 0x7ffffec, 0x7ffffed, 0x7ffffee, 0x7ffffef, 0x7fffff0, 0x3ffffee,
0x3fffffff
]
HUFFMAN_LENGTHS = [
0xd, 0x17, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x18, 0x1e, 0x1c, 0x1c, 0x1e, 0x1c, 0x1c, 0x1c, 0x1c,
0x1c, 0x1c, 0x1c, 0x1c, 0x1e, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x6, 0xa, 0xa, 0xc, 0xd,
0x6, 0x8, 0xb, 0xa, 0xa, 0x8, 0xb, 0x8, 0x6, 0x6, 0x6, 0x5, 0x5, 0x5, 0x6, 0x6, 0x6, 0x6, 0x6, 0x6, 0x6, 0x7,
0x8, 0xf, 0x6, 0xc, 0xa, 0xd, 0x6, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7,
0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x8, 0x7, 0x8, 0xd, 0x13, 0xd, 0xe, 0x6, 0xf, 0x5, 0x6, 0x5, 0x6, 0x5, 0x6,
0x6, 0x6, 0x5, 0x7, 0x7, 0x6, 0x6, 0x6, 0x5, 0x6, 0x7, 0x6, 0x5, 0x5, 0x6, 0x7, 0x7, 0x7, 0x7, 0x7, 0xf, 0xb,
0xe, 0xd, 0x1c, 0x14, 0x16, 0x14, 0x14, 0x16, 0x16, 0x16, 0x17, 0x16, 0x17, 0x17, 0x17, 0x17, 0x17, 0x18,
0x17, 0x18, 0x18, 0x16, 0x17, 0x18, 0x17, 0x17, 0x17, 0x17, 0x15, 0x16, 0x17, 0x16, 0x17, 0x17, 0x18, 0x16,
0x15, 0x14, 0x16, 0x16, 0x17, 0x17, 0x15, 0x17, 0x16, 0x16, 0x18, 0x15, 0x16, 0x17, 0x17, 0x15, 0x15, 0x16,
0x15, 0x17, 0x16, 0x17, 0x17, 0x14, 0x16, 0x16, 0x16, 0x17, 0x16, 0x16, 0x17, 0x1a, 0x1a, 0x14, 0x13, 0x16,
0x17, 0x16, 0x19, 0x1a, 0x1a, 0x1a, 0x1b, 0x1b, 0x1a, 0x18, 0x19, 0x13, 0x15, 0x1a, 0x1b, 0x1b, 0x1a, 0x1b,
0x18, 0x15, 0x15, 0x1a, 0x1a, 0x1c, 0x1b, 0x1b, 0x1b, 0x14, 0x18, 0x14, 0x15, 0x16, 0x15, 0x15, 0x17, 0x16,
0x16, 0x19, 0x19, 0x18, 0x18, 0x1a, 0x17, 0x1a, 0x1b, 0x1a, 0x1a, 0x1b, 0x1b, 0x1b, 0x1b, 0x1b, 0x1c, 0x1b,
0x1b, 0x1b, 0x1b, 0x1b, 0x1a, 0x1e
]
STATIC_TABLE = (
(b':authority', b''),
(b':method', b'GET'),
(b':method', b'POST'),
(b':path', b'/'),
(b':path', b'/index.html'),
(b':scheme', b'http'),
(b':scheme', b'https'),
(b':status', b'200'),
(b':status', b'204'),
(b':status', b'206'),
(b':status', b'304'),
(b':status', b'400'),
(b':status', b'404'),
(b':status', b'500'),
(b'accept-charset', b''),
(b'accept-encoding', b'gzip, deflate'),
(b'accept-language', b''),
(b'accept-ranges', b''),
(b'accept', b''),
(b'access-control-allow-origin', b''),
(b'age', b''),
(b'allow', b''),
(b'authorization', b''),
(b'cache-control', b''),
(b'content-disposition', b''),
(b'content-encoding', b''),
(b'content-language', b''),
(b'content-length', b''),
(b'content-location', b''),
(b'content-range', b''),
(b'content-type', b''),
(b'cookie', b''),
(b'date', b''),
(b'etag', b''),
(b'expect', b''),
(b'expires', b''),
(b'from', b''),
(b'host', b''),
(b'if-match', b''),
(b'if-modified-since', b''),
(b'if-none-match', b''),
(b'if-range', b''),
(b'if-unmodified-since', b''),
(b'last-modified', b''),
(b'link', b''),
(b'location', b''),
(b'max-forwards', b''),
(b'proxy-authenticate', b''),
(b'proxy-authorization', b''),
(b'range', b''),
(b'referer', b''),
(b'refresh', b''),
(b'retry-after', b''),
(b'server', b''),
(b'set-cookie', b''),
(b'strict-transport-security', b''),
(b'transfer-encoding', b''),
(b'user-agent', b''),
(b'vary', b''),
(b'via', b''),
(b'www-authenticate', b''),
)
STATIC_LEN = len(STATIC_TABLE)
# HTTP/2 frame codec and client implementation (RFC 9113; HPACK remains RFC 7541).
# The public API is compatible with the original module. Optional `verify` and
# `ssl_context` arguments were added so TLS verification is opt-in. By default
# verification is disabled (the tool is used against test/self-signed targets).
try:
text_type = unicode
except NameError:
text_type = str
try:
integer_types = (int, long)
except NameError:
integer_types = (int,)
# Frame types
DATA = 0x0
HEADERS = 0x1
PRIORITY = 0x2
RST_STREAM = 0x3
SETTINGS = 0x4
PUSH_PROMISE = 0x5
PING = 0x6
GOAWAY = 0x7
WINDOW_UPDATE = 0x8
CONTINUATION = 0x9
# Flags
FLAG_END_STREAM = 0x1
FLAG_ACK = 0x1
FLAG_END_HEADERS = 0x4
FLAG_PADDED = 0x8
FLAG_PRIORITY = 0x20
# Settings
SETTINGS_HEADER_TABLE_SIZE = 0x1
SETTINGS_ENABLE_PUSH = 0x2
SETTINGS_MAX_CONCURRENT_STREAMS = 0x3
SETTINGS_INITIAL_WINDOW_SIZE = 0x4
SETTINGS_MAX_FRAME_SIZE = 0x5
SETTINGS_MAX_HEADER_LIST_SIZE = 0x6
# Error codes
NO_ERROR = 0x0
PROTOCOL_ERROR = 0x1
INTERNAL_ERROR = 0x2
FLOW_CONTROL_ERROR = 0x3
SETTINGS_TIMEOUT = 0x4
STREAM_CLOSED = 0x5
FRAME_SIZE_ERROR = 0x6
REFUSED_STREAM = 0x7
CANCEL = 0x8
COMPRESSION_ERROR = 0x9
CONNECT_ERROR = 0xa
ENHANCE_YOUR_CALM = 0xb
INADEQUATE_SECURITY = 0xc
HTTP_1_1_REQUIRED = 0xd
CONNECTION_PREFACE = b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
DEFAULT_MAX_FRAME_SIZE = 16384
MAX_FRAME_SIZE = 0xffffff
MAX_WINDOW = (1 << 31) - 1
DEFAULT_WINDOW = 65535
LOCAL_INITIAL_WINDOW = 16 * 1024 * 1024
MAX_RESPONSE_SIZE = 100 * 1024 * 1024
MAX_HEADER_BLOCK_SIZE = 1024 * 1024
MAX_HEADER_LIST_SIZE = 1024 * 1024
MAX_HPACK_INTEGER = MAX_WINDOW
MAX_STREAM_ID = (1 << 31) - 1
H2_POOL_LIMIT = 32
MAX_INFORMATIONAL_RESPONSES = 32
MAX_INFORMATIONAL_HEADER_BYTES = 1024 * 1024
MAX_RESPONSE_FRAMES = 100000
_TOKEN_BYTES = frozenset(bytearray(b"!#$%&'*+-.^_`|~0123456789abcdefghijklmnopqrstuvwxyz"))
_METHOD_TOKEN_BYTES = frozenset(bytearray(
b"!#$%&'*+-.^_`|~0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
))
_CONNECTION_FIELDS = frozenset((
b"connection", b"proxy-connection", b"keep-alive", b"transfer-encoding", b"upgrade"
))
_SENSITIVE_REDIRECT_FIELDS = frozenset((b"authorization", b"cookie", b"proxy-authorization"))
_CONTENT_REDIRECT_FIELDS = frozenset((b"content-length", b"content-type", b"content-encoding"))
class H2Error(IOError):
pass
class H2ProtocolError(H2Error):
pass
class H2CompressionError(H2ProtocolError):
pass
class H2TransportError(H2Error):
pass
class _UnprocessedStream(H2Error):
"""The peer explicitly guaranteed that the request stream was not processed."""
def _as_bytes(value):
if isinstance(value, bytes):
return value
if isinstance(value, bytearray):
return bytes(value)
if isinstance(value, text_type):
return value.encode("latin-1")
raise TypeError("expected bytes/text value, got %s" % type(value).__name__)
def encode_frame(ftype, flags, stream_id, payload=b""):
"""Serialize an HTTP/2 frame.
>>> decode_frame_header(encode_frame(HEADERS, FLAG_END_HEADERS, 1, b'abc')[:9])
(3, 1, 4, 1)
"""
payload = _as_bytes(payload)
if not (0 <= ftype <= 0xff and 0 <= flags <= 0xff):
raise ValueError("frame type and flags must fit in one octet")
if not (0 <= stream_id <= MAX_STREAM_ID):
raise ValueError("stream id must be between 0 and 2^31-1")
if len(payload) > MAX_FRAME_SIZE:
raise ValueError("frame payload exceeds 24-bit length")
header = struct.pack("!I", len(payload))[1:]
header += struct.pack("!BBI", ftype, flags, stream_id)
return header + payload
def decode_frame_header(nine):
"""Parse a 9-byte frame header into (length, type, flags, stream_id).
>>> decode_frame_header(encode_frame(DATA, 0, 1, b'')[:9])
(0, 0, 0, 1)
"""
if len(nine) != 9:
raise ValueError("frame header must be exactly 9 bytes")
length = struct.unpack("!I", b"\x00" + nine[:3])[0]
ftype, flags, stream_id = struct.unpack("!BBI", nine[3:9])
return length, ftype, flags, stream_id & MAX_STREAM_ID
# ---------- Huffman ----------
def huffman_encode(data):
"""Huffman-encode a byte string using the RFC 7541 static code.
>>> huffman_decode(huffman_encode(b'www.example.com')) == b'www.example.com'
True
"""
data = _as_bytes(data)
if not data:
return b""
acc = 0
nbits = 0
for item in bytearray(data):
acc = (acc << HUFFMAN_LENGTHS[item]) | HUFFMAN_CODES[item]
nbits += HUFFMAN_LENGTHS[item]
pad = (8 - nbits % 8) % 8
acc = (acc << pad) | ((1 << pad) - 1)
total = (nbits + pad) // 8
out = bytearray()
for index in range(total - 1, -1, -1):
out.append((acc >> (8 * index)) & 0xff)
return bytes(out)
_HUFF_ROOT = {}
def _build_huffman_trie():
for sym in range(256):
code = HUFFMAN_CODES[sym]
length = HUFFMAN_LENGTHS[sym]
node = _HUFF_ROOT
for index in range(length - 1, -1, -1):
bit = (code >> index) & 1
if index == 0:
node[bit] = sym
else:
child = node.get(bit)
if child is None:
child = {}
node[bit] = child
elif not isinstance(child, dict):
raise AssertionError("invalid prefix Huffman table")
node = child
_build_huffman_trie()
def huffman_decode(data):
data = _as_bytes(data)
out = bytearray()
node = _HUFF_ROOT
consumed = 0
tail = 0
for item in bytearray(data):
for index in range(7, -1, -1):
bit = (item >> index) & 1
nxt = node.get(bit)
if nxt is None:
raise H2CompressionError("invalid Huffman sequence")
consumed += 1
tail = (tail << 1) | bit
if isinstance(nxt, dict):
node = nxt
else:
out.append(nxt)
node = _HUFF_ROOT
consumed = 0
tail = 0
if node is not _HUFF_ROOT:
if consumed > 7 or tail != (1 << consumed) - 1:
raise H2CompressionError("invalid Huffman EOS padding")
return bytes(out)
# ---------- HPACK integer/string ----------
def encode_integer(value, prefix_bits, first_byte=0):
if not isinstance(value, integer_types) or value < 0:
raise ValueError("HPACK integer must be non-negative")
if not 1 <= prefix_bits <= 8:
raise ValueError("invalid HPACK prefix width")
mask = (1 << prefix_bits) - 1
if value < mask:
return bytearray([first_byte | value])
out = bytearray([first_byte | mask])
value -= mask
while value >= 0x80:
out.append((value & 0x7f) | 0x80)
value >>= 7
out.append(value)
return out
def decode_integer(data, pos, prefix_bits, max_value=MAX_HPACK_INTEGER):
if not 1 <= prefix_bits <= 8:
raise H2CompressionError("invalid HPACK prefix width")
if pos < 0 or pos >= len(data):
raise H2CompressionError("truncated HPACK integer")
mask = (1 << prefix_bits) - 1
value = data[pos] & mask
pos += 1
if value < mask:
return value, pos
shift = 0
while True:
if pos >= len(data):
raise H2CompressionError("truncated HPACK integer")
item = data[pos]
pos += 1
addend = (item & 0x7f) << shift
if addend > max_value - value:
raise H2CompressionError("HPACK integer exceeds implementation limit")
value += addend
if not (item & 0x80):
return value, pos
shift += 7
if shift > 28:
raise H2CompressionError("HPACK integer encoding is too long")
def encode_string(value, huffman=True):
value = _as_bytes(value)
if huffman:
encoded = huffman_encode(value)
if len(encoded) < len(value):
return encode_integer(len(encoded), 7, 0x80) + bytearray(encoded)
return encode_integer(len(value), 7, 0x00) + bytearray(value)
def decode_string(data, pos):
if pos < 0 or pos >= len(data):
raise H2CompressionError("truncated HPACK string")
use_huffman = bool(data[pos] & 0x80)
length, pos = decode_integer(data, pos, 7, max_value=MAX_HEADER_BLOCK_SIZE)
end = pos + length
if end > len(data):
raise H2CompressionError("truncated HPACK string payload")
raw = bytes(data[pos:end])
return (huffman_decode(raw) if use_huffman else raw), end
class Decoder(object):
def __init__(self, max_size=4096, max_header_list_size=MAX_HEADER_LIST_SIZE):
self.max_allowed_size = max_size
self.max_size = max_size
self.max_header_list_size = max_header_list_size
self.dynamic = []
self._size = 0
@staticmethod
def _entry_size(name, value):
return 32 + len(name) + len(value)
def _add(self, name, value):
entry_size = self._entry_size(name, value)
if entry_size > self.max_size:
self.dynamic = []
self._size = 0
return
self.dynamic.insert(0, (name, value))
self._size += entry_size
self._evict()
def _evict(self):
while self._size > self.max_size and self.dynamic:
name, value = self.dynamic.pop()
self._size -= self._entry_size(name, value)
def _get(self, index):
if index <= 0:
raise H2CompressionError("invalid HPACK header index 0")
if index <= STATIC_LEN:
return STATIC_TABLE[index - 1]
dynamic_index = index - STATIC_LEN - 1
if dynamic_index < 0 or dynamic_index >= len(self.dynamic):
raise H2CompressionError("HPACK dynamic index out of range")
return self.dynamic[dynamic_index]
def decode(self, data):
data = bytearray(_as_bytes(data))
pos = 0
headers = []
list_size = 0
saw_header = False
while pos < len(data):
first = data[pos]
if first & 0x80:
saw_header = True
index, pos = decode_integer(data, pos, 7)
name, value = self._get(index)
headers.append((name, value))
elif first & 0x40:
saw_header = True
index, pos = decode_integer(data, pos, 6)
if index:
name = self._get(index)[0]
else:
name, pos = decode_string(data, pos)
value, pos = decode_string(data, pos)
self._add(name, value)
headers.append((name, value))
elif first & 0x20:
if saw_header:
raise H2CompressionError("dynamic table size update is not at block start")
new_size, pos = decode_integer(data, pos, 5)
if new_size > self.max_allowed_size:
raise H2CompressionError("dynamic table size exceeds advertised maximum")
self.max_size = new_size
self._evict()
continue
else:
saw_header = True
index, pos = decode_integer(data, pos, 4)
if index:
name = self._get(index)[0]
else:
name, pos = decode_string(data, pos)
value, pos = decode_string(data, pos)
headers.append((name, value))
name, value = headers[-1]
list_size += self._entry_size(name, value)
if list_size > self.max_header_list_size:
raise H2CompressionError("decoded header list exceeds configured limit")
return headers
class Encoder(object):
"""A stateless, literal-without-indexing HPACK encoder."""
def encode(self, headers):
out = bytearray()
for name, value in headers:
name = _as_bytes(name)
value = _as_bytes(value)
out += encode_integer(0, 4, 0x00)
out += encode_string(name)
out += encode_string(value)
return bytes(out)
def _recv_exact(sock, count):
if count < 0:
raise ValueError("negative receive size")
chunks = []
received = 0
while received < count:
chunk = sock.recv(count - received)
if not chunk:
raise H2TransportError("connection closed by peer")
chunks.append(chunk)
received += len(chunk)
return b"".join(chunks)
def _read_frame(sock, max_frame_size=DEFAULT_MAX_FRAME_SIZE):
length, ftype, flags, sid = decode_frame_header(_recv_exact(sock, 9))
if length > max_frame_size:
raise H2ProtocolError("received frame exceeds local maximum frame size")
payload = _recv_exact(sock, length) if length else b""
return ftype, flags, sid, payload
def _idna_host(host):
if isinstance(host, bytes):
try:
host = host.decode("ascii")
except UnicodeDecodeError:
try:
host = host.decode("utf-8")
except UnicodeDecodeError:
raise H2Error("hostname bytes are neither ASCII nor UTF-8")
if not isinstance(host, text_type):
host = text_type(host)
if host.startswith("[") and host.endswith("]"):
host = host[1:-1]
if not host:
raise H2Error("hostname is required")
if ":" in host: # IPv6 literal, optionally with a zone identifier
return host
try:
return host.encode("idna").decode("ascii")
except (UnicodeError, AttributeError):
raise H2Error("invalid internationalized hostname %r" % host)
def _format_authority_host(host):
host = _idna_host(host)
if ":" in host:
return "[%s]" % host
return host
def _authority_for_host(host, port):
authority = _format_authority_host(host)
if port != 443:
authority += ":%d" % port
return authority
def _connect_socket(host, port, proxy, timeout):
if not proxy:
return socket.create_connection((host, port), timeout=timeout)
proxy_host, proxy_port, proxy_cred = proxy
raw = socket.create_connection((proxy_host, proxy_port), timeout=timeout)
try:
tunnel_host = _format_authority_host(host)
authority = "%s:%d" % (tunnel_host, port)
request = "CONNECT %s HTTP/1.1\r\nHost: %s\r\n" % (authority, authority)
if proxy_cred:
token = base64.b64encode(_as_bytes(proxy_cred)).decode("ascii")
request += "Proxy-Authorization: Basic %s\r\n" % token
request += "\r\n"
raw.sendall(request.encode("latin-1"))
# Read only through the CONNECT header terminator. A large recv() can consume
# bytes from the tunneled TLS handshake that immediately follows the proxy
# response, and those bytes cannot be pushed back before wrap_socket().
response = bytearray()
while not bytes(response).endswith(b"\r\n\r\n"):
chunk = raw.recv(1)
if not chunk:
raise H2Error("proxy closed connection during CONNECT")
response.extend(chunk)
if len(response) > 65536:
raise H2Error("oversized proxy CONNECT response")
status_line = bytes(response).split(b"\r\n", 1)[0].decode("latin-1", "replace")
fields = status_line.split(None, 2)
code = int(fields[1]) if len(fields) >= 2 and fields[1].isdigit() else 0
if not 200 <= code < 300:
raise H2Error("proxy CONNECT failed: %s" % status_line)
return raw
except Exception:
try:
raw.close()
except Exception:
pass
raise
def _make_ssl_context(verify, ssl_context):
context = ssl_context
if context is None:
if verify:
creator = getattr(ssl, "create_default_context", None)
if creator is None:
raise H2Error("certificate verification requires ssl.create_default_context")
context = creator()
else:
creator = getattr(ssl, "_create_unverified_context", None)
if creator is not None:
context = creator()
else:
context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
context.verify_mode = ssl.CERT_NONE
tls_version = getattr(ssl, "TLSVersion", None)
if tls_version is not None and hasattr(context, "minimum_version"):
if context.minimum_version < tls_version.TLSv1_2:
context.minimum_version = tls_version.TLSv1_2
else:
for option_name in ("OP_NO_SSLv2", "OP_NO_SSLv3", "OP_NO_TLSv1", "OP_NO_TLSv1_1"):
option = getattr(ssl, option_name, 0)
if option:
context.options |= option
no_compression = getattr(ssl, "OP_NO_COMPRESSION", 0)
if no_compression:
context.options |= no_compression
setter = getattr(context, "set_alpn_protocols", None)
if setter is None:
raise H2Error("this Python/OpenSSL build does not support ALPN")
setter(["h2"])
return context
def _iter_header_items(headers):
if headers is None:
return []
if hasattr(headers, "items"):
return list(headers.items())
return list(headers)
def _validate_regular_name(name):
if not name or name.startswith(b":"):
raise H2ProtocolError("invalid regular field name")
for item in bytearray(name):
if item not in _TOKEN_BYTES:
raise H2ProtocolError("invalid HTTP field name %r" % name)
def _validate_field_value(value):
if b"\x00" in value or b"\r" in value or b"\n" in value:
raise H2ProtocolError("invalid control character in HTTP field value")
if value[:1] in (b" ", b"\t") or value[-1:] in (b" ", b"\t"):
raise H2ProtocolError("HTTP/2 field values may not have leading/trailing whitespace")
def _validate_pseudo_value(name, value):
_validate_field_value(value)
for item in bytearray(value):
if item < 0x21 or item > 0x7e:
raise H2ProtocolError("invalid octet in %s pseudo-field" % name)
def _validate_method(method):
if not method:
raise H2ProtocolError("method is required")
for item in bytearray(method):
if item not in _METHOD_TOKEN_BYTES:
raise H2ProtocolError("invalid HTTP method token")
def _normalize_request_headers(headers, body):
raw_items = []
connection_tokens = set()
for key, value in _iter_header_items(headers):
name = _as_bytes(key).lower()
val = _as_bytes(value)
if name.startswith(b":"):
raise H2ProtocolError("callers may not inject HTTP/2 pseudo-fields")
_validate_regular_name(name)
_validate_field_value(val)
raw_items.append((name, val))
if name == b"connection":
for token in val.split(b","):
token = token.strip().lower()
if token:
_validate_regular_name(token)
connection_tokens.add(token)
normalized = []
content_lengths = []
content_length_added = False
for name, value in raw_items:
if name == b"host" or name == b"proxy-authorization":
continue
if name in _CONNECTION_FIELDS or name in connection_tokens:
continue
if name == b"te" and value.strip().lower() != b"trailers":
raise H2ProtocolError("HTTP/2 permits TE only with value 'trailers'")
if name == b"content-length":
content_lengths.append(value.strip())
if content_length_added:
continue
content_length_added = True
normalized.append((name, value))
if content_lengths:
if len(set(content_lengths)) != 1 or not content_lengths[0].isdigit():
raise H2ProtocolError("invalid or conflicting content-length fields")
expected = len(body)
if int(content_lengths[0]) != expected:
raise H2ProtocolError("content-length does not match request body")
return normalized
def _request_header_list(method, path, authority, headers, body):
method = _as_bytes(method)
authority = _as_bytes(authority)
path = _as_bytes(path)
_validate_method(method)
if not authority:
raise H2ProtocolError("authority is required")
_validate_pseudo_value(":authority", authority)
if method == b"CONNECT":
request = [(b":method", method), (b":authority", authority)]
else:
if not path:
raise H2ProtocolError("non-CONNECT requests require a non-empty :path")
_validate_pseudo_value(":path", path)
request = [(b":method", method), (b":scheme", b"https"), (b":path", path), (b":authority", authority)]
request.extend(_normalize_request_headers(headers, body))
return request
def _field_section_size(headers):
return sum(32 + len(name) + len(value) for name, value in headers)
def _header_frames(stream_id, block, max_frame_size, end_stream):
if max_frame_size < DEFAULT_MAX_FRAME_SIZE:
raise H2ProtocolError("peer advertised an invalid maximum frame size")
if not block:
return [encode_frame(HEADERS, FLAG_END_HEADERS | (FLAG_END_STREAM if end_stream else 0), stream_id, b"")]
frames = []
pos = 0
first = True
while pos < len(block):
chunk = block[pos:pos + max_frame_size]
pos += len(chunk)
final = pos == len(block)
if first:
flags = (FLAG_END_STREAM if end_stream else 0) | (FLAG_END_HEADERS if final else 0)
frames.append(encode_frame(HEADERS, flags, stream_id, chunk))
first = False
else:
flags = FLAG_END_HEADERS if final else 0
frames.append(encode_frame(CONTINUATION, flags, stream_id, chunk))
return frames
def _data_frames(stream_id, body, max_frame_size):
frames = []
pos = 0
while pos < len(body):
chunk = body[pos:pos + max_frame_size]
pos += len(chunk)
flags = FLAG_END_STREAM if pos == len(body) else 0
frames.append(encode_frame(DATA, flags, stream_id, chunk))
return frames
def _extract_header_fragment(ftype, flags, payload):
if ftype == CONTINUATION:
return payload
pos = 0
pad_length = 0
if flags & FLAG_PADDED:
if not payload:
raise H2ProtocolError("padded HEADERS frame has no pad length")
pad_length = bytearray(payload)[0]
pos = 1
if flags & FLAG_PRIORITY:
if len(payload) - pos < 5:
raise H2ProtocolError("truncated HEADERS priority fields")
pos += 5
if pad_length > len(payload) - pos:
raise H2ProtocolError("HEADERS padding exceeds frame payload")
end = len(payload) - pad_length
return payload[pos:end]
def _extract_data(flags, payload):
if not (flags & FLAG_PADDED):
return payload
if not payload:
raise H2ProtocolError("padded DATA frame has no pad length")
pad_length = bytearray(payload)[0]
if pad_length > len(payload) - 1:
raise H2ProtocolError("DATA padding exceeds frame payload")
return payload[1:len(payload) - pad_length]
def _parse_content_length(headers, trailers=False):
values = []
for name, value in headers:
if _as_bytes(name).lower() != b"content-length":
continue
if trailers:
raise H2ProtocolError("content-length is not permitted in trailers")
for part in _as_bytes(value).split(b","):
part = part.strip()
if not part or not part.isdigit():
raise H2ProtocolError("invalid content-length field")
values.append(int(part))
if not values:
return None
expected = values[0]
if any(value != expected for value in values[1:]):
raise H2ProtocolError("conflicting content-length fields")
return expected
def _validate_response_field_section(headers, trailers=False):
regular_seen = False
status = None
for name, value in headers:
name = _as_bytes(name)
value = _as_bytes(value)
_validate_field_value(value)
if name.startswith(b":"):
if trailers or regular_seen or name != b":status" or status is not None:
raise H2ProtocolError("invalid response pseudo-field ordering or duplication")
if len(value) != 3 or not value.isdigit():
raise H2ProtocolError("invalid :status pseudo-field")
status = int(value)
if not 100 <= status <= 599:
raise H2ProtocolError("response status is outside the valid range")
if status == 101:
raise H2ProtocolError("101 Switching Protocols is not valid in HTTP/2")
else:
regular_seen = True
if name.lower() != name:
raise H2ProtocolError("uppercase HTTP/2 field name")
_validate_regular_name(name)
if name in _CONNECTION_FIELDS:
raise H2ProtocolError("connection-specific field in HTTP/2 response")
content_length = _parse_content_length(headers, trailers=trailers)
if trailers:
return None, None
if status is None:
raise H2ProtocolError("response field section lacks :status")
return status, content_length
class _ResponseState(object):
def __init__(self, stream_id, request_method):
self.stream_id = stream_id
self.request_method = _as_bytes(request_method)
self.block = None
self.block_end_stream = False
self.headers = None
self.trailers = []
self.informational_count = 0
self.informational_bytes = 0
self.status = None
self.expected_content_length = None
self.body_forbidden = False
self.body = bytearray()
self.recv_window = LOCAL_INITIAL_WINDOW
self.complete = False
class _H2Connection(object):
"""A sequential HTTP/2 connection with an explicit two-stream timing primitive."""
def __init__(self, host, port, proxy, timeout, verify=False, ssl_context=None):
self.host = _idna_host(host)
self.port = port
self.proxy = proxy
self.dec = Decoder()
self.next_sid = 1
self.usable = True
self.sock = None
self.peer_max_frame_size = DEFAULT_MAX_FRAME_SIZE
self.peer_initial_window = DEFAULT_WINDOW
self.peer_max_concurrent_streams = None
self.peer_max_header_list_size = None
self.conn_send_window = DEFAULT_WINDOW
self.conn_recv_window = LOCAL_INITIAL_WINDOW
self.stream_send_windows = {}
self.local_max_frame_size = DEFAULT_MAX_FRAME_SIZE
self._our_settings_acked = False
context = _make_ssl_context(verify, ssl_context)
raw = _connect_socket(self.host, port, proxy, timeout)
try:
try:
raw.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
except (OSError, socket.error):
pass
self.sock = context.wrap_socket(raw, server_hostname=self.host)
except Exception:
try:
raw.close()
except Exception:
pass
raise
try:
selected = getattr(self.sock, "selected_alpn_protocol", lambda: None)()
if selected != "h2":
raise H2Error("server did not negotiate h2 (ALPN=%r)" % selected)
tls_version = getattr(self.sock, "version", lambda: None)()
if tls_version not in ("TLSv1.2", "TLSv1.3"):
raise H2Error("HTTP/2 requires TLS 1.2 or newer (negotiated %r)" % tls_version)
compression = getattr(self.sock, "compression", lambda: None)()
if compression is not None:
raise H2Error("TLS compression is not permitted for HTTP/2")
self.sock.settimeout(timeout)
settings_payload = struct.pack(
"!HIHIHI",
SETTINGS_ENABLE_PUSH, 0,
SETTINGS_INITIAL_WINDOW_SIZE, LOCAL_INITIAL_WINDOW,
SETTINGS_MAX_HEADER_LIST_SIZE, MAX_HEADER_LIST_SIZE,
)
self.sock.sendall(CONNECTION_PREFACE + encode_frame(SETTINGS, 0, 0, settings_payload))
self.sock.sendall(encode_frame(
WINDOW_UPDATE, 0, 0, struct.pack("!I", LOCAL_INITIAL_WINDOW - DEFAULT_WINDOW)
))
self._receive_initial_settings()
except Exception:
self.close()
raise
def close(self):
self.usable = False
sock = getattr(self, "sock", None)
if sock is not None:
try:
sock.close()
except Exception:
pass
self.sock = None
def __del__(self):
self.close()
def _connection_error(self, message):
self.usable = False
raise H2ProtocolError(message)
def _receive_initial_settings(self):
ftype, flags, sid, payload = _read_frame(self.sock, self.local_max_frame_size)
if ftype != SETTINGS or sid != 0 or (flags & FLAG_ACK):
self._connection_error("the server connection preface did not start with SETTINGS")
self._handle_settings(flags, sid, payload)
def _handle_settings(self, flags, sid, payload):
if sid != 0:
self._connection_error("SETTINGS frame used a non-zero stream id")
if flags & FLAG_ACK:
if payload:
self._connection_error("SETTINGS acknowledgement carried a payload")
self._our_settings_acked = True
return
if len(payload) % 6:
self._connection_error("SETTINGS payload length is not a multiple of 6")
data = bytearray(payload)
for pos in range(0, len(data), 6):
setting_id, value = struct.unpack("!HI", bytes(data[pos:pos + 6]))
if setting_id == SETTINGS_ENABLE_PUSH:
if value != 0:
self._connection_error("a server may set SETTINGS_ENABLE_PUSH only to 0")
elif setting_id == SETTINGS_MAX_CONCURRENT_STREAMS:
self.peer_max_concurrent_streams = value
elif setting_id == SETTINGS_INITIAL_WINDOW_SIZE:
if value > MAX_WINDOW:
self._connection_error("invalid SETTINGS_INITIAL_WINDOW_SIZE")
delta = value - self.peer_initial_window
for stream_id in list(self.stream_send_windows):
updated = self.stream_send_windows[stream_id] + delta
if updated > MAX_WINDOW:
self._connection_error("stream send window overflow after SETTINGS")
self.stream_send_windows[stream_id] = updated
self.peer_initial_window = value
elif setting_id == SETTINGS_MAX_FRAME_SIZE:
if not DEFAULT_MAX_FRAME_SIZE <= value <= MAX_FRAME_SIZE:
self._connection_error("invalid SETTINGS_MAX_FRAME_SIZE")
self.peer_max_frame_size = value
elif setting_id == SETTINGS_MAX_HEADER_LIST_SIZE:
self.peer_max_header_list_size = value
elif setting_id == SETTINGS_HEADER_TABLE_SIZE:
# Our encoder never uses the dynamic table, so every value is safe.
pass
self.sock.sendall(encode_frame(SETTINGS, FLAG_ACK, 0, b""))
def _handle_ping(self, flags, sid, payload):
if sid != 0 or len(payload) != 8:
self._connection_error("invalid PING frame")
if not (flags & FLAG_ACK):
self.sock.sendall(encode_frame(PING, FLAG_ACK, 0, payload))
def _handle_window_update(self, sid, payload):
if len(payload) != 4:
self._connection_error("invalid WINDOW_UPDATE length")
increment = struct.unpack("!I", payload)[0] & MAX_WINDOW
if increment == 0:
self._connection_error("WINDOW_UPDATE increment is zero")
if sid == 0:
if self.conn_send_window > MAX_WINDOW - increment:
self._connection_error("connection flow-control window overflow")
self.conn_send_window += increment
elif sid in self.stream_send_windows:
current = self.stream_send_windows[sid]
if current > MAX_WINDOW - increment:
self._connection_error("stream flow-control window overflow")
self.stream_send_windows[sid] = current + increment
elif sid % 2 == 0 or sid >= self.next_sid:
self._connection_error("WINDOW_UPDATE referenced an idle stream")
def _allocate_stream_ids(self, count):
if count <= 0:
raise ValueError("stream count must be positive")
last = self.next_sid + 2 * (count - 1)
if last > MAX_STREAM_ID:
self.usable = False
raise H2Error("HTTP/2 stream-id space exhausted")
sids = [self.next_sid + 2 * index for index in range(count)]
self.next_sid += 2 * count
if self.next_sid > MAX_STREAM_ID:
self.usable = False
return sids
def _send_headers(self, sid, request_headers, end_stream):
if self.peer_max_header_list_size is not None:
if _field_section_size(request_headers) > self.peer_max_header_list_size:
raise H2Error("request header list exceeds peer-advertised limit")
block = Encoder().encode(request_headers)
return b"".join(_header_frames(sid, block, self.peer_max_frame_size, end_stream))
def _finish_response(self, state):
actual = len(state.body)
if state.body_forbidden and actual:
self._connection_error("response semantics forbid a message body")
if (not state.body_forbidden and state.expected_content_length is not None and
actual != state.expected_content_length):
self._connection_error(
"response content-length mismatch (expected %d, received %d)" %
(state.expected_content_length, actual)
)
state.complete = True
def _complete_header_block(self, state):
try:
headers = self.dec.decode(bytes(state.block))
except H2CompressionError:
self.usable = False
raise
end_stream = state.block_end_stream
state.block = None
state.block_end_stream = False
if state.headers is None:
status, content_length = _validate_response_field_section(headers, trailers=False)
if 100 <= status < 200:
if end_stream:
self._connection_error("informational response ended the stream")
if content_length is not None:
self._connection_error("informational response carried content-length")
state.informational_count += 1
state.informational_bytes += _field_section_size(headers)
if state.informational_count > MAX_INFORMATIONAL_RESPONSES:
self.usable = False
raise H2Error("too many informational responses")
if state.informational_bytes > MAX_INFORMATIONAL_HEADER_BYTES:
self.usable = False
raise H2Error("informational response headers exceed configured limit")
return
state.headers = headers
state.status = status
state.expected_content_length = content_length
state.body_forbidden = state.request_method == b"HEAD" or status in (204, 205, 304)
if status == 204 and content_length is not None:
self._connection_error("204 response must not carry content-length")
if status == 205 and content_length not in (None, 0):
self._connection_error("205 response content-length must be zero")
if state.request_method == b"CONNECT" and 200 <= status < 300 and content_length is not None:
self._connection_error("successful CONNECT response must not carry content-length")
if (not state.body_forbidden and content_length is not None and
content_length > MAX_RESPONSE_SIZE):
self.usable = False
raise H2Error("declared response body exceeds configured limit")
if end_stream:
self._finish_response(state)
else:
_validate_response_field_section(headers, trailers=True)
if not end_stream:
self._connection_error("trailing field section did not end the stream")
if _field_section_size(state.headers) + _field_section_size(headers) > MAX_HEADER_LIST_SIZE:
self.usable = False
raise H2Error("aggregate response headers exceed configured limit")
state.trailers.extend(headers)
state.headers.extend(headers)
self._finish_response(state)
def _process_header_frame(self, ftype, flags, sid, payload, states, continuation_sid):
if continuation_sid is not None:
if ftype != CONTINUATION or sid != continuation_sid:
self._connection_error("frame interleaved inside a CONTINUATION sequence")
elif ftype == CONTINUATION:
self._connection_error("unexpected CONTINUATION frame")
state = states.get(sid)
if state is None:
self._connection_error("header frame for an unknown stream")
if state.complete:
self._connection_error("header frame received after END_STREAM")
if ftype == HEADERS:
if state.block is not None:
self._connection_error("new HEADERS before the prior field section completed")
state.block = bytearray()
state.block_end_stream = bool(flags & FLAG_END_STREAM)
state.block.extend(_extract_header_fragment(ftype, flags, payload))
if len(state.block) > MAX_HEADER_BLOCK_SIZE:
self.usable = False
raise H2Error("compressed response header block exceeds configured limit")
if flags & FLAG_END_HEADERS:
self._complete_header_block(state)
return None
return sid
def _process_data_frame(self, flags, sid, payload, states):
state = states.get(sid)
if state is None:
self._connection_error("DATA frame for an unknown stream")
if state.headers is None:
self._connection_error("DATA received before final response headers")
if state.complete:
self._connection_error("DATA received after END_STREAM")
flow_len = len(payload)
if flow_len > self.conn_recv_window:
self._connection_error("peer exceeded the connection receive window")
if flow_len > state.recv_window:
self._connection_error("peer exceeded the stream receive window")
self.conn_recv_window -= flow_len
state.recv_window -= flow_len
data = _extract_data(flags, payload)
if state.body_forbidden and data:
self._connection_error("response semantics forbid a message body")
if (state.expected_content_length is not None and not state.body_forbidden and
len(state.body) + len(data) > state.expected_content_length):
self._connection_error("response body exceeds declared content-length")
if len(state.body) + len(data) > MAX_RESPONSE_SIZE:
self.usable = False
raise H2Error("response body exceeds configured limit")
state.body.extend(data)
end_stream = bool(flags & FLAG_END_STREAM)
if flow_len:
if end_stream:
try:
self.sock.sendall(encode_frame(WINDOW_UPDATE, 0, 0, struct.pack("!I", flow_len)))
self.conn_recv_window += flow_len
except (socket.error, ssl.SSLError, IOError):
self.usable = False
else:
self.sock.sendall(encode_frame(WINDOW_UPDATE, 0, 0, struct.pack("!I", flow_len)))
self.sock.sendall(encode_frame(WINDOW_UPDATE, 0, sid, struct.pack("!I", flow_len)))
self.conn_recv_window += flow_len
state.recv_window += flow_len
if end_stream:
self._finish_response(state)
def _process_frame(self, frame, states, continuation_sid):
ftype, flags, sid, payload = frame
if continuation_sid is not None and not (ftype == CONTINUATION and sid == continuation_sid):
self._connection_error("non-CONTINUATION frame interleaved in a field section")
if ftype == SETTINGS:
self._handle_settings(flags, sid, payload)
elif ftype == PING:
self._handle_ping(flags, sid, payload)
elif ftype == WINDOW_UPDATE:
self._handle_window_update(sid, payload)
elif ftype == GOAWAY:
if sid != 0 or len(payload) < 8:
self._connection_error("invalid GOAWAY frame")
self.usable = False
last_sid = struct.unpack("!I", payload[:4])[0] & MAX_STREAM_ID
active = sorted(states)
unprocessed = [stream_id for stream_id in active if stream_id > last_sid]
if len(unprocessed) == len(active):
raise _UnprocessedStream("GOAWAY confirms active stream(s) were not processed")
if unprocessed:
raise H2Error("GOAWAY partially processed a multi-stream exchange")
elif ftype == RST_STREAM:
if sid == 0 or len(payload) != 4:
self._connection_error("invalid RST_STREAM frame")
if sid in states:
error_code = struct.unpack("!I", payload)[0]
if len(states) == 1 and error_code == REFUSED_STREAM:
raise _UnprocessedStream("peer refused the stream before processing")
raise H2Error("stream %d reset by peer (error %d)" % (sid, error_code))
if sid % 2 == 0 or sid >= self.next_sid:
self._connection_error("RST_STREAM referenced an idle stream")
elif ftype == PUSH_PROMISE:
self._connection_error("server push received despite SETTINGS_ENABLE_PUSH=0")
elif ftype in (HEADERS, CONTINUATION):
continuation_sid = self._process_header_frame(
ftype, flags, sid, payload, states, continuation_sid
)
elif ftype == DATA:
if sid == 0:
self._connection_error("DATA frame used stream 0")
self._process_data_frame(flags, sid, payload, states)
elif ftype == PRIORITY:
if sid == 0 or len(payload) != 5:
self._connection_error("invalid PRIORITY frame")
# Unknown extension frames are ignored, as required by HTTP/2.
return continuation_sid
def exchange(self, method, path, authority, headers, body, timeout):
if not self.usable:
raise H2Error("HTTP/2 connection is not usable")
if self.peer_max_concurrent_streams == 0:
raise H2Error("peer currently permits no new streams")
self.sock.settimeout(timeout)
body = b"" if body is None else _as_bytes(body)
method = _as_bytes(method)
sid = self._allocate_stream_ids(1)[0]
self.stream_send_windows[sid] = self.peer_initial_window
try:
state = _ResponseState(sid, method)
states = {sid: state}
request_headers = _request_header_list(method, path, authority, headers, body)
self.sock.sendall(self._send_headers(sid, request_headers, not body))
body_pos = 0
continuation_sid = None
frames_seen = 0
while not state.complete:
while body_pos < len(body):
available = min(
self.peer_max_frame_size,
self.conn_send_window,
self.stream_send_windows[sid],
len(body) - body_pos,
)
if available <= 0:
break
chunk = body[body_pos:body_pos + available]
body_pos += len(chunk)
flags = FLAG_END_STREAM if body_pos == len(body) else 0
self.sock.sendall(encode_frame(DATA, flags, sid, chunk))
self.conn_send_window -= len(chunk)
self.stream_send_windows[sid] -= len(chunk)
if state.complete:
break
frame = _read_frame(self.sock, self.local_max_frame_size)
frames_seen += 1
if frames_seen > MAX_RESPONSE_FRAMES:
self.usable = False
raise H2Error("response exceeded the frame-count limit")
continuation_sid = self._process_frame(frame, states, continuation_sid)
if continuation_sid is not None:
self._connection_error("stream ended with an incomplete field section")
if body_pos < len(body):
self.usable = False
try:
self.sock.sendall(encode_frame(RST_STREAM, 0, sid, struct.pack("!I", CANCEL)))
except Exception:
pass
if state.status is None:
self._connection_error("stream ended without final response headers")
return state.status, state.headers, bytes(state.body)
finally:
self.stream_send_windows.pop(sid, None)
def exchange_pair(self, requests, timeout):
"""Send exactly two small requests in one application-level write.
A single sendall() reduces userspace scheduling skew, but TCP/TLS may still split
the bytes into multiple records or packets; packet-level coalescing is not guaranteed.
"""
if not self.usable:
raise H2Error("HTTP/2 connection is not usable")
requests = list(requests)
if len(requests) != 2:
raise ValueError("exchange_pair requires exactly two requests")
if self.peer_max_concurrent_streams is not None and self.peer_max_concurrent_streams < 2:
raise H2Error("peer does not permit two concurrent streams")
self.sock.settimeout(timeout)
sids = self._allocate_stream_ids(2)
for sid in sids:
self.stream_send_windows[sid] = self.peer_initial_window
try:
states = {}
outbound = []
total_body = 0
for sid, request in zip(sids, requests):
body = b"" if request.get("body") is None else _as_bytes(request.get("body"))
method = _as_bytes(request.get("method", "GET"))
states[sid] = _ResponseState(sid, method)
if len(body) > self.peer_initial_window:
raise H2Error("timeless-pair body exceeds the peer stream window")
total_body += len(body)
request_headers = _request_header_list(
method,
request["path"],
request.get("authority") or _authority_for_host(self.host, self.port),
request.get("headers"),
body,
)
outbound.append(self._send_headers(sid, request_headers, not body))
if body:
outbound.extend(_data_frames(sid, body, self.peer_max_frame_size))
self.stream_send_windows[sid] -= len(body)
if total_body > self.conn_send_window:
raise H2Error("timeless-pair bodies exceed the connection flow-control window")
self.conn_send_window -= total_body
self.sock.sendall(b"".join(outbound))
finish_order = []
continuation_sid = None
frames_seen = 0
while len(finish_order) < 2:
before = dict((sid, states[sid].complete) for sid in sids)
frame = _read_frame(self.sock, self.local_max_frame_size)
frames_seen += 1
if frames_seen > MAX_RESPONSE_FRAMES:
self.usable = False
raise H2Error("timeless pair exceeded the frame-count limit")
continuation_sid = self._process_frame(frame, states, continuation_sid)
for sid in sids:
if states[sid].complete and not before[sid] and sid not in finish_order:
finish_order.append(sid)
if continuation_sid is not None:
self._connection_error("pair ended with an incomplete field section")
results = {}
for sid in sids:
state = states[sid]
if state.status is None:
self._connection_error("pair stream ended without final response headers")
results[sid] = (state.status, state.headers, bytes(state.body))
return finish_order, results
finally:
for sid in sids:
self.stream_send_windows.pop(sid, None)
_h2_pool = threading.local()
def _pool_key(host, port, proxy, verify, ssl_context):
proxy_key = tuple(proxy) if isinstance(proxy, list) else proxy
context_key = ssl_context if ssl_context is not None else bool(verify)
return _idna_host(host), port, proxy_key, context_key
def _get_pool():
pool = getattr(_h2_pool, "connections", None)
if pool is None or not isinstance(pool, OrderedDict):
pool = OrderedDict()
_h2_pool.connections = pool
return pool
def _pool_put(pool, key, conn):
old = pool.pop(key, None)
if old is not None and old is not conn:
old.close()
pool[key] = conn
while len(pool) > H2_POOL_LIMIT:
_, stale = pool.popitem(last=False)
stale.close()
def close_pooled_connections():
pool = _get_pool()
for conn in list(pool.values()):
conn.close()
pool.clear()
def _retryable_after_stale_connection(method):
return _as_bytes(method) in (b"GET", b"HEAD")
def _pooledExchange(host, port, proxy, method, path, authority, headers, body, timeout,
verify=False, ssl_context=None):
pool = _get_pool()
key = _pool_key(host, port, proxy, verify, ssl_context)
conn = pool.pop(key, None)
reused = conn is not None and conn.usable
if conn is None or not conn.usable:
if conn is not None:
conn.close()
conn = _H2Connection(host, port, proxy, timeout, verify=verify, ssl_context=ssl_context)
_pool_put(pool, key, conn)
try:
result = conn.exchange(method, path, authority, headers, body, timeout)
except _UnprocessedStream:
conn.close()
pool.pop(key, None)
replacement = _H2Connection(host, port, proxy, timeout, verify=verify, ssl_context=ssl_context)
_pool_put(pool, key, replacement)
try:
result = replacement.exchange(method, path, authority, headers, body, timeout)
except Exception:
replacement.close()
pool.pop(key, None)
raise
conn = replacement
except (socket.error, ssl.SSLError, H2TransportError):
conn.close()
pool.pop(key, None)
if not (reused and _retryable_after_stale_connection(method)):
raise
replacement = _H2Connection(host, port, proxy, timeout, verify=verify, ssl_context=ssl_context)
_pool_put(pool, key, replacement)
try:
result = replacement.exchange(method, path, authority, headers, body, timeout)
except Exception:
replacement.close()
pool.pop(key, None)
raise
conn = replacement
except (H2Error, IOError):
conn.close()
pool.pop(key, None)
raise
if not conn.usable:
conn.close()
pool.pop(key, None)
return result
def h2_request(host, port=443, method="GET", path="/", authority=None, headers=None, body=None,
timeout=30, proxy=None, verify=False, ssl_context=None):
conn = _H2Connection(host, port, proxy, timeout, verify=verify, ssl_context=ssl_context)
try:
return conn.exchange(method, path, authority or _authority_for_host(host, port), headers, body, timeout)
finally:
conn.close()
class H2Response(object):
"""A small urllib-response-compatible wrapper for a buffered HTTP/2 response."""
def __init__(self, url, status, headers, body):
self.url = url
self.code = self.status = status
self.msg = _HTTP_RESPONSES.get(status, "")
self.http_version = "HTTP/2.0"
self._body = body
self._offset = 0
self._info = _Message()
for name, value in headers or []:
name = name.decode("latin-1") if isinstance(name, bytes) else name
value = value.decode("latin-1") if isinstance(value, bytes) else value
if not name.startswith(":"):
self._info[name] = value
self._info.headers = ["%s: %s\r\n" % item for item in self._info.items()]
def info(self):
return self._info
def geturl(self):
return self.url
def getcode(self):
return self.code
def read(self, amt=None):
if amt is None or amt < 0:
data = self._body[self._offset:]
self._offset = len(self._body)
else:
data = self._body[self._offset:self._offset + amt]
self._offset += len(data)
return data
def close(self):
pass
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.close()
return False
def _headers_without(headers, names):
names = set(names)
result = []
for key, value in _iter_header_items(headers):
name = _as_bytes(key).lower()
if name not in names:
result.append((key, value))
return result
def _url_port(parts):
try:
return parts.port or 443
except ValueError as ex:
raise H2Error("invalid URL port: %s" % ex)
def _origin_tuple(parts):
host = _idna_host(parts.hostname or "").lower()
return parts.scheme.lower(), host, _url_port(parts)
def _quote_target_component(value, safe):
if isinstance(value, text_type):
try:
value.encode("ascii")
except UnicodeEncodeError:
value = value.encode("utf-8")
return quote(value, safe=safe)
def open_url(url, method="GET", headers=None, body=None, timeout=30, follow_redirects=True,
max_redirects=10, proxy=None, verify=False, ssl_context=None):
req_headers = headers or {}
current_method = method
current_body = body
for redirect_count in range(max_redirects + 1):
parts = urlsplit(url)
if parts.scheme.lower() != "https":
raise H2Error("native HTTP/2 client supports only https URLs")
if not parts.hostname:
raise H2Error("URL has no hostname")
host = _idna_host(parts.hostname)
port = _url_port(parts)
path = _quote_target_component(parts.path or "/", "/%:@!$&'()*+,;=-._~")
if parts.query:
path += "?" + _quote_target_component(parts.query, "/?%:@!$&'()*+,;=-._~")
authority = _authority_for_host(host, port)
status, response_headers, response_body = _pooledExchange(
host, port, proxy, current_method, path, authority,
req_headers, current_body, timeout, verify=verify, ssl_context=ssl_context
)
if not (follow_redirects and status in REDIRECT_CODES):
return H2Response(url, status, response_headers, response_body)
location = None
for name, value in response_headers or []:
if _as_bytes(name).lower() == b"location":
location = value.decode("latin-1") if isinstance(value, bytes) else value
break
if not location:
return H2Response(url, status, response_headers, response_body)
if redirect_count == max_redirects:
break
new_url = urljoin(url, location)
old_parts = parts
new_parts = urlsplit(new_url)
if _origin_tuple(old_parts) != _origin_tuple(new_parts):
req_headers = _headers_without(req_headers, _SENSITIVE_REDIRECT_FIELDS)
method_bytes = _as_bytes(current_method)
change_to_get = status == 303 and method_bytes != b"HEAD"
change_to_get = change_to_get or (status in (301, 302) and method_bytes == b"POST")
if change_to_get:
current_method = "GET"
current_body = None
req_headers = _headers_without(req_headers, _CONTENT_REDIRECT_FIELDS)
url = new_url
raise H2Error("too many HTTP/2 redirects")
Reference in New Issue View Git Blame Copy Permalink