Skip to content
This repository was archived by the owner on Jan 13, 2021. It is now read-only.

Implement HTTPS proxy support #322

Merged
Lukasa merged 2 commits into from
Jun 21, 2017
Merged

Implement HTTPS proxy support #322

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fa238d8
Add HTTPS proxy support
KostyaEsmukov May 6, 2017
2223f0d
Merge remote-tracking branch 'upstream/development' into secure_proxy
KostyaEsmukov Jun 21, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions hyper/common/connection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,9 @@ class HTTPConnection(object):
:param proxy_host: (optional) The proxy to connect to. This can be an IP
address or a host name and may include a port.
:param proxy_port: (optional) The proxy port to connect to. If not provided
and one also isn't provided in the ``proxy`` parameter, defaults to
8080.
and one also isn't provided in the ``proxy_host`` parameter, defaults
to 8080.
:param proxy_headers: (optional) The headers to send to a proxy.
"""
def __init__(self,
host,
Expand All @@ -56,19 +57,21 @@ def __init__(self,
ssl_context=None,
proxy_host=None,
proxy_port=None,
proxy_headers=None,
**kwargs):

self._host = host
self._port = port
self._h1_kwargs = {
'secure': secure, 'ssl_context': ssl_context,
'proxy_host': proxy_host, 'proxy_port': proxy_port,
'enable_push': enable_push
'proxy_headers': proxy_headers, 'enable_push': enable_push
}
self._h2_kwargs = {
'window_manager': window_manager, 'enable_push': enable_push,
'secure': secure, 'ssl_context': ssl_context,
'proxy_host': proxy_host, 'proxy_port': proxy_port
'proxy_host': proxy_host, 'proxy_port': proxy_port,
'proxy_headers': proxy_headers
}

# Add any unexpected kwargs to both dictionaries.
Expand Down
19 changes: 19 additions & 0 deletions hyper/common/exceptions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,3 +71,22 @@ class MissingCertFile(Exception):
The certificate file could not be found.
"""
pass


# Create our own ConnectionError.
try: # pragma: no cover
ConnectionError = ConnectionError
except NameError: # pragma: no cover
class ConnectionError(Exception):
"""
An error occurred during connection to a host.
"""

@Lukasa Lukasa Jun 20, 2017

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hrm. Here's a question: on Pythons that have ConnectionError, probably we shouldn't create hyper.common.exceptions.ConnectionError as an alias of that exception but as a subclass instead. Mind having this inherit instead?

@KostyaEsmukov KostyaEsmukov Jun 21, 2017 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why?

Here's the ConnectionResetError done the same way in master: https://github.com/Lukasa/hyper/blob/master/hyper/common/exceptions.py#L41

@Lukasa Lukasa Jun 21, 2017

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hrm. Given we have prior art let's keep doing this then. I have mixed feelings about it, but if we're going to fix it we should fix all of it. 😁



class ProxyError(ConnectionError):
"""
An error occurred during connection to a proxy.
"""
def __init__(self, message, response):
self.response = response
super(ProxyError, self).__init__(message)
41 changes: 33 additions & 8 deletions hyper/contrib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@
from requests.adapters import HTTPAdapter
from requests.models import Response
from requests.structures import CaseInsensitiveDict
from requests.utils import get_encoding_from_headers
from requests.utils import (
get_encoding_from_headers, select_proxy, prepend_scheme_if_needed
)
from requests.cookies import extract_cookies_to_jar
except ImportError: # pragma: no cover
HTTPAdapter = object
Expand All @@ -29,7 +31,8 @@ def __init__(self, *args, **kwargs):
#: A mapping between HTTP netlocs and ``HTTP20Connection`` objects.
self.connections = {}

def get_connection(self, host, port, scheme, cert=None, verify=True):
def get_connection(self, host, port, scheme, cert=None, verify=True,
proxy=None):
"""
Gets an appropriate HTTP/2 connection object based on
host/port/scheme/cert tuples.
Expand All @@ -50,29 +53,51 @@ def get_connection(self, host, port, scheme, cert=None, verify=True):
elif verify is not True:
ssl_context = init_context(cert_path=verify, cert=cert)

if proxy:
proxy_headers = self.proxy_headers(proxy)
proxy_netloc = urlparse(proxy).netloc
else:
proxy_headers = None
proxy_netloc = None

# We put proxy headers in the connection_key, because
# ``proxy_headers`` method might be overridden, so we can't
# rely on proxy headers being the same for the same proxies.
proxy_headers_key = (frozenset(proxy_headers.items())
if proxy_headers else None)
connection_key = (host, port, scheme, cert, verify,
proxy_netloc, proxy_headers_key)
try:
conn = self.connections[(host, port, scheme, cert, verify)]
conn = self.connections[connection_key]
except KeyError:
conn = HTTPConnection(
host,
port,
secure=secure,
ssl_context=ssl_context)
self.connections[(host, port, scheme, cert, verify)] = conn
ssl_context=ssl_context,
proxy_host=proxy_netloc,
proxy_headers=proxy_headers)
self.connections[connection_key] = conn

return conn

def send(self, request, stream=False, cert=None, verify=True, **kwargs):
def send(self, request, stream=False, cert=None, verify=True, proxies=None,
**kwargs):
"""
Sends a HTTP message to the server.
"""
proxy = select_proxy(request.url, proxies)
if proxy:
proxy = prepend_scheme_if_needed(proxy, 'http')

parsed = urlparse(request.url)
conn = self.get_connection(
parsed.hostname,
parsed.port,
parsed.scheme,
cert=cert,
verify=verify)
verify=verify,
proxy=proxy)

# Build the selector.
selector = parsed.path
Expand All @@ -97,7 +122,7 @@ def send(self, request, stream=False, cert=None, verify=True, **kwargs):
def build_response(self, request, resp):
"""
Builds a Requests' response object. This emulates most of the logic of
the standard fuction but deals with the lack of the ``.headers``
the standard function but deals with the lack of the ``.headers``
property on the HTTP20Response object.

Additionally, this function builds in a number of features that are
Expand Down
134 changes: 100 additions & 34 deletions hyper/http11/connection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@
from .response import HTTP11Response
from ..tls import wrap_socket, H2C_PROTOCOL
from ..common.bufsocket import BufferedSocket
from ..common.exceptions import TLSUpgrade, HTTPUpgrade
from ..common.exceptions import TLSUpgrade, HTTPUpgrade, ProxyError
from ..common.headers import HTTPHeaderMap
from ..common.util import to_bytestring, to_host_port_tuple, HTTPVersion
from ..common.util import (
to_bytestring, to_host_port_tuple, to_native_string, HTTPVersion
)
from ..compat import bytes

# We prefer pycohttpparser to the pure-Python interpretation
Expand All @@ -36,6 +38,43 @@
BODY_FLAT = 2


def _create_tunnel(proxy_host, proxy_port, target_host, target_port,
proxy_headers=None):
"""
Sends CONNECT method to a proxy and returns a socket with established
connection to the target.

:returns: socket
"""
conn = HTTP11Connection(proxy_host, proxy_port)
conn.request('CONNECT', '%s:%d' % (target_host, target_port),
headers=proxy_headers)

resp = conn.get_response()
if resp.status != 200:
raise ProxyError(
"Tunnel connection failed: %d %s" %
(resp.status, to_native_string(resp.reason)),
response=resp
)
return conn._sock


def _headers_to_http_header_map(headers):
# TODO turn this to a classmethod of HTTPHeaderMap
headers = headers or {}
if not isinstance(headers, HTTPHeaderMap):
if isinstance(headers, Mapping):
headers = HTTPHeaderMap(headers.items())
elif isinstance(headers, Iterable):
headers = HTTPHeaderMap(headers)
else:
raise ValueError(
'Header argument must be a dictionary or an iterable'
)
return headers


class HTTP11Connection(object):
"""
An object representing a single HTTP/1.1 connection to a server.
Expand All @@ -53,14 +92,16 @@ class HTTP11Connection(object):
:param proxy_host: (optional) The proxy to connect to. This can be an IP
address or a host name and may include a port.
:param proxy_port: (optional) The proxy port to connect to. If not provided
and one also isn't provided in the ``proxy`` parameter,
and one also isn't provided in the ``proxy_host`` parameter,
defaults to 8080.
:param proxy_headers: (optional) The headers to send to a proxy.
"""

version = HTTPVersion.http11

def __init__(self, host, port=None, secure=None, ssl_context=None,
proxy_host=None, proxy_port=None, **kwargs):
proxy_host=None, proxy_port=None, proxy_headers=None,
**kwargs):
if port is None:
self.host, self.port = to_host_port_tuple(host, default_port=80)
else:
Expand All @@ -83,17 +124,21 @@ def __init__(self, host, port=None, secure=None, ssl_context=None,
self.ssl_context = ssl_context
self._sock = None

# Keep the current request method in order to be able to know
# in get_response() what was the request verb.
self._current_request_method = None

# Setup proxy details if applicable.
if proxy_host:
if proxy_port is None:
self.proxy_host, self.proxy_port = to_host_port_tuple(
proxy_host, default_port=8080
)
else:
self.proxy_host, self.proxy_port = proxy_host, proxy_port
if proxy_host and proxy_port is None:
self.proxy_host, self.proxy_port = to_host_port_tuple(
proxy_host, default_port=8080
)
elif proxy_host:
self.proxy_host, self.proxy_port = proxy_host, proxy_port
else:
self.proxy_host = None
self.proxy_port = None
self.proxy_headers = proxy_headers

#: The size of the in-memory buffer used to store data from the
#: network. This is used as a performance optimisation. Increase buffer
Expand All @@ -113,19 +158,28 @@ def connect(self):
:returns: Nothing.
"""
if self._sock is None:
if not self.proxy_host:
host = self.host
port = self.port
else:
host = self.proxy_host
port = self.proxy_port

sock = socket.create_connection((host, port), 5)
if self.proxy_host and self.secure:
# Send http CONNECT method to a proxy and acquire the socket
sock = _create_tunnel(
self.proxy_host,
self.proxy_port,
self.host,
self.port,
proxy_headers=self.proxy_headers
)
elif self.proxy_host:
# Simple http proxy
sock = socket.create_connection(
(self.proxy_host, self.proxy_port),
5
)
else:
sock = socket.create_connection((self.host, self.port), 5)
proto = None

if self.secure:
assert not self.proxy_host, "Proxy with HTTPS not supported."
sock, proto = wrap_socket(sock, host, self.ssl_context)
sock, proto = wrap_socket(sock, self.host, self.ssl_context)

log.debug("Selected protocol: %s", proto)
sock = BufferedSocket(sock, self.network_buffer_size)
Expand Down Expand Up @@ -154,33 +208,37 @@ def request(self, method, url, body=None, headers=None):
:returns: Nothing.
"""

headers = headers or {}

method = to_bytestring(method)
is_connect_method = b'CONNECT' == method.upper()
self._current_request_method = method

if self.proxy_host and not self.secure:
# As per https://tools.ietf.org/html/rfc2068#section-5.1.2:
# The absoluteURI form is required when the request is being made
# to a proxy.
url = self._absolute_http_url(url)
url = to_bytestring(url)

if not isinstance(headers, HTTPHeaderMap):
if isinstance(headers, Mapping):
headers = HTTPHeaderMap(headers.items())
elif isinstance(headers, Iterable):
headers = HTTPHeaderMap(headers)
else:
raise ValueError(
'Header argument must be a dictionary or an iterable'
)
headers = _headers_to_http_header_map(headers)

# Append proxy headers.
if self.proxy_host and not self.secure:
headers.update(
_headers_to_http_header_map(self.proxy_headers).items()
)

if self._sock is None:
self.connect()

if self._send_http_upgrade:
if not is_connect_method and self._send_http_upgrade:
self._add_upgrade_headers(headers)
self._send_http_upgrade = False

# We may need extra headers.
if body:
body_type = self._add_body_headers(headers, body)

if b'host' not in headers:
if not is_connect_method and b'host' not in headers:
headers[b'host'] = self.host

# Begin by emitting the header block.
Expand All @@ -192,13 +250,20 @@ def request(self, method, url, body=None, headers=None):

return

def _absolute_http_url(self, url):
port_part = ':%d' % self.port if self.port != 80 else ''
return 'http://%s%s%s' % (self.host, port_part, url)

def get_response(self):
"""
Returns a response object.

This is an early beta, so the response object is pretty stupid. That's
ok, we'll fix it later.
"""
method = self._current_request_method

@Lukasa Lukasa May 10, 2017

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It'd be nice to None out self._current_request_method here as well.

self._current_request_method = None

headers = HTTPHeaderMap()

response = None
Expand Down Expand Up @@ -228,7 +293,8 @@ def get_response(self):
response.msg.tobytes(),
headers,
self._sock,
self
self,
method
)

def _send_headers(self, method, url, headers):
Expand Down
Loading