From 58aac6ee12d1ac2172f9596491976f1fdc5a8357 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 2 Mar 2021 12:22:38 +0000 Subject: [PATCH 1/5] Update supported Python versions Remove references to, and handling of, Python 2.7 in our project scaffolding: - updated python_requires in setup.py to state our intent to support Python 3.6 and above (but not Python 4, yet) - Drop no longer required dependencies in setup.py, and requirements-*.txt (further refinement of requirements files will be handled in #1161) - Remove Python 2.7 from our tox environments Signed-off-by: Joshua Lock --- requirements-pinned.txt | 6 +----- requirements-test.txt | 5 +---- setup.py | 7 +------ tox.ini | 2 +- 4 files changed, 4 insertions(+), 16 deletions(-) diff --git a/requirements-pinned.txt b/requirements-pinned.txt index 621f2039c7..988b1edb35 100644 --- a/requirements-pinned.txt +++ b/requirements-pinned.txt @@ -1,15 +1,11 @@ certifi==2020.12.5 # via requests cffi==1.14.5 # via cryptography, pynacl chardet==4.0.0 # via requests -cryptography==3.4.6 ; python_version >= '3' # via securesystemslib -cryptography==3.3.2 ; python_version < '3' # via securesystemslib -enum34==1.1.10 ; python_version < '3' # via cryptography +cryptography==3.4.6 # via securesystemslib idna==2.10 # via requests -ipaddress==1.0.23 ; python_version < '3' # via cryptography pycparser==2.20 # via cffi pynacl==1.4.0 # via securesystemslib requests==2.25.1 securesystemslib[crypto,pynacl]==0.20.0 six==1.15.0 -subprocess32==3.5.4 ; python_version < '3' # via securesystemslib urllib3==1.26.3 # via requests diff --git a/requirements-test.txt b/requirements-test.txt index fc83f41a6e..80a7b09904 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,13 +3,10 @@ # pinned tuf runtime dependencies (should auto-update and -trigger ci/cd) -r requirements-pinned.txt -# test runtime dependencies (see 'tests_require' field in setup.py) -mock; python_version < "3.3" - # tuf.api tests use python-dateutil python-dateutil # additional test tools for linting and coverage measurement coverage pylint -bandit; python_version >= "3.5" +bandit diff --git a/setup.py b/setup.py index 7cfb24cc19..c65a794645 100755 --- a/setup.py +++ b/setup.py @@ -97,8 +97,6 @@ 'Operating System :: MacOS :: MacOS X', 'Operating System :: Microsoft :: Windows', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', @@ -112,15 +110,12 @@ 'Source': 'https://github.com/theupdateframework/tuf', 'Issues': 'https://github.com/theupdateframework/tuf/issues' }, - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4", + python_requires="~=3.6", install_requires = [ 'requests>=2.19.1', 'six>=1.11.0', 'securesystemslib>=0.18.0' ], - tests_require = [ - 'mock; python_version < "3.3"' - ], packages = find_packages(exclude=['tests']), scripts = [ 'tuf/scripts/repo.py', diff --git a/tox.ini b/tox.ini index acf804419d..9ab6dee135 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = lint,py{27,36,37,38,39} +envlist = lint,py{36,37,38,39} skipsdist = true [testenv] From 16bd3c2358f9557b79275ad3876655282f4955f7 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 2 Mar 2021 12:27:22 +0000 Subject: [PATCH 2/5] Remove Python 2.7 from GitHub CI configuration - Drop Python 2.7 from GitHub Actions workflows. Note: There is likely additional cleanup that can be done to the workflow now we no longer care about supporting Python 2.7. - No longer tell dependabot to ignore idna updates. Signed-off-by: Joshua Lock --- .github/dependabot.yml | 5 ----- .github/workflows/ci.yml | 10 ++-------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 556eb87ecc..f4952bab42 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,8 +6,3 @@ updates: interval: daily time: "10:00" open-pull-requests-limit: 10 - ignore: - # New 'idna' (see 'requests') releases break Python 2.7 builds. Ignore here - # to avoid listing/pinning transitive dependencies in requirements.txt. - # FIXME: Un-ignore when dropping Python 2.7 or resolving #1249 - - dependency-name: "idna" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 450d809b68..d001cee190 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: # Run regular TUF tests on each OS/Python combination, plus special tests # (sslib master) and linters on Linux/Python3.x only. matrix: - python-version: [2.7, 3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9] os: [ubuntu-latest, macos-latest, windows-latest] toxenv: [py] include: @@ -32,7 +32,7 @@ jobs: # NOTE: The Python 2.7 runner has two Python versions on the path (see # setup-python below), so we tell tox explicitly to use the 'py27' # testenv. For all other runners the toxenv configured above suffices. - TOXENV: ${{ matrix.python-version == '2.7' && 'py27' || matrix.toxenv }} + TOXENV: ${{ matrix.toxenv }} runs-on: ${{ matrix.os }} @@ -45,12 +45,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Set up service Python 3.x (on 2.7 only) - uses: actions/setup-python@v2 - if: ${{ matrix.python-version == 2.7 }} - with: - python-version: 3.x - - name: Find pip cache dir id: pip-cache run: echo "::set-output name=dir::$(pip cache dir)" From 8ee8e862af5f3d652d3ffeb3e7bbf11d3dc2a824 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 2 Mar 2021 12:30:33 +0000 Subject: [PATCH 3/5] updater: remove magic number Remove the magic number, a whence value of 2 for file.seek(), and instead use the io.SEEK_END constant from the io module. Signed-off-by: Joshua Lock --- tuf/client/updater.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 3496889089..1897b1f599 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -128,6 +128,7 @@ import fnmatch import copy import warnings +import io import tuf import tuf.download @@ -1245,9 +1246,7 @@ def _check_file_length(self, file_object, trusted_file_length): None. """ - # seek to the end of the file; that is offset 0 from the end of the file, - # represented by a whence value of 2 - file_object.seek(0, 2) + file_object.seek(0, io.SEEK_END) observed_length = file_object.tell() # Return and log a message if the length 'file_object' is equal to From 13b085712f2b4b83fc2e7014c34016d55061ab7f Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 2 Mar 2021 14:44:19 +0000 Subject: [PATCH 4/5] tests: remove some Python 2 specific tests Signed-off-by: Joshua Lock --- tests/proxy_server.py | 498 ---------------------------------------- tests/test_proxy_use.py | 374 ------------------------------ tests/test_utils.py | 30 --- tests/utils.py | 19 +- 4 files changed, 1 insertion(+), 920 deletions(-) delete mode 100644 tests/proxy_server.py delete mode 100755 tests/test_proxy_use.py diff --git a/tests/proxy_server.py b/tests/proxy_server.py deleted file mode 100644 index 7dec621f57..0000000000 --- a/tests/proxy_server.py +++ /dev/null @@ -1,498 +0,0 @@ -#!/usr/bin/env python - -# This code is taken from: github.com/inaz2/proxy2 -# Credit goes to the author. It has been very slightly modified here to use -# IPv4 instead of IPv6, and to only attempt interception of HTTPS traffic -# (instead of relaying via HTTP CONNECT) if new global variable INTERCEPT is -# set to True. (Modified sections are marked '# MODIFIED'.) -# -# Because this is a helper module for a test, the style is less important, and -# so to minimize changes from the source, it has NOT been changed to match the -# TUF project's code style outside of rewritten sections. - -""" - - proxy_server.py - - - Taken from a repository set to BSD 3-Clause "New" or "Revised" License. See: - https://github.com/inaz2/proxy2/blob/b2bab648173ac69f0a10421750125517accdfe26/LICENSE - - - Serves as an HTTP, HTTP CONNECT (TCP), and HTTPS proxy, for testing purposes. - This is used by test_proxy_use.py. - - In Python versions < 2.7.9, this proxy does not perform certificate - validation of the target server. As that is not part of what the current - tests using this script require, that is currently OK. In Python - versions > 2.7.9 (SSLContext was added in 2.7.9), the same code actually does - check the certificate, using the system's trusted CAs. As a result, since we - are using custom certificates, we need to either disable certificate - checking in 2.7.9 or load the specific CA for target test server, using the - SSLContext and create_default_context functionality also added in 2.7.9. It - is easier to do the latter, so the behavior in 2.7.9+ is to check the cert - and below 2.7.9 is not to. Note that we do not support Python < 2.7. - SSLContext is also available in all Python3 versions that we support. - - This module requires Python2.7 and does not support Python3. - - Note that this is not thread-safe, in part due to its use of globals. -""" - -import sys -import os -import socket -import ssl -import select -import httplib -import urlparse -import threading -import gzip -import zlib -import time -import json -import re -from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler -from SocketServer import ThreadingMixIn -from cStringIO import StringIO -from subprocess import Popen, PIPE -from HTMLParser import HTMLParser - -# MODIFIED: (added) three globals -# INTERCEPT: A boolean: -# False: normal HTTP proxy. Support HTTP & HTTPS connections to target server -# True: intercepting MITM transparent HTTPS proxy. Makes own TLS connections -# and has its own cert; must be trusted by the client and is able to -# modify requests. -# TARGET_SERVER_CA_FILEPATH: location of certificate to use as CA for -# connections to target servers (to constrain certs to trust from target -# servers). -# The remaining globals define the certs and keys to be used in communications -# with the client, with the proxy's CA signing new certs for individual hosts -# the client wishes to connect to, and placing them in dir PROXY_CERTS_DIR. -INTERCEPT = False -CERTS_DIR = os.path.join( - os.path.dirname(os.path.abspath(__file__)), 'ssl_certs') -TARGET_SERVER_CA_FILEPATH = os.path.join(CERTS_DIR, 'ssl_cert.crt') -PROXY_CA_KEY = os.path.join(CERTS_DIR, 'proxy_ca.key') # was cakey -PROXY_CA_CERT = os.path.join(CERTS_DIR, 'proxy_ca.crt') # was cacert -PROXY_CERTS_KEY = os.path.join(CERTS_DIR, 'proxy_cert.key') # was certkey -PROXY_CERTS_DIR = os.path.join(CERTS_DIR, 'proxy_certs') # was certdir - - -def with_color(c, s): - return "\x1b[%dm%s\x1b[0m" % (c, s) - -# MODIFIED: removed join_with_script_dir -# def get_cert_filepath(path): -# return os.path.join(CERTS_DIR, path) - - -class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): - address_family = socket.AF_INET # MODIFIED to use IPv4 instead of IPv6 - daemon_threads = True - - def handle_error(self, request, client_address): - # suppress socket/ssl related errors - cls, e = sys.exc_info()[:2] - if cls is socket.error or cls is ssl.SSLError: - pass - else: - return HTTPServer.handle_error(self, request, client_address) - - -class ProxyRequestHandler(BaseHTTPRequestHandler): - # MODIFIED: Variables here made into globals. - #Calls below modified: filenames changed, function changed to - # include ssl_certs directory. - timeout = 5 - lock = threading.Lock() - - def __init__(self, *args, **kwargs): - self.tls = threading.local() - self.tls.conns = {} - - BaseHTTPRequestHandler.__init__(self, *args, **kwargs) - - def log_error(self, format, *args): - # suppress "Request timed out: timeout('timed out',)" - if isinstance(args[0], socket.timeout): - return - - self.log_message(format, *args) - - def do_CONNECT(self): - # MODIFIED: This function has been modified to use new global INTERCEPT - # and to issue an error if the necessary certificate/key files are - # missing for interception attempts. - if not INTERCEPT: - print('\n\nRELAYING\n\n') - self.connect_relay() - - else: - assert os.path.isfile(PROXY_CA_KEY) \ - and os.path.isfile(PROXY_CA_CERT) \ - and os.path.isfile(PROXY_CERTS_KEY) \ - and os.path.isdir(PROXY_CERTS_DIR), \ - '\nMissing key or certificate files; unable to perform TLS ' \ - 'handshake with client to intercept traffic.\n' - print('\n\nINTERCEPTING\n\n') - self.connect_intercept() - - def connect_intercept(self): - hostname = self.path.split(':')[0] - certpath = os.path.join(PROXY_CERTS_DIR, hostname + '.crt') # MODIFIED for Windows compatibility and to use new globals - - with self.lock: - if not os.path.isfile(certpath): - epoch = "%d" % (time.time() * 1000) - p1 = Popen(["openssl", "req", "-new", "-key", PROXY_CERTS_KEY, "-subj", "/CN=%s" % hostname], stdout=PIPE) - p2 = Popen(["openssl", "x509", "-req", "-days", "3650", "-CA", PROXY_CA_CERT, "-CAkey", PROXY_CA_KEY, "-set_serial", epoch, "-out", certpath], stdin=p1.stdout, stderr=PIPE) # MODIFIED to use the new globals - p2.communicate() - - self.wfile.write("%s %d %s\r\n" % (self.protocol_version, 200, 'Connection Established')) - self.end_headers() - - self.connection = ssl.wrap_socket(self.connection, keyfile=PROXY_CERTS_KEY, certfile=certpath, server_side=True) # MODIFIED: Updated to use new globals - self.rfile = self.connection.makefile("rb", self.rbufsize) - self.wfile = self.connection.makefile("wb", self.wbufsize) - - conntype = self.headers.get('Proxy-Connection', '') - if self.protocol_version == "HTTP/1.1" and conntype.lower() != 'close': - self.close_connection = 0 - else: - self.close_connection = 1 - - def connect_relay(self): - address = self.path.split(':', 1) - address[1] = int(address[1]) or 443 - try: - s = socket.create_connection(address, timeout=self.timeout) - except Exception as e: - self.send_error(502) - return - self.send_response(200, 'Connection Established') - self.end_headers() - - conns = [self.connection, s] - self.close_connection = 0 - while not self.close_connection: - rlist, wlist, xlist = select.select(conns, [], conns, self.timeout) - if xlist or not rlist: - break - for r in rlist: - other = conns[1] if r is conns[0] else conns[0] - data = r.recv(8192) - if not data: - self.close_connection = 1 - break - other.sendall(data) - - def do_GET(self): - if self.path == 'http://proxy2.test/': - self.send_cacert() - return - - req = self - content_length = int(req.headers.get('Content-Length', 0)) - req_body = self.rfile.read(content_length) if content_length else None - - if req.path[0] == '/': - if isinstance(self.connection, ssl.SSLSocket): - req.path = "https://%s%s" % (req.headers['Host'], req.path) - else: - req.path = "http://%s%s" % (req.headers['Host'], req.path) - - req_body_modified = self.request_handler(req, req_body) - if req_body_modified is False: - self.send_error(403) - return - elif req_body_modified is not None: - req_body = req_body_modified - req.headers['Content-length'] = str(len(req_body)) - - u = urlparse.urlsplit(req.path) - scheme, netloc, path = u.scheme, u.netloc, (u.path + '?' + u.query if u.query else u.path) - assert scheme in ('http', 'https') - if netloc: - req.headers['Host'] = netloc - setattr(req, 'headers', self.filter_headers(req.headers)) - - try: - origin = (scheme, netloc) - if not origin in self.tls.conns: - if scheme == 'https': - # MODIFIED: Added Python version checking and changed behavior - # in Python2.7.9+ to use custom certificate for target server - # inherited from command line argument. - # In Python versions < 2.7.9, there is no certificate - # validation through this method of the target server. - # In supported Python versions > 2.7.9, we check the target - # server's certificate against our expected custom cert. - # See this script's docstring. - if sys.version_info.major == 2 \ - and sys.version_info.minor == 7 \ - and sys.version_info.micro < 9: - self.tls.conns[origin] = httplib.HTTPSConnection( - netloc, timeout=self.timeout) - else: - self.tls.conns[origin] = httplib.HTTPSConnection( - netloc, timeout=self.timeout, - context=ssl.create_default_context( # reqs Python2.7.9+ - cafile=TARGET_SERVER_CA_FILEPATH)) - else: - self.tls.conns[origin] = httplib.HTTPConnection(netloc, timeout=self.timeout) - conn = self.tls.conns[origin] - conn.request(self.command, path, req_body, dict(req.headers)) - res = conn.getresponse() - - version_table = {10: 'HTTP/1.0', 11: 'HTTP/1.1'} - setattr(res, 'headers', res.msg) - setattr(res, 'response_version', version_table[res.version]) - - # support streaming - if not 'Content-Length' in res.headers and 'no-store' in res.headers.get('Cache-Control', ''): - self.response_handler(req, req_body, res, '') - setattr(res, 'headers', self.filter_headers(res.headers)) - self.relay_streaming(res) - with self.lock: - self.save_handler(req, req_body, res, '') - return - - res_body = res.read() - except Exception as e: - if origin in self.tls.conns: - del self.tls.conns[origin] - self.send_error(502) - return - - content_encoding = res.headers.get('Content-Encoding', 'identity') - res_body_plain = self.decode_content_body(res_body, content_encoding) - - res_body_modified = self.response_handler(req, req_body, res, res_body_plain) - if res_body_modified is False: - self.send_error(403) - return - elif res_body_modified is not None: - res_body_plain = res_body_modified - res_body = self.encode_content_body(res_body_plain, content_encoding) - res.headers['Content-Length'] = str(len(res_body)) - - setattr(res, 'headers', self.filter_headers(res.headers)) - - self.wfile.write("%s %d %s\r\n" % (self.protocol_version, res.status, res.reason)) - for line in res.headers.headers: - self.wfile.write(line) - self.end_headers() - self.wfile.write(res_body) - self.wfile.flush() - - with self.lock: - self.save_handler(req, req_body, res, res_body_plain) - - def relay_streaming(self, res): - self.wfile.write("%s %d %s\r\n" % (self.protocol_version, res.status, res.reason)) - for line in res.headers.headers: - self.wfile.write(line) - self.end_headers() - try: - while True: - chunk = res.read(8192) - if not chunk: - break - self.wfile.write(chunk) - self.wfile.flush() - except socket.error: - # connection closed by client - pass - - do_HEAD = do_GET - do_POST = do_GET - do_PUT = do_GET - do_DELETE = do_GET - do_OPTIONS = do_GET - - def filter_headers(self, headers): - # http://tools.ietf.org/html/rfc2616#section-13.5.1 - hop_by_hop = ('connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', 'upgrade') - for k in hop_by_hop: - del headers[k] - - # accept only supported encodings - if 'Accept-Encoding' in headers: - ae = headers['Accept-Encoding'] - filtered_encodings = [x for x in re.split(r',\s*', ae) if x in ('identity', 'gzip', 'x-gzip', 'deflate')] - headers['Accept-Encoding'] = ', '.join(filtered_encodings) - - return headers - - def encode_content_body(self, text, encoding): - if encoding == 'identity': - data = text - elif encoding in ('gzip', 'x-gzip'): - io = StringIO() - with gzip.GzipFile(fileobj=io, mode='wb') as f: - f.write(text) - data = io.getvalue() - elif encoding == 'deflate': - data = zlib.compress(text) - else: - raise Exception("Unknown Content-Encoding: %s" % encoding) - return data - - def decode_content_body(self, data, encoding): - if encoding == 'identity': - text = data - elif encoding in ('gzip', 'x-gzip'): - io = StringIO(data) - with gzip.GzipFile(fileobj=io) as f: - text = f.read() - elif encoding == 'deflate': - try: - text = zlib.decompress(data) - except zlib.error: - text = zlib.decompress(data, -zlib.MAX_WBITS) - else: - raise Exception("Unknown Content-Encoding: %s" % encoding) - return text - - def send_cacert(self): - with open(PROXY_CA_CERT, 'rb') as f: # MODIFIED to use new globals - data = f.read() - - self.wfile.write("%s %d %s\r\n" % (self.protocol_version, 200, 'OK')) - self.send_header('Content-Type', 'application/x-x509-ca-cert') - self.send_header('Content-Length', len(data)) - self.send_header('Connection', 'close') - self.end_headers() - self.wfile.write(data) - - def print_info(self, req, req_body, res, res_body): - def parse_qsl(s): - return '\n'.join("%-20s %s" % (k, v) for k, v in urlparse.parse_qsl(s, keep_blank_values=True)) - - req_header_text = "%s %s %s\n%s" % (req.command, req.path, req.request_version, req.headers) - res_header_text = "%s %d %s\n%s" % (res.response_version, res.status, res.reason, res.headers) - - print with_color(33, req_header_text) - - u = urlparse.urlsplit(req.path) - if u.query: - query_text = parse_qsl(u.query) - print with_color(32, "==== QUERY PARAMETERS ====\n%s\n" % query_text) - - cookie = req.headers.get('Cookie', '') - if cookie: - cookie = parse_qsl(re.sub(r';\s*', '&', cookie)) - print with_color(32, "==== COOKIE ====\n%s\n" % cookie) - - auth = req.headers.get('Authorization', '') - if auth.lower().startswith('basic'): - token = auth.split()[1].decode('base64') - print with_color(31, "==== BASIC AUTH ====\n%s\n" % token) - - if req_body is not None: - req_body_text = None - content_type = req.headers.get('Content-Type', '') - - if content_type.startswith('application/x-www-form-urlencoded'): - req_body_text = parse_qsl(req_body) - elif content_type.startswith('application/json'): - try: - json_obj = json.loads(req_body) - json_str = json.dumps(json_obj, indent=2) - if json_str.count('\n') < 50: - req_body_text = json_str - else: - lines = json_str.splitlines() - req_body_text = "%s\n(%d lines)" % ('\n'.join(lines[:50]), len(lines)) - except ValueError: - req_body_text = req_body - elif len(req_body) < 1024: - req_body_text = req_body - - if req_body_text: - print with_color(32, "==== REQUEST BODY ====\n%s\n" % req_body_text) - - print with_color(36, res_header_text) - - cookies = res.headers.getheaders('Set-Cookie') - if cookies: - cookies = '\n'.join(cookies) - print with_color(31, "==== SET-COOKIE ====\n%s\n" % cookies) - - if res_body is not None: - res_body_text = None - content_type = res.headers.get('Content-Type', '') - - if content_type.startswith('application/json'): - try: - json_obj = json.loads(res_body) - json_str = json.dumps(json_obj, indent=2) - if json_str.count('\n') < 50: - res_body_text = json_str - else: - lines = json_str.splitlines() - res_body_text = "%s\n(%d lines)" % ('\n'.join(lines[:50]), len(lines)) - except ValueError: - res_body_text = res_body - elif content_type.startswith('text/html'): - m = re.search(r']*>\s*([^<]+?)\s*', res_body, re.I) - if m: - h = HTMLParser() - print with_color(32, "==== HTML TITLE ====\n%s\n" % h.unescape(m.group(1).decode('utf-8'))) - elif content_type.startswith('text/') and len(res_body) < 1024: - res_body_text = res_body - - if res_body_text: - print with_color(32, "==== RESPONSE BODY ====\n%s\n" % res_body_text) - - def request_handler(self, req, req_body): - pass - - def response_handler(self, req, req_body, res, res_body): - pass - - def save_handler(self, req, req_body, res, res_body): - self.print_info(req, req_body, res, res_body) - - -def test(HandlerClass=ProxyRequestHandler, ServerClass=ThreadingHTTPServer, protocol="HTTP/1.1"): - # MODIFIED: Added these globals. - global INTERCEPT - global TARGET_SERVER_CA_FILEPATH - - server_address = ('localhost', 0) - - # MODIFIED: Argument added, conditional below added to control INTERCEPT - # setting. - if len(sys.argv) > 1: - if sys.argv[1].lower() == 'intercept': - INTERCEPT = True - - # MODIFIED: Argument added to control certificate(s) the proxy expects of - # the target server(s), and added default value. - if len(sys.argv) > 2: - if os.path.exists(sys.argv[2]): - TARGET_SERVER_CA_FILEPATH = sys.argv[2] - else: - raise Exception('Target server cert file not found: ' + sys.argv[2]) - - # MODIFIED: Create the target-host-specific proxy certificates directory if - # it doesn't already exist. - if not os.path.exists(PROXY_CERTS_DIR): - os.mkdir(PROXY_CERTS_DIR) - - - HandlerClass.protocol_version = protocol - httpd = ServerClass(server_address, HandlerClass) - sa = httpd.socket.getsockname() - port_message = 'bind succeeded, server port is: ' + str(sa[1]) - print(port_message) - print("Serving HTTP Proxy on", sa[0], "port", sa[1], "...") - httpd.serve_forever() - - - -if __name__ == '__main__': - test() diff --git a/tests/test_proxy_use.py b/tests/test_proxy_use.py deleted file mode 100755 index 8731b407a6..0000000000 --- a/tests/test_proxy_use.py +++ /dev/null @@ -1,374 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018, New York University and the TUF contributors -# SPDX-License-Identifier: MIT OR Apache-2.0 - -""" - - test_proxy_use.py - - - See LICENSE-MIT OR LICENSE for licensing information. - - - Integration/regression test of TUF downloads through proxies. - - NOTE: Make sure test_proxy_use.py is run in 'tuf/tests/' directory. - Otherwise, test data or scripts may not be found. - - THIS module requires Python2.7 (not 2.8.x, not 3.x, just 2.7.x) as the test - proxy it uses only supports Python2.7. - - So long as the tests succeed in Python 2.7, it is unlikely that TUF - behaves differently with respect to proxies when it runs in other Python - versions. - - As a result of this dependency, this test is only run by aggregate_tests.py - when the Python version is 2.7.x. -""" - -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - -import logging -import os -import unittest -import sys - -import tuf -import tuf.download as download -import tuf.log -import tuf.unittest_toolbox as unittest_toolbox -import tuf.exceptions - -from tests import utils - -import six - -logger = logging.getLogger(__name__) - -IS_PY_VERSION_SUPPORTED = sys.version_info == (2, 7) - -# Use setUpModule to tell unittest runner to skip this test module gracefully. -def setUpModule(): - if not IS_PY_VERSION_SUPPORTED: - raise unittest.SkipTest('requires Python 2.7') - -class TestWithProxies(unittest_toolbox.Modified_TestCase): - - @classmethod - def setUpClass(cls): - """ - Setup performed before the first test function (TestWithProxies class - method) runs. - Launch HTTP, HTTPS, and proxy servers in the current working directory. - We'll set up four servers: - - HTTP server (simple_server.py) - - HTTPS server (simple_https_server.py) - - HTTP proxy server (proxy_server.py) - (that supports HTTP CONNECT to funnel HTTPS connections) - - HTTPS proxy server (proxy_server.py) - (trusted by the client to intercept and resign connections) - """ - - unittest_toolbox.Modified_TestCase.setUpClass() - - if not six.PY2: - raise NotImplementedError("TestWithProxies only works with Python 2" - " (proxy_server.py is Python2 only)") - - # Launch a simple HTTP server (serves files in the current dir). - cls.http_server_handler = utils.TestServerProcess(log=logger) - - # Launch an HTTPS server (serves files in the current dir). - cls.https_server_handler = utils.TestServerProcess(log=logger, - server='simple_https_server.py') - - # Launch an HTTP proxy server derived from inaz2/proxy2. - # This one is able to handle HTTP CONNECT requests, and so can pass HTTPS - # requests on to the target server. - cls.http_proxy_handler = utils.TestServerProcess(log=logger, - server='proxy_server.py') - - # Note that the HTTP proxy server's address uses http://, regardless of the - # type of connection used with the target server. - cls.http_proxy_addr = 'http://' + utils.TEST_HOST_ADDRESS + ':' + \ - str(cls.http_proxy_handler.port) - - - # Launch an HTTPS proxy server, also derived from inaz2/proxy2. - # (An HTTPS proxy performs its own TLS connection with the client and must - # be trusted by it, and is capable of tampering.) - # We instruct the proxy server to expect certain certificates from the - # target server. - # 1st arg: port - # 2nd arg: whether to intercept (HTTPS proxy) or relay (TCP tunnel using - # HTTP CONNECT verb, to facilitate an HTTPS connection between the client - # and server which the proxy cannot inspect) - # 3rd arg: (optional) certificate file for telling the proxy what target - # server certs to accept in its HTTPS connection to the target server. - # This is only relevant if the proxy is in intercept mode. - good_cert_fpath = os.path.join('ssl_certs', 'ssl_cert.crt') - cls.https_proxy_handler = utils.TestServerProcess(log=logger, - server='proxy_server.py', extra_cmd_args=['intercept', - good_cert_fpath]) - - # Note that the HTTPS proxy server's address uses https://, regardless of - # the type of connection used with the target server. - cls.https_proxy_addr = 'https://localhost:' + str(cls.https_proxy_handler.port) - - # Initialize the default fetcher for the download - self.fetcher = tuf.fetcher.RequestsFetcher() - - - - @classmethod - def tearDownClass(cls): - """ - Cleanup performed after the last of the tests (TestWithProxies methods) - has been run. - Stop server process and perform clean up. - """ - unittest_toolbox.Modified_TestCase.tearDownClass() - - for proc_handler in [ - cls.http_server_handler, - cls.https_server_handler, - cls.http_proxy_handler, - cls.https_proxy_handler, - ]: - - # Kill the SimpleHTTPServer process. - proc_handler.clean() - - - - def setUp(self): - """ - Setup performed before EACH test function (TestWithProxies class method) - runs. - """ - unittest_toolbox.Modified_TestCase.setUp(self) - - # Dictionary for saving environment values to restore. - self.old_env_values = {} - - # Make a temporary file to serve on the server, and determine its length, - # and its url on the server. - current_dir = os.getcwd() - target_filepath = self.make_temp_data_file(directory=current_dir) - - with open(target_filepath, 'r') as target_file_object: - self.target_data_length = len(target_file_object.read()) - - suffix = '/' + os.path.basename(target_filepath) - self.url = \ - 'http://' + utils.TEST_HOST_ADDRESS + ':' + \ - str(self.http_server_handler.port) + suffix - - self.url_https = \ - 'https://' + utils.TEST_HOST_ADDRESS + ':' + \ - str(self.https_server_handler.port) + suffix - - - - - - def tearDown(self): - """ - Cleanup performed after each test (each TestWithProxies method). - Reset environment variables (for next test, etc.). - """ - unittest_toolbox.Modified_TestCase.tearDown(self) - - self.restore_all_modified_env_values() - - for proc_handler in [ - self.http_server_handler, - self.https_server_handler, - self.http_proxy_handler, - self.https_proxy_handler, - ]: - - # Logs stdout and stderr from the sever subprocess. - proc_handler.flush_log() - - - - - def test_baseline_no_proxy(self): - """ - Test a length-validating TUF download of a file through a proxy. Use an - HTTP proxy, and perform an HTTP connection with the final server. - """ - - logger.info('Trying HTTP download with no proxy: ' + self.url) - download.safe_download(self.url, self.target_data_length, self.fetcher).close() - download.unsafe_download(self.url, self.target_data_length, self.fetcher).close() - - - - - - def test_http_dl_via_smart_http_proxy(self): - """ - Test a length-validating TUF download of a file through a proxy. Use an - HTTP proxy normally, and make an HTTP connection with the final server. - """ - - self.set_env_value('HTTP_PROXY', self.http_proxy_addr) - - logger.info('Trying HTTP download via HTTP proxy: ' + self.url) - download.safe_download(self.url, self.target_data_length, self.fetcher).close() - download.unsafe_download(self.url, self.target_data_length, self.fetcher).close() - - - - - - def test_https_dl_via_smart_http_proxy(self): - """ - Test a length-validating TUF download of a file through a proxy. Use an - HTTP proxy that supports HTTP CONNECT (which essentially causes it to act - as a TCP proxy), and perform an HTTPS connection through with the final - server. - - Note that the proxy address is still http://... even though the connection - with the target server is an HTTPS connection. The proxy itself will act as - a TCP proxy via HTTP CONNECT. - """ - self.set_env_value('HTTP_PROXY', self.http_proxy_addr) # http as intended - self.set_env_value('HTTPS_PROXY', self.http_proxy_addr) # http as intended - - self.set_env_value('REQUESTS_CA_BUNDLE', - os.path.join('ssl_certs', 'ssl_cert.crt')) - # Clear sessions to ensure that the certificate we just specified is used. - # TODO: Confirm necessity of this session clearing and lay out mechanics. - self.fetcher._sessions = {} - - logger.info('Trying HTTPS download via HTTP proxy: ' + self.url_https) - download.safe_download(self.url_https, self.target_data_length, self.fetcher).close() - download.unsafe_download(self.url_https, self.target_data_length, self.fetcher).close() - - - - - - def test_http_dl_via_https_proxy(self): - """ - Test a length-validating TUF download of a file through a proxy. Use an - HTTPS proxy, and perform an HTTP connection with the final server. - """ - self.set_env_value('HTTP_PROXY', self.https_proxy_addr) - self.set_env_value('HTTPS_PROXY', self.https_proxy_addr) # unnecessary - - # We're making an HTTPS connection with the proxy. The proxy will make a - # plain HTTP connection to the target server. - self.set_env_value('REQUESTS_CA_BUNDLE', - os.path.join('ssl_certs', 'proxy_ca.crt')) - # Clear sessions to ensure that the certificate we just specified is used. - # TODO: Confirm necessity of this session clearing and lay out mechanics. - self.fetcher._sessions = {} - - logger.info('Trying HTTP download via HTTPS proxy: ' + self.url_https) - download.safe_download(self.url, self.target_data_length, self.fetcher).close() - download.unsafe_download(self.url, self.target_data_length, self.fetcher).close() - - - - - - def test_https_dl_via_https_proxy(self): - """ - Test a length-validating TUF download of a file through a proxy. Use an - HTTPS proxy, and perform an HTTPS connection with the final server. - """ - self.set_env_value('HTTP_PROXY', self.https_proxy_addr) # unnecessary - self.set_env_value('HTTPS_PROXY', self.https_proxy_addr) - - # We're making an HTTPS connection with the proxy. The proxy will make its - # own HTTPS connection with the target server, and will have to know what - # certificate to trust. It was told what certs to trust when it was - # started in setUpClass(). - self.set_env_value('REQUESTS_CA_BUNDLE', - os.path.join('ssl_certs', 'proxy_ca.crt')) - # Clear sessions to ensure that the certificate we just specified is used. - # TODO: Confirm necessity of this session clearing and lay out mechanics. - self.fetcher._sessions = {} - - logger.info('Trying HTTPS download via HTTPS proxy: ' + self.url_https) - download.safe_download(self.url_https, self.target_data_length, self.fetcher).close() - download.unsafe_download(self.url_https, self.target_data_length, self.fetcher).close() - - - - - - def set_env_value(self, key, value): - """ - Set an environment variable after noting what the original value was, if it - was set, and add it to the queue for restoring to its original value / lack - of a value after the test finishes. - - Safe for multiple uses in one test: does not overwrite original saved value - with new saved values. - """ - - # Only save the current value if we have not previously saved an older - # value. The original one is the one we'll restore to, not whatever we - # most recently overwrote. - if key not in self.old_env_values: - # If the value was previously unset in os.environ, save the old value - # as None so that we know to unset it. - self.old_env_values[key] = os.environ.get(key, None) - - # Actually set the new value. - os.environ[key] = value - - - - - - def restore_env_value(self, key): - # Save old values for environment variables for restoration after the test. - # Save the pre-existing value of the environment variables HTTP_PROXY and - # HTTPS_PROXY so that we can restore them in tearDown() after the test. - # If the value was not originally set at all, we'll try to unset it again, - # too. - assert key in self.old_env_values, 'Test coding mistake: something is ' \ - 'trying to restore environment variable ' + key + ', but that ' \ - 'variable does not appear in the list of values to restore. ' \ - 'Please make sure to use set_env_value().' - - if self.old_env_values[key] is None: - # If it was not previously set, try to unset it. - # If the platform provides a way to unset environment variables, - # del os.environ[key] should unset the variable. Otherwise, we'll just - # have to settle for setting it to an empty string. - # See os.environ in: - # https://docs.python.org/2/library/os.html#process-parameters - os.environ[key] = '' - del os.environ[key] - - else: - # If it was previously set, restore the original value from when the - # test was being set up. - os.environ[key] = self.old_env_values[key] - - - - def restore_all_modified_env_values(self): - for key in self.old_env_values: - self.restore_env_value(key) - - - -# Run unit test. -if __name__ == '__main__': - utils.configure_test_logging(sys.argv) - unittest.main() diff --git a/tests/test_utils.py b/tests/test_utils.py index 28d974944c..63589fb516 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -89,36 +89,6 @@ def test_simple_https_server_startup(self): self.assertTrue(self.can_connect()) - @unittest.skipIf(sys.version_info.major != 2, "Test for Python 2.X") - def test_proxy_server_startup(self): - # Test normal case - self.server_process_handler = utils.TestServerProcess(log=logger, - server='proxy_server.py') - - # Make sure we can connect to the server. - self.assertTrue(self.can_connect()) - - self.server_process_handler.clean() - - # Test start proxy_server using certificate files. - good_cert_fpath = os.path.join('ssl_certs', 'ssl_cert.crt') - self.server_process_handler = utils.TestServerProcess(log=logger, - server='proxy_server.py', extra_cmd_args=['intercept', - good_cert_fpath]) - - # Make sure we can connect to the server. - self.assertTrue(self.can_connect()) - self.server_process_handler.clean() - - # Test with a non existing cert file. - non_existing_cert_path = os.path.join('ssl_certs', 'non_existing.crt') - self.server_process_handler = utils.TestServerProcess(log=logger, - server='proxy_server.py', extra_cmd_args=[non_existing_cert_path]) - - # Make sure we can connect to the server. - self.assertTrue(self.can_connect()) - - def test_slow_retrieval_server_startup(self): # Test normal case self.server_process_handler = utils.TestServerProcess(log=logger, diff --git a/tests/utils.py b/tests/utils.py index 10a12436ac..caccb18086 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -29,11 +29,7 @@ import subprocess import threading import warnings - -try: - import queue -except ImportError: - import Queue as queue # python2 +import queue import tuf.log @@ -42,19 +38,6 @@ # Used when forming URLs on the client side TEST_HOST_ADDRESS = '127.0.0.1' -try: - # is defined in Python 3 - TimeoutError -except NameError: - # Define for Python 2 - class TimeoutError(Exception): - - def __init__(self, value="Timeout"): - self.value = value - - def __str__(self): - return repr(self.value) - class TestServerProcessError(Exception): From d144141ec72316c234417e579f9f3e20f3a94c76 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 2 Mar 2021 21:13:12 +0000 Subject: [PATCH 5/5] tests: remove check for python >= 3.6 in test_api Signed-off-by: Joshua Lock --- tests/test_api.py | 42 ++++++++++++++++-------------------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 5d2dce0269..ec7d182b79 100755 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -20,32 +20,22 @@ from tests import utils -# TODO: Remove case handling when fully dropping support for versions >= 3.6 -IS_PY_VERSION_SUPPORTED = sys.version_info >= (3, 6) - -# Use setUpModule to tell unittest runner to skip this test module gracefully. -def setUpModule(): - if not IS_PY_VERSION_SUPPORTED: - raise unittest.SkipTest('requires Python 3.6 or higher') - -# Since setUpModule is called after imports we need to import conditionally. -if IS_PY_VERSION_SUPPORTED: - import tuf.exceptions - from tuf.api.metadata import ( - Metadata, - Snapshot, - Timestamp, - Targets - ) - - from securesystemslib.interface import ( - import_ed25519_publickey_from_file, - import_ed25519_privatekey_from_file - ) - - from securesystemslib.keys import ( - format_keyval_to_metadata - ) +import tuf.exceptions +from tuf.api.metadata import ( + Metadata, + Snapshot, + Timestamp, + Targets +) + +from securesystemslib.interface import ( + import_ed25519_publickey_from_file, + import_ed25519_privatekey_from_file +) + +from securesystemslib.keys import ( + format_keyval_to_metadata +) logger = logging.getLogger(__name__)