From 96041b16ff2d3ca56c0ed5f68d8fb91c1be67104 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 6 Mar 2018 19:25:34 -0800 Subject: [PATCH 1/6] Add docs for python-executable --- docs/source/command_line.rst | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index a723908799110..01b6036f69930 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -360,11 +360,27 @@ Here are some more useful flags: updates the cache, but regular incremental mode ignores cache files written by quick mode. +- ``--python-executable EXECUTABLE`` will have mypy collect type information + from `PEP 561`_ compliant packages installed for the Python executable + ``EXECUTABLE``. If not provided, mypy will use PEP 561 compliant packages + installed for the Python executable running mypy. See + :ref:`installed-packages` for more on making PEP 561 compliant packages. This + flag will attempt to set ``--python-version`` if not already set. + - ``--python-version X.Y`` will make mypy typecheck your code as if it were run under Python version X.Y. Without this option, mypy will default to using whatever version of Python is running mypy. Note that the ``-2`` and ``--py2`` flags are aliases for ``--python-version 2.7``. See - :ref:`version_and_platform_checks` for more about this feature. + :ref:`version_and_platform_checks` for more about this feature. This flag + will attempt to find a Python executable of the corresponding version to + search for `PEP 561`_ compliant packages. If you'd like to disable this, see + ``--no-site-packages`` below. + +- ``--no-site-packages`` will disable searching for `PEP 561`_ compliant + packages. This will also disable searching for a usable Python executable. + Use this flag if mypy cannot find a Python executable for the version of + Python being checked, and you don't need to use PEP 561 typed packages. + Otherwise, use ``--python-executable``. - ``--platform PLATFORM`` will make mypy typecheck your code as if it were run under the the given operating system. Without this option, mypy will @@ -447,6 +463,8 @@ For the remaining flags you can read the full ``mypy -h`` output. Command line flags are liable to change between releases. +.. _PEP 561: https://www.python.org/dev/peps/pep-0561/ + .. _integrating-mypy: Integrating mypy into another Python application From 4483b3eb3119bfa3f86d467c503804fe27dba2b1 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 6 Mar 2018 22:53:15 -0800 Subject: [PATCH 2/6] Add --python-executable and --no-site-packages --- mypy/main.py | 79 ++++++++++++++++++++++++++++++++++++++++++-- mypy/options.py | 1 + mypy/test/helpers.py | 1 + 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index b7c1117ea0293..512297b18fbea 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1,10 +1,12 @@ """Mypy type checker command line tool.""" import argparse +import ast import configparser import fnmatch import os import re +import subprocess import sys import time @@ -205,6 +207,45 @@ def invert_flag_name(flag: str) -> str: return '--no-{}'.format(flag[2:]) +class PythonExecutableInferenceError(Exception): + """Represents a failure to infer the version or executable while searching.""" + + +if sys.platform == 'win32': + def python_executable_prefix(v: str) -> List[str]: + return ['py', '-{}'.format(v)] +else: + def python_executable_prefix(v: str) -> List[str]: + return ['python{}'.format(v)] + + +def _python_version_from_executable(python_executable: str) -> Tuple[int, int]: + try: + check = subprocess.check_output([python_executable, '-c', + 'import sys; print(repr(sys.version_info[:2]))'], + stderr=subprocess.STDOUT).decode() + return ast.literal_eval(check) + except (subprocess.CalledProcessError, FileNotFoundError): + raise PythonExecutableInferenceError( + 'Error: invalid Python executable {}'.format(python_executable)) + + +def _python_executable_from_version(python_version: Tuple[int, int]) -> str: + if sys.version_info[:2] == python_version: + return sys.executable + str_ver = '.'.join(map(str, python_version)) + print(str_ver) + try: + sys_exe = subprocess.check_output(python_executable_prefix(str_ver) + + ['-c', 'import sys; print(sys.executable)'], + stderr=subprocess.STDOUT).decode().strip() + return sys_exe + except (subprocess.CalledProcessError, FileNotFoundError): + raise PythonExecutableInferenceError( + 'Error: failed to find a Python executable matching version {},' + ' perhaps try --python-executable, or --no-site-packages?'.format(python_version)) + + def process_options(args: List[str], require_targets: bool = True, server_options: bool = False, @@ -255,10 +296,16 @@ def add_invertible_flag(flag: str, parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__) parser.add_argument('--python-version', type=parse_version, metavar='x.y', - help='use Python x.y') + help='use Python x.y', dest='special-opts:python_version') + parser.add_argument('--python-executable', action='store', metavar='EXECUTABLE', + help="Python executable whose installed packages will be" + " used in typechecking.", dest='special-opts:python_executable') + parser.add_argument('--no-site-packages', action='store_true', + dest='special-opts:no_site_packages', + help="Do not search for PEP 561 packages in the package directory.") parser.add_argument('--platform', action='store', metavar='PLATFORM', help="typecheck special-cased code for the given OS platform " - "(defaults to sys.platform).") + "(defaults to sys.platform).") parser.add_argument('-2', '--py2', dest='python_version', action='store_const', const=defaults.PYTHON2_VERSION, help="use Python 2 mode") parser.add_argument('--ignore-missing-imports', action='store_true', @@ -482,6 +529,34 @@ def add_invertible_flag(flag: str, print("Warning: --no-fast-parser no longer has any effect. The fast parser " "is now mypy's default and only parser.") + try: + # Infer Python version and/or executable if one is not given + if special_opts.python_executable is not None and special_opts.python_version is not None: + py_exe_ver = _python_version_from_executable(special_opts.python_executable) + if py_exe_ver != special_opts.python_version: + parser.error( + 'Python version {} did not match executable {}, got version {}.'.format( + special_opts.python_version, special_opts.python_executable, py_exe_ver + )) + else: + options.python_version = special_opts.python_version + options.python_executable = special_opts.python_executable + elif special_opts.python_executable is None and special_opts.python_version is not None: + options.python_version = special_opts.python_version + py_exe = None + if not special_opts.no_site_packages: + py_exe = _python_executable_from_version(special_opts.python_version) + options.python_executable = py_exe + elif special_opts.python_version is None and special_opts.python_executable is not None: + options.python_version = _python_version_from_executable( + special_opts.python_executable) + options.python_executable = special_opts.python_executable + except PythonExecutableInferenceError as e: + parser.error(str(e)) + + if special_opts.no_site_packages: + options.python_executable = None + # Check for invalid argument combinations. if require_targets: code_methods = sum(bool(c) for c in [special_opts.modules, diff --git a/mypy/options.py b/mypy/options.py index 5ea251df2c9d2..c3d07df08191e 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -53,6 +53,7 @@ def __init__(self) -> None: # -- build options -- self.build_type = BuildType.STANDARD self.python_version = sys.version_info[:2] # type: Tuple[int, int] + self.python_executable = sys.executable # type: Optional[str] self.platform = sys.platform self.custom_typing_module = None # type: Optional[str] self.custom_typeshed_dir = None # type: Optional[str] diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index 267f99e5586bf..ad17d8387acea 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -316,6 +316,7 @@ def parse_options(program_text: str, testcase: DataDrivenTestCase, flag_list = None if flags: flag_list = flags.group(1).split() + flag_list.append('--no-site-packages') # the tests shouldn't need an installed Python targets, options = process_options(flag_list, require_targets=False) if targets: # TODO: support specifying targets via the flags pragma From 53ff42c936fa47dd6db6cd200ce4a0eea022df8e Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 6 Mar 2018 23:09:28 -0800 Subject: [PATCH 3/6] Split inference out of process_options --- mypy/main.py | 49 ++++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 512297b18fbea..1b5863b226ee5 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -246,6 +246,33 @@ def _python_executable_from_version(python_version: Tuple[int, int]) -> str: ' perhaps try --python-executable, or --no-site-packages?'.format(python_version)) +def infer_python_version_and_executable(options: Options, + special_opts: argparse.Namespace + ) -> Options: + # Infer Python version and/or executable if one is not given + if special_opts.python_executable is not None and special_opts.python_version is not None: + py_exe_ver = _python_version_from_executable(special_opts.python_executable) + if py_exe_ver != special_opts.python_version: + raise PythonExecutableInferenceError( + 'Python version {} did not match executable {}, got version {}.'.format( + special_opts.python_version, special_opts.python_executable, py_exe_ver + )) + else: + options.python_version = special_opts.python_version + options.python_executable = special_opts.python_executable + elif special_opts.python_executable is None and special_opts.python_version is not None: + options.python_version = special_opts.python_version + py_exe = None + if not special_opts.no_site_packages: + py_exe = _python_executable_from_version(special_opts.python_version) + options.python_executable = py_exe + elif special_opts.python_version is None and special_opts.python_executable is not None: + options.python_version = _python_version_from_executable( + special_opts.python_executable) + options.python_executable = special_opts.python_executable + return options + + def process_options(args: List[str], require_targets: bool = True, server_options: bool = False, @@ -530,27 +557,7 @@ def add_invertible_flag(flag: str, "is now mypy's default and only parser.") try: - # Infer Python version and/or executable if one is not given - if special_opts.python_executable is not None and special_opts.python_version is not None: - py_exe_ver = _python_version_from_executable(special_opts.python_executable) - if py_exe_ver != special_opts.python_version: - parser.error( - 'Python version {} did not match executable {}, got version {}.'.format( - special_opts.python_version, special_opts.python_executable, py_exe_ver - )) - else: - options.python_version = special_opts.python_version - options.python_executable = special_opts.python_executable - elif special_opts.python_executable is None and special_opts.python_version is not None: - options.python_version = special_opts.python_version - py_exe = None - if not special_opts.no_site_packages: - py_exe = _python_executable_from_version(special_opts.python_version) - options.python_executable = py_exe - elif special_opts.python_version is None and special_opts.python_executable is not None: - options.python_version = _python_version_from_executable( - special_opts.python_executable) - options.python_executable = special_opts.python_executable + options = infer_python_version_and_executable(options, special_opts) except PythonExecutableInferenceError as e: parser.error(str(e)) From f523efa7ab7b8146fad4d55787f066c1be8a8391 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 6 Mar 2018 23:25:26 -0800 Subject: [PATCH 4/6] Change --no-site-packages to --no-infer-executable --- docs/source/command_line.rst | 20 +++++++------------- mypy/main.py | 14 +++++++------- mypy/test/helpers.py | 2 +- 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 01b6036f69930..0dda5af9c6372 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -360,27 +360,21 @@ Here are some more useful flags: updates the cache, but regular incremental mode ignores cache files written by quick mode. -- ``--python-executable EXECUTABLE`` will have mypy collect type information - from `PEP 561`_ compliant packages installed for the Python executable - ``EXECUTABLE``. If not provided, mypy will use PEP 561 compliant packages - installed for the Python executable running mypy. See - :ref:`installed-packages` for more on making PEP 561 compliant packages. This - flag will attempt to set ``--python-version`` if not already set. +- ``--python-executable EXECUTABLE`` This flag will attempt to set + ``--python-version`` if not already set based on the interpreter given. - ``--python-version X.Y`` will make mypy typecheck your code as if it were run under Python version X.Y. Without this option, mypy will default to using whatever version of Python is running mypy. Note that the ``-2`` and ``--py2`` flags are aliases for ``--python-version 2.7``. See :ref:`version_and_platform_checks` for more about this feature. This flag - will attempt to find a Python executable of the corresponding version to - search for `PEP 561`_ compliant packages. If you'd like to disable this, see - ``--no-site-packages`` below. + will attempt to find a Python executable of the corresponding version. If + you'd like to disable this, see ``--no-infer-executable`` below. -- ``--no-site-packages`` will disable searching for `PEP 561`_ compliant - packages. This will also disable searching for a usable Python executable. +- ``--no-infer-executable`` will disable searching for a usable Python + executable based on the Python version mypy is using to type check code. Use this flag if mypy cannot find a Python executable for the version of - Python being checked, and you don't need to use PEP 561 typed packages. - Otherwise, use ``--python-executable``. + Python being checked, and don't need mypy to use an executable. - ``--platform PLATFORM`` will make mypy typecheck your code as if it were run under the the given operating system. Without this option, mypy will diff --git a/mypy/main.py b/mypy/main.py index 1b5863b226ee5..887dd106e8d54 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -263,7 +263,7 @@ def infer_python_version_and_executable(options: Options, elif special_opts.python_executable is None and special_opts.python_version is not None: options.python_version = special_opts.python_version py_exe = None - if not special_opts.no_site_packages: + if not special_opts.no_executable: py_exe = _python_executable_from_version(special_opts.python_version) options.python_executable = py_exe elif special_opts.python_version is None and special_opts.python_executable is not None: @@ -325,11 +325,11 @@ def add_invertible_flag(flag: str, parser.add_argument('--python-version', type=parse_version, metavar='x.y', help='use Python x.y', dest='special-opts:python_version') parser.add_argument('--python-executable', action='store', metavar='EXECUTABLE', - help="Python executable whose installed packages will be" - " used in typechecking.", dest='special-opts:python_executable') - parser.add_argument('--no-site-packages', action='store_true', - dest='special-opts:no_site_packages', - help="Do not search for PEP 561 packages in the package directory.") + help="Python executable which will be used in typechecking.", + dest='special-opts:python_executable') + parser.add_argument('--no-infer-executable', action='store_true', + dest='special-opts:no_executable', + help="Do not infer a Python executable based on the version.") parser.add_argument('--platform', action='store', metavar='PLATFORM', help="typecheck special-cased code for the given OS platform " "(defaults to sys.platform).") @@ -561,7 +561,7 @@ def add_invertible_flag(flag: str, except PythonExecutableInferenceError as e: parser.error(str(e)) - if special_opts.no_site_packages: + if special_opts.no_executable: options.python_executable = None # Check for invalid argument combinations. diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index ad17d8387acea..adce623d8af05 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -316,7 +316,7 @@ def parse_options(program_text: str, testcase: DataDrivenTestCase, flag_list = None if flags: flag_list = flags.group(1).split() - flag_list.append('--no-site-packages') # the tests shouldn't need an installed Python + flag_list.append('--no-infer-executable') # the tests shouldn't need an installed Python targets, options = process_options(flag_list, require_targets=False) if targets: # TODO: support specifying targets via the flags pragma From 4d668272d918251c543e96ebca9747fbf43073b4 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 6 Mar 2018 23:43:53 -0800 Subject: [PATCH 5/6] Add --no-infer-executable to subprocessed tests --- mypy/main.py | 2 +- mypy/test/testcmdline.py | 1 + mypy/test/testpythoneval.py | 2 +- runtests.py | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 887dd106e8d54..4f45aba8e3a90 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -243,7 +243,7 @@ def _python_executable_from_version(python_version: Tuple[int, int]) -> str: except (subprocess.CalledProcessError, FileNotFoundError): raise PythonExecutableInferenceError( 'Error: failed to find a Python executable matching version {},' - ' perhaps try --python-executable, or --no-site-packages?'.format(python_version)) + ' perhaps try --python-executable, or --no-infer-executable?'.format(python_version)) def infer_python_version_and_executable(options: Options, diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py index 57910c1a1dc0d..7a3016991cce2 100644 --- a/mypy/test/testcmdline.py +++ b/mypy/test/testcmdline.py @@ -47,6 +47,7 @@ def test_python_cmdline(testcase: DataDrivenTestCase) -> None: file.write('{}\n'.format(s)) args = parse_args(testcase.input[0]) args.append('--show-traceback') + args.append('--no-infer-executable') # Type check the program. fixed = [python3_path, os.path.join(testcase.old_cwd, 'scripts', 'mypy')] diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py index 0634442f172ae..871753b833de4 100644 --- a/mypy/test/testpythoneval.py +++ b/mypy/test/testpythoneval.py @@ -49,7 +49,7 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None: version. """ assert testcase.old_cwd is not None, "test was not properly set up" - mypy_cmdline = ['--show-traceback'] + mypy_cmdline = ['--show-traceback', '--no-infer-executable'] py2 = testcase.name.lower().endswith('python2') if py2: mypy_cmdline.append('--py2') diff --git a/runtests.py b/runtests.py index a2a24c29a7caa..0714cf88cabf0 100755 --- a/runtests.py +++ b/runtests.py @@ -73,6 +73,7 @@ def add_mypy_cmd(self, name: str, mypy_args: List[str], cwd: Optional[str] = Non return args = [sys.executable, self.mypy] + mypy_args args.append('--show-traceback') + args.append('--no-infer-executable') self.waiter.add(LazySubprocess(full_name, args, cwd=cwd, env=self.env)) def add_mypy(self, name: str, *args: str, cwd: Optional[str] = None) -> None: From 1582692c54fb303e4580097cb0f50197e865d295 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 7 Mar 2018 00:24:18 -0800 Subject: [PATCH 6/6] Add test for python-executable and no-infer-executable --- mypy/test/testargs.py | 51 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/mypy/test/testargs.py b/mypy/test/testargs.py index 20db610cda186..337b591fb0dd7 100644 --- a/mypy/test/testargs.py +++ b/mypy/test/testargs.py @@ -4,10 +4,15 @@ defaults, and that argparse doesn't assign any new members to the Options object it creates. """ +import argparse +import sys + +import pytest # type: ignore from mypy.test.helpers import Suite, assert_equal from mypy.options import Options -from mypy.main import process_options +from mypy.main import (process_options, PythonExecutableInferenceError, + infer_python_version_and_executable) class ArgSuite(Suite): @@ -17,3 +22,47 @@ def test_coherence(self) -> None: # FIX: test this too. Requires changing working dir to avoid finding 'setup.cfg' options.config_file = parsed_options.config_file assert_equal(options, parsed_options) + + def test_executable_inference(self) -> None: + """Test the --python-executable flag with --python-version""" + sys_ver_str = '.'.join(map(str, sys.version_info[:2])) + + base = ['file.py'] # dummy file + + # test inference given one (infer the other) + matching_version = base + ['--python-version={}'.format(sys_ver_str)] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + matching_version = base + ['--python-executable={}'.format(sys.executable)] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + # test inference given both + matching_version = base + ['--python-version={}'.format(sys_ver_str), + '--python-executable={}'.format(sys.executable)] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + # test that we error if the version mismatch + # argparse sys.exits on a parser.error, we need to check the raw inference function + options = Options() + + special_opts = argparse.Namespace() + special_opts.python_executable = sys.executable + special_opts.python_version = (2, 10) # obviously wrong + special_opts.no_executable = None + with pytest.raises(PythonExecutableInferenceError) as e: + options = infer_python_version_and_executable(options, special_opts) + assert str(e.value) == 'Python version (2, 10) did not match executable {}, got' \ + ' version {}.'.format(sys.executable, str(sys.version_info[:2])) + + # test that --no-infer-executable will disable executable inference + matching_version = base + ['--python-version={}'.format(sys_ver_str), + '--no-infer-executable'] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable is None