Add support for prebuilt wheels

pythonforandroid/build.py

@@ -11,6 +11,7 @@
 import re
 import shutil
 import subprocess
+import sys
 
 import sh
 
@@ -21,7 +22,7 @@
 from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86, Archx86_64
 from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint, Out_Style, Out_Fore)
 from pythonforandroid.pythonpackage import get_package_name
-from pythonforandroid.recipe import CythonRecipe, Recipe
+from pythonforandroid.recipe import CythonRecipe, Recipe, PyProjectRecipe
 from pythonforandroid.recommendations import (
     check_ndk_version, check_target_api, check_ndk_api,
     RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API)
@@ -101,6 +102,14 @@ class Context:
 
     java_build_tool = 'auto'
 
+    skip_prebuilt = False
+
+    extra_index_urls = []
+
+    use_prebuilt_version_for = []
+
+    save_wheel_dir = ''
+
     @property
     def packages_path(self):
         '''Where packages are downloaded before being unpacked'''
@@ -503,6 +512,10 @@ def build_recipes(build_order, python_modules, ctx, project_dir,
             recipe.prepare_build_dir(arch.arch)
 
         info_main('# Prebuilding recipes')
+        # ensure we have `ctx.python_recipe` and `ctx.hostpython`
+        Recipe.get_recipe("python3", ctx).prebuild_arch(arch)
+        ctx.hostpython = Recipe.get_recipe("hostpython3", ctx).python_exe
+
         # 2) prebuild packages
         for recipe in recipes:
             info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch))
@@ -667,7 +680,17 @@ def is_wheel_platform_independent(whl_name):
     return all(tag.platform == "any" for tag in tags)
 
 
-def process_python_modules(ctx, modules):
+def is_wheel_compatible(whl_name, arch, ctx):
+    name, version, build, tags = parse_wheel_filename(whl_name)
+    supported_tags = PyProjectRecipe.get_wheel_platform_tags(arch.arch, ctx)
+    supported_tags.append("any")
+    result = all(tag.platform in supported_tags for tag in tags)
+    if not result:
+        warning(f"Incompatible module : {whl_name}")
+    return result
+
+
+def process_python_modules(ctx, modules, arch):
     """Use pip --dry-run to resolve dependencies and filter for pure-Python packages
     """
     modules = list(modules)
@@ -702,6 +725,7 @@ def process_python_modules(ctx, modules):
 
     # setup hostpython recipe
     env = environ.copy()
+    host_recipe = None
     try:
         host_recipe = Recipe.get_recipe("hostpython3", ctx)
         _python_path = host_recipe.get_path_to_python()
@@ -710,14 +734,32 @@ def process_python_modules(ctx, modules):
             _python_path, "Modules") + ":" + (libdir[0] if libdir else "")
         pip = host_recipe.pip
     except Exception:
-        # hostpython3 non available so we use system pip (like in tests)
+        # hostpython3 is unavailable, so fall back to system pip
         pip = sh.Command("pip")
 
+    # add platform tags
+    platforms = []
+    tags = PyProjectRecipe.get_wheel_platform_tags(arch.arch, ctx)
+    for tag in tags:
+        platforms.append(f"--platform={tag}")
+
+    if host_recipe is not None:
+        platforms.extend(["--python-version", host_recipe.version])
+    else:
+        # use the version of the currently running Python interpreter
+        current_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
+        platforms.extend(["--python-version", current_version])
+
+    indices = []
+    # add extra index urls
+    for index in ctx.extra_index_urls:
+        indices.extend(["--extra-index-url", index])
     try:
         shprint(
             pip, 'install', *modules,
             '--dry-run', '--break-system-packages', '--ignore-installed',
-            '--report', path, '-q', _env=env
+            '--disable-pip-version-check', '--only-binary=:all:',
+            '--report', path, '-q', *platforms, *indices, _env=env
         )
     except Exception as e:
         warning(f"Auto module resolution failed: {e}")
@@ -751,7 +793,9 @@ def process_python_modules(ctx, modules):
         filename = basename(module["download_info"]["url"])
         pure_python = True
 
-        if (filename.endswith(".whl") and not is_wheel_platform_independent(filename)):
+        if (
+                filename.endswith(".whl") and not is_wheel_compatible(filename, arch, ctx)
+        ):
             any_not_pure_python = True
             pure_python = False
 
@@ -769,7 +813,7 @@ def process_python_modules(ctx, modules):
         )
 
         if pure_python:
-            processed_modules.append(f"{mname}=={mver}")
+            processed_modules.append(module["download_info"]["url"])
     info(" ")
 
     if any_not_pure_python:
@@ -793,7 +837,7 @@ def run_pymodules_install(ctx, arch, modules, project_dir=None,
 
     info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE FOR ARCH: {} ***'.format(arch))
 
-    modules = process_python_modules(ctx, modules)
+    modules = process_python_modules(ctx, modules, arch)
 
     modules = [m for m in modules if ctx.not_has_package(m, arch)]
 

pythonforandroid/recipe.py

@@ -923,8 +923,7 @@ def real_hostpython_location(self):
         if host_name == 'hostpython3':
             return self._host_recipe.python_exe
         else:
-            python_recipe = self.ctx.python_recipe
-            return 'python{}'.format(python_recipe.version)
+            return 'python{}'.format(self.ctx.python_recipe.version)
 
     @property
     def hostpython_location(self):
@@ -1248,6 +1247,59 @@ class PyProjectRecipe(PythonRecipe):
     extra_build_args = []
     call_hostpython_via_targetpython = False
 
+    def get_pip_name(self):
+        name_str = self.name
+        if self.name not in self.ctx.use_prebuilt_version_for and self.version is not None:
+            # Like: v2.3.0 -> 2.3.0
+            cleaned_version = self.version.replace("v", "")
+            name_str += f"=={cleaned_version}"
+        return name_str
+
+    def get_pip_install_args(self, arch):
+        python_recipe = Recipe.get_recipe("python3", self.ctx)
+        opts = [
+            "install",
+            self.get_pip_name(),
+            "--ignore-installed",
+            "--disable-pip-version-check",
+            "--python-version",
+            python_recipe.version,
+            "--only-binary=:all:",
+            "--no-deps",
+        ]
+        # add platform tags
+        tags = PyProjectRecipe.get_wheel_platform_tags(arch.arch, self.ctx)
+        for tag in tags:
+            opts.append(f"--platform={tag}")
+
+        # add extra index urls
+        for index in self.ctx.extra_index_urls:
+            opts.extend(["--extra-index-url", index])
+
+        return opts
+
+    def lookup_prebuilt(self, arch):
+        pip_options = self.get_pip_install_args(arch)
+        # do not install
+        pip_options.extend(["--dry-run", "-q"])
+        pip_env = self.get_hostrecipe_env()
+        try:
+            shprint(self._host_recipe.pip, *pip_options, _env=pip_env)
+        except Exception:
+            return False
+        return True
+
+    def check_prebuilt(self, arch, msg=""):
+        if self.ctx.skip_prebuilt:
+            return False
+
+        if self.lookup_prebuilt(arch):
+            if msg != "":
+                info(f"Prebuilt pip wheel found, {msg}")
+            return True
+
+        return False
+
     def get_recipe_env(self, arch, **kwargs):
         # Custom hostpython
         self.ctx.python_recipe.python_exe = join(
@@ -1259,24 +1311,42 @@ def get_recipe_env(self, arch, **kwargs):
 
         with open(build_opts, "w") as file:
             file.write("[bdist_wheel]\nplat_name={}".format(
-                self.get_wheel_platform_tag(arch)
+                self.get_wheel_platform_tag(arch.arch)
             ))
             file.close()
 
         env["DIST_EXTRA_CONFIG"] = build_opts
         return env
 
-    def get_wheel_platform_tag(self, arch):
+    @staticmethod
+    def get_wheel_platform_tags(arch, ctx):
         # https://peps.python.org/pep-0738/#packaging
         # official python only supports 64 bit:
         # android_21_arm64_v8a
         # android_21_x86_64
-        return f"android_{self.ctx.ndk_api}_" + {
-            "arm64-v8a": "arm64_v8a",
-            "x86_64": "x86_64",
-            "armeabi-v7a": "arm",
-            "x86": "i686",
-        }[arch.arch]
+        _suffix = {
+            "arm64-v8a": ["arm64_v8a", "aarch64"],
+            "x86_64": ["x86_64"],
+            "armeabi-v7a": ["arm"],
+            "x86": ["i686"],
+        }[arch]
+        return [f"android_{ctx.ndk_api}_" + _ for _ in _suffix]
+
+    def get_wheel_platform_tag(self, arch):
+        return PyProjectRecipe.get_wheel_platform_tags(arch, self.ctx)[0]
+
+    def install_prebuilt_wheel(self, arch):
+        info("Installing prebuilt built wheel")
+        destination = self.ctx.get_python_install_dir(arch.arch)
+        pip_options = self.get_pip_install_args(arch)
+        pip_options.extend(["--target", destination])
+        pip_options.append("--upgrade")
+        pip_env = self.get_hostrecipe_env()
+        try:
+            shprint(self._host_recipe.pip, *pip_options, _env=pip_env)
+        except Exception:
+            return False
+        return True
 
     def install_wheel(self, arch, built_wheels):
         with patch_wheel_setuptools_logging():
@@ -1287,16 +1357,18 @@ def install_wheel(self, arch, built_wheels):
         # Fix wheel platform tag
         wheel_tag = wheel_tags(
             _wheel,
-            platform_tags=self.get_wheel_platform_tag(arch),
+            platform_tags=self.get_wheel_platform_tag(arch.arch),
             remove=True,
         )
         selected_wheel = join(built_wheel_dir, wheel_tag)
-
         _dev_wheel_dir = environ.get("P4A_WHEEL_DIR", False)
         if _dev_wheel_dir:
             ensure_dir(_dev_wheel_dir)
             shprint(sh.cp, selected_wheel, _dev_wheel_dir)
 
+        if exists(self.ctx.save_wheel_dir):
+            shprint(sh.cp, selected_wheel, self.ctx.save_wheel_dir)
+
         info(f"Installing built wheel: {wheel_tag}")
         destination = self.ctx.get_python_install_dir(arch.arch)
         with WheelFile(selected_wheel) as wf:
@@ -1305,6 +1377,11 @@ def install_wheel(self, arch, built_wheels):
             wf.close()
 
     def build_arch(self, arch):
+        if self.check_prebuilt(arch, "skipping build_arch"):
+            result = self.install_prebuilt_wheel(arch)
+            if result:
+                return
+            warning("Failed to install prebuilt wheel, falling back to build_arch")
 
         build_dir = self.get_build_dir(arch.arch)
         if not (isfile(join(build_dir, "pyproject.toml")) or isfile(join(build_dir, "setup.py"))):

pythonforandroid/recipes/hostpython3/__init__.py

@@ -6,7 +6,7 @@
 from os.path import join
 
 from packaging.version import Version
-from pythonforandroid.logger import shprint
+from pythonforandroid.logger import shprint, error
 from pythonforandroid.recipe import Recipe
 from pythonforandroid.util import (
     BuildInterruptingException,
@@ -48,6 +48,16 @@ class HostPython3Recipe(Recipe):
 
     patches = ["fix_ensurepip.patch"]
 
+    # apply version guard
+    def download(self):
+        python_recipe = Recipe.get_recipe("python3", self.ctx)
+        if python_recipe.version != self.version:
+            error(
+                f"python3 should have same version as hostpython3, {python_recipe.version} != {self.version}"
+            )
+            exit(1)
+        super().download()
+
     @property
     def _exe_name(self):
         '''

pythonforandroid/toolchain.py

@@ -188,6 +188,7 @@ class NoAbbrevParser(argparse.ArgumentParser):
     This subclass alternative is follows the suggestion at
     https://bugs.python.org/issue14910.
     """
+
     def _get_option_tuples(self, option_string):
         return []
 
@@ -267,6 +268,44 @@ def __init__(self):
             '--arch', help='The archs to build for.',
             action='append', default=[])
 
+        generic_parser.add_argument(
+            '--extra-index-url',
+            help=(
+                'Extra package indexes to look for prebuilt Android wheels. '
+                'Can be used multiple times.'
+            ),
+            action='append',
+            default=[],
+            dest="extra_index_urls",
+        )
+
+        generic_parser.add_argument(
+            '--skip-prebuilt',
+            help='Always build from source; do not use prebuilt wheels.',
+            action='store_true',
+            default=False,
+            dest="skip_prebuilt",
+        )
+
+        generic_parser.add_argument(
+            '--use-prebuilt-version-for',
+            help=(
+                'For these packages, ignore pinned versions and use the latest '
+                'prebuilt version from the extra index if available.'
+                'Only applies to packages with a recipe.'
+            ),
+            action='append',
+            default=[],
+            dest="use_prebuilt_version_for",
+        )
+
+        generic_parser.add_argument(
+            '--save-wheel-dir',
+            dest='save_wheel_dir',
+            default='',
+            help='Directory to store wheels built by PyProjectRecipe.',
+        )
+
         # Options for specifying the Distribution
         generic_parser.add_argument(
             '--dist-name', '--dist_name',
@@ -672,6 +711,11 @@ def add_parser(subparsers, *args, **kwargs):
         self.ctx.activity_class_name = args.activity_class_name
         self.ctx.service_class_name = args.service_class_name
 
+        self.ctx.extra_index_urls = args.extra_index_urls
+        self.ctx.skip_prebuilt = args.skip_prebuilt
+        self.ctx.use_prebuilt_version_for = args.use_prebuilt_version_for
+        self.ctx.save_wheel_dir = args.save_wheel_dir
+
         # Each subparser corresponds to a method
         command = args.subparser_name.replace('-', '_')
         getattr(self, command)(args)