diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..b6b4bef9 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,37 @@ +# This file is for use as a devcontainer and a runtime container +# +# The devcontainer should use the build target and run as root with podman +# or docker with user namespaces. +# +FROM python:3.11 as build + +ARG PIP_OPTIONS + +# Add any system dependencies for the developer/build environment here e.g. +# RUN apt-get update && apt-get upgrade -y && \ +# apt-get install -y --no-install-recommends \ +# desired-packages \ +# && rm -rf /var/lib/apt/lists/* + +# set up a virtual environment and put it in PATH +RUN python -m venv /venv +ENV PATH=/venv/bin:$PATH + +# Copy any required context for the pip install over +COPY . /context +WORKDIR /context + +# install python package into /venv +RUN pip install ${PIP_OPTIONS} + +FROM python:3.11-slim as runtime + +# Add apt-get system dependecies for runtime here if needed + +# copy the virtual environment from the build stage and put it in PATH +COPY --from=build /venv/ /venv/ +ENV PATH=/venv/bin:$PATH + +# change this entrypoint if it is not the same as the repo +ENTRYPOINT ["python3-pip-skeleton"] +CMD ["--version"] diff --git a/.devcontainer.json b/.devcontainer/devcontainer.json similarity index 81% rename from .devcontainer.json rename to .devcontainer/devcontainer.json index d0921df6..1046ef9e 100644 --- a/.devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,16 +4,17 @@ "build": { "dockerfile": "Dockerfile", "target": "build", - "context": ".", - "args": {} + // Only upgrade pip, we will install the project below + "args": { + "PIP_OPTIONS": "--upgrade pip" + } }, "remoteEnv": { "DISPLAY": "${localEnv:DISPLAY}" }, // Set *default* container specific settings.json values on container create. "settings": { - "python.defaultInterpreterPath": "/venv/bin/python", - "python.linting.enabled": true + "python.defaultInterpreterPath": "/venv/bin/python" }, // Add the IDs of extensions you want installed when the container is created. "extensions": [ @@ -24,6 +25,7 @@ "initializeCommand": "bash -c 'for i in $HOME/.inputrc; do [ -f $i ] || touch $i; done'", "runArgs": [ "--net=host", + "--security-opt=label=type:container_runtime_t", "-v=${localEnv:HOME}/.ssh:/root/.ssh", "-v=${localEnv:HOME}/.inputrc:/root/.inputrc" ], @@ -35,5 +37,5 @@ "workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind", "workspaceFolder": "${localWorkspaceFolder}", // After the container is created, install the python project in editable form - "postCreateCommand": "pip install $([ -f requirements_dev.txt ] && echo -r requirements_dev.txt ) -e .[dev]" + "postCreateCommand": "pip install -e .[dev]" } \ No newline at end of file diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index a6fab0ea..00000000 --- a/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -build/ -dist/ -.mypy_cache -.tox -.venv* -venv* diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml new file mode 100644 index 00000000..25a146d1 --- /dev/null +++ b/.github/actions/install_requirements/action.yml @@ -0,0 +1,58 @@ +name: Install requirements +description: Run pip install with requirements and upload resulting requirements +inputs: + requirements_file: + description: Name of requirements file to use and upload + required: true + install_options: + description: Parameters to pass to pip install + required: true + python_version: + description: Python version to install + default: "3.x" + +runs: + using: composite + + steps: + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + + - name: Pip install + run: | + touch ${{ inputs.requirements_file }} + # -c uses requirements.txt as constraints, see 'Validate requirements file' + pip install -c ${{ inputs.requirements_file }} ${{ inputs.install_options }} + shell: bash + + - name: Create lockfile + run: | + mkdir -p lockfiles + pip freeze --exclude-editable > lockfiles/${{ inputs.requirements_file }} + # delete the self referencing line and make sure it isn't blank + sed -i '/file:/d' lockfiles/${{ inputs.requirements_file }} + shell: bash + + - name: Upload lockfiles + uses: actions/upload-artifact@v3 + with: + name: lockfiles + path: lockfiles + + # This eliminates the class of problems where the requirements being given no + # longer match what the packages themselves dictate. E.g. In the rare instance + # where I install some-package which used to depend on vulnerable-dependency + # but now uses good-dependency (despite being nominally the same version) + # pip will install both if given a requirements file with -r + - name: If requirements file exists, check it matches pip installed packages + run: | + if [ -s ${{ inputs.requirements_file }} ]; then + if ! diff -u ${{ inputs.requirements_file }} lockfiles/${{ inputs.requirements_file }}; then + echo "Error: ${{ inputs.requirements_file }} need the above changes to be exhaustive" + exit 1 + fi + fi + shell: bash + diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index d3413464..3016431c 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -4,8 +4,11 @@ on: push: pull_request: schedule: - # Run every Monday at 8am to check latest versions of dependencies + # Run weekly to check latest versions of dependencies - cron: "0 8 * * WED" +env: + # The target python version, which must match the Dockerfile version + CONTAINER_PYTHON: "3.11" jobs: lint: @@ -17,16 +20,14 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Setup python - uses: actions/setup-python@v4 + - name: Install python packages + uses: ./.github/actions/install_requirements with: - python-version: "3.10" + requirements_file: requirements-dev-3.x.txt + install_options: -e .[dev] - name: Lint - run: | - touch requirements_dev.txt requirements.txt - pip install -r requirements.txt -r requirements_dev.txt -e .[dev] - tox -e pre-commit,mypy + run: tox -e pre-commit,mypy test: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository @@ -34,7 +35,13 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] # can add windows-latest, macos-latest - python: ["3.8", "3.9", "3.10"] + python: ["3.9", "3.10", "3.11"] + install: ["-e .[dev]"] + # Make one version be non-editable to test both paths of version code + include: + - os: "ubuntu-latest" + python: "3.8" + install: ".[dev]" runs-on: ${{ matrix.os }} env: @@ -50,18 +57,21 @@ jobs: - name: Checkout uses: actions/checkout@v3 with: + # Need this to get version number from last tag fetch-depth: 0 - - name: Setup python ${{ matrix.python }} - uses: actions/setup-python@v4 + - name: Install python packages + uses: ./.github/actions/install_requirements with: - python-version: ${{ matrix.python }} + python_version: ${{ matrix.python }} + requirements_file: requirements-test-${{ matrix.os }}-${{ matrix.python }}.txt + install_options: ${{ matrix.install }} - - name: Install with latest dependencies - run: pip install .[dev] + - name: List dependency tree + run: pipdeptree - name: Run tests - run: pytest tests + run: pytest - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 @@ -69,9 +79,46 @@ jobs: name: ${{ matrix.python }}/${{ matrix.os }} files: cov.xml - container: + dist: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: "ubuntu-latest" + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + # Need this to get version number from last tag + fetch-depth: 0 + + - name: Build sdist and wheel + run: | + export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ + pipx run build + + - name: Upload sdist and wheel as artifacts + uses: actions/upload-artifact@v3 + with: + name: dist + path: dist + + - name: Check for packaging errors + run: pipx run twine check dist/* + + - name: Install python packages + uses: ./.github/actions/install_requirements + with: + python_version: ${{env.CONTAINER_PYTHON}} + requirements_file: requirements.txt + install_options: dist/*.whl + + - name: Test module --version works using the installed wheel + # If more than one module in src/ replace with module name to test + run: python -m $(ls src | head -1) --version + + container: + needs: [lint, dist, test] runs-on: ubuntu-latest + permissions: contents: read packages: write @@ -79,8 +126,15 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + + # image names must be all lower case + - name: Generate image repo name + run: echo IMAGE_REPOSITORY=ghcr.io/$(tr '[:upper:]' '[:lower:]' <<< "${{ github.repository }}") >> $GITHUB_ENV + + - name: Download wheel and lockfiles + uses: actions/download-artifact@v3 with: - fetch-depth: 0 + path: .devcontainer - name: Log in to GitHub Docker Registry if: github.event_name != 'pull_request' @@ -94,75 +148,45 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ghcr.io/${{ github.repository }} + images: ${{ env.IMAGE_REPOSITORY }} tags: | - type=ref,event=branch type=ref,event=tag + type=raw,value=latest - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v2 - - name: Build developer image for testing - uses: docker/build-push-action@v3 - with: - tags: build:latest - context: . - target: build - load: true - - - name: Run tests in the container locked with requirements_dev.txt - run: | - docker run --name test build bash /project/.github/workflows/container_tests.sh - docker cp test:/project/dist . - docker cp test:/project/lockfiles . - docker cp test:/project/cov.xml . - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - name: 3.10-locked/ubuntu-latest - files: cov.xml - - name: Build runtime image uses: docker/build-push-action@v3 with: - push: ${{ github.event_name != 'pull_request' }} + build-args: | + PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl + push: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }} + load: ${{ ! (github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) }} tags: ${{ steps.meta.outputs.tags }} - context: . - labels: ${{ steps.meta.outputs.labels }} + context: .devcontainer + # If you have a long docker build, uncomment the following to turn on caching + # For short build times this makes it a little slower + #cache-from: type=gha + #cache-to: type=gha,mode=max - name: Test cli works in runtime image - # check that the first tag can run with --version parameter - run: docker run $(echo ${{ steps.meta.outputs.tags }} | head -1) --version - - - name: Test cli works in sdist installed in local python - # ${GITHUB_REPOSITORY##*/} is the repo name without org - # Replace this with the cli command if different to the repo name - # (python3-pip-skeleton-cli replaces this with python3-pip-skeleton) - run: pip install dist/*.gz && python3-pip-skeleton --version - - - name: Upload build files - uses: actions/upload-artifact@v3 - with: - name: dist - path: dist - - - name: Upload lock files - uses: actions/upload-artifact@v3 - with: - name: lockfiles - path: lockfiles + run: docker run ${{ env.IMAGE_REPOSITORY }} --version release: # upload to PyPI and make a release on every tag - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - needs: container + needs: [lint, dist, test] + if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }} runs-on: ubuntu-latest steps: - uses: actions/download-artifact@v3 + - name: Fixup blank lockfiles + # Github release artifacts can't be blank + run: for f in lockfiles/*; do [ -s $f ] || echo '# No requirements' >> $f; done + - name: Github Release # We pin to the SHA, not the tag, for security reasons. # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions diff --git a/.github/workflows/container_tests.sh b/.github/workflows/container_tests.sh deleted file mode 100644 index b581612c..00000000 --- a/.github/workflows/container_tests.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -x - -cd /project -source /venv/bin/activate - -touch requirements_dev.txt -pip install -r requirements_dev.txt -e .[dev] -mkdir -p lockfiles -pip freeze --exclude-editable > lockfiles/requirements_dev.txt - -pipdeptree - -git config --global user.email "you@example.com" -git config --global user.name "Your Name" - -pytest tests diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f0e7ebb6..94fa2151 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -7,11 +7,6 @@ on: jobs: docs: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository - strategy: - fail-fast: false - matrix: - python: ["3.10"] - runs-on: ubuntu-latest steps: @@ -19,24 +14,21 @@ jobs: if: startsWith(github.ref, 'refs/tags') run: sleep 60 - - name: Install python version - uses: actions/setup-python@v4 + - name: Checkout + uses: actions/checkout@v3 with: - python-version: ${{ matrix.python }} + # Need this to get version number from last tag + fetch-depth: 0 - - name: Install Packages + - name: Install system packages # Can delete this if you don't use graphviz in your docs run: sudo apt-get install graphviz - - name: checkout - uses: actions/checkout@v3 + - name: Install python packages + uses: ./.github/actions/install_requirements with: - fetch-depth: 0 - - - name: Install dependencies - run: | - touch requirements_dev.txt - pip install -r requirements_dev.txt -e .[dev] + requirements_file: requirements-dev-3.x.txt + install_options: -e .[dev] - name: Build docs run: tox -e docs @@ -51,10 +43,10 @@ jobs: run: python .github/pages/make_switcher.py --add $DOCS_VERSION ${{ github.repository }} .github/pages/switcher.json - name: Publish Docs to gh-pages - if: github.event_name == 'push' + if: github.event_name == 'push' && github.actor != 'dependabot[bot]' # We pin to the SHA, not the tag, for security reasons. # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions - uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0 + uses: peaceiris/actions-gh-pages@de7ea6f8efb354206b205ef54722213d99067935 # v3.9.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: .github/pages diff --git a/.github/workflows/docs_clean.yml b/.github/workflows/docs_clean.yml index d5425e42..a67e1881 100644 --- a/.github/workflows/docs_clean.yml +++ b/.github/workflows/docs_clean.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - - name: checkout + - name: Checkout uses: actions/checkout@v3 with: ref: gh-pages @@ -35,7 +35,7 @@ jobs: - name: update index and push changes run: | - rm -r ${{ env.DOCS_VERSION }} + rm -r $DOCS_VERSION python make_switcher.py --remove $DOCS_VERSION ${{ github.repository }} switcher.json git config --global user.name 'GitHub Actions Docs Cleanup CI' git config --global user.email 'GithubActionsCleanup@noreply.github.com' diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index e6838560..42d199c4 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -1,8 +1,9 @@ name: Link Check on: + workflow_dispatch: schedule: - # Run every Monday at 8am to check URL links still resolve + # Run weekly to check URL links still resolve - cron: "0 8 * * WED" jobs: @@ -17,18 +18,12 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Install python version - uses: actions/setup-python@v4 + - name: Install python packages + uses: ./.github/actions/install_requirements with: - python-version: ${{ matrix.python }} - - - name: Install dependencies - run: | - touch requirements_dev.txt - pip install -r requirements_dev.txt -e .[dev] + requirements_file: requirements-dev-3.x.txt + install_options: -e .[dev] - name: Check links - run: tox -e docs -- -b linkcheck + run: tox -e docs build -- -b linkcheck diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 11b87346..00000000 --- a/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -# This file is for use as a devcontainer and a runtime container -# -# The devcontainer should use the build target and run as root with podman -# or docker with user namespaces. -# -FROM python:3.10 as build - -# Add any system dependencies for the developer/build environment here -RUN apt-get update && apt-get upgrade -y && \ - apt-get install -y --no-install-recommends \ - build-essential \ - busybox \ - git \ - graphviz \ - net-tools \ - vim \ - && rm -rf /var/lib/apt/lists/* \ - && busybox --install - -COPY . /project -WORKDIR /project - -# make the wheel outside of the venv so 'build' does not dirty requirements.txt -RUN pip install --upgrade pip build && \ - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ - git diff && \ - python -m build && \ - touch requirements.txt - -# set up a virtual environment and put it in PATH -RUN python -m venv /venv -ENV PATH=/venv/bin:$PATH -ENV TOX_DIRECT=1 - -# install the wheel and generate the requirements file -RUN pip install --upgrade pip && \ - pip install -r requirements.txt dist/*.whl && \ - mkdir -p lockfiles && \ - pip freeze > lockfiles/requirements.txt && \ - # we don't want to include our own wheel in requirements - remove with sed - # and replace with a comment to avoid a zero length asset upload later - sed -i '/file:/s/^/# Requirements for /' lockfiles/requirements.txt - -FROM python:3.10-slim as runtime - -# Add apt-get system dependecies for runtime here if needed - -# copy the virtual environment from the build stage and put it in PATH -COPY --from=build /venv/ /venv/ -ENV PATH=/venv/bin:$PATH - -# change this entrypoint if it is not the same as the repo -ENTRYPOINT ["python3-pip-skeleton"] -CMD ["--version"] diff --git a/setup.cfg b/setup.cfg index 43e8f168..4a9fcd14 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,6 +12,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 [options] python_requires = >=3.8 @@ -50,8 +51,6 @@ dev = [options.packages.find] where = src -# Don't include our tests directory in the distribution -exclude = tests # Specify any package data to be included in the wheel below. # [options.package_data] @@ -106,24 +105,21 @@ source = # NOTE that we pre-install all tools in the dev dependencies (including tox). # Hence the use of allowlist_externals instead of using the tox virtualenvs. # This ensures a match between developer time tools in the IDE and tox tools. -# Setting TOX_DIRECT=1 in the environment will make this even faster [tox:tox] skipsdist = True -[testenv:pytest] -allowlist_externals = pytest -commands = pytest {posargs} - -[testenv:mypy] -allowlist_externals = mypy -commands = mypy src tests {posargs} - -[testenv:pre-commit] -allowlist_externals = pre-commit -commands = pre-commit run --all-files {posargs} - -[testenv:docs] -allowlist_externals = +[testenv:{pre-commit,mypy,pytest,docs}] +# Don't create a virtualenv for the command, requires tox-direct plugin +direct = True +passenv = * +allowlist_externals = + pytest + pre-commit + mypy sphinx-build sphinx-autobuild -commands = sphinx-{posargs:build -EW --keep-going} -T docs build/html +commands = + pytest: pytest {posargs} + mypy: mypy src tests {posargs} + pre-commit: pre-commit run --all-files {posargs} + docs: sphinx-{posargs:build -EW --keep-going} -T docs build/html