From 4853958ec6078d8bab4fdf57067820fed5c7df8c Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 31 Jan 2024 09:22:01 +0000 Subject: [PATCH 1/4] Split CI and replace files with links --- .../actions/install_requirements/action.yml | 34 +++ .github/workflows/_container.yml | 56 ++++ .github/workflows/_dist.yml | 36 +++ .github/workflows/_docs.yml | 11 +- .github/workflows/_pypi.yml | 19 ++ .github/workflows/_test.yml | 43 ++++ .github/workflows/_tox.yml | 9 +- .github/workflows/ci.yml | 45 +++- .github/workflows/periodic.yml | 13 + Dockerfile | 11 +- copier.yml | 10 +- template/.devcontainer/devcontainer.json | 16 +- ...{% if docker %}Dockerfile{% endif %}.jinja | 34 +-- .../actions" | 1 + .../actions/install_requirements/action.yml" | 60 ----- .../dependabot.yml" | 8 +- .../pages" | 1 + .../pages/index.html" | 11 - .../pages/make_switcher.py" | 99 ------- .../workflows/_check.yml" | 1 + .../workflows/_dist.yml" | 1 + .../workflows/_pypi.yml" | 1 + .../workflows/_release.yml" | 1 + .../workflows/_test.yml" | 1 + .../workflows/_tox.yml" | 1 + .../workflows/ci.yml.jinja" | 65 +++++ .../workflows/code.yml" | 243 ------------------ .../workflows/docs.yml" | 54 ---- .../workflows/docs_clean.yml" | 43 ---- .../workflows/linkcheck.yml" | 28 -- .../{% if docker %}_container.yml{% endif %}" | 1 + .../{% if sphinx %}_docs.yml{% endif %}" | 1 + .../{% if sphinx %}periodic.yml{% endif %}" | 1 + .../conf.py.jinja | 0 .../developer/explanations/decisions.rst | 0 .../0001-record-architecture-decisions.rst | 0 .../0002-switched-to-pip-skeleton.rst.jinja | 0 .../developer/how-to/build-docs.rst | 0 .../developer/how-to/contribute.rst | 0 .../developer/how-to/lint.rst | 0 .../developer/how-to/make-release.rst.jinja | 0 .../developer/how-to/pin-requirements.rst | 0 .../developer/how-to/run-tests.rst | 0 .../developer/how-to/static-analysis.rst | 0 .../developer/how-to/test-container.rst | 0 .../developer/how-to/update-tools.rst | 0 .../developer/index.rst | 0 .../developer/reference/standards.rst | 0 .../developer/tutorials/dev-install.rst.jinja | 0 .../genindex.rst | 0 .../images/dls-favicon.ico | Bin .../images/dls-logo.svg | 0 .../index.rst.jinja | 0 .../user/explanations/docs-structure.rst | 0 .../user/how-to/run-container.rst.jinja | 0 .../user/index.rst | 0 .../user/reference/api.rst.jinja | 0 .../user/tutorials/installation.rst.jinja | 0 58 files changed, 359 insertions(+), 600 deletions(-) create mode 100644 .github/actions/install_requirements/action.yml create mode 100644 .github/workflows/_container.yml create mode 100644 .github/workflows/_dist.yml create mode 100644 .github/workflows/_pypi.yml create mode 100644 .github/workflows/_test.yml create mode 100644 .github/workflows/periodic.yml create mode 120000 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/actions" delete mode 100644 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/actions/install_requirements/action.yml" create mode 120000 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/pages" delete mode 100644 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/pages/index.html" delete mode 100755 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/pages/make_switcher.py" create mode 120000 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_check.yml" create mode 120000 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_dist.yml" create mode 120000 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_pypi.yml" create mode 120000 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_release.yml" create mode 120000 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_test.yml" create mode 120000 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_tox.yml" create mode 100644 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/ci.yml.jinja" delete mode 100644 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/code.yml" delete mode 100644 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/docs.yml" delete mode 100644 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/docs_clean.yml" delete mode 100644 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/linkcheck.yml" create mode 120000 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/{% if docker %}_container.yml{% endif %}" create mode 120000 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/{% if sphinx %}_docs.yml{% endif %}" create mode 120000 "template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/{% if sphinx %}periodic.yml{% endif %}" rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/conf.py.jinja" => template/{% if sphinx %}docs{% endif %}/conf.py.jinja (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/explanations/decisions.rst" => template/{% if sphinx %}docs{% endif %}/developer/explanations/decisions.rst (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/explanations/decisions/0001-record-architecture-decisions.rst" => template/{% if sphinx %}docs{% endif %}/developer/explanations/decisions/0001-record-architecture-decisions.rst (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst.jinja" => template/{% if sphinx %}docs{% endif %}/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst.jinja (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/build-docs.rst" => template/{% if sphinx %}docs{% endif %}/developer/how-to/build-docs.rst (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/contribute.rst" => template/{% if sphinx %}docs{% endif %}/developer/how-to/contribute.rst (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/lint.rst" => template/{% if sphinx %}docs{% endif %}/developer/how-to/lint.rst (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/make-release.rst.jinja" => template/{% if sphinx %}docs{% endif %}/developer/how-to/make-release.rst.jinja (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/pin-requirements.rst" => template/{% if sphinx %}docs{% endif %}/developer/how-to/pin-requirements.rst (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/run-tests.rst" => template/{% if sphinx %}docs{% endif %}/developer/how-to/run-tests.rst (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/static-analysis.rst" => template/{% if sphinx %}docs{% endif %}/developer/how-to/static-analysis.rst (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/test-container.rst" => template/{% if sphinx %}docs{% endif %}/developer/how-to/test-container.rst (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/update-tools.rst" => template/{% if sphinx %}docs{% endif %}/developer/how-to/update-tools.rst (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/index.rst" => template/{% if sphinx %}docs{% endif %}/developer/index.rst (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/reference/standards.rst" => template/{% if sphinx %}docs{% endif %}/developer/reference/standards.rst (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/tutorials/dev-install.rst.jinja" => template/{% if sphinx %}docs{% endif %}/developer/tutorials/dev-install.rst.jinja (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/genindex.rst" => template/{% if sphinx %}docs{% endif %}/genindex.rst (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/images/dls-favicon.ico" => template/{% if sphinx %}docs{% endif %}/images/dls-favicon.ico (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/images/dls-logo.svg" => template/{% if sphinx %}docs{% endif %}/images/dls-logo.svg (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/index.rst.jinja" => template/{% if sphinx %}docs{% endif %}/index.rst.jinja (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/user/explanations/docs-structure.rst" => template/{% if sphinx %}docs{% endif %}/user/explanations/docs-structure.rst (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/user/how-to/run-container.rst.jinja" => template/{% if sphinx %}docs{% endif %}/user/how-to/run-container.rst.jinja (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/user/index.rst" => template/{% if sphinx %}docs{% endif %}/user/index.rst (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/user/reference/api.rst.jinja" => template/{% if sphinx %}docs{% endif %}/user/reference/api.rst.jinja (100%) rename "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/user/tutorials/installation.rst.jinja" => template/{% if sphinx %}docs{% endif %}/user/tutorials/installation.rst.jinja (100%) diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml new file mode 100644 index 00000000..aab283a7 --- /dev/null +++ b/.github/actions/install_requirements/action.yml @@ -0,0 +1,34 @@ +name: Install requirements +description: Install a version of python then call pip install and report what was installed +inputs: + python-version: + description: Python version to install, default is from Dockerfile + default: "dev" + pip-install: + description: Parameters to pass to pip install + default: "-e .[dev]" + +runs: + using: composite + steps: + - name: Get version of python + run: | + PYTHON_VERSION="${{ inputs.python-version }}" + if [ $PYTHON_VERSION == "dev" ]; then + PYTHON_VERSION=$(sed -n "s/ARG PYTHON_VERSION=//p" Dockerfile) + fi + echo "PYTHON_VERSION=$PYTHON_VERSION" >> "$GITHUB_ENV" + shell: bash + + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install packages + run: pip install ${{ inputs.pip-install }} + shell: bash + + - name: Report what was installed + run: pip freeze + shell: bash diff --git a/.github/workflows/_container.yml b/.github/workflows/_container.yml new file mode 100644 index 00000000..4857ee9e --- /dev/null +++ b/.github/workflows/_container.yml @@ -0,0 +1,56 @@ +on: + workflow_call: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # Need this to get version number from last tag + fetch-depth: 0 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Docker Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and export to Docker local cache + uses: docker/build-push-action@v5 + with: + context: . + # Need load and tags so we can test it below + load: true + tags: tag_for_testing + + - name: Test cli works in cached runtime image + run: docker run --rm tag_for_testing --version + + - name: Create tags for publishing image + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=ref,event=tag + type=raw,value=latest + + - name: Push cached image to container registry + if: github.ref_type == 'tag' + uses: docker/build-push-action@v5 + # This does not build the image again, it will find the image in the + # Docker cache and publish it + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/_dist.yml b/.github/workflows/_dist.yml new file mode 100644 index 00000000..b1c4c93c --- /dev/null +++ b/.github/workflows/_dist.yml @@ -0,0 +1,36 @@ +on: + workflow_call: + +jobs: + build: + runs-on: "ubuntu-latest" + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # Need this to get version number from last tag + fetch-depth: 0 + + - name: Build sdist and wheel + run: > + export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && + pipx run build + + - name: Upload sdist and wheel as artifacts + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist + + - name: Check for packaging errors + run: pipx run twine check --strict dist/* + + - name: Install produced wheel + uses: ./.github/actions/install_requirements + with: + pip-install: dist/*.whl + + - name: Test module --version works using the installed wheel + # If more than one module in src/ replace with module name to test + run: python -m $(ls --hide='*.egg-info' src | head -1) --version diff --git a/.github/workflows/_docs.yml b/.github/workflows/_docs.yml index 6b56fbec..72d07e6b 100644 --- a/.github/workflows/_docs.yml +++ b/.github/workflows/_docs.yml @@ -9,18 +9,15 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - submodules: true + # Need this to get version number from last tag + fetch-depth: 0 - name: Install system packages + # Can delete this if you don't use graphviz in your docs run: sudo apt-get install graphviz - - name: Setup python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - name: Install python packages - run: pip install -c requirements/constraints.txt -e .[dev] + uses: ./.github/actions/install_requirements - name: Build docs run: tox -e docs diff --git a/.github/workflows/_pypi.yml b/.github/workflows/_pypi.yml new file mode 100644 index 00000000..69103d1e --- /dev/null +++ b/.github/workflows/_pypi.yml @@ -0,0 +1,19 @@ +on: + workflow_call: + +jobs: + upload: + runs-on: ubuntu-latest + environment: release + + steps: + - name: Download dist artifact + uses: actions/download-artifact@v4 + with: + name: dist + path: dist + + - name: Publish to PyPI using trusted publishing + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml new file mode 100644 index 00000000..d4a67b25 --- /dev/null +++ b/.github/workflows/_test.yml @@ -0,0 +1,43 @@ +on: + workflow_call: + inputs: + python-version: + type: string + description: The version of python to install + required: true + runs-on: + type: string + description: The runner to run this job on + required: true + +env: + # https://github.com/pytest-dev/pytest/issues/2042 + PY_IGNORE_IMPORTMISMATCH: "1" + +jobs: + run: + runs-on: ${{ inputs.runs-on }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - if: inputs.python-version == 'dev' + name: Install dev versions of python packages + uses: ./.github/actions/install_requirements + + - if: inputs.python-version != 'dev' + name: Install latest versions of python packages + uses: ./.github/actions/install_requirements + with: + python-version: ${{ inputs.python-version }} + pip-install: ".[dev]" + + - name: Run tests + run: tox -e pytest + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + name: ${{ inputs.python-version }}/${{ inputs.runs-on }} + files: cov.xml diff --git a/.github/workflows/_tox.yml b/.github/workflows/_tox.yml index d9701fc0..a13536d3 100644 --- a/.github/workflows/_tox.yml +++ b/.github/workflows/_tox.yml @@ -14,16 +14,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - name: Install python packages - run: pip install -c requirements/constraints.txt -e .[dev] + uses: ./.github/actions/install_requirements - name: Run tox run: tox -e ${{ inputs.tox }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5011033c..5de3018f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,28 +6,59 @@ on: jobs: check: - # TODO: Use the CI straight from the template uses: ./.github/workflows/_check.yml - # TODO: Use the CI straight from the template - test: + lint: needs: check if: ${{ ! needs.check.outputs.branch-pr }} uses: ./.github/workflows/_tox.yml with: - tox: pre-commit,pytest + tox: pre-commit,pyright + + test: + needs: check + if: ${{ ! needs.check.outputs.branch-pr }} + strategy: + matrix: + runs-on: ["ubuntu-latest", "windows-latest", "macos-latest"] + python-version: ["3.8", "3.9", "3.10", "3.11"] + include: + # Include one that runs in the dev environment + - runs-on: "ubuntu-latest" + python-version: "dev" + fail-fast: false + uses: ./.github/workflows/_test.yml + with: + runs-on: ${{ matrix.runs-on }} + python-version: ${{ matrix.python-version }} + + container: + needs: check + if: ${{ ! needs.check.outputs.branch-pr }} + uses: ./.github/workflows/_container.yml + permissions: + packages: write docs: needs: check if: ${{ ! needs.check.outputs.branch-pr }} uses: ./.github/workflows/_docs.yml + + dist: + needs: check + if: ${{ ! needs.check.outputs.branch-pr }} + uses: ./.github/workflows/_dist.yml + + pypi: + if: github.ref_type == 'tag' + needs: dist + uses: ./.github/workflows/_pypi.yml permissions: - contents: write + id-token: write release: if: github.ref_type == 'tag' - needs: docs - # TODO: Use the CI straight from the template + needs: [dist, docs] uses: ./.github/workflows/_release.yml permissions: contents: write diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml new file mode 100644 index 00000000..e2a0fd1b --- /dev/null +++ b/.github/workflows/periodic.yml @@ -0,0 +1,13 @@ +name: Periodic + +on: + workflow_dispatch: + schedule: + # Run weekly to check URL links still resolve + - cron: "0 8 * * WED" + +jobs: + linkcheck: + uses: ./.github/workflows/_tox.yml + with: + tox: docs build -- -b linkcheck diff --git a/Dockerfile b/Dockerfile index 8dd09a45..b8d33308 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,13 @@ -# This file is for use as a devcontainer -# # The devcontainer should use the developer target and run as root with podman # or docker with user namespaces. ARG PYTHON_VERSION=3.11 FROM python:${PYTHON_VERSION} as developer -RUN apt-get update && apt-get upgrade -y && \ - apt-get install -y --no-install-recommends \ + +# Add any system dependencies for the developer/build environment here +RUN apt-get update && apt-get install -y --no-install-recommends \ graphviz \ && rm -rf /var/lib/apt/lists/* + +# Set up a virtual environment and put it in PATH RUN python -m venv /venv -ENV PATH=/venv/bin:$PATH +ENV PATH=/venv/bin:$PATH \ No newline at end of file diff --git a/copier.yml b/copier.yml index ed2188c5..2faae9f9 100644 --- a/copier.yml +++ b/copier.yml @@ -76,8 +76,7 @@ docs_type: - README - sphinx -_subdirectory: "template" - +# Internal variables repo_url: type: str default: "https://github.com/{{github_org}}/{{repo_name}}" @@ -88,5 +87,12 @@ docs_url: default: "https://{{github_org | lower}}.github.io/{{repo_name}}" when: false +sphinx: + type: bool + default: "{{ docs_type == 'sphinx' }}" + when: false + +_subdirectory: "template" + _tasks: - "git init --initial-branch=main" diff --git a/template/.devcontainer/devcontainer.json b/template/.devcontainer/devcontainer.json index 44de8d36..96427ee4 100644 --- a/template/.devcontainer/devcontainer.json +++ b/template/.devcontainer/devcontainer.json @@ -3,11 +3,7 @@ "name": "Python 3 Developer Container", "build": { "dockerfile": "../Dockerfile", - "target": "build", - // Only upgrade pip, we will install the project below - "args": { - "PIP_OPTIONS": "--upgrade pip" - } + "target": "developer" }, "remoteEnv": { "DISPLAY": "${localEnv:DISPLAY}" @@ -19,11 +15,11 @@ "upgradePackages": false } }, - // Set *default* container specific settings.json values on container create. - "settings": { - "python.defaultInterpreterPath": "/venv/bin/python" - }, "customizations": { + // Set *default* container specific settings.json values on container create. + "settings": { + "python.defaultInterpreterPath": "/venv/bin/python" + }, "vscode": { // Add the IDs of extensions you want installed when the container is created. "extensions": [ @@ -51,4 +47,4 @@ "workspaceFolder": "${localWorkspaceFolder}", // After the container is created, install the python project in editable form "postCreateCommand": "pip install -e '.[dev]'" -} +} \ No newline at end of file diff --git a/template/{% if docker %}Dockerfile{% endif %}.jinja b/template/{% if docker %}Dockerfile{% endif %}.jinja index 3fd17db0..cae59efb 100644 --- a/template/{% if docker %}Dockerfile{% endif %}.jinja +++ b/template/{% if docker %}Dockerfile{% endif %}.jinja @@ -1,34 +1,26 @@ -# This file is for use as a devcontainer and a runtime container -# -# The devcontainer should use the build target and run as root with podman +# The devcontainer should use the developer target and run as root with podman # or docker with user namespaces. -# -FROM python:3.11 as build +ARG PYTHON_VERSION=3.11 +FROM python:${PYTHON_VERSION} as developer -ARG PIP_OPTIONS=. +# Add any system dependencies for the developer/build environment here +RUN apt-get update && apt-get install -y --no-install-recommends \ + graphviz \ + && rm -rf /var/lib/apt/lists/* -# Add any system dependencies for the developer/build environment here e.g. -# RUN apt-get update && apt-get upgrade -y && \ -# apt-get install -y --no-install-recommends \ -# desired-packages \ -# && rm -rf /var/lib/apt/lists/* - -# set up a virtual environment and put it in PATH +# Set up a virtual environment and put it in PATH RUN python -m venv /venv ENV PATH=/venv/bin:$PATH -# Copy any required context for the pip install over +# The build stage installs the context into the venv +FROM developer as build COPY . /context WORKDIR /context +RUN pip install . -# install python package into /venv -RUN pip install ${PIP_OPTIONS} - -FROM python:3.11-slim as runtime - +# The runtime stage copies the built venv into a slim runtime container +FROM python:${PYTHON_VERSION}-slim as runtime # Add apt-get system dependecies for runtime here if needed - -# copy the virtual environment from the build stage and put it in PATH COPY --from=build /venv/ /venv/ ENV PATH=/venv/bin:$PATH diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/actions" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/actions" new file mode 120000 index 00000000..c18123eb --- /dev/null +++ "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/actions" @@ -0,0 +1 @@ +../../.github/actions/ \ No newline at end of file diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/actions/install_requirements/action.yml" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/actions/install_requirements/action.yml" deleted file mode 100644 index 79d1a71e..00000000 --- "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/actions/install_requirements/action.yml" +++ /dev/null @@ -1,60 +0,0 @@ -name: Install requirements -description: Run pip install with requirements and upload resulting requirements -inputs: - requirements_file: - description: Name of requirements file to use and upload - required: true - install_options: - description: Parameters to pass to pip install - required: true - artifact_name: - description: A user friendly name to give the produced artifacts - required: true - python_version: - description: Python version to install - default: "3.x" - -runs: - using: composite - - steps: - - name: Setup python - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python_version }} - - - name: Pip install - run: | - touch ${{ inputs.requirements_file }} - # -c uses requirements.txt as constraints, see 'Validate requirements file' - pip install -c ${{ inputs.requirements_file }} ${{ inputs.install_options }} - shell: bash - - - name: Create lockfile - run: | - mkdir -p lockfiles - pip freeze --exclude-editable > lockfiles/${{ inputs.requirements_file }} - # delete the self referencing line and make sure it isn't blank - sed -i'' -e '/file:/d' lockfiles/${{ inputs.requirements_file }} - shell: bash - - - name: Upload lockfiles - uses: actions/upload-artifact@v4.0.0 - with: - name: lockfiles-${{ inputs.python_version }}-${{ inputs.artifact_name }}-${{ github.sha }} - path: lockfiles - - # This eliminates the class of problems where the requirements being given no - # longer match what the packages themselves dictate. E.g. In the rare instance - # where I install some-package which used to depend on vulnerable-dependency - # but now uses good-dependency (despite being nominally the same version) - # pip will install both if given a requirements file with -r - - name: If requirements file exists, check it matches pip installed packages - run: | - if [ -s ${{ inputs.requirements_file }} ]; then - if ! diff -u ${{ inputs.requirements_file }} lockfiles/${{ inputs.requirements_file }}; then - echo "Error: ${{ inputs.requirements_file }} need the above changes to be exhaustive" - exit 1 - fi - fi - shell: bash diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/dependabot.yml" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/dependabot.yml" index 2d1af873..184ba363 100644 --- "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/dependabot.yml" +++ "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/dependabot.yml" @@ -10,11 +10,15 @@ updates: schedule: interval: "weekly" groups: - github-artifacts: + actions: patterns: - - actions/*-artifact + - "*" - package-ecosystem: "pip" directory: "/" schedule: interval: "weekly" + groups: + dev-dependencies: + patterns: + - "*" diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/pages" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/pages" new file mode 120000 index 00000000..765622eb --- /dev/null +++ "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/pages" @@ -0,0 +1 @@ +../../.github/pages \ No newline at end of file diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/pages/index.html" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/pages/index.html" deleted file mode 100644 index c495f39f..00000000 --- "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/pages/index.html" +++ /dev/null @@ -1,11 +0,0 @@ - - - - - Redirecting to main branch - - - - - - diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/pages/make_switcher.py" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/pages/make_switcher.py" deleted file mode 100755 index ae227ab7..00000000 --- "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/pages/make_switcher.py" +++ /dev/null @@ -1,99 +0,0 @@ -import json -import logging -from argparse import ArgumentParser -from pathlib import Path -from subprocess import CalledProcessError, check_output -from typing import List, Optional - - -def report_output(stdout: bytes, label: str) -> List[str]: - ret = stdout.decode().strip().split("\n") - print(f"{label}: {ret}") - return ret - - -def get_branch_contents(ref: str) -> List[str]: - """Get the list of directories in a branch.""" - stdout = check_output(["git", "ls-tree", "-d", "--name-only", ref]) - return report_output(stdout, "Branch contents") - - -def get_sorted_tags_list() -> List[str]: - """Get a list of sorted tags in descending order from the repository.""" - stdout = check_output(["git", "tag", "-l", "--sort=-v:refname"]) - return report_output(stdout, "Tags list") - - -def get_versions(ref: str, add: Optional[str], remove: Optional[str]) -> List[str]: - """Generate the file containing the list of all GitHub Pages builds.""" - # Get the directories (i.e. builds) from the GitHub Pages branch - try: - builds = set(get_branch_contents(ref)) - except CalledProcessError: - builds = set() - logging.warning(f"Cannot get {ref} contents") - - # Add and remove from the list of builds - if add: - builds.add(add) - if remove: - assert remove in builds, f"Build '{remove}' not in {sorted(builds)}" - builds.remove(remove) - - # Get a sorted list of tags - tags = get_sorted_tags_list() - - # Make the sorted versions list from main branches and tags - versions: List[str] = [] - for version in ["master", "main"] + tags: - if version in builds: - versions.append(version) - builds.remove(version) - - # Add in anything that is left to the bottom - versions += sorted(builds) - print(f"Sorted versions: {versions}") - return versions - - -def write_json(path: Path, repository: str, versions: str): - org, repo_name = repository.split("/") - struct = [ - {"version": version, "url": f"https://{org}.github.io/{repo_name}/{version}/"} - for version in versions - ] - text = json.dumps(struct, indent=2) - print(f"JSON switcher:\n{text}") - path.write_text(text, encoding="utf-8") - - -def main(args=None): - parser = ArgumentParser( - description="Make a versions.txt file from gh-pages directories" - ) - parser.add_argument( - "--add", - help="Add this directory to the list of existing directories", - ) - parser.add_argument( - "--remove", - help="Remove this directory from the list of existing directories", - ) - parser.add_argument( - "repository", - help="The GitHub org and repository name: ORG/REPO", - ) - parser.add_argument( - "output", - type=Path, - help="Path of write switcher.json to", - ) - args = parser.parse_args(args) - - # Write the versions file - versions = get_versions("origin/gh-pages", args.add, args.remove) - write_json(args.output, args.repository, versions) - - -if __name__ == "__main__": - main() diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_check.yml" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_check.yml" new file mode 120000 index 00000000..9c0f79a5 --- /dev/null +++ "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_check.yml" @@ -0,0 +1 @@ +../../../.github/workflows/_check.yml \ No newline at end of file diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_dist.yml" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_dist.yml" new file mode 120000 index 00000000..a821da81 --- /dev/null +++ "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_dist.yml" @@ -0,0 +1 @@ +../../../.github/workflows/_dist.yml \ No newline at end of file diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_pypi.yml" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_pypi.yml" new file mode 120000 index 00000000..157ae6d1 --- /dev/null +++ "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_pypi.yml" @@ -0,0 +1 @@ +../../../.github/workflows/_pypi.yml \ No newline at end of file diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_release.yml" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_release.yml" new file mode 120000 index 00000000..d5d5c173 --- /dev/null +++ "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_release.yml" @@ -0,0 +1 @@ +../../../.github/workflows/_release.yml \ No newline at end of file diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_test.yml" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_test.yml" new file mode 120000 index 00000000..75b1ba09 --- /dev/null +++ "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_test.yml" @@ -0,0 +1 @@ +../../../.github/workflows/_test.yml \ No newline at end of file diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_tox.yml" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_tox.yml" new file mode 120000 index 00000000..c781ae05 --- /dev/null +++ "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/_tox.yml" @@ -0,0 +1 @@ +../../../.github/workflows/_tox.yml \ No newline at end of file diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/ci.yml.jinja" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/ci.yml.jinja" new file mode 100644 index 00000000..e3d7b84d --- /dev/null +++ "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/ci.yml.jinja" @@ -0,0 +1,65 @@ +name: CI + +on: + push: + pull_request: + +jobs: + check: + uses: ./.github/workflows/_check.yml + + lint: + needs: check + if: ${{ ! needs.check.outputs.branch-pr }} + uses: ./.github/workflows/_tox.yml + with: + tox: pre-commit,mypy + + test: + needs: check + if: ${{ ! needs.check.outputs.branch-pr }} + strategy: + matrix: + runs-on: ["ubuntu-latest"] # can add windows-latest, macos-latest + python-version: ["3.8", "3.9", "3.10", "3.11"] + include: + # Include one that runs in the dev environment + - runs-on: "ubuntu-latest" + python-version: "dev" + fail-fast: false + uses: ./.github/workflows/_test.yml + with: + runs-on: ${{ matrix.runs-on }} + python-version: ${{ matrix.python-version }} +{% if docker %} + container: + needs: check + if: ${{ ! needs.check.outputs.branch-pr }} + uses: ./.github/workflows/_container.yml + permissions: + packages: write +{% endif %} +{% if sphinx %} + docs: + needs: check + if: ${{ ! needs.check.outputs.branch-pr }} + uses: ./.github/workflows/_docs.yml +{% endif %} + dist: + needs: check + if: ${{ ! needs.check.outputs.branch-pr }} + uses: ./.github/workflows/_dist.yml + + pypi: + if: github.ref_type == 'tag' + needs: dist + uses: ./.github/workflows/_pypi.yml + permissions: + id-token: write + + release: + if: github.ref_type == 'tag' + needs: [dist{% if sphinx %}, docs{% endif %}] + uses: ./.github/workflows/_release.yml + permissions: + contents: write diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/code.yml" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/code.yml" deleted file mode 100644 index 364ff43d..00000000 --- "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/code.yml" +++ /dev/null @@ -1,243 +0,0 @@ -name: Code CI - -on: - push: - pull_request: -env: - # The target python version, which must match the Dockerfile version - CONTAINER_PYTHON: "3.11" - DIST_WHEEL_PATH: dist-${{ github.sha }} - -jobs: - lint: - # pull requests are a duplicate of a branch push if within the same repo. - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install python packages - uses: ./.github/actions/install_requirements - with: - requirements_file: requirements-dev-3.x.txt - install_options: -e .[dev] - artifact_name: lint - - - name: Lint - run: tox -e pre-commit,mypy - - test: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] # can add windows-latest, macos-latest - python: ["3.8", "3.9", "3.10", "3.11"] - install: ["-e .[dev]"] - # Make one version be non-editable to test both paths of version code - include: - - os: "ubuntu-latest" - python: "3.7" - install: ".[dev]" - - runs-on: ${{ matrix.os }} - env: - # https://github.com/pytest-dev/pytest/issues/2042 - PY_IGNORE_IMPORTMISMATCH: "1" - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # Need this to get version number from last tag - fetch-depth: 0 - - - name: Install python packages - uses: ./.github/actions/install_requirements - with: - python_version: ${{ matrix.python }} - requirements_file: requirements-test-${{ matrix.os }}-${{ matrix.python }}.txt - install_options: ${{ matrix.install }} - artifact_name: tests - - - name: List dependency tree - run: pipdeptree - - - name: Run tests - run: tox -e pytest - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - name: ${{ matrix.python }}/${{ matrix.os }} - files: cov.xml - - dist: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository - runs-on: "ubuntu-latest" - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # Need this to get version number from last tag - fetch-depth: 0 - - - name: Build sdist and wheel - run: | - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ - pipx run build - - - name: Upload sdist and wheel as artifacts - uses: actions/upload-artifact@v4.0.0 - with: - name: ${{ env.DIST_WHEEL_PATH }} - path: dist - - - name: Check for packaging errors - run: pipx run twine check --strict dist/* - - - name: Install python packages - uses: ./.github/actions/install_requirements - with: - python_version: ${{env.CONTAINER_PYTHON}} - requirements_file: requirements.txt - install_options: dist/*.whl - artifact_name: dist - - - name: Test module --version works using the installed wheel - # If more than one module in src/ replace with module name to test - run: python -m $(ls --hide='*.egg-info' src | head -1) --version - - container: - needs: [lint, dist, test] - runs-on: ubuntu-latest - - permissions: - contents: read - packages: write - - env: - TEST_TAG: "testing" - - steps: - - name: Checkout - uses: actions/checkout@v4 - - # image names must be all lower case - - name: Generate image repo name - run: echo IMAGE_REPOSITORY=ghcr.io/$(tr '[:upper:]' '[:lower:]' <<< "${{ github.repository }}") >> $GITHUB_ENV - - - name: Set lockfile location in environment - run: | - echo "DIST_LOCKFILE_PATH=lockfiles-${{ env.CONTAINER_PYTHON }}-dist-${{ github.sha }}" >> $GITHUB_ENV - - - name: Download wheel and lockfiles - uses: actions/download-artifact@v4.1.0 - with: - path: artifacts/ - pattern: "*dist*" - - - name: Log in to GitHub Docker Registry - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and export to Docker local cache - uses: docker/build-push-action@v5 - with: - # Note build-args, context, file, and target must all match between this - # step and the later build-push-action, otherwise the second build-push-action - # will attempt to build the image again - build-args: | - PIP_OPTIONS=-r ${{ env.DIST_LOCKFILE_PATH }}/requirements.txt ${{ env.DIST_WHEEL_PATH }}/*.whl - context: artifacts/ - file: ./Dockerfile - target: runtime - load: true - tags: ${{ env.TEST_TAG }} - # If you have a long docker build (2+ minutes), uncomment the - # following to turn on caching. For short build times this - # makes it a little slower - #cache-from: type=gha - #cache-to: type=gha,mode=max - - - name: Test cli works in cached runtime image - run: docker run docker.io/library/${{ env.TEST_TAG }} --version - - - name: Create tags for publishing image - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.IMAGE_REPOSITORY }} - tags: | - type=ref,event=tag - type=raw,value=latest, enable=${{ github.ref_type == 'tag' }} - # type=edge,branch=main - # Add line above to generate image for every commit to given branch, - # and uncomment the end of if clause in next step - - - name: Push cached image to container registry - if: github.ref_type == 'tag' # || github.ref_name == 'main' - uses: docker/build-push-action@v5 - # This does not build the image again, it will find the image in the - # Docker cache and publish it - with: - # Note build-args, context, file, and target must all match between this - # step and the previous build-push-action, otherwise this step will - # attempt to build the image again - build-args: | - PIP_OPTIONS=-r ${{ env.DIST_LOCKFILE_PATH }}/requirements.txt ${{ env.DIST_WHEEL_PATH }}/*.whl - context: artifacts/ - file: ./Dockerfile - target: runtime - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - release: - # upload to PyPI and make a release on every tag - needs: [lint, dist, test] - if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }} - runs-on: ubuntu-latest - env: - HAS_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN != '' }} - - steps: - - name: Download wheel and lockfiles - uses: actions/download-artifact@v4.1.0 - with: - path: artifacts/ - pattern: "*dist*" - - - name: Fixup blank lockfiles - # Github release artifacts can't be blank - run: for f in ${{ env.DIST_LOCKFILE_PATH }}/*; do [ -s $f ] || echo '# No requirements' >> $f; done - - - name: Github Release - # We pin to the SHA, not the tag, for security reasons. - # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions - uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15 - with: - prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }} - files: | - ${{ env.DIST_WHEEL_PATH }}/* - ${{ env.DIST_LOCKFILE_PATH }}/* - generate_release_notes: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Publish to PyPI - if: ${{ env.HAS_PYPI_TOKEN }} - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_TOKEN }} diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/docs.yml" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/docs.yml" deleted file mode 100644 index 3c29ff94..00000000 --- "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/docs.yml" +++ /dev/null @@ -1,54 +0,0 @@ -name: Docs CI - -on: - push: - pull_request: - -jobs: - docs: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository - runs-on: ubuntu-latest - - steps: - - name: Avoid git conflicts when tag and branch pushed at same time - if: startsWith(github.ref, 'refs/tags') - run: sleep 60 - - - name: Checkout - uses: actions/checkout@v4 - with: - # Need this to get version number from last tag - fetch-depth: 0 - - - name: Install system packages - # Can delete this if you don't use graphviz in your docs - run: sudo apt-get install graphviz - - - name: Install python packages - uses: ./.github/actions/install_requirements - with: - requirements_file: requirements-dev-3.x.txt - install_options: -e .[dev] - artifact_name: docs - - - name: Build docs - run: tox -e docs - - - name: Sanitize ref name for docs version - run: echo "DOCS_VERSION=${GITHUB_REF_NAME//[^A-Za-z0-9._-]/_}" >> $GITHUB_ENV - - - name: Move to versioned directory - run: mv build/html .github/pages/$DOCS_VERSION - - - name: Write switcher.json - run: python .github/pages/make_switcher.py --add $DOCS_VERSION ${{ github.repository }} .github/pages/switcher.json - - - name: Publish Docs to gh-pages - if: github.event_name == 'push' && github.actor != 'dependabot[bot]' - # We pin to the SHA, not the tag, for security reasons. - # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions - uses: peaceiris/actions-gh-pages@64b46b4226a4a12da2239ba3ea5aa73e3163c75b # v3.9.1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: .github/pages - keep_files: true diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/docs_clean.yml" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/docs_clean.yml" deleted file mode 100644 index e324640e..00000000 --- "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/docs_clean.yml" +++ /dev/null @@ -1,43 +0,0 @@ -name: Docs Cleanup CI - -# delete branch documentation when a branch is deleted -# also allow manually deleting a documentation version -on: - delete: - workflow_dispatch: - inputs: - version: - description: "documentation version to DELETE" - required: true - type: string - -jobs: - remove: - if: github.event.ref_type == 'branch' || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: gh-pages - - - name: removing documentation for branch ${{ github.event.ref }} - if: ${{ github.event_name != 'workflow_dispatch' }} - run: echo "REF_NAME=${{ github.event.ref }}" >> $GITHUB_ENV - - - name: manually removing documentation version ${{ github.event.inputs.version }} - if: ${{ github.event_name == 'workflow_dispatch' }} - run: echo "REF_NAME=${{ github.event.inputs.version }}" >> $GITHUB_ENV - - - name: Sanitize ref name for docs version - run: echo "DOCS_VERSION=${REF_NAME//[^A-Za-z0-9._-]/_}" >> $GITHUB_ENV - - - name: update index and push changes - run: | - rm -r $DOCS_VERSION - python make_switcher.py --remove $DOCS_VERSION ${{ github.repository }} switcher.json - git config --global user.name 'GitHub Actions Docs Cleanup CI' - git config --global user.email 'GithubActionsCleanup@noreply.github.com' - git commit -am "Removing redundant docs version $DOCS_VERSION" - git push diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/linkcheck.yml" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/linkcheck.yml" deleted file mode 100644 index 7f651a27..00000000 --- "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/linkcheck.yml" +++ /dev/null @@ -1,28 +0,0 @@ -name: Link Check - -on: - workflow_dispatch: - schedule: - # Run weekly to check URL links still resolve - - cron: "0 8 * * WED" - -jobs: - docs: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install python packages - uses: ./.github/actions/install_requirements - with: - requirements_file: requirements-dev-3.x.txt - install_options: -e .[dev] - artifact_name: link_check - - - name: Check links - run: tox -e docs build -- -b linkcheck - - - name: Keepalive Workflow - uses: gautamkrishnar/keepalive-workflow@v1 diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/{% if docker %}_container.yml{% endif %}" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/{% if docker %}_container.yml{% endif %}" new file mode 120000 index 00000000..fa91b318 --- /dev/null +++ "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/{% if docker %}_container.yml{% endif %}" @@ -0,0 +1 @@ +../../../.github/workflows/_container.yml \ No newline at end of file diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/{% if sphinx %}_docs.yml{% endif %}" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/{% if sphinx %}_docs.yml{% endif %}" new file mode 120000 index 00000000..6f47d301 --- /dev/null +++ "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/{% if sphinx %}_docs.yml{% endif %}" @@ -0,0 +1 @@ +../../../.github/workflows/_docs.yml \ No newline at end of file diff --git "a/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/{% if sphinx %}periodic.yml{% endif %}" "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/{% if sphinx %}periodic.yml{% endif %}" new file mode 120000 index 00000000..7d361caf --- /dev/null +++ "b/template/{% if git_platform==\"github.com\" %}.github{% endif %}/workflows/{% if sphinx %}periodic.yml{% endif %}" @@ -0,0 +1 @@ +../../../.github/workflows/periodic.yml \ No newline at end of file diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/conf.py.jinja" b/template/{% if sphinx %}docs{% endif %}/conf.py.jinja similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/conf.py.jinja" rename to template/{% if sphinx %}docs{% endif %}/conf.py.jinja diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/explanations/decisions.rst" b/template/{% if sphinx %}docs{% endif %}/developer/explanations/decisions.rst similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/explanations/decisions.rst" rename to template/{% if sphinx %}docs{% endif %}/developer/explanations/decisions.rst diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/explanations/decisions/0001-record-architecture-decisions.rst" b/template/{% if sphinx %}docs{% endif %}/developer/explanations/decisions/0001-record-architecture-decisions.rst similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/explanations/decisions/0001-record-architecture-decisions.rst" rename to template/{% if sphinx %}docs{% endif %}/developer/explanations/decisions/0001-record-architecture-decisions.rst diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst.jinja" b/template/{% if sphinx %}docs{% endif %}/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst.jinja similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst.jinja" rename to template/{% if sphinx %}docs{% endif %}/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst.jinja diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/build-docs.rst" b/template/{% if sphinx %}docs{% endif %}/developer/how-to/build-docs.rst similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/build-docs.rst" rename to template/{% if sphinx %}docs{% endif %}/developer/how-to/build-docs.rst diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/contribute.rst" b/template/{% if sphinx %}docs{% endif %}/developer/how-to/contribute.rst similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/contribute.rst" rename to template/{% if sphinx %}docs{% endif %}/developer/how-to/contribute.rst diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/lint.rst" b/template/{% if sphinx %}docs{% endif %}/developer/how-to/lint.rst similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/lint.rst" rename to template/{% if sphinx %}docs{% endif %}/developer/how-to/lint.rst diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/make-release.rst.jinja" b/template/{% if sphinx %}docs{% endif %}/developer/how-to/make-release.rst.jinja similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/make-release.rst.jinja" rename to template/{% if sphinx %}docs{% endif %}/developer/how-to/make-release.rst.jinja diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/pin-requirements.rst" b/template/{% if sphinx %}docs{% endif %}/developer/how-to/pin-requirements.rst similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/pin-requirements.rst" rename to template/{% if sphinx %}docs{% endif %}/developer/how-to/pin-requirements.rst diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/run-tests.rst" b/template/{% if sphinx %}docs{% endif %}/developer/how-to/run-tests.rst similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/run-tests.rst" rename to template/{% if sphinx %}docs{% endif %}/developer/how-to/run-tests.rst diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/static-analysis.rst" b/template/{% if sphinx %}docs{% endif %}/developer/how-to/static-analysis.rst similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/static-analysis.rst" rename to template/{% if sphinx %}docs{% endif %}/developer/how-to/static-analysis.rst diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/test-container.rst" b/template/{% if sphinx %}docs{% endif %}/developer/how-to/test-container.rst similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/test-container.rst" rename to template/{% if sphinx %}docs{% endif %}/developer/how-to/test-container.rst diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/update-tools.rst" b/template/{% if sphinx %}docs{% endif %}/developer/how-to/update-tools.rst similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/how-to/update-tools.rst" rename to template/{% if sphinx %}docs{% endif %}/developer/how-to/update-tools.rst diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/index.rst" b/template/{% if sphinx %}docs{% endif %}/developer/index.rst similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/index.rst" rename to template/{% if sphinx %}docs{% endif %}/developer/index.rst diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/reference/standards.rst" b/template/{% if sphinx %}docs{% endif %}/developer/reference/standards.rst similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/reference/standards.rst" rename to template/{% if sphinx %}docs{% endif %}/developer/reference/standards.rst diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/tutorials/dev-install.rst.jinja" b/template/{% if sphinx %}docs{% endif %}/developer/tutorials/dev-install.rst.jinja similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/developer/tutorials/dev-install.rst.jinja" rename to template/{% if sphinx %}docs{% endif %}/developer/tutorials/dev-install.rst.jinja diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/genindex.rst" b/template/{% if sphinx %}docs{% endif %}/genindex.rst similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/genindex.rst" rename to template/{% if sphinx %}docs{% endif %}/genindex.rst diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/images/dls-favicon.ico" b/template/{% if sphinx %}docs{% endif %}/images/dls-favicon.ico similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/images/dls-favicon.ico" rename to template/{% if sphinx %}docs{% endif %}/images/dls-favicon.ico diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/images/dls-logo.svg" b/template/{% if sphinx %}docs{% endif %}/images/dls-logo.svg similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/images/dls-logo.svg" rename to template/{% if sphinx %}docs{% endif %}/images/dls-logo.svg diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/index.rst.jinja" b/template/{% if sphinx %}docs{% endif %}/index.rst.jinja similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/index.rst.jinja" rename to template/{% if sphinx %}docs{% endif %}/index.rst.jinja diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/user/explanations/docs-structure.rst" b/template/{% if sphinx %}docs{% endif %}/user/explanations/docs-structure.rst similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/user/explanations/docs-structure.rst" rename to template/{% if sphinx %}docs{% endif %}/user/explanations/docs-structure.rst diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/user/how-to/run-container.rst.jinja" b/template/{% if sphinx %}docs{% endif %}/user/how-to/run-container.rst.jinja similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/user/how-to/run-container.rst.jinja" rename to template/{% if sphinx %}docs{% endif %}/user/how-to/run-container.rst.jinja diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/user/index.rst" b/template/{% if sphinx %}docs{% endif %}/user/index.rst similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/user/index.rst" rename to template/{% if sphinx %}docs{% endif %}/user/index.rst diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/user/reference/api.rst.jinja" b/template/{% if sphinx %}docs{% endif %}/user/reference/api.rst.jinja similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/user/reference/api.rst.jinja" rename to template/{% if sphinx %}docs{% endif %}/user/reference/api.rst.jinja diff --git "a/template/{% if docs_type==\"sphinx\" %}docs{% endif %}/user/tutorials/installation.rst.jinja" b/template/{% if sphinx %}docs{% endif %}/user/tutorials/installation.rst.jinja similarity index 100% rename from "template/{% if docs_type==\"sphinx\" %}docs{% endif %}/user/tutorials/installation.rst.jinja" rename to template/{% if sphinx %}docs{% endif %}/user/tutorials/installation.rst.jinja From 714fab11ff414222e2fc2c80e6acbb4e344fd30f Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 31 Jan 2024 09:49:14 +0000 Subject: [PATCH 2/4] Use split CI in the root --- .devcontainer/devcontainer.json | 7 +- .github/workflows/_check.yml | 9 +-- .github/workflows/_docs.yml | 1 - .github/workflows/ci.yml | 46 ++----------- .pre-commit-config.yaml | 7 -- requirements/constraints.txt | 68 ------------------- requirements/regenerate.sh | 26 ------- ...file{% endif %}.jinja => Dockerfile.jinja} | 3 +- .../workflows/ci.yml.jinja" | 17 +++-- 9 files changed, 24 insertions(+), 160 deletions(-) delete mode 100644 requirements/constraints.txt delete mode 100755 requirements/regenerate.sh rename template/{{% if docker %}Dockerfile{% endif %}.jinja => Dockerfile.jinja} (97%) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index cf2215a2..9781d9d9 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -7,7 +7,7 @@ }, "remoteEnv": { // Allow X11 apps to run inside the container - "DISPLAY": "${localEnv:DISPLAY}", + "DISPLAY": "${localEnv:DISPLAY}" }, "customizations": { "vscode": { @@ -22,7 +22,8 @@ "tamasfe.even-better-toml", "redhat.vscode-yaml", "ryanluker.vscode-coverage-gutters", - "charliermarsh.ruff" + "charliermarsh.ruff", + "ms-azuretools.vscode-docker" ] } }, @@ -41,5 +42,5 @@ // Mount the parent as /workspaces so we can pip install peers as editable "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind", // After the container is created, install the python project in editable form - "postCreateCommand": "pip install -c requirements/constraints.txt -e '.[dev]'" + "postCreateCommand": "pip install -e '.[dev]'" } \ No newline at end of file diff --git a/.github/workflows/_check.yml b/.github/workflows/_check.yml index a6139c19..b26d72a5 100644 --- a/.github/workflows/_check.yml +++ b/.github/workflows/_check.yml @@ -1,15 +1,15 @@ on: workflow_call: outputs: - branch-pr: + not-in-pr: description: The PR number if the branch is in one - value: ${{ jobs.pr.outputs.branch-pr }} + value: ${{ jobs.pr.outputs.not-in-pr }} jobs: pr: runs-on: "ubuntu-latest" outputs: - branch-pr: ${{ steps.script.outputs.result }} + not-in-pr: ${{ steps.script.outputs.result }} steps: - uses: actions/github-script@v7 id: script @@ -23,5 +23,6 @@ jobs: }) if (prs.data.length) { console.log(`::notice ::Skipping CI on branch push as it is already run in PR #${prs.data[0]["number"]}`) - return prs.data[0]["number"] + } else { + return "not-in-pr" } diff --git a/.github/workflows/_docs.yml b/.github/workflows/_docs.yml index 72d07e6b..4d9d7ad1 100644 --- a/.github/workflows/_docs.yml +++ b/.github/workflows/_docs.yml @@ -13,7 +13,6 @@ jobs: fetch-depth: 0 - name: Install system packages - # Can delete this if you don't use graphviz in your docs run: sudo apt-get install graphviz - name: Install python packages diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5de3018f..ea1f3562 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,57 +8,21 @@ jobs: check: uses: ./.github/workflows/_check.yml - lint: - needs: check - if: ${{ ! needs.check.outputs.branch-pr }} - uses: ./.github/workflows/_tox.yml - with: - tox: pre-commit,pyright - test: needs: check - if: ${{ ! needs.check.outputs.branch-pr }} - strategy: - matrix: - runs-on: ["ubuntu-latest", "windows-latest", "macos-latest"] - python-version: ["3.8", "3.9", "3.10", "3.11"] - include: - # Include one that runs in the dev environment - - runs-on: "ubuntu-latest" - python-version: "dev" - fail-fast: false - uses: ./.github/workflows/_test.yml + if: needs.check.outputs.not-in-pr + uses: ./.github/workflows/_tox.yml with: - runs-on: ${{ matrix.runs-on }} - python-version: ${{ matrix.python-version }} - - container: - needs: check - if: ${{ ! needs.check.outputs.branch-pr }} - uses: ./.github/workflows/_container.yml - permissions: - packages: write + tox: pre-commit,pytest docs: needs: check - if: ${{ ! needs.check.outputs.branch-pr }} + if: needs.check.outputs.not-in-pr uses: ./.github/workflows/_docs.yml - dist: - needs: check - if: ${{ ! needs.check.outputs.branch-pr }} - uses: ./.github/workflows/_dist.yml - - pypi: - if: github.ref_type == 'tag' - needs: dist - uses: ./.github/workflows/_pypi.yml - permissions: - id-token: write - release: if: github.ref_type == 'tag' - needs: [dist, docs] + needs: docs uses: ./.github/workflows/_release.yml permissions: contents: write diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3143ae60..5a4cbf7b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,10 +21,3 @@ repos: entry: ruff format --force-exclude types: [python] require_serial: true - - - id: constraints - name: check constraints match installed - language: system - entry: ./requirements/regenerate.sh --if-non-empty - files: ^(pyproject\.toml|requirements/constraints\.txt)$ - pass_filenames: false diff --git a/requirements/constraints.txt b/requirements/constraints.txt deleted file mode 100644 index 81e84b6e..00000000 --- a/requirements/constraints.txt +++ /dev/null @@ -1,68 +0,0 @@ -accessible-pygments==0.0.4 -alabaster==0.7.16 -annotated-types==0.6.0 -Babel==2.14.0 -beautifulsoup4==4.12.2 -certifi==2023.11.17 -cfgv==3.4.0 -charset-normalizer==3.3.2 -colorama==0.4.6 -copier==9.1.1 -decorator==5.1.1 -distlib==0.3.8 -docutils==0.20.1 -dunamai==1.19.0 -filelock==3.13.1 -funcy==2.0 -identify==2.5.33 -idna==3.6 -imagesize==1.4.1 -iniconfig==2.0.0 -Jinja2==3.1.3 -jinja2-ansible-filters==1.3.2 -livereload==2.6.3 -markdown-it-py==3.0.0 -MarkupSafe==2.1.4 -mdit-py-plugins==0.4.0 -mdurl==0.1.2 -myst-parser==2.0.0 -nodeenv==1.8.0 -packaging==23.2 -pathlib2==2.3.7.post1 -pathspec==0.12.1 -platformdirs==4.1.0 -pluggy==1.3.0 -plumbum==1.8.2 -pre-commit==3.6.0 -prompt-toolkit==3.0.36 -py==1.11.0 -pydantic==2.5.3 -pydantic_core==2.14.6 -pydata-sphinx-theme==0.15.2 -Pygments==2.17.2 -pytest==7.4.4 -PyYAML==6.0.1 -pyyaml-include==1.3.2 -questionary==2.0.1 -requests==2.31.0 -ruff==0.1.14 -six==1.16.0 -snowballstemmer==2.2.0 -soupsieve==2.5 -Sphinx==7.2.6 -sphinx-autobuild==2021.3.14 -sphinx-copybutton==0.5.2 -sphinx_design==0.5.0 -sphinxcontrib-applehelp==1.0.8 -sphinxcontrib-devhelp==1.0.6 -sphinxcontrib-htmlhelp==2.0.5 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.7 -sphinxcontrib-serializinghtml==1.1.10 -tornado==6.4 -tox==3.28.0 -tox-direct==0.4 -typing_extensions==4.9.0 -urllib3==2.1.0 -virtualenv==20.25.0 -wcwidth==0.2.13 diff --git a/requirements/regenerate.sh b/requirements/regenerate.sh deleted file mode 100755 index 9540e7b3..00000000 --- a/requirements/regenerate.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -CONSTRAINTS="$(dirname $0)/constraints.txt" -if [ "$#" -eq 0 ]; then - # Run with no args then always generate - pip freeze --exclude-editable > $CONSTRAINTS -elif [ "$1" == "--if-non-empty" ]; then - # Under pre-commit only generate if non empty - [ -s "$CONSTRAINTS" ] && pip freeze --exclude-editable > $CONSTRAINTS -else - # Not recognised, provide help - cat < Date: Wed, 31 Jan 2024 10:03:53 +0000 Subject: [PATCH 3/4] Make test not use shallow clone --- .github/workflows/_test.yml | 3 +++ .github/workflows/ci.yml | 12 ++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index d4a67b25..9f164afe 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -21,6 +21,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + # Need this to get version number from last tag + fetch-depth: 0 - if: inputs.python-version == 'dev' name: Install dev versions of python packages diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea1f3562..48a7f332 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,12 +8,20 @@ jobs: check: uses: ./.github/workflows/_check.yml - test: + lint: needs: check if: needs.check.outputs.not-in-pr uses: ./.github/workflows/_tox.yml with: - tox: pre-commit,pytest + tox: pre-commit + + test: + needs: check + if: needs.check.outputs.not-in-pr + uses: ./.github/workflows/_test.yml + with: + python-version: dev + runs-on: ubuntu-latest docs: needs: check From 79fc8c757d54c40a6d2330866e0dcc03a25bc191 Mon Sep 17 00:00:00 2001 From: Tom Cobb Date: Wed, 31 Jan 2024 14:25:27 +0000 Subject: [PATCH 4/4] Add some more ADRs --- docs/explanations/decisions.md | 12 ++++ docs/explanations/decisions.rst | 17 ----- .../0001-record-architecture-decisions.md | 20 ++++++ .../0001-record-architecture-decisions.rst | 24 ------- .../decisions/0002-make-skeleton.md | 21 ++++++ ...s-structure.rst => 0003-docs-structure.md} | 29 ++++---- docs/explanations/decisions/0004-why-src.md | 43 +++++++++++ docs/explanations/decisions/0004-why-src.rst | 50 ------------- .../decisions/0005-pyproject-toml.md | 19 +++++ .../decisions/0005-pyproject-toml.rst | 25 ------- ...uptools-scm.rst => 0006-setuptools-scm.md} | 17 ++--- ...endencies.rst => 0007-dev-dependencies.md} | 21 +++--- .../{0008-use-tox.rst => 0008-use-tox.md} | 52 +++++++------- .../decisions/0009-sphinx-theme.md | 19 +++++ .../decisions/0009-sphinx-theme.rst | 24 ------- ...e-settings.rst => 0010-vscode-settings.md} | 17 ++--- ...ments-txt.rst => 0011-requirements-txt.md} | 19 ++--- ...0012-containers.rst => 0012-containers.md} | 15 ++-- .../decisions/0013-switch-to-copier.md | 19 +++++ .../decisions/0014-split-ci-yaml.md | 26 +++++++ docs/tutorials/adopt-existing.md | 3 + docs/tutorials/adopt-existing.rst | 4 -- docs/tutorials/create-new.md | 29 ++++++++ docs/tutorials/create-new.rst | 34 --------- docs/tutorials/installation.md | 48 +++++++++++++ docs/tutorials/installation.rst | 47 ------------ template/.gitignore | 72 +------------------ 27 files changed, 329 insertions(+), 397 deletions(-) create mode 100644 docs/explanations/decisions.md delete mode 100644 docs/explanations/decisions.rst create mode 100644 docs/explanations/decisions/0001-record-architecture-decisions.md delete mode 100644 docs/explanations/decisions/0001-record-architecture-decisions.rst create mode 100644 docs/explanations/decisions/0002-make-skeleton.md rename docs/explanations/decisions/{0003-docs-structure.rst => 0003-docs-structure.md} (63%) create mode 100644 docs/explanations/decisions/0004-why-src.md delete mode 100644 docs/explanations/decisions/0004-why-src.rst create mode 100644 docs/explanations/decisions/0005-pyproject-toml.md delete mode 100644 docs/explanations/decisions/0005-pyproject-toml.rst rename docs/explanations/decisions/{0006-setuptools-scm.rst => 0006-setuptools-scm.md} (68%) rename docs/explanations/decisions/{0007-dev-dependencies.rst => 0007-dev-dependencies.md} (70%) rename docs/explanations/decisions/{0008-use-tox.rst => 0008-use-tox.md} (80%) create mode 100644 docs/explanations/decisions/0009-sphinx-theme.md delete mode 100644 docs/explanations/decisions/0009-sphinx-theme.rst rename docs/explanations/decisions/{0010-vscode-settings.rst => 0010-vscode-settings.md} (74%) rename docs/explanations/decisions/{0011-requirements-txt.rst => 0011-requirements-txt.md} (72%) rename docs/explanations/decisions/{0012-containers.rst => 0012-containers.md} (84%) create mode 100644 docs/explanations/decisions/0013-switch-to-copier.md create mode 100644 docs/explanations/decisions/0014-split-ci-yaml.md create mode 100644 docs/tutorials/adopt-existing.md delete mode 100644 docs/tutorials/adopt-existing.rst create mode 100644 docs/tutorials/create-new.md delete mode 100644 docs/tutorials/create-new.rst create mode 100644 docs/tutorials/installation.md delete mode 100644 docs/tutorials/installation.rst mode change 100644 => 120000 template/.gitignore diff --git a/docs/explanations/decisions.md b/docs/explanations/decisions.md new file mode 100644 index 00000000..0533b98d --- /dev/null +++ b/docs/explanations/decisions.md @@ -0,0 +1,12 @@ +# Architectural Decision Records + +Architectural decisions are made throughout a project's lifetime. As a way of keeping track of these decisions, we record these decisions in Architecture Decision Records (ADRs) listed below. + +```{toctree} +:glob: true +:maxdepth: 1 + +decisions/* +``` + +For more information on ADRs see this [blog by Michael Nygard](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions). diff --git a/docs/explanations/decisions.rst b/docs/explanations/decisions.rst deleted file mode 100644 index 5841e6ea..00000000 --- a/docs/explanations/decisions.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. This Source Code Form is subject to the terms of the Mozilla Public -.. License, v. 2.0. If a copy of the MPL was not distributed with this -.. file, You can obtain one at http://mozilla.org/MPL/2.0/. - -Architectural Decision Records -============================== - -We record major architectural decisions in Architecture Decision Records (ADRs), -as `described by Michael Nygard -`_. -Below is the list of our current ADRs. - -.. toctree:: - :maxdepth: 1 - :glob: - - decisions/* \ No newline at end of file diff --git a/docs/explanations/decisions/0001-record-architecture-decisions.md b/docs/explanations/decisions/0001-record-architecture-decisions.md new file mode 100644 index 00000000..d496b718 --- /dev/null +++ b/docs/explanations/decisions/0001-record-architecture-decisions.md @@ -0,0 +1,20 @@ +# 1. Record architecture decisions + +Date: 2022-02-18 + +## Status + +Accepted + +## Context + +We need to record the architectural decisions made on this project. + +## Decision + +We will use Architecture Decision Records, as [described by Michael Nygard](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions). + +## Consequences + +See Michael Nygard's article, linked above. To create new ADRs we will copy and +paste from existing ones. diff --git a/docs/explanations/decisions/0001-record-architecture-decisions.rst b/docs/explanations/decisions/0001-record-architecture-decisions.rst deleted file mode 100644 index 0604062c..00000000 --- a/docs/explanations/decisions/0001-record-architecture-decisions.rst +++ /dev/null @@ -1,24 +0,0 @@ -1. Record architecture decisions -================================ - -Status ------- - -Accepted - -Context -------- - -We need to record the architectural decisions made on this project. - -Decision --------- - -We will use Architecture Decision Records, as `described by Michael Nygard -`_. - -Consequences ------------- - -See Michael Nygard's article, linked above. To create new ADRs we will copy and -paste from existing ones. diff --git a/docs/explanations/decisions/0002-make-skeleton.md b/docs/explanations/decisions/0002-make-skeleton.md new file mode 100644 index 00000000..02805f88 --- /dev/null +++ b/docs/explanations/decisions/0002-make-skeleton.md @@ -0,0 +1,21 @@ +# 2. Make a skeleton repository + +Date: 2022-06-18 + +## Status + +Accepted + +## Context + +Many projects start from some kind of template. These define some basic structure, customized with project specific variables, that developers can add their code into. One example of this is [cookiecutter](https://cookiecutter.readthedocs.io). + +The problem with this approach is that it is difficult to apply changes to the template into projects that have been cut from it. Individual changes have to be copy/pasted into the code, leading to partially applied fixes and missed updates. + +## Decision + +We will use a skeleton structure as defined in [Jaraco's blog](https://blog.jaraco.com/a-project-skeleton-for-python-projects/), using git to keep the downstream projects up to date. + +## Consequences + +We will need a cli module to ease the adoption of this diff --git a/docs/explanations/decisions/0003-docs-structure.rst b/docs/explanations/decisions/0003-docs-structure.md similarity index 63% rename from docs/explanations/decisions/0003-docs-structure.rst rename to docs/explanations/decisions/0003-docs-structure.md index 65989185..2cf0976c 100644 --- a/docs/explanations/decisions/0003-docs-structure.rst +++ b/docs/explanations/decisions/0003-docs-structure.md @@ -1,30 +1,26 @@ -.. _documentation-structure: +(documentation-structure)= -3. Standard documentation structure -=================================== +# 3. Standard documentation structure Date: 2023-01-18 -Status ------- +## Status Accepted -Context -------- +## Context Skeleton based project's documentation requires organizing. -Decision --------- +## Decision Use the approach proposed by David Laing. - :material-regular:`format_quote;2em` - - The Grand Unified Theory of Documentation - - -- David Laing +> {material-regular}`format_quote;2em` +> +> The Grand Unified Theory of Documentation +> +>

-David Laing

There is a secret that needs to be understood in order to write good software documentation: there isn't one thing called *documentation*, there are four. @@ -34,9 +30,8 @@ They represent four different purposes or functions, and require four different approaches to their creation. Understanding the implications of this will help improve most documentation - often immensely. -`More information on this topic. `_ +[More information on this topic.](https://documentation.divio.com) -Consequences ------------- +## Consequences See article linked above. diff --git a/docs/explanations/decisions/0004-why-src.md b/docs/explanations/decisions/0004-why-src.md new file mode 100644 index 00000000..e4485930 --- /dev/null +++ b/docs/explanations/decisions/0004-why-src.md @@ -0,0 +1,43 @@ +(src)= + +# 4. Use a source directory + +Date: 2023-01-18 + +## Status + +Accepted + +## Context + +We need to decide how to structure the source code in skeleton based projects. + +## Decision + +As per [Hynek's article] and summarized below. + +The main advantage is that the source directory cannot shadow installed packages +as it would if it was in the root of the repository. This means that if you +install the package, then run the tests, they will run against the installed +package and not the source directory, testing for packaging bugs. + +A secondary advantage is symmetry, sources go in `src/`, tests go in +`tests\`, docs go in `docs`. + +This is tested in CI in the following way: + +- `wheel` job creates a wheel, then installs it in an isolated environment and + runs the cli. This checks `install_requirements` are sufficient to run the + cli. +- `test` job with `lock: true` does an [editable install] of the + package. This is the mode that is used at development time, as modifications + to sources can be tested without reinstalling. +- `test` job with `lock: false` does a regular install, which + checks that all files needed for the tests are packaged with the distribution. + +## Consequences + +See the article linked above. + +[editable install]: https://pip.pypa.io/en/stable/cli/pip_install/#editable-installs +[hynek's article]: https://hynek.me/articles/testing-packaging/ diff --git a/docs/explanations/decisions/0004-why-src.rst b/docs/explanations/decisions/0004-why-src.rst deleted file mode 100644 index b09ea944..00000000 --- a/docs/explanations/decisions/0004-why-src.rst +++ /dev/null @@ -1,50 +0,0 @@ -.. _src: - -4. Use a source directory -========================= - -Date: 2023-01-18 - -Status ------- - -Accepted - -Context -------- - -We need to decide how to structure the source code in skeleton based projects. - - -Decision --------- - -As per `Hynek's article`_ and summarized below. - -.. _Hynek's article: https://hynek.me/articles/testing-packaging/ - -The main advantage is that the source directory cannot shadow installed packages -as it would if it was in the root of the repository. This means that if you -install the package, then run the tests, they will run against the installed -package and not the source directory, testing for packaging bugs. - -A secondary advantage is symmetry, sources go in ``src/``, tests go in -``tests\``, docs go in ``docs``. - -This is tested in CI in the following way: - -- ``wheel`` job creates a wheel, then installs it in an isolated environment and - runs the cli. This checks ``install_requirements`` are sufficient to run the - cli. -- ``test`` job with ``lock: true`` does an `editable install`_ of the - package. This is the mode that is used at development time, as modifications - to sources can be tested without reinstalling. -- ``test`` job with ``lock: false`` does a regular install, which - checks that all files needed for the tests are packaged with the distribution. - -.. _editable install: https://pip.pypa.io/en/stable/cli/pip_install/#editable-installs - -Consequences ------------- - -See the article linked above. diff --git a/docs/explanations/decisions/0005-pyproject-toml.md b/docs/explanations/decisions/0005-pyproject-toml.md new file mode 100644 index 00000000..bad000da --- /dev/null +++ b/docs/explanations/decisions/0005-pyproject-toml.md @@ -0,0 +1,19 @@ +# 5. Use pyproject.toml + +Date: 2023-01-18 + +## Status + +Accepted + +## Context + +Need a decision on where to put the python project configuration. + +## Decision + +Use pyproject.toml as per + +## Consequences + +See article linked above. diff --git a/docs/explanations/decisions/0005-pyproject-toml.rst b/docs/explanations/decisions/0005-pyproject-toml.rst deleted file mode 100644 index ee8a1177..00000000 --- a/docs/explanations/decisions/0005-pyproject-toml.rst +++ /dev/null @@ -1,25 +0,0 @@ -5. Use pyproject.toml -===================== - -Date: 2023-01-18 - -Status ------- - -Accepted - -Context -------- - -Need a decision on where to put the python project configuration. - -Decision --------- - -Use pyproject.toml as per https://peps.python.org/pep-0518/ - - -Consequences ------------- - -See article linked above. \ No newline at end of file diff --git a/docs/explanations/decisions/0006-setuptools-scm.rst b/docs/explanations/decisions/0006-setuptools-scm.md similarity index 68% rename from docs/explanations/decisions/0006-setuptools-scm.rst rename to docs/explanations/decisions/0006-setuptools-scm.md index 3f6f33a0..ac934e5c 100644 --- a/docs/explanations/decisions/0006-setuptools-scm.rst +++ b/docs/explanations/decisions/0006-setuptools-scm.md @@ -1,27 +1,22 @@ -6. Use setuptools_scm -===================== +# 6. Use setuptools_scm Date: 2023-01-18 -Status ------- +## Status Accepted -Context -------- +## Context We require a mechanism for generating version numbers in python. -Decision --------- +## Decision Generate the version number from git tags using setuptools scm. -See https://github.com/pypa/setuptools_scm/ +See -Consequences ------------- +## Consequences Versions are generated automatically from git tags. This means you can can verify if you are running a released version of the code as diff --git a/docs/explanations/decisions/0007-dev-dependencies.rst b/docs/explanations/decisions/0007-dev-dependencies.md similarity index 70% rename from docs/explanations/decisions/0007-dev-dependencies.rst rename to docs/explanations/decisions/0007-dev-dependencies.md index 507551b4..2a8cff4e 100644 --- a/docs/explanations/decisions/0007-dev-dependencies.rst +++ b/docs/explanations/decisions/0007-dev-dependencies.md @@ -1,37 +1,32 @@ -7. Installing developer environment -=================================== +# 7. Installing developer environment Date: 2023-01-18 -Status ------- +## Status Accepted -Context -------- +## Context We need to provide a way to setup a developer environment for a skeleton based project. -Decision --------- +## Decision Use optional dependencies in pyproject.toml. PEP 621 provides a mechanism for adding optional dependencies in pyproject.toml -https://peps.python.org/pep-0621/#dependencies-optional-dependencies. +. We supply a list of developer dependencies under the title "dev". These developer dependencies enable building and testing the project and its documentation. -Consequences ------------- +## Consequences Any developer can update their virtual environment in order to work on a skeleton based project with the command: -```bash +`` `bash pip install -e .[dev] -``` \ No newline at end of file +` `` diff --git a/docs/explanations/decisions/0008-use-tox.rst b/docs/explanations/decisions/0008-use-tox.md similarity index 80% rename from docs/explanations/decisions/0008-use-tox.rst rename to docs/explanations/decisions/0008-use-tox.md index 3e259ce2..3172aa1f 100644 --- a/docs/explanations/decisions/0008-use-tox.rst +++ b/docs/explanations/decisions/0008-use-tox.md @@ -1,32 +1,27 @@ -8. Use tox and pre-commit -========================= +# 8. Use tox and pre-commit Date: 2023-01-18 -Status ------- +## Status Accepted -Context -------- +## Context We require an easy way to locally run the same checks as CI. This provides a rapid inner-loop developer experience. -Decision --------- +## Decision Use tox and pre-commit. tox is an automation tool that we use to run all checks in parallel, -see https://tox.wiki/en/latest/. +see . pre-commit provides a hook into git commit which runs some of the checks against the changes you are about to commit. -Decision detail ---------------- +## Decision detail There are a number of things that CI needs to run: @@ -40,6 +35,7 @@ There are a number of things that CI needs to run: The initial approach this module took was to integrate everything under pytest that had a plugin, and isort under flake8: +```{eval-rst} .. digraph:: initial bgcolor=transparent @@ -55,15 +51,16 @@ under pytest that had a plugin, and isort under flake8: "flake8-isort" } } +``` -This had the advantage that a ``pytest tests`` run in CI would catch and +This had the advantage that a `pytest tests` run in CI would catch and report all test failures, but made each run take longer than it needed to. Also, -flake8 states that it `does not have a public, stable, Python API -`_ so did not +flake8 states that it [does not have a public, stable, Python API](https://flake8.pycqa.org/en/latest/user/python-api.html) so did not recommend the approach taken by pytest-flake8. To address this, the tree was rearranged: +```{eval-rst} .. digraph:: rearranged bgcolor=transparent @@ -77,25 +74,31 @@ To address this, the tree was rearranged: label = "flake8" "flake8-isort" } +``` If using VSCode, this will still run black, flake8 and mypy on file save, but for those using other editors and for CI another solution was needed. Enter -`pre-commit `_. This allows hooks to be run at ``git -commit`` time on just the files that have changed, as well as on all tracked -files by CI. All that is needed is a one time install of the git commit hook:: +[pre-commit](https://pre-commit.com/). This allows hooks to be run at `git +commit` time on just the files that have changed, as well as on all tracked +files by CI. All that is needed is a one time install of the git commit hook: - $ pre-commit install +``` +$ pre-commit install +``` Finally tox was added to run all of the CI checks including the documentation build. mypy was moved out of the pre-commit and into tox because it was quite long running and therefore intrusive. tox can be invoked to run all the checks in -parallel with:: +parallel with: - $ tox -p +``` +$ tox -p +``` The graph now looks like: +```{eval-rst} .. digraph:: rearranged bgcolor=transparent @@ -117,6 +120,7 @@ The graph now looks like: } } } +``` Now the workflow looks like this: @@ -127,12 +131,10 @@ Now the workflow looks like this: - Push to remote and CI runs black, flake8, mypy once on all files (using tox), then pytest multiple times in a test matrix +## Consequences -Consequences ------------- - -Running ``tox -p`` before pushing to GitHub verifies that the CI will *most +Running `tox -p` before pushing to GitHub verifies that the CI will *most likely* succeed. Committing changes to git will run all of the non-time critical checks and -help avoid some of the most common mistakes. \ No newline at end of file +help avoid some of the most common mistakes. diff --git a/docs/explanations/decisions/0009-sphinx-theme.md b/docs/explanations/decisions/0009-sphinx-theme.md new file mode 100644 index 00000000..54aea6d8 --- /dev/null +++ b/docs/explanations/decisions/0009-sphinx-theme.md @@ -0,0 +1,19 @@ +# 9. Sphinx theme + +Date: 2023-01-18 + +## Status + +Accepted + +## Context + +Documentation requires a theme as well as a structure. + +## Decision + +Use the pydata theme defined here + +## Consequences + +The documentation looks nice and has good navigation features. diff --git a/docs/explanations/decisions/0009-sphinx-theme.rst b/docs/explanations/decisions/0009-sphinx-theme.rst deleted file mode 100644 index b80fee08..00000000 --- a/docs/explanations/decisions/0009-sphinx-theme.rst +++ /dev/null @@ -1,24 +0,0 @@ -9. Sphinx theme -=============== - -Date: 2023-01-18 - -Status ------- - -Accepted - -Context -------- - -Documentation requires a theme as well as a structure. - -Decision --------- - -Use the pydata theme defined here https://pydata-sphinx-theme.readthedocs.io/ - -Consequences ------------- - -The documentation looks nice and has good navigation features. diff --git a/docs/explanations/decisions/0010-vscode-settings.rst b/docs/explanations/decisions/0010-vscode-settings.md similarity index 74% rename from docs/explanations/decisions/0010-vscode-settings.rst rename to docs/explanations/decisions/0010-vscode-settings.md index a676ace4..c21124ac 100644 --- a/docs/explanations/decisions/0010-vscode-settings.rst +++ b/docs/explanations/decisions/0010-vscode-settings.md @@ -1,21 +1,17 @@ -10. Include vscode settings -=========================== +# 10. Include vscode settings Date: 2023-01-18 -Status ------- +## Status Accepted -Context -------- +## Context For vscode users a couple of settings are required for neat integration with the IDE. -Decision --------- +## Decision Include a .vscode folder in the repo with some json files that enable: @@ -24,8 +20,7 @@ Include a .vscode folder in the repo with some json files that enable: - settings to make code verification match the tools in CI - a task to launch the tox tests from the IDE -Consequences ------------- +## Consequences Users of vscode should find that their development environment just works. -Users of other editors will be unaffected. \ No newline at end of file +Users of other editors will be unaffected. diff --git a/docs/explanations/decisions/0011-requirements-txt.rst b/docs/explanations/decisions/0011-requirements-txt.md similarity index 72% rename from docs/explanations/decisions/0011-requirements-txt.rst rename to docs/explanations/decisions/0011-requirements-txt.md index 0247512e..318c8301 100644 --- a/docs/explanations/decisions/0011-requirements-txt.rst +++ b/docs/explanations/decisions/0011-requirements-txt.md @@ -1,23 +1,19 @@ -11. Pinning requirements -======================== +# 11. Pinning requirements Date: 2023-01-18 -Status ------- +## Status Accepted -Context -------- +## Context Require the ability to pin requirements for a guaranteed rebuild. By default CI builds against the latest version of all dependencies, but we need a mechanism for overriding this behaviour with a lock file when there are issues. -Decision --------- +## Decision Have every release generate requirements.txt files using pip freeze and publish them as release assets. @@ -25,12 +21,7 @@ publish them as release assets. Request that the user download the asset and commit it into the repo in order to lock dependencies for the next CI build. -TODO: link to the How-To on pinning requirements to be written in -python3-pip-skeleton developer documentation. - -Consequences ------------- +## Consequences There is less overhead in managing lock files. Incoming issues with dependencies will be highlighted early but can be worked around quickly if needed. - diff --git a/docs/explanations/decisions/0012-containers.rst b/docs/explanations/decisions/0012-containers.md similarity index 84% rename from docs/explanations/decisions/0012-containers.rst rename to docs/explanations/decisions/0012-containers.md index e53dd541..9facedd8 100644 --- a/docs/explanations/decisions/0012-containers.rst +++ b/docs/explanations/decisions/0012-containers.md @@ -1,20 +1,16 @@ -12. Use containers -================== +# 12. Use containers Date: 2023-01-18 -Status ------- +## Status Accepted -Context -------- +## Context Allow developers and users to take advantage of containers. -Decision --------- +## Decision Provide a single Dockerfile that can build two kinds of container: @@ -28,7 +24,6 @@ CI builds the runtime container and publishes it to ghcr.io. A .devcontainer folder provides the means to build and launch the developer container using vscode. -Consequences ------------- +## Consequences We can label projects as cloud native. diff --git a/docs/explanations/decisions/0013-switch-to-copier.md b/docs/explanations/decisions/0013-switch-to-copier.md new file mode 100644 index 00000000..55a68636 --- /dev/null +++ b/docs/explanations/decisions/0013-switch-to-copier.md @@ -0,0 +1,19 @@ +# 13. Switch to copier + +Date: 2024-01-18 + +## Status + +Accepted + +## Context + +The previous attempt, [python3-pip-skeleton](https://github.com/DiamondLightSource/python3-pip-skeleton) was a repo that can be forked and updates merged into projects tracking it. This was initially encouraging, but led to downstream confusion on who had actually contributed to the repository, as well as messy merges when text that contained the project name (like documentation) was changed upstream. + +## Decision + +Use [copier](https://copier.readthedocs.io/) which gives the best of both worlds, a templating engine to expand the template, then an update mechanism to apply diffs of the updated template to the project. + +## Consequences + +We will need to create and document an adoption process for existing skeleton projects to move to this copier template diff --git a/docs/explanations/decisions/0014-split-ci-yaml.md b/docs/explanations/decisions/0014-split-ci-yaml.md new file mode 100644 index 00000000..b0843f1e --- /dev/null +++ b/docs/explanations/decisions/0014-split-ci-yaml.md @@ -0,0 +1,26 @@ +# 14. Split up CI YAML + +Date: 2024-01-31 + +## Status + +Accepted + +## Context + +The existing monolithic CI had some problems: + +- We want some parts to be optional, which required templated CI which then couldn't be easily tested +- We wanted to reuse some parts of the CI in the template repo +- It was long and hard to read +- It was split into multiple top level jobs + +## Decision + +Break it into multiple [reusable workflows](https://docs.github.com/en/actions/using-workflows/reusing-workflows) that are called from: +- A single `ci` workflow that runs on push, and tests, builds, and adds artifacts to releases +- A single `periodic` workflow that runs once a week and checks links aren't broken + +## Consequences + +We can reuse workflows in the template repo diff --git a/docs/tutorials/adopt-existing.md b/docs/tutorials/adopt-existing.md new file mode 100644 index 00000000..f1712368 --- /dev/null +++ b/docs/tutorials/adopt-existing.md @@ -0,0 +1,3 @@ +# Adopt the template into an existing repo + +TODO diff --git a/docs/tutorials/adopt-existing.rst b/docs/tutorials/adopt-existing.rst deleted file mode 100644 index d9c6497d..00000000 --- a/docs/tutorials/adopt-existing.rst +++ /dev/null @@ -1,4 +0,0 @@ -Adopting the template into an existing repo -=========================================== - -TODO \ No newline at end of file diff --git a/docs/tutorials/create-new.md b/docs/tutorials/create-new.md new file mode 100644 index 00000000..2aafd06c --- /dev/null +++ b/docs/tutorials/create-new.md @@ -0,0 +1,29 @@ +# Create a new repo from the template + +Once you have followed the [](./installation) tutorial, you can use `copier` to make a new project from the template: + +``` +$ mkdir /path/to/my-project +$ copier copy gh:DiamondLightSource/python-copier-template /path/to/my-project +``` + +This will: + +- Ask some questions about the project to be created +- Expand the template with the answers give +- Record the answers in the project so they can be used in later updates + +## Uploading to GitHub + +You can now [create a new blank project on GitHub](https://github.com/new). Choose the same GitHub owner and repo name that you answered in the questions earlier. GitHub will now give you the commands needed to upload your repo from GitHub + +## Setings + +You can now go to the `Settings` and set: + +- The project description to match the answer you gave +- Enable Pages if you chose to use sphinx for your documentation + +## Getting started with your new repo + +You can now [](../how-to/dev-install), and then follow some of the other [](../how-to). diff --git a/docs/tutorials/create-new.rst b/docs/tutorials/create-new.rst deleted file mode 100644 index 4c42bd27..00000000 --- a/docs/tutorials/create-new.rst +++ /dev/null @@ -1,34 +0,0 @@ -Creating a new repo from the template -===================================== - -Once you have followed the `./installation` tutorial, you can use ``copier`` -to make a new project from the template:: - - $ mkdir /path/to/my-project - $ copier copy gh:DiamondLightSource/python-copier-template /path/to/my-project - -This will: - -- Ask some questions about the project to be created -- Expand the template with the answers give -- Record the answers in the project so they can be used in later updates - -Uploading to GitHub -------------------- - -You can now create a new blank project at . Choose the same -GitHub owner and repo name that you answered in the questions earlier. GitHub will -now give you the commands needed to upload your repo from GitHub - -Setings -------- - -You can now go to the ``Settings`` and set: - -- The project description to match the answer you gave -- Enable Pages if you chose to use sphinx for your documentation - -Getting started with your new repo ----------------------------------- - -You can now `../how-to/dev-install`, and then follow some of the other `../how-to`. diff --git a/docs/tutorials/installation.md b/docs/tutorials/installation.md new file mode 100644 index 00000000..e1bfa5ab --- /dev/null +++ b/docs/tutorials/installation.md @@ -0,0 +1,48 @@ +# Install template pre-requisites + +This tutorial will take you through installing copier, the templating engine that will allow you +to create new projects from the template, update existing projects in line with it, and keep projects in sync with changes to it. + +## Check your version of python + +You will need python 3.8 or later. You can check your version of python by +typing into a terminal: + +``` +$ python3 --version +``` + +:::{note} +At Diamond you can use `module load python` to get a more recent version of python on your path +::: + +## Create a virtual environment + +It is recommended that you install into a “virtual environment” so this +installation will not interfere with any existing Python software: + +``` +$ python3 -m venv /path/to/venv +$ source /path/to/venv/bin/activate +``` + +:::{note} +You may wish to deactivate any existing virual environments before sourcing the new +environment. Deactivation can be performed by executing: + +- `conda deactivate` for conda +- `deactivate` for venv or virtualenv +- `exit` for pipenv +::: + +## Installing copier + +You can now use `pip` to install copier so you can make your project from the template: + +``` +$ python3 -m pip install copier +``` + +## Conclusion + +You now have the pre-requisites to allow you to [](./create-new) and [](./adopt-existing). diff --git a/docs/tutorials/installation.rst b/docs/tutorials/installation.rst deleted file mode 100644 index fcb505a7..00000000 --- a/docs/tutorials/installation.rst +++ /dev/null @@ -1,47 +0,0 @@ -Installing template pre-requisites -================================== - -This tutorial will take you through installing copier, the templating engine that will allow you -to create new projects from the template, update existing projects in line with it, and keep projects in sync with changes to it. - -Check your version of python ----------------------------- - -You will need python 3.8 or later. You can check your version of python by -typing into a terminal:: - - $ python3 --version - -.. note:: - - At Diamond you can use ``module load python`` to get a more recent version of python on your path - -Create a virtual environment ----------------------------- - -It is recommended that you install into a “virtual environment” so this -installation will not interfere with any existing Python software:: - - $ python3 -m venv /path/to/venv - $ source /path/to/venv/bin/activate - -.. note:: - - You may wish to deactivate any existing virual environments before sourcing the new - environment. Deactivation can be performed by executing: - - - :code:`conda deactivate` for conda - - :code:`deactivate` for venv or virtualenv - - :code:`exit` for pipenv - -Installing copier ------------------ - -You can now use ``pip`` to install copier so you can make your project from the template:: - - $ python3 -m pip install copier - -Conclusion ----------- - -You now have the pre-requisites to allow you to `./create-new` and `./adopt-existing`. diff --git a/template/.gitignore b/template/.gitignore deleted file mode 100644 index a37be99b..00000000 --- a/template/.gitignore +++ /dev/null @@ -1,71 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -.venv -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg -**/_version.py - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -cov.xml -.pytest_cache/ -.mypy_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# likely venv names -.venv* -venv* - -# further build artifacts -lockfiles/ - -# ruff cache -.ruff_cache/ diff --git a/template/.gitignore b/template/.gitignore new file mode 120000 index 00000000..5a19b83f --- /dev/null +++ b/template/.gitignore @@ -0,0 +1 @@ +../.gitignore \ No newline at end of file