From f79852076e675cc2947075956c55eb7dd64b8d8a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Dec 2022 16:10:46 +0000 Subject: [PATCH 01/18] Bump softprops/action-gh-release from 0.1.14 to 0.1.15 Bumps [softprops/action-gh-release](https://github.com/softprops/action-gh-release) from 0.1.14 to 0.1.15. - [Release notes](https://github.com/softprops/action-gh-release/releases) - [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md) - [Commits](https://github.com/softprops/action-gh-release/compare/1e07f4398721186383de40550babbdf2b84acfc5...de2c0eb89ae2a093876385947365aca7b0e5f844) --- updated-dependencies: - dependency-name: softprops/action-gh-release dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index d4d047a3..26fb71a7 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -193,7 +193,7 @@ jobs: - name: Github Release # We pin to the SHA, not the tag, for security reasons. # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions - uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 # v0.1.14 + uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15 with: prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }} files: | From b2a9327170e3f0f83de5138fea7e1299b2c5e200 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 16:06:56 +0000 Subject: [PATCH 02/18] Bump peaceiris/actions-gh-pages from 3.9.0 to 3.9.1 Bumps [peaceiris/actions-gh-pages](https://github.com/peaceiris/actions-gh-pages) from 3.9.0 to 3.9.1. - [Release notes](https://github.com/peaceiris/actions-gh-pages/releases) - [Changelog](https://github.com/peaceiris/actions-gh-pages/blob/main/CHANGELOG.md) - [Commits](https://github.com/peaceiris/actions-gh-pages/compare/de7ea6f8efb354206b205ef54722213d99067935...64b46b4226a4a12da2239ba3ea5aa73e3163c75b) --- updated-dependencies: - dependency-name: peaceiris/actions-gh-pages dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 94fa2151..c510d577 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'push' && github.actor != 'dependabot[bot]' # We pin to the SHA, not the tag, for security reasons. # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions - uses: peaceiris/actions-gh-pages@de7ea6f8efb354206b205ef54722213d99067935 # v3.9.0 + uses: peaceiris/actions-gh-pages@64b46b4226a4a12da2239ba3ea5aa73e3163c75b # v3.9.1 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: .github/pages From 5b999f0427349836e9e4d43e7c3fa96af0838324 Mon Sep 17 00:00:00 2001 From: tizayi Date: Mon, 9 Jan 2023 14:04:56 +0000 Subject: [PATCH 03/18] simplify local container workflow --- .devcontainer/devcontainer.json | 2 +- .github/workflows/code.yml | 5 +++-- .devcontainer/Dockerfile => Dockerfile | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) rename .devcontainer/Dockerfile => Dockerfile (98%) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f5e71fe9..7a30e9be 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ { "name": "Python 3 Developer Container", "build": { - "dockerfile": "Dockerfile", + "dockerfile": "../Dockerfile", "target": "build", // Only upgrade pip, we will install the project below "args": { diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 26fb71a7..c728ed8b 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -134,7 +134,7 @@ jobs: - name: Download wheel and lockfiles uses: actions/download-artifact@v3 with: - path: .devcontainer + path: artifacts/ - name: Log in to GitHub Docker Registry if: github.event_name != 'pull_request' @@ -166,7 +166,8 @@ jobs: load: ${{ ! (github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - context: .devcontainer + context: artifacts/ + file: ./Dockerfile # If you have a long docker build, uncomment the following to turn on caching # For short build times this makes it a little slower #cache-from: type=gha diff --git a/.devcontainer/Dockerfile b/Dockerfile similarity index 98% rename from .devcontainer/Dockerfile rename to Dockerfile index b6b4bef9..31d05606 100644 --- a/.devcontainer/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ # FROM python:3.11 as build -ARG PIP_OPTIONS +ARG PIP_OPTIONS=. # Add any system dependencies for the developer/build environment here e.g. # RUN apt-get update && apt-get upgrade -y && \ From b99329f9cf307715ac8c06b9a870e33cec29a8c3 Mon Sep 17 00:00:00 2001 From: Garry O'Donnell Date: Fri, 13 Jan 2023 11:30:00 +0000 Subject: [PATCH 04/18] Add note on deactivating other virtual environment --- docs/user/tutorials/installation.rst | 9 +++++++++ docs/user/tutorials/new.rst | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/docs/user/tutorials/installation.rst b/docs/user/tutorials/installation.rst index 43435328..77e96ca3 100644 --- a/docs/user/tutorials/installation.rst +++ b/docs/user/tutorials/installation.rst @@ -19,6 +19,15 @@ installation will not interfere with any existing Python software:: $ python3 -m venv /path/to/venv $ source /path/to/venv/bin/activate +.. note:: + + You may wish to deactivate any existing virual environments before sourcing the new + environment. Deactivation can be performed by executing: + + - :code:`conda deactivate` for conda + - :code:`deactivate` for venv or virtualenv + - :code:`exit` for pipenv + Installing the library ---------------------- diff --git a/docs/user/tutorials/new.rst b/docs/user/tutorials/new.rst index 2b5d5ed8..91bade31 100644 --- a/docs/user/tutorials/new.rst +++ b/docs/user/tutorials/new.rst @@ -24,6 +24,15 @@ pip to install packages in a virtual environment:: source .venv/bin/activate pip install -e .[dev] +.. note:: + + You may wish to deactivate any existing virual environments before sourcing the new + environment. Deactivation can be performed by executing: + + - :code:`conda deactivate` for conda + - :code:`deactivate` for venv or virtualenv + - :code:`exit` for pipenv + You can then run any entry points declared in setup.cfg e.g.:: python3-pip-skeleton --version From 6c1d129a3d18c74cd0ae0c768268279a45da39c0 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 18 Jan 2023 09:02:47 +0000 Subject: [PATCH 05/18] set check switcher to true --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index d0f2611c..a89bc5d5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -166,7 +166,7 @@ json_url=switcher_json, version_match=version, ), - check_switcher=False, + check_switcher=True, navbar_end=["theme-switcher", "icon-links", "version-switcher"], external_links=[ dict( From 72cf2217c84e384ad470b429f67706f6804b9ebe Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 18 Jan 2023 12:43:01 +0000 Subject: [PATCH 06/18] adding some ADRs --- docs/developer/index.rst | 2 - .../explanations/decisions.rst | 0 .../0001-record-architecture-decisions.rst | 0 .../0003-docs-structure.rst} | 26 ++++++++++++- .../0004-why-src.rst} | 28 ++++++++++++-- .../decisions/0005-pyproject-toml.rst | 25 +++++++++++++ .../decisions/0006-setuptools-scm.rst | 28 ++++++++++++++ .../decisions/0007-dev-dependencies.rst | 37 +++++++++++++++++++ .../explanations/decisions/0008-use-tox.rst | 36 ++++++++++++++++++ .../decisions/0009-sphinx-theme.rst | 24 ++++++++++++ .../decisions/0010-vscode-settings.rst | 31 ++++++++++++++++ .../decisions/0011-requirements-txt.rst | 32 ++++++++++++++++ docs/user/index.rst | 5 +-- 13 files changed, 263 insertions(+), 11 deletions(-) rename docs/{developer => user}/explanations/decisions.rst (100%) rename docs/{developer => user}/explanations/decisions/0001-record-architecture-decisions.rst (100%) rename docs/user/explanations/{docs-structure.rst => decisions/0003-docs-structure.rst} (66%) rename docs/user/explanations/{why-src.rst => decisions/0004-why-src.rst} (77%) create mode 100644 docs/user/explanations/decisions/0005-pyproject-toml.rst create mode 100644 docs/user/explanations/decisions/0006-setuptools-scm.rst create mode 100644 docs/user/explanations/decisions/0007-dev-dependencies.rst create mode 100644 docs/user/explanations/decisions/0008-use-tox.rst create mode 100644 docs/user/explanations/decisions/0009-sphinx-theme.rst create mode 100644 docs/user/explanations/decisions/0010-vscode-settings.rst create mode 100644 docs/user/explanations/decisions/0011-requirements-txt.rst diff --git a/docs/developer/index.rst b/docs/developer/index.rst index bf291875..3a7b65d0 100644 --- a/docs/developer/index.rst +++ b/docs/developer/index.rst @@ -43,8 +43,6 @@ side-bar. :caption: Explanations :maxdepth: 1 - explanations/decisions - +++ Explanations of how and why the architecture is why it is. diff --git a/docs/developer/explanations/decisions.rst b/docs/user/explanations/decisions.rst similarity index 100% rename from docs/developer/explanations/decisions.rst rename to docs/user/explanations/decisions.rst diff --git a/docs/developer/explanations/decisions/0001-record-architecture-decisions.rst b/docs/user/explanations/decisions/0001-record-architecture-decisions.rst similarity index 100% rename from docs/developer/explanations/decisions/0001-record-architecture-decisions.rst rename to docs/user/explanations/decisions/0001-record-architecture-decisions.rst diff --git a/docs/user/explanations/docs-structure.rst b/docs/user/explanations/decisions/0003-docs-structure.rst similarity index 66% rename from docs/user/explanations/docs-structure.rst rename to docs/user/explanations/decisions/0003-docs-structure.rst index f25a09ba..2032a766 100644 --- a/docs/user/explanations/docs-structure.rst +++ b/docs/user/explanations/decisions/0003-docs-structure.rst @@ -1,5 +1,22 @@ -About the documentation ------------------------ +3. Standard documentation structure +=================================== + +Date: 2023-01-18 + +Status +------ + +Accepted + +Context +------- + +Skeleton based project's documentation requires organizing. + +Decision +-------- + +Use the approach proposed by David Laing. :material-regular:`format_quote;2em` @@ -16,3 +33,8 @@ approaches to their creation. Understanding the implications of this will help improve most documentation - often immensely. `More information on this topic. `_ + +Consequences +------------ + +See article linked above. diff --git a/docs/user/explanations/why-src.rst b/docs/user/explanations/decisions/0004-why-src.rst similarity index 77% rename from docs/user/explanations/why-src.rst rename to docs/user/explanations/decisions/0004-why-src.rst index e12548ca..5ad095ef 100644 --- a/docs/user/explanations/why-src.rst +++ b/docs/user/explanations/decisions/0004-why-src.rst @@ -1,8 +1,23 @@ -Why use a source directory -========================== +4. Use a source directory +========================= -This skeleton repo has made the decision to use a source directory. The reasons -for this are set out in `Hynek's article`_ and summarized below. +Date: 2023-01-18 + +Status +------ + +Accepted + +Context +------- + +We need to decide how to structure the source code in skeleton based projects. + + +Decision +-------- + +As per `Hynek's article`_ and summarized below. .. _Hynek's article: https://hynek.me/articles/testing-packaging/ @@ -26,3 +41,8 @@ This is tested in CI in the following way: checks that all files needed for the tests are packaged with the distribution. .. _editable install: https://pip.pypa.io/en/stable/cli/pip_install/#editable-installs + +Consequences +------------ + +See the article linked above. diff --git a/docs/user/explanations/decisions/0005-pyproject-toml.rst b/docs/user/explanations/decisions/0005-pyproject-toml.rst new file mode 100644 index 00000000..ee8a1177 --- /dev/null +++ b/docs/user/explanations/decisions/0005-pyproject-toml.rst @@ -0,0 +1,25 @@ +5. Use pyproject.toml +===================== + +Date: 2023-01-18 + +Status +------ + +Accepted + +Context +------- + +Need a decision on where to put the python project configuration. + +Decision +-------- + +Use pyproject.toml as per https://peps.python.org/pep-0518/ + + +Consequences +------------ + +See article linked above. \ No newline at end of file diff --git a/docs/user/explanations/decisions/0006-setuptools-scm.rst b/docs/user/explanations/decisions/0006-setuptools-scm.rst new file mode 100644 index 00000000..3f6f33a0 --- /dev/null +++ b/docs/user/explanations/decisions/0006-setuptools-scm.rst @@ -0,0 +1,28 @@ +6. Use setuptools_scm +===================== + +Date: 2023-01-18 + +Status +------ + +Accepted + +Context +------- + +We require a mechanism for generating version numbers in python. + +Decision +-------- + +Generate the version number from git tags using setuptools scm. + +See https://github.com/pypa/setuptools_scm/ + +Consequences +------------ + +Versions are generated automatically from git tags. This means you can +can verify if you are running a released version of the code as +setup tools scm adds a suffix to untagged commits. diff --git a/docs/user/explanations/decisions/0007-dev-dependencies.rst b/docs/user/explanations/decisions/0007-dev-dependencies.rst new file mode 100644 index 00000000..507551b4 --- /dev/null +++ b/docs/user/explanations/decisions/0007-dev-dependencies.rst @@ -0,0 +1,37 @@ +7. Installing developer environment +=================================== + +Date: 2023-01-18 + +Status +------ + +Accepted + +Context +------- + +We need to provide a way to setup a developer environment for a skeleton based +project. + +Decision +-------- + +Use optional dependencies in pyproject.toml. + +PEP 621 provides a mechanism for adding optional dependencies in pyproject.toml +https://peps.python.org/pep-0621/#dependencies-optional-dependencies. + +We supply a list of developer dependencies under the title "dev". These +developer dependencies enable building and testing the project and +its documentation. + +Consequences +------------ + +Any developer can update their virtual environment in order to work on +a skeleton based project with the command: + +```bash +pip install -e .[dev] +``` \ No newline at end of file diff --git a/docs/user/explanations/decisions/0008-use-tox.rst b/docs/user/explanations/decisions/0008-use-tox.rst new file mode 100644 index 00000000..19a83f02 --- /dev/null +++ b/docs/user/explanations/decisions/0008-use-tox.rst @@ -0,0 +1,36 @@ +8. Use tox and pre-commit +========================= + +Date: 2023-01-18 + +Status +------ + +Accepted + +Context +------- + +We require an easy way to locally run the same checks as CI. This provides a +rapid inner-loop developer experience. + +Decision +-------- + +Use tox and pre-commit. + +tox is an automation tool that we use to run all checks in parallel, +see https://tox.wiki/en/latest/. + +pre-commit provides a hook into git commit which runs some of the checks +against the changes you are about to commit. + + +Consequences +------------ + +Running ``tox -p`` before pushing to GitHub verifies that the CI will *most +likely* succeed. + +Committing changes to git will run all of the non-time critical checks and +help avoid some of the most common mistakes. \ No newline at end of file diff --git a/docs/user/explanations/decisions/0009-sphinx-theme.rst b/docs/user/explanations/decisions/0009-sphinx-theme.rst new file mode 100644 index 00000000..7a56abbe --- /dev/null +++ b/docs/user/explanations/decisions/0009-sphinx-theme.rst @@ -0,0 +1,24 @@ +9. Sphinx Theme +=============== + +Date: 2023-01-18 + +Status +------ + +Accepted + +Context +------- + +Documentation requires a theme as well as a structure. + +Decision +-------- + +Use the pydata theme defined here https://pydata-sphinx-theme.readthedocs.io/ + +Consequences +------------ + +The documentation looks nice and has good navigation features. diff --git a/docs/user/explanations/decisions/0010-vscode-settings.rst b/docs/user/explanations/decisions/0010-vscode-settings.rst new file mode 100644 index 00000000..a676ace4 --- /dev/null +++ b/docs/user/explanations/decisions/0010-vscode-settings.rst @@ -0,0 +1,31 @@ +10. Include vscode settings +=========================== + +Date: 2023-01-18 + +Status +------ + +Accepted + +Context +------- + +For vscode users a couple of settings are required for neat integration with +the IDE. + +Decision +-------- + +Include a .vscode folder in the repo with some json files that enable: + +- recommended extension for best experience +- launcher to make debugging of python code override the coverage settings +- settings to make code verification match the tools in CI +- a task to launch the tox tests from the IDE + +Consequences +------------ + +Users of vscode should find that their development environment just works. +Users of other editors will be unaffected. \ No newline at end of file diff --git a/docs/user/explanations/decisions/0011-requirements-txt.rst b/docs/user/explanations/decisions/0011-requirements-txt.rst new file mode 100644 index 00000000..b8effa20 --- /dev/null +++ b/docs/user/explanations/decisions/0011-requirements-txt.rst @@ -0,0 +1,32 @@ +5. Use pyproject.toml +===================== + +Date: 2023-01-18 + +Status +------ + +Accepted + +Context +------- + +Require the ability to pin requirements for a guaranteed rebuild. +By default CI builds against the latest version of all dependencies, but we +need a mechanism for overriding this behaviour with a lock file +when there are issues. + +Decision +-------- + +Have every release generate requirements.txt files using pip freeze and +publish them as release assets. + +Request that the user to download the asset and commit it into the repo in order +to lock dependencies for the next CI build. + +Consequences +------------ + +There is less overhead managing fg lock files. + diff --git a/docs/user/index.rst b/docs/user/index.rst index ffd1af32..f846c921 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -43,11 +43,10 @@ side-bar. :caption: Explanations :maxdepth: 1 - explanations/docs-structure + explanations/decisions explanations/why-use-skeleton - explanations/why-src explanations/why-pre-commit - explanations/skeleton + explanations/skeleton +++ From d780a97591274d5f12773704ad60b40f34d18d54 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 18 Jan 2023 16:36:33 +0000 Subject: [PATCH 07/18] remaining ADRs --- .../explanations/decisions/0008-use-tox.rst | 102 ++++++++++++++++++ .../decisions/0011-requirements-txt.rst | 8 +- .../decisions/0012-containers.rst | 35 ++++++ docs/user/explanations/why-pre-commit.rst | 86 --------------- docs/user/index.rst | 1 - 5 files changed, 143 insertions(+), 89 deletions(-) create mode 100644 docs/user/explanations/decisions/0012-containers.rst delete mode 100644 docs/user/explanations/why-pre-commit.rst diff --git a/docs/user/explanations/decisions/0008-use-tox.rst b/docs/user/explanations/decisions/0008-use-tox.rst index 19a83f02..3e259ce2 100644 --- a/docs/user/explanations/decisions/0008-use-tox.rst +++ b/docs/user/explanations/decisions/0008-use-tox.rst @@ -25,6 +25,108 @@ see https://tox.wiki/en/latest/. pre-commit provides a hook into git commit which runs some of the checks against the changes you are about to commit. +Decision detail +--------------- + +There are a number of things that CI needs to run: + +- pytest +- black +- mypy +- flake8 +- isort +- build documentation + +The initial approach this module took was to integrate everything +under pytest that had a plugin, and isort under flake8: + +.. digraph:: initial + + bgcolor=transparent + graph [fontname="Lato" fontsize=10 style=filled fillcolor="#8BC4E9"] + node [fontname="Lato" fontsize=10 shape=box style=filled fillcolor="#8BC4E9"] + + subgraph cluster_0 { + label = "pytest" + "pytest-black" + "pytest-mypy" + subgraph cluster_1 { + label = "pytest-flake8" + "flake8-isort" + } + } + +This had the advantage that a ``pytest tests`` run in CI would catch and +report all test failures, but made each run take longer than it needed to. Also, +flake8 states that it `does not have a public, stable, Python API +`_ so did not +recommend the approach taken by pytest-flake8. + +To address this, the tree was rearranged: + +.. digraph:: rearranged + + bgcolor=transparent + graph [fontname="Lato" fontsize=10 style=filled fillcolor="#8BC4E9"] + node [fontname="Lato" fontsize=10 shape=box style=filled fillcolor="#8BC4E9"] + + pytest + black + mypy + subgraph cluster_1 { + label = "flake8" + "flake8-isort" + } + +If using VSCode, this will still run black, flake8 and mypy on file save, but +for those using other editors and for CI another solution was needed. Enter +`pre-commit `_. This allows hooks to be run at ``git +commit`` time on just the files that have changed, as well as on all tracked +files by CI. All that is needed is a one time install of the git commit hook:: + + $ pre-commit install + +Finally tox was added to run all of the CI checks including +the documentation build. mypy was moved out of the pre-commit and into tox +because it was quite long running and +therefore intrusive. tox can be invoked to run all the checks in +parallel with:: + + $ tox -p + +The graph now looks like: + +.. digraph:: rearranged + + bgcolor=transparent + graph [fontname="Lato" fontsize=10 style=filled fillcolor="#8BC4E9"] + node [fontname="Lato" fontsize=10 shape=box style=filled fillcolor="#8BC4E9"] + + subgraph cluster_0 + { + label = "tox -p" + pytest + mypy + "sphinx-build" + subgraph cluster_1 { + label = "pre-commit" + black + subgraph cluster_2 { + label = "flake8" + "flake8-isort" + } + } + } + +Now the workflow looks like this: + +- Save file, VSCode runs black, flake8 and mypy on it +- Run 'tox -p' and fix issues until it succeeds +- Commit files and pre-commit runs black and flake8 on them (if the + developer had not run tox then this catches some of the most common issues) +- Push to remote and CI runs black, flake8, mypy once on all files + (using tox), then pytest multiple times in a test matrix + Consequences ------------ diff --git a/docs/user/explanations/decisions/0011-requirements-txt.rst b/docs/user/explanations/decisions/0011-requirements-txt.rst index b8effa20..4196e32d 100644 --- a/docs/user/explanations/decisions/0011-requirements-txt.rst +++ b/docs/user/explanations/decisions/0011-requirements-txt.rst @@ -22,11 +22,15 @@ Decision Have every release generate requirements.txt files using pip freeze and publish them as release assets. -Request that the user to download the asset and commit it into the repo in order +Request that the user download the asset and commit it into the repo in order to lock dependencies for the next CI build. +TODO: link to the How-To on pinning requirements to be written in +python3-pip-skeleton developer documentation. + Consequences ------------ -There is less overhead managing fg lock files. +There is less overhead in managing lock files. Incoming issues with dependencies +will be highlighted early but can be worked around quickly if needed. diff --git a/docs/user/explanations/decisions/0012-containers.rst b/docs/user/explanations/decisions/0012-containers.rst new file mode 100644 index 00000000..d6b6b593 --- /dev/null +++ b/docs/user/explanations/decisions/0012-containers.rst @@ -0,0 +1,35 @@ +5. Use containers +================= + +Date: 2023-01-18 + +Status +------ + +Accepted + +Context +------- + +Allow developers and users to take advantage of containers. + +Decision +-------- + +Provide a single Dockerfile that can build two kinds of container: + +- a minimal runtime container that can be used to execute the application in + isolation without setting up a virtual environment or installing system + dependencies +- a devcontainer for working on the project with the same isolation as above + +CI builds the runtime container and publishes it to ghcr.io. + +a .devcontainer folder provides the means to build and launch the developer +container using vscode. + +Consequences +------------ + +We can label projects as cloud native. + diff --git a/docs/user/explanations/why-pre-commit.rst b/docs/user/explanations/why-pre-commit.rst deleted file mode 100644 index 8b487b90..00000000 --- a/docs/user/explanations/why-pre-commit.rst +++ /dev/null @@ -1,86 +0,0 @@ -Why use pre-commit -================== - -There are a number of things that CI needs to run: - -- pytest -- black -- mypy -- flake8 -- isort - -The initial approach this module took was to integrate everything -under pytest that had a plugin, and isort under flake8: - -.. digraph:: initial - - bgcolor=transparent - graph [fontname="Lato" fontsize=10 style=filled fillcolor="#8BC4E9"] - node [fontname="Lato" fontsize=10 shape=box style=filled fillcolor="#8BC4E9"] - - subgraph cluster_0 { - label = "pytest" - "pytest-black" - "pytest-mypy" - subgraph cluster_1 { - label = "pytest-flake8" - "flake8-isort" - } - } - -This had the advantage that a ``pytest tests`` run in CI would catch and -report all test failures, but made each run take longer than it needed to. Also, -flake8 states that it `does not have a public, stable, Python API -`_ so did not -recommend the approach taken by pytest-flake8. - -To address this, the tree was rearranged: - -.. digraph:: rearranged - - bgcolor=transparent - graph [fontname="Lato" fontsize=10 style=filled fillcolor="#8BC4E9"] - node [fontname="Lato" fontsize=10 shape=box style=filled fillcolor="#8BC4E9"] - - pytest - black - mypy - subgraph cluster_1 { - label = "flake8" - "flake8-isort" - } - -If using VSCode, this will still run black, flake8 and mypy on file save, but -for those using other editors and for CI another solution was needed. Enter -`pre-commit `_. This allows hooks to be run at ``git -commit`` time on just the files that have changed, as well as on all tracked -files by CI. All that is needed is a one time install of the git commit hook:: - - $ pre-commit install - -The graph now looks like: - -.. digraph:: rearranged - - bgcolor=transparent - graph [fontname="Lato" fontsize=10 style=filled fillcolor="#8BC4E9"] - node [fontname="Lato" fontsize=10 shape=box style=filled fillcolor="#8BC4E9"] - - pytest - subgraph cluster_0 { - label = "pre-commit" - black - mypy - subgraph cluster_1 { - label = "flake8" - "flake8-isort" - } - } - -Now the workflow looks like this: - -- Save file, VSCode runs black, flake8 and mypy on it -- Run pytest until tests pass -- Commit files and pre-commit runs black, flake8 and mypy on them -- Push to remote and CI runs black, flake8, mypy once on all files, then pytest - multiple times in a test matrix diff --git a/docs/user/index.rst b/docs/user/index.rst index f846c921..245e2aee 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -45,7 +45,6 @@ side-bar. explanations/decisions explanations/why-use-skeleton - explanations/why-pre-commit explanations/skeleton +++ From 13fdef31a7db8ab3811bce81b4d48c6971b22236 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 18 Jan 2023 16:50:55 +0000 Subject: [PATCH 08/18] fix ADR titles for 11,12 --- docs/user/explanations/decisions/0011-requirements-txt.rst | 4 ++-- docs/user/explanations/decisions/0012-containers.rst | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/user/explanations/decisions/0011-requirements-txt.rst b/docs/user/explanations/decisions/0011-requirements-txt.rst index 4196e32d..05e80105 100644 --- a/docs/user/explanations/decisions/0011-requirements-txt.rst +++ b/docs/user/explanations/decisions/0011-requirements-txt.rst @@ -1,5 +1,5 @@ -5. Use pyproject.toml -===================== +11. Pinning Requirements +======================== Date: 2023-01-18 diff --git a/docs/user/explanations/decisions/0012-containers.rst b/docs/user/explanations/decisions/0012-containers.rst index d6b6b593..e53dd541 100644 --- a/docs/user/explanations/decisions/0012-containers.rst +++ b/docs/user/explanations/decisions/0012-containers.rst @@ -1,5 +1,5 @@ -5. Use containers -================= +12. Use containers +================== Date: 2023-01-18 @@ -25,11 +25,10 @@ Provide a single Dockerfile that can build two kinds of container: CI builds the runtime container and publishes it to ghcr.io. -a .devcontainer folder provides the means to build and launch the developer +A .devcontainer folder provides the means to build and launch the developer container using vscode. Consequences ------------ We can label projects as cloud native. - From 61f868175e6fbb9a58ab009f52df8c553ea0d8ab Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 19 Jan 2023 09:09:06 +0000 Subject: [PATCH 09/18] move "Working on Skeleton" to dev guide --- docs/{user => developer}/explanations/skeleton.rst | 0 docs/developer/index.rst | 2 ++ docs/user/explanations/decisions/0009-sphinx-theme.rst | 2 +- docs/user/explanations/decisions/0011-requirements-txt.rst | 2 +- docs/user/index.rst | 1 - 5 files changed, 4 insertions(+), 3 deletions(-) rename docs/{user => developer}/explanations/skeleton.rst (100%) diff --git a/docs/user/explanations/skeleton.rst b/docs/developer/explanations/skeleton.rst similarity index 100% rename from docs/user/explanations/skeleton.rst rename to docs/developer/explanations/skeleton.rst diff --git a/docs/developer/index.rst b/docs/developer/index.rst index 3a7b65d0..bf1f79a2 100644 --- a/docs/developer/index.rst +++ b/docs/developer/index.rst @@ -43,6 +43,8 @@ side-bar. :caption: Explanations :maxdepth: 1 + explanations/skeleton + +++ Explanations of how and why the architecture is why it is. diff --git a/docs/user/explanations/decisions/0009-sphinx-theme.rst b/docs/user/explanations/decisions/0009-sphinx-theme.rst index 7a56abbe..b80fee08 100644 --- a/docs/user/explanations/decisions/0009-sphinx-theme.rst +++ b/docs/user/explanations/decisions/0009-sphinx-theme.rst @@ -1,4 +1,4 @@ -9. Sphinx Theme +9. Sphinx theme =============== Date: 2023-01-18 diff --git a/docs/user/explanations/decisions/0011-requirements-txt.rst b/docs/user/explanations/decisions/0011-requirements-txt.rst index 05e80105..0247512e 100644 --- a/docs/user/explanations/decisions/0011-requirements-txt.rst +++ b/docs/user/explanations/decisions/0011-requirements-txt.rst @@ -1,4 +1,4 @@ -11. Pinning Requirements +11. Pinning requirements ======================== Date: 2023-01-18 diff --git a/docs/user/index.rst b/docs/user/index.rst index 245e2aee..5d7d16a1 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -45,7 +45,6 @@ side-bar. explanations/decisions explanations/why-use-skeleton - explanations/skeleton +++ From 048058d4fa10695ce79f6fdfedaf059f96ff79d4 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 19 Jan 2023 12:54:02 +0000 Subject: [PATCH 10/18] add structure.rst, reformat README --- README.rst | 33 +++++++----- docs/developer/how-to/run-tests.rst | 2 + .../decisions/0003-docs-structure.rst | 2 + .../explanations/decisions/0004-why-src.rst | 2 + docs/user/explanations/structure.rst | 51 +++++++++++++++++++ docs/user/explanations/why-use-skeleton.rst | 2 +- docs/user/index.rst | 3 +- 7 files changed, 79 insertions(+), 16 deletions(-) create mode 100644 docs/user/explanations/structure.rst diff --git a/README.rst b/README.rst index 676fa084..f9deec9e 100644 --- a/README.rst +++ b/README.rst @@ -3,10 +3,9 @@ python3-pip-skeleton-cli |code_ci| |docs_ci| |coverage| |pypi_version| |license| -This skeleton module (inspired by `jaraco/skeleton -`_) is a generic Python project structure -which provides a means to keep tools and techniques in sync between multiple -Python projects. +``python3-pip-skeleton-cli`` provides the documentation +and a command line tool to enable the adoption of python3-pip-skeleton_ +into a new or existing Python project. ============== ============================================================== PyPI ``pip install python3-pip-skeleton`` @@ -15,7 +14,16 @@ Documentation https://DiamondLightSource.github.io/python3-pip-skeleton-cli Releases https://github.com/DiamondLightSource/python3-pip-skeleton-cli/releases ============== ============================================================== -It integrates the following tools: +The related python3-pip-skeleton_ repository contains the source +code that can be merged into new or existing projects, and pulled from to +keep them up to date. It can also serve as a working example for those who +would prefer to cherry-pick. + +python3-pip-skeleton_ is inspired by `jaraco/skeleton +`_. +It provides a generic Python project structure +and allows developers to keep tools and techniques in sync between multiple +Python projects. It integrates the following tools: - pip and setuptools_scm for version management - Pre-commit with black, flake8 and isort for static analysis @@ -26,20 +34,17 @@ It integrates the following tools: - which verifies all the things that CI does - If you use VSCode, it will run black, flake8, isort and mypy on save -The related skeleton_ repo for this module contains the source -code that can be merged into new or existing projects, and pulled from to -keep them up to date. It can also serve as a working example for those who -would prefer to cherry-pick. -.. _skeleton: https://github.com/DiamondLightSource/python3-pip-skeleton +.. _python3-pip-skeleton: https://github.com/DiamondLightSource/python3-pip-skeleton + +Quick start +----------- -This ``python3-pip-skeleton-cli`` repo contains the -docs and a command line tool to ease the adoption of this skeleton into a -new project like this:: +To create a new project based on skeleton:: python3-pip-skeleton new /path/to/be/created --org my_github_user_or_org -and existing projects:: +or to adopt skeleton into existing projects:: python3-pip-skeleton existing /path/to/existing/repo --org my_github_user_or_org diff --git a/docs/developer/how-to/run-tests.rst b/docs/developer/how-to/run-tests.rst index d2e03644..6350ad84 100644 --- a/docs/developer/how-to/run-tests.rst +++ b/docs/developer/how-to/run-tests.rst @@ -1,3 +1,5 @@ +.. _using pytest: + Run the tests using pytest ========================== diff --git a/docs/user/explanations/decisions/0003-docs-structure.rst b/docs/user/explanations/decisions/0003-docs-structure.rst index 2032a766..6c42b0ff 100644 --- a/docs/user/explanations/decisions/0003-docs-structure.rst +++ b/docs/user/explanations/decisions/0003-docs-structure.rst @@ -1,3 +1,5 @@ +.. _documentation structure: + 3. Standard documentation structure =================================== diff --git a/docs/user/explanations/decisions/0004-why-src.rst b/docs/user/explanations/decisions/0004-why-src.rst index 5ad095ef..b09ea944 100644 --- a/docs/user/explanations/decisions/0004-why-src.rst +++ b/docs/user/explanations/decisions/0004-why-src.rst @@ -1,3 +1,5 @@ +.. _src: + 4. Use a source directory ========================= diff --git a/docs/user/explanations/structure.rst b/docs/user/explanations/structure.rst new file mode 100644 index 00000000..66766c2d --- /dev/null +++ b/docs/user/explanations/structure.rst @@ -0,0 +1,51 @@ +Skeleton Project Structure +========================== + +The skeleton project has the following folders at the root level. + +src +--- + +This folder contains the source code for the project. Typically this +contains a single folder with the package name for the project and the +folder contains python modules files. + +See `src` for details. + +tests +----- + +This folder holds all of the tests that will be run by pytest, both locally +and in CI. + +See `using pytest` + +docs +---- + +This folder contains the source for sphinx documentation. + +See `documentation structure` for details. + +.github +------- + +Configuration for the Continuous Integration Workflow on +github + +VSCode specific folders +----------------------- + +.devcontainer +~~~~~~~~~~~~~ + +Configuration for running the developer container for this project +in VSCode. + +.vscode +~~~~~~~ + +VSCode settings for this project. + +- enable static analysis in the editor +- enables python debugging. diff --git a/docs/user/explanations/why-use-skeleton.rst b/docs/user/explanations/why-use-skeleton.rst index f0a429e5..796e55ef 100644 --- a/docs/user/explanations/why-use-skeleton.rst +++ b/docs/user/explanations/why-use-skeleton.rst @@ -1,4 +1,4 @@ -Why use a skeleton structure? +Why Use a Skeleton Structure? ============================= Many projects start from some kind of template. These define some basic diff --git a/docs/user/index.rst b/docs/user/index.rst index 5d7d16a1..fbf1917c 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -43,8 +43,9 @@ side-bar. :caption: Explanations :maxdepth: 1 - explanations/decisions + explanations/structure explanations/why-use-skeleton + explanations/decisions +++ From 7dd199d4c79212c539cf1e76269d3b8e3eb35b7a Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 19 Jan 2023 13:24:15 +0000 Subject: [PATCH 11/18] add local container testing docs --- docs/developer/how-to/test-container.rst | 25 ++++++++++++++++++++++++ docs/developer/index.rst | 1 + 2 files changed, 26 insertions(+) create mode 100644 docs/developer/how-to/test-container.rst diff --git a/docs/developer/how-to/test-container.rst b/docs/developer/how-to/test-container.rst new file mode 100644 index 00000000..a4a43a6f --- /dev/null +++ b/docs/developer/how-to/test-container.rst @@ -0,0 +1,25 @@ +Container Local Build and Test +============================== + +CI builds a runtime container for the project. The local tests +checks available via ``tox -p`` do not verify this because not +all developers will have docker installed locally. + +If CI is failing to build the container, then it is best to fix and +test the problem locally. This would require that you have docker +or podman installed on your local workstation. + +In the following examples the command ``docker`` is interchangeable with +``podman`` depending on which container cli you have installed. + +To build the container and call it ``test``:: + + cd + docker build -t test . + +To verify that the container runs:: + + docker run -it test --help + +You can pass any other command line parameters to your application +instead of --help. diff --git a/docs/developer/index.rst b/docs/developer/index.rst index bf1f79a2..f7aa265d 100644 --- a/docs/developer/index.rst +++ b/docs/developer/index.rst @@ -32,6 +32,7 @@ side-bar. how-to/lint how-to/update-tools how-to/make-release + how-to/test-container +++ From c70f4efcb1301f33392f0c521b51351089de1a15 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 19 Jan 2023 14:33:54 +0000 Subject: [PATCH 12/18] remove conda howto --- docs/user/how-to/conda.rst | 23 ----------------------- docs/user/index.rst | 1 - 2 files changed, 24 deletions(-) delete mode 100644 docs/user/how-to/conda.rst diff --git a/docs/user/how-to/conda.rst b/docs/user/how-to/conda.rst deleted file mode 100644 index ce0c3ce6..00000000 --- a/docs/user/how-to/conda.rst +++ /dev/null @@ -1,23 +0,0 @@ -Creating an Anaconda Token -========================== - -To publish your package on Anaconda.org requires an Anaconda account and for GitHub -Actions to have an Anaconda token authorizing access to that account. - - -The simplest approach is to set up an token that is scoped to your Anaconda account -and add it to the secrets for your GitHub Organization (or user). This means -that all new projects created in the Organization will automatically gain -permission to publish to Anaconda.org. - -If you do not already have a Anaconda account use this link: create_account_. -You may wish to additionally create an organisation, using this link: create_organisation_. - -To learn how to create a token see: creating_a_token_. -To learn how to add this token to a github repository see: creating_a_secret_. -Note that skeleton uses ``ANACONDA_TOKEN`` as the secret name. - -.. _create_account: https://anaconda.org/account/register -.. _create_organisation: https://anaconda.org/garryod/organizations -.. _creating_a_token: https://docs.anaconda.com/anacondaorg/user-guide/tasks/work-with-accounts/#creating-access-tokens -.. _creating_a_secret: https://docs.github.com/en/actions/security-guides/encrypted-secrets#creating-encrypted-secrets-for-a-repository \ No newline at end of file diff --git a/docs/user/index.rst b/docs/user/index.rst index fbf1917c..bd7a717b 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -31,7 +31,6 @@ side-bar. how-to/update how-to/excalidraw how-to/pypi - how-to/conda +++ From 3963fdee1f69830c2749f9b1cc877d0c50613ed4 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Wed, 18 Jan 2023 14:16:09 +0000 Subject: [PATCH 13/18] add how to do pin requirements doc --- docs/developer/how-to/pin-requirements.rst | 74 ++++++++++++++++++++++ docs/developer/index.rst | 2 +- 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 docs/developer/how-to/pin-requirements.rst diff --git a/docs/developer/how-to/pin-requirements.rst b/docs/developer/how-to/pin-requirements.rst new file mode 100644 index 00000000..89639623 --- /dev/null +++ b/docs/developer/how-to/pin-requirements.rst @@ -0,0 +1,74 @@ +Pinning Requirements +==================== + +Introduction +------------ + +By design this project only defines dependencies in one place, i.e. in +the ``requires`` table in ``pyproject.toml``. + +In the ``requires`` table it is possible to pin versions of some dependencies +as needed. For library projects it is best to leave pinning to a minimum so +that your library can be used by the widest range of applications. + +When CI builds the project it will use the latest compatible set of +dependencies available (after applying your pins and any dependencies' pins). + +This approach means that there is a possibility that a future build may +break because an updated release of a dependency has made a breaking change. + +The correct way to fix such an issue is to work out the minimum pinning in +``requires`` that will resolve the problem. However this can be quite hard to +do and may be time consuming when simply trying to release a minor update. + +For this reason we provide a mechanism for locking all dependencies to +the same version as a previous successful release. This is a quick fix that +should guarantee a successful CI build. + +Finding the lock files +---------------------- + +Every release of the project will have a set of requirements files published +as release assets. + +For example take a look at the release page for python3-pip-skeleton-cli here: +https://github.com/DiamondLightSource/python3-pip-skeleton-cli/releases/tag/3.3.0 + +There is a list of requirements*.txt files showing as assets on the release. + +There is one file for each time the CI installed the project into a virtual +environment. There are multiple of these as the CI creates a number of +different environments. + +The files are created using ``pip freeze`` and will contain a full list +of the dependencies and sub-dependencies with pinned versions. + +You can download any of these files by clicking on them. It is best to use +the one that ran with the lowest Python version as this is more likely to +be compatible with all the versions of Python in the test matrix. +i.e. ``requirements-test-ubuntu-latest-3.8.txt`` in this example. + +Applying the lock file +---------------------- + +To apply a lockfile: + +- copy the requirements file you have downloaded to the root of your + repository +- rename it to requirements.txt +- commit it into the repo +- push the changes + +The CI looks for a requirements.txt in the root and will pass it to pip +when installing each of the test environments. pip will then install exactly +the same set of packages as the previous release. + +Removing dependency locking from CI +----------------------------------- + +Once the reasons for locking the build have been resolved it is a good idea +to go back to an unlocked build. This is because you get an early indication +of any incoming problems. + +To restore unlocked builds in CI simply remove requirements.txt from the root +of the repo and push. diff --git a/docs/developer/index.rst b/docs/developer/index.rst index f7aa265d..8e1850b4 100644 --- a/docs/developer/index.rst +++ b/docs/developer/index.rst @@ -32,7 +32,7 @@ side-bar. how-to/lint how-to/update-tools how-to/make-release - how-to/test-container + how-to/pin-requirements +++ From 619b2945885a731d95ce00026e1f30f69e27dcb1 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 19 Jan 2023 14:23:01 +0000 Subject: [PATCH 14/18] add local container testing docs --- docs/developer/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/developer/index.rst b/docs/developer/index.rst index 8e1850b4..4a8dd02d 100644 --- a/docs/developer/index.rst +++ b/docs/developer/index.rst @@ -33,6 +33,7 @@ side-bar. how-to/update-tools how-to/make-release how-to/pin-requirements + how-to/test-container +++ From 71b9101e457691be4fbde732fdffdcc62aa511ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 17:10:57 +0000 Subject: [PATCH 15/18] Bump peaceiris/actions-gh-pages from 3.9.1 to 3.9.2 Bumps [peaceiris/actions-gh-pages](https://github.com/peaceiris/actions-gh-pages) from 3.9.1 to 3.9.2. - [Release notes](https://github.com/peaceiris/actions-gh-pages/releases) - [Changelog](https://github.com/peaceiris/actions-gh-pages/blob/main/CHANGELOG.md) - [Commits](https://github.com/peaceiris/actions-gh-pages/compare/64b46b4226a4a12da2239ba3ea5aa73e3163c75b...bd8c6b06eba6b3d25d72b7a1767993c0aeee42e7) --- updated-dependencies: - dependency-name: peaceiris/actions-gh-pages dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c510d577..7c8dfb5c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -46,7 +46,7 @@ jobs: if: github.event_name == 'push' && github.actor != 'dependabot[bot]' # We pin to the SHA, not the tag, for security reasons. # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions - uses: peaceiris/actions-gh-pages@64b46b4226a4a12da2239ba3ea5aa73e3163c75b # v3.9.1 + uses: peaceiris/actions-gh-pages@bd8c6b06eba6b3d25d72b7a1767993c0aeee42e7 # v3.9.2 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: .github/pages From 8da867ad943135af6389e2a54678464f0fd47492 Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Wed, 1 Feb 2023 15:20:16 +0000 Subject: [PATCH 16/18] Added unit tests to boost coverage, and tomli for parsing pyproject.toml --- pyproject.toml | 2 +- src/python3_pip_skeleton/__main__.py | 135 +++++++++++++++++++++++---- tests/test_adopt.py | 64 +++++++++++++ 3 files changed, 184 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 964e9cdb..5359c2d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] description = "One line description of your module" -dependencies = [] # Add project dependencies here, e.g. ["click", "numpy"] +dependencies = ["tomli"] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] license.file = "LICENSE" readme = "README.rst" diff --git a/src/python3_pip_skeleton/__main__.py b/src/python3_pip_skeleton/__main__.py index bc6be1a8..a2b40bcf 100644 --- a/src/python3_pip_skeleton/__main__.py +++ b/src/python3_pip_skeleton/__main__.py @@ -7,6 +7,8 @@ from tempfile import TemporaryDirectory from typing import List +import tomli + from . import __version__ __all__ = ["main"] @@ -52,6 +54,7 @@ def merge_skeleton( org: str, full_name: str, email: str, + from_branch: str, package, ): path = path.resolve() @@ -63,6 +66,7 @@ def replace_text(text: str) -> str: text = text.replace("python3_pip_skeleton", package) text = text.replace("Firstname Lastname", full_name) text = text.replace("email@address.com", email) + text = text.replace("main", from_branch) return text branches = list_branches(path) @@ -82,7 +86,7 @@ def replace_text(text: str) -> str: # will do the wrong thing shutil.rmtree(git_tmp / "src", ignore_errors=True) # Merge in the skeleton commits - git_tmp("pull", "--rebase=false", SKELETON, "main") + git_tmp("pull", "--rebase=false", SKELETON, from_branch) # Move things around if package != "python3_pip_skeleton": git_tmp("mv", "src/python3_pip_skeleton", f"src/{package}") @@ -138,6 +142,17 @@ def verify_not_adopted(root: Path): ) +def obtain_git_author_email(path : Path): + author = str( + git("--git-dir", path / ".git", "config", "--get", "user.name").strip() + ) + author_email = str( + git("--git-dir", path / ".git", "config", "--get", "user.email").strip() + ) + + return author, author_email + + def new(args): path: Path = args.path @@ -149,22 +164,100 @@ def new(args): path.mkdir(parents=True) package = validate_package(args) + + if args.full_name and args.email: + author, author_email = args.full_name, args.email + else: + author, author_email = obtain_git_author_email(Path(".")) + git("init", "-b", "main", cwd=path) print(f"Created git repo in {path}") merge_skeleton( path=path, org=args.org, - full_name=args.full_name or git("config", "--get", "user.name").strip(), - email=args.email or git("config", "--get", "user.email").strip(), + full_name=author, + email=author_email, + from_branch=args.from_branch or "main", package=package, ) cfg_issue = """Missing parameter in setup.cfg. Expected format: -[metadata] -name = example -author = Firstname Lastname -author_email = email@address.com""" + [metadata] + name = example + author = Firstname Lastname + author_email = email@address.com + + ------- pyproject.toml + [[project.authors]] + name = "Firstname Lastname" + email = "email@address.com" +""" + + +def obtain_author_name_email(path: Path) -> tuple: + author: str = "" + author_email: str = "" + file_path_setup_cfg: Path = path / "setup.cfg" + file_path_pyproject_toml: Path = path / "pyproject.toml" + + # Parse for an author name, email. The order of preference used is + # setup.cfg -> pyproject.toml -> .git -> user input. + # Author and Email are recieved together to avoid mismatches from + # obtaining in different places. + + if file_path_setup_cfg.exists(): + try: + conf_cfg = ConfigParser() + conf_cfg.read(file_path_setup_cfg) + + if "metadata" in conf_cfg: + if "author" in conf_cfg["metadata"]: + author = conf_cfg["metadata"]["author"] + if "author_email" in conf_cfg["metadata"]: + author_email = conf_cfg["metadata"]["author_email"] + except Exception as exception: + print( + "\033[1mUnable to parse setup.cfg because of the following error, " + "will try other sources:\033[0m" + ) + print(exception) + print() + + if (not author or not author_email) and file_path_pyproject_toml.exists(): + file = open(file_path_pyproject_toml, "rb") + try: + conf_toml = tomli.load(file) + if "project" in conf_toml and "authors" in conf_toml["project"]: + if "author" in conf_toml["project"]["authors"][0]: + author = conf_toml["project"]["authors"][0]["author"] + if "email" in conf_toml["project"]["authors"][0]: + author_email = conf_toml["project"]["authors"][0]["email"] + except Exception as exception: + # We want to use something else if the pyproject.toml has some errors. + print( + "\033[1mUnable to parse project.toml because of the following error, " + "will try other sources:\033[0m" + ) + print(exception) + print() + file.close() + + if not author or not author_email: + author, author_email = obtain_git_author_email(path) + + # If all else fails, just ask the user. + if not author or not author_email: + print(cfg_issue) + print("Enter author name manually:") + author = str(input()) + print("Enter author email manually:") + author_email = str(input()) + + assert author, "Inputted no author" + assert author_email, "Inputted no author_email" + + return author, author_email def existing(args): @@ -173,21 +266,21 @@ def existing(args): assert path.is_dir(), f"Expected {path} to be an existing directory" package = validate_package(args) - file_path: Path = path / "setup.cfg" - assert file_path.is_file(), "Expected a setup.cfg file in the directory." + if not args.force: verify_not_adopted(args.path) - conf = ConfigParser() - conf.read(path / "setup.cfg") - assert "metadata" in conf, cfg_issue - assert "author" in conf["metadata"], cfg_issue - assert "author_email" in conf["metadata"], cfg_issue + if args.full_name and args.email: + author, author_email = args.full_name, args.email + else: + author, author_email = obtain_author_name_email(path) + merge_skeleton( path=args.path, org=args.org, - full_name=conf["metadata"]["author"], - email=conf["metadata"]["author_email"], + full_name=author, + email=author_email, + from_branch=args.from_branch or "main", package=package, ) @@ -225,6 +318,11 @@ def main(args=None): sub.add_argument( "--email", default=None, help="Email address, defaults to git config user.email" ) + sub.add_argument( + "--from-branch", + default=None, + help="Merge from skeleton branch, defaults to main", + ) # Add a command for adopting in existing repo sub = subparsers.add_parser("existing", help="Adopt skeleton in existing repo") sub.set_defaults(func=existing) @@ -234,6 +332,11 @@ def main(args=None): sub.add_argument( "--package", default=None, help="Package name, defaults to directory name" ) + sub.add_argument( + "--from-branch", + default=None, + help="Merge from skeleton branch, defaults to main", + ) # Add a command for cleaning an existing repo of skeleton code sub = subparsers.add_parser( "clean", help="Clean up branch from failed skeleton merge" diff --git a/tests/test_adopt.py b/tests/test_adopt.py index aaece404..dcc8b41f 100644 --- a/tests/test_adopt.py +++ b/tests/test_adopt.py @@ -51,6 +51,7 @@ def test_new_module(tmp_path: Path): assert (module / "src" / "my_module").is_dir() assert check_output("git", "branch", cwd=module).strip() == "* main" check_output("python", "-m", "venv", "venv", cwd=module) + check_output("venv/bin/pip", "install", "--upgrade", "pip", cwd=module) check_output("venv/bin/pip", "install", ".[dev]", cwd=module) check_output( "venv/bin/python", @@ -89,6 +90,45 @@ def test_new_module_existing_dir(tmp_path: Path): assert "to not exist, or be an empty dir" in str(excinfo.value) +def test_new_module_merge_from_valid_branch(tmp_path: Path): + module = tmp_path / "my-module" + check_output( + sys.executable, + "-m", + "python3_pip_skeleton", + "new", + "--org=myorg", + "--package=my_module", + "--full-name=Firstname Lastname", + "--email=me@myaddress.com", + "--from-branch=main", + str(module), + ) + # Test basic functionality + assert (module / "src" / "my_module").is_dir() + check_output("python", "-m", "venv", "venv", cwd=module) + check_output("venv/bin/pip", "install", ".[dev]", cwd=module) + + +def test_new_module_merge_from_invalid_branch(tmp_path: Path): + module = tmp_path / "my-module" + + with pytest.raises(ValueError) as excinfo: + check_output( + sys.executable, + "-m", + "python3_pip_skeleton", + "new", + "--org=myorg", + "--package=my_module", + "--full-name=Firstname Lastname", + "--email=me@myaddress.com", + "--from-branch=fail", + str(module), + ) + assert "couldn't find remote ref fail" in str(excinfo.value) + + def test_existing_module(tmp_path: Path): module = tmp_path / "scanspec" __main__.git( @@ -169,3 +209,27 @@ def test_existing_module_already_adopted(tmp_path: Path): str(module), ) assert "already adopted skeleton" in str(excinfo.value) + + +def test_existing_module_merge_from_invalid_branch(tmp_path: Path): + module = tmp_path / "scanspec" + __main__.git( + "clone", + "--depth", + "1", + "--branch", + "0.5.3", + "https://github.com/dls-controls/scanspec", + str(module), + ) + with pytest.raises(ValueError) as excinfo: + check_output( + sys.executable, + "-m", + "python3_pip_skeleton", + "existing", + "--org=epics-containers", + "--from-branch=fail", + str(module), + ) + assert "couldn't find remote ref fail" in str(excinfo.value) From ff7b6a8b76af6f3427672b57fd27119a8f859067 Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Mon, 20 Feb 2023 16:06:52 +0000 Subject: [PATCH 17/18] Added unit tests --- src/python3_pip_skeleton/__main__.py | 35 ++++++++++++++--- tests/test_adopt.py | 59 ++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 5 deletions(-) diff --git a/src/python3_pip_skeleton/__main__.py b/src/python3_pip_skeleton/__main__.py index 14f14dae..cb3c23bb 100644 --- a/src/python3_pip_skeleton/__main__.py +++ b/src/python3_pip_skeleton/__main__.py @@ -142,7 +142,16 @@ def verify_not_adopted(root: Path): ) -def obtain_git_author_email(path: Path): +def obtain_git_author_email(path: Path, force_local=True): + # If we force local then we require there to be a local .git we can look for + # the username and password on. + # If we don't force local then we will try to look for a local .git, if not found + # git will use the global user.[name, email]. + if force_local and not (path / ".git").exists(): + raise FileNotFoundError( + ".git could not be found when searching " + f"for a username and password in {path}" + ) author = str( git("--git-dir", path / ".git", "config", "--get", "user.name").strip() ) @@ -168,7 +177,7 @@ def new(args): if args.full_name and args.email: author, author_email = args.full_name, args.email else: - author, author_email = obtain_git_author_email(Path(".")) + author, author_email = obtain_git_author_email(Path("."), force_local=False) git("init", "-b", "main", cwd=path) print(f"Created git repo in {path}") @@ -229,8 +238,12 @@ def obtain_author_name_email(path: Path) -> tuple: try: conf_toml = tomli.load(file) if "project" in conf_toml and "authors" in conf_toml["project"]: - if "author" in conf_toml["project"]["authors"][0]: - author = conf_toml["project"]["authors"][0]["author"] + # pyproject.toml will use "author" or "name" so we look for both + for author_variable_name in ["author", "name"]: + if author_variable_name in conf_toml["project"]["authors"][0]: + author = conf_toml["project"]["authors"][0][ + author_variable_name + ] if "email" in conf_toml["project"]["authors"][0]: author_email = conf_toml["project"]["authors"][0]["email"] except Exception as exception: @@ -244,7 +257,13 @@ def obtain_author_name_email(path: Path) -> tuple: file.close() if not author or not author_email: - author, author_email = obtain_git_author_email(path) + try: + author, author_email = obtain_git_author_email(path) + except FileNotFoundError: + print( + "\033[1mUnable to find a .git in the repo," + "will try other sources\033[0m" + ) # If all else fails, just ask the user. if not author or not author_email: @@ -332,6 +351,12 @@ def main(args=None): sub.add_argument( "--package", default=None, help="Package name, defaults to directory name" ) + sub.add_argument( + "--full-name", default=None, help="Full name, defaults to git config user.name" + ) + sub.add_argument( + "--email", default=None, help="Email address, defaults to git config user.email" + ) sub.add_argument( "--from-branch", default=None, diff --git a/tests/test_adopt.py b/tests/test_adopt.py index dcc8b41f..bfb37094 100644 --- a/tests/test_adopt.py +++ b/tests/test_adopt.py @@ -2,6 +2,7 @@ import sys from os import makedirs from pathlib import Path +from unittest.mock import patch import pytest import toml @@ -233,3 +234,61 @@ def test_existing_module_merge_from_invalid_branch(tmp_path: Path): str(module), ) assert "couldn't find remote ref fail" in str(excinfo.value) + + +def test_obtain_git_author_email(tmp_path): + __main__.git("--git-dir", tmp_path / ".git", "init") + __main__.git("--git-dir", tmp_path / ".git", "config", "user.name", "Foo Bar") + __main__.git("--git-dir", tmp_path / ".git", "config", "user.email", "Foo@Bar") + assert __main__.obtain_git_author_email(tmp_path) == ("Foo Bar", "Foo@Bar") + + +def test_obtain_author_name_email_setup_cfg(tmp_path): + cfg_str = """ + [metadata] + author = Foo Bar + author_email = Foo@Bar + """ + with open(tmp_path / "setup.cfg", "w+") as cfg_file: + cfg_file.write(cfg_str) + assert __main__.obtain_author_name_email(tmp_path) == ("Foo Bar", "Foo@Bar") + + +def test_obtain_author_name_email_pyproject_toml(tmp_path): + toml_str = """ + [[project.authors]] + email = "Foo@Bar" + name = "Foo Bar" + """ + with open(tmp_path / "pyproject.toml", "w+") as toml_file: + toml_file.write(toml_str) + assert __main__.obtain_author_name_email(tmp_path) == ("Foo Bar", "Foo@Bar") + + +@patch("python3_pip_skeleton.__main__.input", return_value="Foo") +def test_obtain_author_name_email_botched_cfg_toml(input, tmp_path): + toml_str = """ + email + name = "Foo Bar" + """ + cfg_str = """ + author = Foo Bar + author_email = Foo@Bar + """ + with open(tmp_path / "setup.cfg", "w+") as cfg_file: + cfg_file.write(cfg_str) + with open(tmp_path / "pyproject.toml", "w+") as toml_file: + toml_file.write(toml_str) + assert __main__.obtain_author_name_email(tmp_path) == ("Foo", "Foo") + + +def test_obtain_author_name_email_git(tmp_path): + __main__.git("--git-dir", tmp_path / ".git", "init") + __main__.git("--git-dir", tmp_path / ".git", "config", "user.name", "Foo Bar") + __main__.git("--git-dir", tmp_path / ".git", "config", "user.email", "Foo@Bar") + assert __main__.obtain_author_name_email(tmp_path) == ("Foo Bar", "Foo@Bar") + + +@patch("python3_pip_skeleton.__main__.input", return_value="Foo") +def test_obtain_author_name_email_terminal_output(input, tmp_path): + assert __main__.obtain_author_name_email(tmp_path) == ("Foo", "Foo") From 64066608ce3a98135e7bcb968af4d1812a9af07f Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Fri, 17 Mar 2023 09:26:08 +0000 Subject: [PATCH 18/18] Boosted coverage up to 100% --- tests/test_adopt.py | 56 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/tests/test_adopt.py b/tests/test_adopt.py index bfb37094..17aaf273 100644 --- a/tests/test_adopt.py +++ b/tests/test_adopt.py @@ -1,6 +1,6 @@ import subprocess import sys -from os import makedirs +from os import chdir, makedirs from pathlib import Path from unittest.mock import patch @@ -24,7 +24,33 @@ def test_cli_version(): assert output.strip() == __version__ -def test_new_module(tmp_path: Path): +@pytest.mark.parametrize( + "extra_args", [(), ("--full-name=Firstname Lastname", "--email=me@myaddress.com")] +) +def test_new_module(extra_args, tmp_path: Path): + if not extra_args: + original_path = Path(".").absolute() + check_output("git", "init", str(tmp_path), cwd=tmp_path) + check_output( + "git", + "--git-dir", + str(tmp_path / ".git"), + "config", + "user.name", + "Firstname Lastname", + cwd=tmp_path, + ) + check_output( + "git", + "--git-dir", + str(tmp_path / ".git"), + "config", + "user.email", + "me@myaddress.com", + cwd=tmp_path, + ) + chdir(tmp_path) + module = tmp_path / "my-module" output = check_output( sys.executable, @@ -33,10 +59,13 @@ def test_new_module(tmp_path: Path): "new", "--org=myorg", "--package=my_module", - "--full-name=Firstname Lastname", - "--email=me@myaddress.com", + *extra_args, str(module), ) + + if not extra_args: + chdir(original_path) + assert output.strip().endswith( "Developer instructions in docs/developer/tutorials/dev-install.rst" ) @@ -73,6 +102,7 @@ def test_new_module(tmp_path: Path): def test_new_module_existing_dir(tmp_path: Path): + print(Path(".").absolute()) module = tmp_path / "my-module" makedirs(module / "existing_dir") @@ -130,8 +160,19 @@ def test_new_module_merge_from_invalid_branch(tmp_path: Path): assert "couldn't find remote ref fail" in str(excinfo.value) -def test_existing_module(tmp_path: Path): +SETUP_CFG = """[metadata] + name = example + author = Firstname Lastname + author_email = email@address.com + """ + + +@pytest.mark.parametrize( + "extra_args", [(), ("--full-name=Firstname Lastname", "--email=me@myaddress.com")] +) +def test_existing_module(extra_args, tmp_path: Path): module = tmp_path / "scanspec" + __main__.git( "clone", "--depth", @@ -141,12 +182,17 @@ def test_existing_module(tmp_path: Path): "https://github.com/dls-controls/scanspec", str(module), ) + + with open(module / "setup.cfg", "w+") as setup_cfg: + setup_cfg.write(SETUP_CFG) + output = check_output( sys.executable, "-m", "python3_pip_skeleton", "existing", "--org=epics-containers", + *extra_args, str(module), ) assert output.endswith(