HDXDSYS-2684 Reimplement munge_url from libhxl in HDX Python API (#106)

* Add munge_url from libhxl as follow_url and remove libhxl dependency
* Upgrade to uv

Author: mcarans

.coveragerc

@@ -13,25 +13,22 @@ jobs:
       url: https://pypi.org/p/hdx-python-api
 
     permissions:
-      id-token: write  # IMPORTANT: mandatory for trusted publishing
+      id-token: write
+      contents: read
 
     steps:
     - uses: actions/checkout@v6
+
     - name: Get history and tags for versioning to work
       run: |
         git fetch --prune --unshallow
         git fetch --depth=1 origin +refs/tags/*:refs/tags/*
-    - name: Set up Python
-      uses: actions/setup-python@v6
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-    - name: Install Hatch
-      uses: pypa/hatch@install
-    - name: Build with hatch
-      run: |
-        hatch build
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+
+    - name: Build with uv
+      run: uv build
+
     - name: Publish distribution 📦 to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
+      run: uv publish

.github/workflows/publish.yaml

@@ -1,60 +1,52 @@
-# This workflow will install Python dependencies, lint and run tests
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
 name: Run tests
 
 on:
-  workflow_dispatch: # add run button in github
+  workflow_dispatch:
   push:
-    branches-ignore:
-      - gh-pages
-      - 'dependabot/**'
+    branches-ignore: [gh-pages, "dependabot/**"]
   pull_request:
-    branches-ignore:
-      - gh-pages
-
-concurrency:
-  group: ${{ github.head_ref || github.ref_name }}
-
+    branches-ignore: [gh-pages]
 
 jobs:
   build:
     runs-on: ubuntu-latest
-
     permissions:
+      contents: read
       checks: write
       pull-requests: write
 
     steps:
-    - uses: actions/checkout@v6
-    - name: Set up Python
-      uses: actions/setup-python@v6
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-    - name: Install Hatch
-      uses: pypa/hatch@install
-    - name: Test with hatch/pytest
-      env:
-        HDX_KEY_TEST: ${{ secrets.HDX_BOT_SCRAPERS_API_TOKEN }}
-        GSHEET_AUTH: ${{ secrets.HDX_PIPELINE_GSHEET_AUTH }}
-      run: |
-        hatch test
-    - name: Check styling
-      if: always()
-      run: |
-        hatch fmt --check
-    - name: Publish Unit Test Results
-      uses: EnricoMi/publish-unit-test-result-action@v2
-      if: always()
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        junit_files: test-results.xml
-    - name: Publish in Coveralls
-      uses: coverallsapp/github-action@v2
-      with:
-        github-token: ${{ secrets.GITHUB_TOKEN }}
-        flag-name: tests
-        format: lcov
+      - uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: "3.13"
+
+      - name: Install dependencies
+        run: uv sync --frozen
+
+      - name: Check styling
+        run: |
+          uv run ruff format --check
+          uv run ruff check
+
+      - name: Test with pytest
+        env:
+          HDX_KEY_TEST: ${{ secrets.HDX_BOT_SCRAPERS_API_TOKEN }}
+          GSHEET_AUTH: ${{ secrets.HDX_PIPELINE_GSHEET_AUTH }}
+        run: uv run pytest
+
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          files: test-results.xml
+
+      - name: Publish in Coveralls
+        uses: coverallsapp/github-action@v2
+        if: always()
+        with:
+          flag-name: tests
+          format: lcov

.github/workflows/run-python-tests.yaml

@@ -1,26 +1,27 @@
 default_language_version:
-    python: python3.13
+  python: python3.13
+
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v6.0.0
     hooks:
       - id: trailing-whitespace
+        exclude: (test_loader.py|pretty-false_sortkeys-false.yaml|pretty-false_sortkeys-true.yaml)
       - id: end-of-file-fixer
+        exclude: (test_csv_processing_blanks.csv|test.txt)
       - id: check-ast
+
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.14.10
+    rev: v0.14.13
     hooks:
       # Run the linter.
       - id: ruff-check
         args: [ --fix ]
       # Run the formatter.
       - id: ruff-format
+
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.9.22
+    rev: 0.9.25
     hooks:
-      # Run the pip compile
-      - id: pip-compile
-        name: pip-compile requirements.txt
-        files: pyproject.toml
-        args: [ pyproject.toml, --resolver=backtracking, --upgrade, -q,
-                -o, requirements.txt ]
+      # Ensure the lockfile is up-to-date with pyproject.toml
+      - id: uv-lock

.pre-commit-config.yaml

@@ -2,67 +2,73 @@
 
 ## Environment
 
-Development is currently done using Python 3.11. We recommend using a virtual
-environment such as ``venv``:
+Development is currently done using Python 3.13. The environment can be created with:
 
-    python3.11 -m venv venv
-    source venv/bin/activate
+```shell
+    uv sync
+```
 
-In your virtual environment, please install all packages for
-development by running:
+This creates a .venv folder with the versions specified in the project's uv.lock file.
 
-    pip install -r requirements.txt
+### Pre-commit
 
-## Pre-Commit
-
-Also be sure to install `pre-commit`, which is run every time
-you make a git commit:
+pre-commit will be installed when syncing uv. It is run every time you make a git
+commit if you call it like this:
 
+```shell
     pre-commit install
-
-The configuration file for this project is in a
-non-standard location. Thus, you will need to edit your
-`.git/hooks/pre-commit` file to reflect this. Change
-the line that begins with `ARGS` to:
-
-    ARGS=(hook-impl --config=.config/pre-commit-config.yaml --hook-type=pre-commit)
+```
 
 With pre-commit, all code is formatted according to
-[ruff](https://github.com/astral-sh/ruff) guidelines.
+[ruff](https://docs.astral.sh/ruff/) guidelines.
 
 To check if your changes pass pre-commit without committing, run:
 
-    pre-commit run --all-files --config=.config/pre-commit-config.yaml
+```shell
+    pre-commit run --all-files
+```
 
-## Environment Variables
+## Packages
 
-For the `test_ckan.py` tests to run successfully some configuration is required:
+[uv](https://github.com/astral-sh/uv) is used for package management.  If
+you’ve introduced a new package to the source code (i.e. anywhere in `src/`),
+please add it to the `project.dependencies` section of `pyproject.toml` with
+any known version constraints.
 
-1. The environment variable `HDX_KEY_TEST` needs to contain a valid key from the HDX demo server at
-https://demo.data-humdata-org.ahconu.org/
-2. Authentication details for Google Sheets need to be obtained from Mike Rans and either saved in a file named `.gsheet_auth.json` in the home directory (~) or placed in an environment variable `GSHEET_AUTH`. The file is preferred for Windows systems since adding such a long text string to an environment variable in Windows is challenging.
+To add packages required only for testing, add them to the
+`[dependency-groups]`.
 
-## Testing
+Any changes to the dependencies will be automatically reflected in
+`uv.lock` with `pre-commit`, but you can re-generate the files without committing by
+executing:
 
-To run the tests and view coverage, execute:
+```shell
+    uv lock --upgrade
+```
 
-    pytest -c .config/pytest.ini --cov hdx --cov-config .config/coveragerc
+## Project
 
-Follow the example set out already in ``documentation/main.md`` as you write the documentation.
+[uv](https://github.com/astral-sh/uv) is used for project management. The project can be
+built using:
 
-## Packages
+```shell
+    uv build
+```
 
-[pip-tools](https://github.com/jazzband/pip-tools) is used for
-package management.  If you’ve introduced a new package to the
-source code (i.e.anywhere in `src/`), please add it to the
-`project.dependencies` section of
-`pyproject.toml` with any known version constraints.
+Linting and syntax checking can be run with:
 
-For adding packages for testing or development, add them to
-the `test` or `dev` sections under `[project.optional-dependencies]`.
+```shell
+    uv run ruff check
+```
 
-Any changes to the dependencies will be automatically reflected in
-`requirements.txt` with `pre-commit`, but you can re-generate
-the file without committing by executing:
+To run the tests and view coverage, execute:
+
+```shell
+    uv run pytest
+```
+
+## Documentation
 
-    pre-commit run pip-compile --all-files --config=.config/pre-commit-config.yaml
+The documentation, including API documentation, is generated using ReadtheDocs and
+MkDocs with Material. As you change the source code, remember to update the
+documentation at `documentation/index.md`.