Merge pull request #749 from Josverl/wf/createstubs

Update createstubs.yml and firmware flashing process

Author: Josverl

.github/actions/get-mpversions/action.yml

@@ -0,0 +1,29 @@
+name: "get-mpversions"
+description: "get the most recent 4 published versions of MicroPython"
+inputs:
+  max:
+    description: "Maximum number of versions to return"
+    required: false
+    default: "4"
+
+
+outputs:
+  versions:
+    description: "Return recent versions of MicroPython"
+    value: ${{ steps.dynamic.outputs.versions }}
+
+runs:
+    using: composite
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+          cache: pip
+
+      - run: pip install pygithub packaging
+        shell: bash
+
+      - run: python .github/actions/get-mpversions/list_versions.py --latest  --preview  --max ${{ inputs.max }}
+        shell: bash
+        id: dynamic
+

.github/actions/get-mpversions/list_versions.py

@@ -0,0 +1,68 @@
+"""
+This module retrieves the versions of Micropython from the Micropython repository on GitHub.
+It is used to generate a matrix of versions to create stubs for using Github Actions.
+
+It provides a function `micropython_versions` that returns a list of versions starting from a specified version.
+The module also includes a main block that generates a matrix of versions based on command-line arguments and environment variables.
+The matrix is printed as JSON and can be optionally written to a file if running in a GitHub Actions workflow.
+"""
+import argparse
+import json
+import os
+
+from github import Auth, Github
+from packaging.version import parse
+
+# Token with no permissions to avoid throttling
+# https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#getting-a-higher-rate-limit
+PAT_NO_ACCESS = (
+    "github_pat" + "_11AAHPVFQ0qAkDnSUaMKSp" + "_ZkDl5NRRwBsUN6EYg9ahp1Dvj4FDDONnXVgimxC2EtpY7Q7BUKBoQ0Jq72X"
+)
+PAT = os.environ.get("GITHUB_TOKEN") or PAT_NO_ACCESS
+
+def micropython_versions(start="v1.10"):
+    g = Github(auth=Auth.Token(PAT))
+    try:
+        repo = g.get_repo("micropython/micropython")
+        # Suppress `v1.22.0-preview` tags
+        tags = sorted([tag.name for tag in repo.get_tags() if "-preview" not in tag.name and parse(tag.name) >= parse(start)], reverse=True)
+    except Exception as e:
+        print(f"Error: {e}")
+        tags = ["stable"]
+    return tags
+
+
+def main():
+    matrix = {}
+    
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--stable", "--latest","-s", action=argparse.BooleanOptionalAction,default=True, help="Add latest version")
+    parser.add_argument("--preview", "-p", action=argparse.BooleanOptionalAction, default=False, help="Add preview version")
+    parser.add_argument("--max", "-m", type=int, default=3, help="Maximum number of versions")
+
+    args = parser.parse_args()
+
+    # only run latests when running in ACT locally for testing
+    if os.environ.get("ACT"):
+        matrix["version"] = micropython_versions(start="v1.20")[:1] # only latest
+    else:
+       matrix["version"] = micropython_versions(start="v1.20")[1:args.max] # last three
+
+
+    print(args)
+    if args.stable:
+        matrix["version"].insert(0, "stable")
+    if args.preview:
+        matrix["version"].insert(0, "preview")
+    matrix["version"]=matrix["version"][:args.max]
+    # GITHUB_OUTPUT is set by github actions
+    if os.getenv('GITHUB_OUTPUT'):
+        with open(os.getenv('GITHUB_OUTPUT'), 'a') as file:   #  type: ignore
+            file.write(f"versions={json.dumps(matrix)}")
+    else:
+        print(json.dumps(matrix, indent=4))
+
+# sourcery skip: assign-if-exp, merge-dict-assign
+if __name__ == "__main__":
+    main()
+

.github/not-active/action-test.yml

@@ -28,7 +28,7 @@ jobs:
 
       # make Python work
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
 

.github/not-active/get-all-frozen.yml

@@ -28,7 +28,7 @@ jobs:
         with:
           fetch-depth: 1
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11 
 
@@ -61,7 +61,7 @@ jobs:
 
       # make Python work
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
 
@@ -108,7 +108,7 @@ jobs:
 
   #     # make Python work
   #     - name: Set up Python
-  #       uses: actions/setup-python@v4
+  #       uses: actions/setup-python@v5
   #       with:
   #         python-version: 3.9
   #         cache: "pip"

.github/not-active/get-doc-stubs.yml

@@ -52,7 +52,7 @@ jobs:
 
       # make Python work
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
           cache: "pip"

.github/not-active/get-loboris-frozen.yml

@@ -31,7 +31,7 @@ jobs:
 
       # make Python work
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
 

.github/not-active/testspace.yml

@@ -23,7 +23,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Python 3.x
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
           cache: "pip"

.github/runner/selfhosted.md

@@ -3,20 +3,84 @@ name: Createstubs
 on: [push, workflow_dispatch]
 
 jobs:
-  build:
-    runs-on: self-hosted
+########################################################################################################################################
+  list-versions-cs:
+    # 'Get a matrix for the latest versions of micropython'
+    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-          python-version: 3.12
-          # cache: pip
-        
-    - name: Install dependencies
-      run: |
-        pip install mpremote
-      
-    - name: list connected devices
-      run: |
-        mpremote devs
+        - uses: actions/checkout@v4
+          with:
+            fetch-depth: 1
+        - uses: ./.github/actions/get-mpversions
+          id: dynamic
+    outputs:
+        versions: ${{ steps.dynamic.outputs.versions }}
+########################################################################################################################################
+  createstubs:
+    needs: list-versions-cs
+    runs-on: [self-hosted, mpremote, ARM64]
+    strategy:
+      matrix: ${{ fromJSON(needs.list-versions-cs.outputs.versions) }}
+      max-parallel: 1
+    #   matrix:
+    #     version: [preview, v1.22.2]
+    steps:
+      - name: Checkout stubs repo
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+      # if: ${{ matrix.os == 'ubuntu-arm' }}
+        uses: deadsnakes/[email protected]
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          pipx install poetry
+          pip install -U mpremote micropython-stubber
+
+      - name: Install mp_downloader
+        run: |
+          # dev versie 
+          pip install -I git+https://github.com/josverl/micropython-stubber.git@action_board_stubber#subdirectory=src/mptool
+
+      - uses: actions/cache@v4
+        with:
+          path: firmware
+          key: ${{ runner.os }}-firmware-${{ matrix.version }}-${{ hashFiles('firmware/firmware.json') }}
+          restore-keys: |
+            ${{ runner.os }}-firmware-${{ matrix.version }}
+            ${{ runner.os }}-firmware-
+
+
+      - name: Get ${{ matrix.version }} firmwares for flashing
+        run: |
+          mptool download --version ${{ matrix.version }}
+
+      - name: Update firmware to ${{ matrix.version }} on all connected boards
+        # cant update stm32 as that requires sudo :-( 
+        run: |
+          mptool flash --version ${{ matrix.version }}
+
+      - name: clone Repos
+        run: |
+          stubber clone
+
+      - name: run board stubber
+        run: |
+          stubber get-mcu-stubs
+
+      - name: check in changes
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git pull
+          git add .
+          git commit -m "create boardstubs"
+          git push
+
+      - name: Build App
+        if: always()
+        run: echo "Done."
+
+

.github/workflows/createstubs.yml

@@ -13,14 +13,6 @@
 from github import Github
 from packaging.version import parse
 
-...
-import json
-import os
-import sys
-
-from github import Github
-from packaging.version import parse
-
 
 def micropython_versions(start="v1.10"):
     g = Github()
@@ -33,22 +25,28 @@ def micropython_versions(start="v1.10"):
         tags = ["v1.19.1"]
     return tags
 
-# sourcery skip: assign-if-exp, merge-dict-assign
-if __name__ == "__main__":
+
+def main():
     matrix = {}
     # only run latests when running in ACT locally for testing
     if os.environ.get("ACT"):
-        matrix["version"] = micropython_versions(start="v1.20")[-1:]
+        matrix["version"] = micropython_versions(start="v1.20")[:1] # only latest
     else:
-        matrix["version"] = micropython_versions(start="v1.20")
+       matrix["version"] = micropython_versions(start="v1.20")[:3] # last three
 
     add_latest = False
     if len(sys.argv) > 1 and (sys.argv[1].lower() in ["--latest", "-l"]):
-        print("Adding latest")
+        # print("Adding latest")
         matrix["version"].insert(0, "latest")
 
-    print(json.dumps(matrix))
     # GITHUB_OUTPUT is set by github actions
     if os.getenv('GITHUB_OUTPUT'):
         with open(os.getenv('GITHUB_OUTPUT'), 'a') as file:   #  type: ignore
             file.write(f"versions={json.dumps(matrix)}")
+    else:
+        print(json.dumps(matrix))
+
+# sourcery skip: assign-if-exp, merge-dict-assign
+if __name__ == "__main__":
+    main()
+