Merge pull request #25 from whi-tw/download_via_torrent

download via torrent (or, rather, don't)
whi-tw · Mar 14, 2023 · 4a2cb11 · 4a2cb11
2 parents ac0871a + 1af1ca5
commit 4a2cb11
Show file tree

Hide file tree

Showing 16 changed files with 547 additions and 49 deletions.
diff --git a/.github/workflows/build-kernel-modules.yml b/.github/workflows/build-kernel-modules.yml
@@ -15,6 +15,10 @@ on:
       - .github/workflows/build-kernel-modules.yml
   workflow_dispatch:
 
+env:
+  APT_DEPS: aria2
+  DOWNLOAD_FROM_TORRENT: false
+
 jobs:
   prepare-build:
     runs-on: ubuntu-22.04
@@ -25,22 +29,40 @@ jobs:
       python: ${{ steps.tool-versions.outputs.python }}
     steps:
       - uses: actions/checkout@v3
+
       - name: Read .tool-versions
         uses: marocchino/tool-versions-action@v1
         id: tool-versions
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ steps.tool-versions.outputs.python }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: latest
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.tool-versions.outputs.python }}-${{ hashFiles('**/poetry.lock') }}
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+
       - name: 🏗️ Build matrix
         id: matrix
-        run: python3 ci/generate_build_matrix.py >> $GITHUB_OUTPUT
+        run: poetry run python3 ci/generate_build_matrix.py
       - name: 🏗️ Collect crosscompiler packages
         id: crosscompilers
-        run: python3 ci/get_all_crosscompilers.py >> $GITHUB_OUTPUT
+        run: poetry run python3 ci/get_all_crosscompilers.py
       - name: Get and cache apt dependencies
         uses: awalsh128/cache-apt-pkgs-action@v1
         with:
-          packages: ${{ steps.crosscompilers.outputs.ALL_CROSSCOMPILERS }}
+          packages: ${{ steps.crosscompilers.outputs.ALL_CROSSCOMPILERS }} ${{ env.APT_DEPS }}
           version: 1.0
       - name: Get version
         id: version
@@ -56,28 +78,66 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ needs.prepare-build.outputs.python }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: latest
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ needs.prepare-build.outputs.python }}-${{ hashFiles('**/poetry.lock') }}
+
       - name: Extract device metadata
         id: get-device-metadata
-        run: python3 ci/get_device_metadata.py ${{ matrix.device }} >> $GITHUB_OUTPUT
+        run: poetry run python3 ci/get_device_metadata.py ${{ matrix.device }}
+
       - name: Extract firmware metadata
         id: get-firmware-metadata
         run: |
-          python3 ci/get_firmware_metadata.py ${{ matrix.device }} ${{ matrix.firmware }} >> $GITHUB_OUTPUT
-          FIRMWARE_DIRECTORY='devices/${{ matrix.device }}/${{ matrix.firmware }}'
-          echo "firmware-directory=${FIRMWARE_DIRECTORY}" >> $GITHUB_OUTPUT
+          poetry run python3 ci/get_firmware_metadata.py ${{ matrix.device }} ${{ matrix.firmware }}
+
+      - name: Read archive info from file
+        id: archive-data
+        run: poetry run python3 ci/parse_archive_info.py ${{ fromJSON(steps.get-firmware-metadata.outputs.json).kernel-archive-path }}
+
+      - name: Restore cached apt dependencies
+        uses: awalsh128/cache-apt-pkgs-action@v1
+        with:
+          packages: ${{ fromJSON(steps.get-device-metadata.outputs.json).cross-compiler-package }} ${{ env.APT_DEPS }}
+          version: 1.0
+
       - name: Restore kernel archive from cache
         id: cache-kernel-archive
         uses: actions/cache@v3
         with:
           path: ./linux-unifios.tar.gz
-          key: ${{ fromJSON(steps.get-firmware-metadata.outputs.json).kernel-url-base64 }}
-      - name: Download kernel archive if required
-        if: steps.cache-kernel-archive.outputs.cache-hit != 'true'
+          key: ${{ fromJSON(steps.get-firmware-metadata.outputs.json).kernel-sha256sum }}
+
+      - name: Download kernel archive if required (http)
+        if: steps.cache-kernel-archive.outputs.cache-hit != 'true' && env.DOWNLOAD_FROM_TORRENT != 'true'
         run: |
-          curl -Lo ./linux-unifios.tar.gz "${{ fromJSON(steps.get-firmware-metadata.outputs.json).kernel-url }}"
+          curl -Lo ./linux-unifios.tar.gz "${{ steps.archive-data.outputs.firmware-http-url }}"
+
+      - name: Download kernel archive if required (torrent)
+        if: steps.cache-kernel-archive.outputs.cache-hit != 'true' &&  env.DOWNLOAD_FROM_TORRENT == 'true'
+        run: |
+          wget -O archive.torrent "${{ steps.archive-data.outputs.archive-torrent-url }}"
+          FILE_INDEX="$(aria2c -d . -S archive.torrent \
+            | grep '${{ fromJSON(steps.get-firmware-metadata.outputs.json).kernel-archive-path }}$' \
+            | cut -d '|' -f1)"
+          aria2c -d . --seed-time=0 --select-file "${FILE_INDEX}" \
+            --bt-stop-timeout=10 --index-out="${FILE_INDEX}=./linux-unifios.tar.gz" \
+            archive.torrent || true
+
       - name: Ensure kernel SHA matches
         run: echo '${{ fromJSON(steps.get-firmware-metadata.outputs.json).kernel-sha256sum }}  linux-unifios.tar.gz' | sha256sum -c
       - name: Extract Kernel
@@ -91,11 +151,6 @@ jobs:
           sed -i "s@%%VERSION%%@${{ needs.prepare-build.outputs.version }}@" ${{ steps.get-firmware-metadata.outputs.firmware-directory }}/patches/*.patch
       - name: Apply firmware-specific patches
         run: find ${{ steps.get-firmware-metadata.outputs.firmware-directory }}/patches -type f -name '*.patch' -print0 | sort -z | xargs -t -0 -n 1 patch -p0 -i
-      - name: Restore cached apt dependencies
-        uses: awalsh128/cache-apt-pkgs-action@v1
-        with:
-          packages: ${{ fromJSON(steps.get-device-metadata.outputs.json).cross-compiler-package }}
-          version: 1.0
       - name: Prepare for building
         run: |
           cd linux-source
@@ -121,13 +176,31 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ needs.prepare-build.outputs.python }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: latest
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ needs.prepare-build.outputs.python }}-${{ hashFiles('**/poetry.lock') }}
+
       - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # 3.0.2
         id: download
         with:
           path: built_modules
       - name: Generate release text files
         run: |
-          python3 ci/build_release_text_files.py v${{ needs.prepare-build.outputs.version }} ${{steps.download.outputs.download-path}}
+          poetry run python3 ci/build_release_text_files.py v${{ needs.prepare-build.outputs.version }} ${{steps.download.outputs.download-path}}
       - name: Create release
         uses: ncipollo/release-action@a2e71bdd4e7dab70ca26a852f29600c98b33153e # 1.12.0
         with:

diff --git a/.gitignore b/.gitignore
@@ -3,3 +3,5 @@ text_files_out/
 linux-udr.tar.gz
 linux-source
 built_modules
+/aria_dir
+archive.torrent
diff --git a/.schemas/device_metadata.schema.json b/.schemas/device_metadata.schema.json
@@ -20,5 +20,10 @@
       "type": "string",
       "pattern": "^[0-9a-z-]+$"
     }
-  }
+  },
+  "required": [
+    "architecture",
+    "cross-compiler",
+    "cross-compiler-package"
+  ]
 }
diff --git a/.schemas/firmware_metadata.schema.json b/.schemas/firmware_metadata.schema.json
@@ -5,15 +5,19 @@
   "description": "Metadata for a specific firmware version",
   "type": "object",
   "properties": {
-    "kernel-url": {
-      "description": "URL to the kernel image",
+    "kernel-archive-path": {
+      "description": "Path to the kernel image within the firmware archive",
       "type": "string",
-      "format": "uri"
+      "pattern": "^([a-zA-Z-]+)\/linux-[a-zA-Z-]+-([0-9.]+).tar.gz$"
     },
     "kernel-sha256sum": {
       "description": "SHA256 hash of the kernel image",
       "type": "string",
       "pattern": "^[0-9a-f]{64}$"
     }
-  }
+  },
+  "required": [
+    "kernel-archive-path",
+    "kernel-sha256sum"
+  ]
 }
diff --git a/ci/generate_build_matrix.py b/ci/generate_build_matrix.py
@@ -37,4 +37,6 @@
         )
 
 matrix = {"include": matrix_combinations}
-print(f"matrix={json.dumps(matrix)}")
+
+with open(os.environ.get("GITHUB_OUTPUT", False), "a") as f:
+    f.write(f"matrix={json.dumps(matrix)}\n")
diff --git a/ci/get_all_crosscompilers.py b/ci/get_all_crosscompilers.py
@@ -29,4 +29,5 @@
         )
 
 
-print(f'ALL_CROSSCOMPILERS={",".join(crosscompiler_packages)}')
+with open(os.environ.get("GITHUB_OUTPUT", False), "a") as f:
+    f.write(f'ALL_CROSSCOMPILERS={",".join(crosscompiler_packages)}\n')
diff --git a/ci/get_device_metadata.py b/ci/get_device_metadata.py
@@ -4,21 +4,27 @@
 import os
 import sys
 
+import jsonschema
 
-def check_attr_in_device_metadata(device, device_metadata, attr):
-    if attr not in device_metadata or not device_metadata[attr]:
+
+def validate_against_schema(data, device):
+    with open("./.schemas/device_metadata.schema.json") as f:
+        schema = json.load(f)
+    try:
+        jsonschema.validate(instance=data, schema=schema)
+    except jsonschema.exceptions.ValidationError as e:
         raise Exception(
-            f"Device {device} has no {attr} specified in device_metadata.json"
-        )
+            f"Invalid device_metadata.json for {device}: {e.message}"
+        ) from e
 
 
 device = sys.argv[1]
 device_dir_path = os.path.abspath(os.path.join("./devices", device))
 
 with open(os.path.join(device_dir_path, "device_metadata.json")) as f:
     device_metadata = json.load(f)
-    check_attr_in_device_metadata(device, device_metadata, "architecture")
-    check_attr_in_device_metadata(device, device_metadata, "cross-compiler")
-    check_attr_in_device_metadata(device, device_metadata, "cross-compiler-package")
+    validate_against_schema(device_metadata, device)
+    device_metadata.pop("$schema", None)
 
-print(f"json={json.dumps(device_metadata)}")
+with open(os.environ.get("GITHUB_OUTPUT", False), "a") as f:
+    f.write(f"json={json.dumps(device_metadata)}\n")
diff --git a/ci/get_firmware_metadata.py b/ci/get_firmware_metadata.py
@@ -5,28 +5,38 @@
 import os
 import sys
 
+import jsonschema
 
-def check_attr_in_firmware_metadata(device, firmware_metadata, attr):
-    if attr not in firmware_metadata or not firmware_metadata[attr]:
+
+def validate_against_schema(data, device, firmware_version):
+    with open("./.schemas/firmware_metadata.schema.json") as f:
+        schema = json.load(f)
+    try:
+        jsonschema.validate(instance=data, schema=schema)
+    except jsonschema.exceptions.ValidationError as e:
         raise Exception(
-            f"Firmware {firmware_version} for device {device} "
-            "has no {attr} specified in firmware_metadata.json"
-        )
+            f"Invalid firmware_metadata.json for {device}/{firmware_version}: "
+            f"{e.message}"
+        ) from e
 
 
-device = sys.argv[1]
-firmware_version = sys.argv[2]
+try:
+    device = sys.argv[1]
+    firmware_version = sys.argv[2]
+except IndexError as e:
+    raise Exception("No device / firmware version specified.") from e
 
 device_dir_path = os.path.abspath(os.path.join("./devices", device))
 firmware_dir_path = os.path.abspath(os.path.join(device_dir_path, firmware_version))
 
 with open(os.path.join(firmware_dir_path, "firmware_metadata.json")) as f:
     firmware_metadata = json.load(f)
-    check_attr_in_firmware_metadata(device, firmware_metadata, "kernel-url")
-    check_attr_in_firmware_metadata(device, firmware_metadata, "kernel-sha256sum")
-    firmware_metadata["kernel-url-base64"] = base64.b64encode(
-        firmware_metadata["kernel-url"].encode("utf-8")
+    validate_against_schema(firmware_metadata, device, firmware_version)
+    firmware_metadata["kernel-archive-path-base64"] = base64.b64encode(
+        firmware_metadata["kernel-archive-path"].encode("utf-8")
     ).decode("utf-8")
+    firmware_metadata.pop("$schema", None)
 
-
-print(f"json={json.dumps(firmware_metadata)}")
+with open(os.environ.get("GITHUB_OUTPUT", False), "a") as f:
+    f.write(f"json={json.dumps(firmware_metadata)}\n")
+    f.write(f"firmware-directory={firmware_dir_path}\n")
diff --git a/ci/parse_archive_info.py b/ci/parse_archive_info.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import sys
+
+DATA_FILE = "./gpl-dump-archive.json"
+
+try:
+    file_path = sys.argv[1]
+except IndexError:
+    file_path = None
+
+try:
+    with open(DATA_FILE) as f:
+        data = json.load(f)
+except FileNotFoundError as e:
+    raise FileNotFoundError(f"data file {DATA_FILE} not found") from e
+
+archive_name = data["name"]
+archive_download_baseurl = f"https://archive.org/download/{archive_name}"
+archive_torrent_url = f"{archive_download_baseurl}/{archive_name}_archive.torrent"
+archive_file_data_url = f"{archive_download_baseurl}/{archive_name}_files.xml"
+
+with open(os.environ.get("GITHUB_OUTPUT", False), "a") as f:
+    f.write(f"archive-name={archive_name}\n")
+    f.write(f"archive-download-baseurl={archive_download_baseurl}\n")
+    f.write(f"archive-torrent-url={archive_torrent_url}\n")
+    f.write(f"archive-file-data-url={archive_file_data_url}\n")
+
+    if file_path:
+        f.write(f"firmware-http-url={archive_download_baseurl}/{file_path}\n")
diff --git a/devices/UDM/2.4.27/firmware_metadata.json b/devices/UDM/2.4.27/firmware_metadata.json
@@ -1,5 +1,5 @@
 {
     "$schema": "../../../.schemas/firmware_metadata.schema.json",
-    "kernel-url": "https://archive.org/download/unifi-udr-gpl-archives/UDM/linux-udm-2.4.27.tar.gz",
+    "kernel-archive-path": "UDM/linux-udm-2.4.27.tar.gz",
     "kernel-sha256sum": "9d1a90640f3a302817466e42bf4232bc4965641db1d442438e59d5086d91349e"
 }
diff --git a/devices/UDR/2.2.12/firmware_metadata.json b/devices/UDR/2.2.12/firmware_metadata.json
@@ -1,5 +1,5 @@
 {
     "$schema": "../../../.schemas/firmware_metadata.schema.json",
-    "kernel-url": "https://archive.org/download/unifi-udr-gpl-archives/UDR/linux-udr-2.2.12.tar.gz",
+    "kernel-archive-path": "UDR/linux-udr-2.2.12.tar.gz",
     "kernel-sha256sum": "d232dec7c148a44b74e1ddfd6d336a32d4a228ee70b3dba586c3a7e43d9f31b7"
 }
diff --git a/devices/UDR/3.0.13/firmware_metadata.json b/devices/UDR/3.0.13/firmware_metadata.json
@@ -1,5 +1,5 @@
 {
     "$schema": "../../../.schemas/firmware_metadata.schema.json",
-    "kernel-url": "https://archive.org/download/unifi-udr-gpl-archives/UDR/linux-udr-3.0.13.tar.gz",
+    "kernel-archive-path": "UDR/linux-udr-3.0.13.tar.gz",
     "kernel-sha256sum": "b28c3737306ae0172d2b99fd5c9af881f0c14367e453481aaa0129a7883301fc"
 }
diff --git a/gpl-dump-archive.json b/gpl-dump-archive.json
@@ -0,0 +1,3 @@
+{
+    "name": "unifi-udr-gpl-archives"
+}
diff --git a/macvlan-unifios-version b/macvlan-unifios-version
@@ -1 +1 @@
-1.5.0
+1.4.1