Add base board config generation from yaml, refactor and add a common.py and typing #214

Author: guysoft

src/base_image_downloader.py

@@ -7,12 +7,13 @@
 import hashlib
 import shutil
 import re
+from typing import Dict, Any, Optional, cast
+from common import get_image_config, read_images
 PRECENT_PROGRESS_SIZE = 5
 
 class ChecksumFailException(Exception):
     pass
 
-IMAGES_CONFIG = os.path.join(os.path.dirname(__file__), "images.yml")
 RETRY = 3
 
 def ensure_dir(d, chmod=0o777):
@@ -26,13 +27,6 @@ def ensure_dir(d, chmod=0o777):
         return False
     return True
 
-def read_images():
-    if not os.path.isfile(IMAGES_CONFIG):
-        raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}")
-    with open(IMAGES_CONFIG,'r') as f:
-        output = yaml.safe_load(f)
-        return output
-
 class DownloadProgress:
     last_precent: float = 0
     def show_progress(self, block_num, block_size, total_size):
@@ -53,7 +47,7 @@ def get_sha256(filename):
         return file_checksum
     return
 
-def download_image_http(board: str, dest_folder: str, redownload: bool = False):
+def download_image_http(board: Dict[str, Any], dest_folder: str, redownload: bool = False):
     url = board["url"]
     checksum = board["checksum"]
 
@@ -118,14 +112,21 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
     base_board = os.environ.get("BASE_BOARD", None)
     base_image_path = os.environ.get("BASE_IMAGE_PATH", None)
 
-    if base_board is not None and base_board in images["images"]:
-        if images["images"][base_board]["type"] == "http":
-            download_image_http(images["images"][base_board], base_image_path)
-        elif images["images"][base_board]["type"] == "torrent":
+    if base_image_path is None:
+        print(f'Error: did not find image config file')
+        exit(1)
+    cast(str, base_image_path)
+
+    image_config = get_image_config()
+    if image_config is not None:
+        if image_config["type"] == "http":
+            download_image_http(image_config, base_image_path)
+        elif image_config["type"] == "torrent":
             print("Error: Torrent not implemented")
             exit(1)
         else:
-            print("Error: Unsupported image download type")
+            print(f'Error: Unsupported image download type: {image_config["type"]}')
             exit(1)
+
 
     print("Done")

src/build

@@ -10,12 +10,16 @@ define(){ IFS='\n' read -r -d '' ${1} || true; }
 
 define SCRIPT <<'EOF'
 BUILD_SCRIPT_PATH=$(dirname $(realpath -s $BASH_SOURCE))
+export EXTRA_BAORD_CONFIG=$(mktemp)
+${BUILD_SCRIPT_PATH}/generate_board_config.py "${EXTRA_BAORD_CONFIG}"
+echo "Temp source file: ${EXTRA_BAORD_CONFIG}"
+
 source ${BUILD_SCRIPT_PATH}/common.sh
 install_cleanup_trap
 
 CUSTOM_OS_PATH=$(dirname $(realpath -s $0))
 
-source ${CUSTOM_PI_OS_PATH}/config ${@}
+source ${CUSTOM_PI_OS_PATH}/config ${1} "${EXTRA_BAORD_CONFIG}" ${@}
 ${CUSTOM_PI_OS_PATH}/config_sanity
 
 [ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios ${@}

src/common.py

@@ -0,0 +1,23 @@
+""" Common functions between CustomPiOS python scripts"""
+from typing import Dict, Any, Optional, cast
+import yaml
+import os
+
+IMAGES_CONFIG = os.path.join(os.path.dirname(__file__), "images.yml")
+
+def read_images():
+    if not os.path.isfile(IMAGES_CONFIG):
+        raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}")
+    with open(IMAGES_CONFIG,'r') as f:
+        output = yaml.safe_load(f)
+        return output
+
+def get_image_config() -> Optional[Dict["str", Any]]:
+    images = read_images()
+
+    base_board = os.environ.get("BASE_BOARD", None)
+    base_image_path = os.environ.get("BASE_IMAGE_PATH", None)
+
+    if base_board is not None and base_board in images["images"]:
+        return images["images"][base_board]
+    return None

src/config

@@ -7,6 +7,8 @@ export BUILD_VARIANT=""
 BUILD_VARIANT="$1"
 : ${BUILD_VARIANT:=default}
 
+EXTRA_BAORD_CONFIG=$2
+
 export BUILD_FLAVOR=""
 # Disable flavor system
 #BUILD_FLAVOR="$1"
@@ -86,6 +88,13 @@ MODULES_LIST="${TMP//)/,}"
 # [ -n "$BASE_CHROOT_SCRIPT_PATH" ] || BASE_CHROOT_SCRIPT_PATH=$BASE_SCRIPT_PATH/chroot_script
 [ -n "$BASE_MOUNT_PATH" ] || BASE_MOUNT_PATH=$BASE_WORKSPACE/mount
 
+# Import remote and submodules config
+if [ -f "${EXTRA_BAORD_CONFIG}" ]; then
+  source "${EXTRA_BAORD_CONFIG}"
+else
+  echo "Note: Not sourceing board config"
+fi
+
 export REMOTE_AND_META_CONFIG="$BASE_WORKSPACE"/remote_and_meta_config
 # Remote modules and meta modulese go in first if they want to change standard behaviour
 if [ -f "${REMOTE_AND_META_CONFIG}" ]; then

src/custompios

@@ -108,6 +108,11 @@ fi
 mkdir -p $BASE_WORKSPACE
 mkdir -p $BASE_MOUNT_PATH
 
+# This is already genrated at "build" sourced in "config", but copying here mostly for debug
+if [ -f "${EXTRA_BAORD_CONFIG}" ]; then
+  mv -v "${EXTRA_BAORD_CONFIG}" "${BASE_WORKSPACE}"/extra_board_config
+fi
+
 # Clean exported artifacts from other builds
 rm -rf "${BASE_WORKSPACE}"/*.tar.gz
 

src/requirements-devel.txt

@@ -0,0 +1 @@
+types-PyYAML