Remote module support #214

Author: guysoft

.github/workflows/docker-build.yml

@@ -6,6 +6,9 @@ on:
       - "master"
       - "devel"
       - "docker-github-actions"
+      - "feature/remote-module-support"
+      - "feature/meta-modules"
+      - "beta"
 
 jobs:
   docker:

README.rst

@@ -63,6 +63,7 @@ Requirements
 #. sudo (the script itself calls it, running as root without sudo won't work)
 #. p7zip-full
 #. Python 3.2+
+#. GitPython
 
 Known to work building configurations
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/Dockerfile

@@ -13,6 +13,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         wget \
         p7zip-full \
         python3 \
+        python3-distutils \
+        python3-dev \
+        python3-git \
         binfmt-support \
         qemu-system \
         qemu-user \

src/execution_order.py

@@ -1,6 +1,7 @@
 #a='base(octopi,a(b,c(a2)),mm)'
 import argparse
 import os
+from get_remote_modules import get_remote_module
 
 def handle(module, state, out):
     out.write("# " + state + "_" + module + "\n")
@@ -9,14 +10,28 @@ def handle(module, state, out):
                       os.path.join(os.environ['CUSTOM_PI_OS_PATH'], "modules", module)
         ]
 
+    found_local = False
+    found_remote = False
     for module_folder in module_folders:
         if os.path.isdir(module_folder):
-            script = os.path.join(module_folder, state + "_chroot_script")
-            if os.path.isfile(script):
-                out.write("execute_chroot_script '" + module_folder + "' '" + script + "'\n")
-            else:
-                print("WARNING: No file at - " + script)
+            found_local = True
             break
+        
+    if not found_local:
+        # TODO: Handle update
+        found_remote, module_folder = get_remote_module(module)
+        
+        
+    if not found_local and not found_remote:
+        print(f"Error: Module {module} does not exist and is not in remote modules list")
+        exit(1)
+            
+    script = os.path.join(module_folder, state + "_chroot_script")
+    if os.path.isfile(script):
+        out.write("execute_chroot_script '" + module_folder + "' '" + script + "'\n")
+    else:
+        print("WARNING: No file at - " + script)
+    
     return
 
 def parse(a, callback):

src/get_remote_modules.py

@@ -0,0 +1,90 @@
+import os
+import yaml
+from pathlib import Path
+from typing import Tuple, Optional
+import git
+from git import RemoteProgress
+            
+# TODO add env var to set this
+REMOTES_DIR = os.path.join(os.path.dirname(__file__), "remotes")
+REMOTE_CONFIG = os.path.join(os.path.dirname(__file__), "modules_remote.yml")
+
+
+class CloneProgress(RemoteProgress):
+    def update(self, op_code, cur_count, max_count=None, message=''):
+        if message:
+            print(message)
+
+
+def ensure_dir(d, chmod=0o777):
+    """
+    Ensures a folder exists.
+    Returns True if the folder already exists
+    """
+    if not os.path.exists(d):
+        os.makedirs(d, chmod)
+        os.chmod(d, chmod)
+        return False
+    return True
+
+
+def read_remotes():
+    if not os.path.isfile(REMOTE_CONFIG):
+        raise Exception(f"Error: Remotes config file not found: {REMOTE_CONFIG}")
+    with open(REMOTE_CONFIG,'r') as f:
+        output = yaml.safe_load(f)
+        return output
+
+def get_remote_module(module: str) -> Tuple[bool, Optional[str]]:
+    """ Gets the remote module and saves it to cache. Returns True if found, else false"""
+    print(f"INFO: Module {module}, looking for remote module and downloading")
+    modules_remotes = read_remotes()
+    print(modules_remotes.keys())
+    
+    if "modules" not in modules_remotes.keys() and module not in modules_remotes["modules"].keys():
+        return False, None
+    
+    ensure_dir(REMOTES_DIR)
+    
+    if "remotes" not in modules_remotes.keys() and module not in modules_remotes["modules"].keys():
+        return False, None
+    
+    module_config = modules_remotes["modules"][module]
+    
+    remote_for_module = module_config["remote"]
+    remote_config = modules_remotes["remotes"][remote_for_module]
+    
+    if remote_config.get("type", "git") == "git":
+        if "repo" not in remote_config.keys():
+            print(f"Error: repo field not set for remote: {remote_for_module} used by remote module {module}")
+            return False, None
+        
+        if "tag" not in remote_config.keys():
+            print(f"Error: repo tag field not set for remote: {remote_for_module} used by remote module {module}")
+            return False, None
+            
+        repo_url = remote_config["repo"]
+        branch = remote_config["tag"]
+        
+        # credentials = base64.b64encode(f"{GHE_TOKEN}:".encode("latin-1")).decode("latin-1")
+        # TODO: Handle update of remote
+        remote_to_path = os.path.join(REMOTES_DIR, remote_for_module)
+        if not os.path.exists(remote_to_path):
+            git.Repo.clone_from(
+                url=repo_url,
+                single_branch=True,
+                depth=1,
+                to_path=f"{remote_to_path}",
+                branch=branch,
+            )
+            
+        if "path" not in module_config.keys():
+            print(f"Error: repo tag field not set for remote: {remote_for_module} used by remote module {module}")
+            return False, None
+        module_path = os.path.join(remote_to_path, module_config["path"])
+        return True, module_path
+
+    else:
+        print(f"Error: unsupported type {modules_remotes[module]['type']} for module {module}")
+        return False, None
+    return False, None

src/make_rpi-imager-snipplet.py

@@ -6,6 +6,7 @@
 import argparse
 from datetime import date
 import glob
+from typing import Optional, Dict, Union
 
 
 
@@ -44,12 +45,13 @@ def handle_arg(key, optional=False):
 
     output_path = os.path.join(workspace_path, "rpi-imager-snipplet.json")
 
-    json_out = {"name": name,
-                "description": description,
-                "url": url,
-                "icon": icon,
-                "release_date": release_date,
-                }
+    json_out: Dict[str, Optional[Union[str, int]]] = {
+        "name": name,
+        "description": description,
+        "url": url,
+        "icon": icon,
+        "release_date": release_date,
+        }
 
     if website is not None:
         json_out["website"] = website
@@ -60,14 +62,14 @@ def handle_arg(key, optional=False):
         json_out["extract_sha256"] = f.read().split()[0]
 
     json_out["extract_size"] = None
-    with zipfile.ZipFile(zip_local) as zipfile:
-        json_out["extract_size"] = zipfile.filelist[0].file_size
+    with zipfile.ZipFile(zip_local) as zip_file:
+        json_out["extract_size"] = zip_file.filelist[0].file_size
 
     json_out["image_download_size"] = os.stat(zip_local).st_size
 
     json_out["image_download_sha256"] = None
-    with open(zip_local,"rb") as f:
-        json_out["image_download_sha256"] = hashlib.sha256(f.read()).hexdigest()
+    with open(zip_local,"rb") as fh:
+        json_out["image_download_sha256"] = hashlib.sha256(fh.read()).hexdigest()
 
     with open(output_path, "w") as w:
         json.dump(json_out, w, indent=2)

src/modules_remote.yml

@@ -0,0 +1,66 @@
+modules:
+    armbian:
+      remote: MainsailOS
+      path: src/modules/armbian
+    
+    armbian_net:
+      remote: MainsailOS
+      path: src/modules/armbian_net
+
+    crowsnest:
+      remote: MainsailOS
+      path: src/modules/crowsnest
+
+    is_req_preinstall:
+      remote: MainsailOS
+      path: src/modules/is_req_preinstall
+
+    klipper:
+      remote: MainsailOS
+      path: src/modules/klipper
+
+    mainsail:
+      remote: MainsailOS
+      path: src/modules/mainsail
+
+    mainsailos:
+      remote: MainsailOS
+      path: src/modules/mainsailos
+
+    moonraker:
+      remote: MainsailOS
+      path: src/modules/moonraker
+
+    orangepi:
+      remote: MainsailOS
+      path: src/modules/orangepi
+
+    orangepi_net:
+      remote: MainsailOS
+      path: src/modules/orangepi_net
+
+    piconfig:
+      remote: MainsailOS
+      path: src/modules/piconfig
+
+    postrename:
+        remote: MainsailOS
+        path: src/modules/postrename
+
+    sonar:
+      remote: MainsailOS
+      path: src/modules/sonar
+
+    timelapse:
+      remote: MainsailOS
+      path: src/modules/timelapse
+
+    udev_fix:
+      remote: MainsailOS
+      path: src/modules/udev_fix
+
+remotes:
+  MainsailOS:
+    repo: https://github.com/mainsail-crew/MainsailOS.git
+    type: git
+    tag: develop

src/requirements.txt

@@ -0,0 +1 @@
+GitPython