Skip to content

Commit 3718f75

Browse files
guysoftguysoft
committed
Inital support for meta modules
1 parent 8e951d0 commit 3718f75

File tree

7 files changed

+156
-37
lines changed

7 files changed

+156
-37
lines changed

‎src/Dockerfile

+1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
1616
python3-distutils \
1717
python3-dev \
1818
python3-git \
19+
python3-yaml \
1920
binfmt-support \
2021
qemu-system \
2122
qemu-user \

‎src/execution_order.py

+107-33
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,41 @@
11
#a='base(octopi,a(b,c(a2)),mm)'
22
import argparse
33
import os
4+
import subprocess
45
from get_remote_modules import get_remote_module
6+
from typing import TextIO, List, Tuple, Dict, Any
7+
8+
9+
def run_command(command: List[str], **kwargs: Dict[str, Any]):
10+
is_timeout = False
11+
p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
12+
try:
13+
stdout, stderr = p.communicate(timeout=5)
14+
except subprocess.TimeoutExpired as e:
15+
p.kill()
16+
stdout,stderr = p.communicate()
17+
is_timeout = True
18+
try:
19+
stdout = stdout.decode("utf-8")
20+
except UnicodeDecodeError as e:
21+
print("Error: can't decode stdout")
22+
print(e)
23+
print(stdout)
24+
stdout = ""
25+
26+
try:
27+
stderr = stderr.decode("utf-8")
28+
except UnicodeDecodeError as e:
29+
print("Error: can't decode stderr")
30+
print(stderr)
31+
print(e)
32+
stderr = ""
33+
34+
return stdout, stderr, is_timeout
535

6-
def handle(module, state, out):
7-
out.write("# " + state + "_" + module + "\n")
8-
9-
module_folders = [os.path.join(os.environ['DIST_PATH'], "modules", module),
10-
os.path.join(os.environ['CUSTOM_PI_OS_PATH'], "modules", module)
11-
]
1236

13-
found_local = False
14-
found_remote = False
15-
for module_folder in module_folders:
16-
if os.path.isdir(module_folder):
17-
found_local = True
18-
break
19-
20-
if not found_local:
21-
# TODO: Handle update
22-
found_remote, module_folder = get_remote_module(module)
23-
24-
25-
if not found_local and not found_remote:
26-
print(f"Error: Module {module} does not exist and is not in remote modules list")
27-
exit(1)
28-
37+
def write_modules_scripts(module: str, state: str, module_folder: str, out: TextIO):
38+
out.write("# " + state + "_" + module + "\n")
2939
script = os.path.join(module_folder, state + "_chroot_script")
3040
if os.path.isfile(script):
3141
out.write("execute_chroot_script '" + module_folder + "' '" + script + "'\n")
@@ -34,38 +44,95 @@ def handle(module, state, out):
3444

3545
return
3646

37-
def parse(a, callback):
47+
def parse(a: str) -> List[Tuple[str,str]]:
3848
stack=[]
49+
return_value = []
3950
token = ""
4051

4152
for char in a:
4253
if char == "(":
4354
stack.append(token)
4455
if token != "":
45-
callback(token, "start")
56+
return_value.append((token, "start"))
4657
token = ""
4758
elif char == ")":
4859
parent = stack.pop()
4960
if token != "":
50-
callback(token, "start")
51-
callback(token, "end")
61+
return_value.append((token, "start"))
62+
return_value.append((token, "end"))
5263
token = ""
5364
if parent != "":
54-
callback(parent, "end")
65+
return_value.append((parent, "end"))
5566
elif char == ",":
5667
if token != "":
57-
callback(token, "start")
58-
callback(token, "end")
68+
return_value.append((token, "start"))
69+
return_value.append((token, "end"))
5970
token = ""
6071
else:
6172
token += char
6273

6374
if token != "":
64-
callback(token, "start")
65-
callback(token, "end")
75+
return_value.append((token, "start"))
76+
return_value.append((token, "end"))
6677
if len(stack) > 0:
6778
raise Exception(str(stack))
68-
return
79+
return return_value
80+
81+
def handle_meta_modules(modules: List[Tuple[str,str]]) -> Tuple[List[Tuple[str,str]],Dict[str,str]]:
82+
return_value = []
83+
modules_to_modules_folder = {}
84+
for module, state in modules:
85+
module_folders = [
86+
os.path.join(os.environ['DIST_PATH'], "modules", module),
87+
os.path.join(os.environ['CUSTOM_PI_OS_PATH'], "modules", module)
88+
]
89+
# In case this is a meta module, order counts
90+
if state == "start":
91+
return_value.append((module, state))
92+
found_local = False
93+
found_remote = False
94+
for module_folder in module_folders:
95+
if os.path.isdir(module_folder):
96+
found_local = True
97+
modules_to_modules_folder[module] = module_folder
98+
break
99+
100+
if not found_local:
101+
# TODO: Handle update
102+
found_remote, module_folder = get_remote_module(module)
103+
modules_to_modules_folder[module] = module_folder
104+
105+
106+
if not found_local and not found_remote:
107+
print(f"Error: Module {module} does not exist and is not in remote modules list")
108+
exit(1)
109+
110+
meta_module_path = os.path.join(module_folder, "meta")
111+
if os.path.isfile(meta_module_path):
112+
# Meta module detected
113+
print(f"Running: {meta_module_path}")
114+
print(f"ENV: {os.environ['BASE_BOARD']}")
115+
submodules, meta_module_errors, is_timeout = run_command(meta_module_path)
116+
submodules = submodules.strip()
117+
print(f"Adding in modules: {submodules}")
118+
if meta_module_errors != "" or is_timeout:
119+
print(meta_module_errors)
120+
print(f"Got error processing meta module at: {meta_module_path}")
121+
exit(1)
122+
if submodules != "":
123+
print(f"Got sub modules: {submodules}")
124+
125+
for sub_module in submodules.split(","):
126+
sub_module = sub_module.strip()
127+
return_value_sub, modules_to_modules_folder_sub = handle_meta_modules([(sub_module, state)])
128+
return_value += return_value_sub
129+
modules_to_modules_folder.update(modules_to_modules_folder_sub)
130+
# In case this is a meta module, order counts
131+
if state == "end":
132+
return_value.append((module, state))
133+
134+
return return_value, modules_to_modules_folder
135+
69136

70137
if __name__ == "__main__":
71138
parser = argparse.ArgumentParser(add_help=True, description='Parse and run CustomPiOS chroot modules')
@@ -80,7 +147,14 @@ def parse(a, callback):
80147
f.write("#!/usr/bin/env bash\n")
81148
f.write("set -x\n")
82149
f.write("set -e\n")
83-
parse(args.modules.replace(" ", ""), lambda module, state: handle(module, state, f))
150+
initial_execution_order = parse(args.modules.replace(" ", ""))
151+
f.write(f"# Defined execution order: {initial_execution_order}\n")
152+
modules_execution_order, modules_to_modules_folder = handle_meta_modules(initial_execution_order)
153+
f.write(f"# With meta modules order: {modules_execution_order}\n")
154+
155+
for module, state in modules_execution_order:
156+
module_folder = modules_to_modules_folder[module]
157+
write_modules_scripts(module, state, module_folder, f)
84158

85159
os.chmod(args.output_script, 0o755)
86160

‎src/get_remote_modules.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ def read_remotes():
3737

3838
def get_remote_module(module) -> Tuple[bool, Optional[str]]:
3939
""" Gets the remote module and saves it to cache. Returns True if found, else false"""
40-
print(f"INFO: Module {module}, looking for remote module and downloading")
40+
print(f'INFO: Module "{module}", looking for remote module and downloading')
4141
modules_remotes = read_remotes()
4242
print(modules_remotes.keys())
4343

@@ -46,7 +46,7 @@ def get_remote_module(module) -> Tuple[bool, Optional[str]]:
4646

4747
ensure_dir(REMOTES_DIR)
4848

49-
if "remotes" not in modules_remotes.keys() and module not in modules_remotes["modules"].keys():
49+
if "remotes" not in modules_remotes.keys() or module not in modules_remotes["modules"].keys():
5050
return False, None
5151

5252
module_config = modules_remotes["modules"][module]
@@ -56,11 +56,11 @@ def get_remote_module(module) -> Tuple[bool, Optional[str]]:
5656

5757
if remote_config.get("type", "git") == "git":
5858
if "repo" not in remote_config.keys():
59-
print(f"Error: repo field not set for remote: {remote_for_module} used by remote module {module}")
59+
print(f'Error: repo field not set for remote: "{remote_for_module}" used by remote module "{module}"')
6060
return False, None
6161

6262
if "tag" not in remote_config.keys():
63-
print(f"Error: repo tag field not set for remote: {remote_for_module} used by remote module {module}")
63+
print(f'Error: repo tag field not set for remote: "{remote_for_module}" used by remote module "{module}"')
6464
return False, None
6565

6666
repo_url = remote_config["repo"]

‎src/modules/base/config

+5
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@ BASE_VERSION=1.5.0
1515
# Distro
1616
[ -n "$BASE_DISTRO" ] || BASE_DISTRO=raspbian
1717

18+
# Board and OS
19+
[ -n "$BASE_BOARD" ] || BASE_BOARD="raspberrypi"
20+
[ -n "$BASE_OS" ] || BASE_OS="debian_bookworm"
21+
22+
1823
# Note: Set BASE_ZIP_IMG relative to the distro/src/workspace directory to pass a custom named file or an already extracted '.img'-file.
1924
if [ "${BASE_DISTRO}" == "ubuntu" ]; then
2025
# Default image ubuntu

‎src/modules/base/meta

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/usr/bin/env bash
2+
# Base script
3+
# Basic and manditory settings for the base of a CustomPiOS build
4+
# Written by Guy Sheffer <guysoft at gmail dot com>
5+
# GPL V3
6+
########
7+
set -e
8+
9+
export LC_ALL=C
10+
11+
FINAL_MODULES=()
12+
13+
if [ "${BASE_BOARD}" == "armbian" ]; then
14+
FINAL_MODULES+=("armbian")
15+
elif [ "${BASE_BOARD}" == "orange" ]; then
16+
FINAL_MODULES+=("orange")
17+
fi
18+
19+
printf '%s\n' "$(IFS=,; printf '%s' "${FINAL_MODULES[*]}")"

‎src/modules/network/meta

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/usr/bin/env bash
2+
# Net script
3+
# Basic and manditory settings for the network of a CustomPiOS build
4+
# Written by Guy Sheffer <guysoft at gmail dot com>
5+
# GPL V3
6+
########
7+
set -e
8+
9+
export LC_ALL=C
10+
11+
FINAL_MODULES=()
12+
13+
if [ "${BASE_BOARD}" == "armbian" ]; then
14+
FINAL_MODULES+=("armbian_net")
15+
elif [ "${BASE_BOARD}" == "orange" ]; then
16+
FINAL_MODULES+=("orange_net")
17+
fi
18+
19+
printf '%s\n' "$(IFS=,; printf '%s' "${FINAL_MODULES[*]}")"

‎src/variants/armbian/config

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
VARIANT_CONFIG_DIR=$(realpath -s $(dirname $(realpath -s $BASH_SOURCE))/../..)
22
BASE_ZIP_IMG=`ls -t ${DIST_PATH}/image-armbian/*.{zip,7z,xz} | head -n 1`
33
BASE_APT_CACHE=no
4+
BASE_BOARD=armbian
45
OCTOPI_INCLUDE_WIRINGPI=no
56
export BASE_DISTRO=armbian
67
# The root partition of the image filesystem, 2 for raspbian, 1 for armbian

0 commit comments

Comments
 (0)