Skip to content

Commit c2539a5

Browse files
authored
Install all tools and toolchain from github via tl-install tool
1 parent 327b40e commit c2539a5

File tree

4 files changed

+222
-33
lines changed

4 files changed

+222
-33
lines changed

.github/workflows/examples.yml

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ jobs:
1212
strategy:
1313
fail-fast: false
1414
matrix:
15-
os: [ubuntu-latest, windows-latest, macos-14]
16-
python-version: ["3.11", "3.12"]
15+
os: [ubuntu-latest, windows-latest, macos-15]
16+
python-version: ["3.11", "3.12", "3.13"]
1717
example:
1818
- "examples/arduino-asyncudp"
1919
- "examples/arduino-blink"
@@ -30,9 +30,8 @@ jobs:
3030
python-version: ${{ matrix.python-version }}
3131
- name: Install dependencies
3232
run: |
33-
python -m pip install --upgrade pip
3433
pip install wheel
35-
pip install -U https://github.com/platformio/platformio/archive/develop.zip
34+
pip install -U https://github.com/pioarduino/platformio-core/archive/pio_github.zip
3635
pio pkg install --global --platform symlink://.
3736
- name: git clone Tasmota and add to examples
3837
if: matrix.example == 'examples/tasmota'

builder/main.py

Lines changed: 47 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,23 @@
1919
import sys
2020
from os.path import join
2121

22+
from SCons.Script import (
23+
ARGUMENTS, COMMAND_LINE_TARGETS, AlwaysBuild, Builder, Default,
24+
DefaultEnvironment)
2225

23-
from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild,
24-
Builder, Default, DefaultEnvironment)
2526

2627
#
2728
# Helpers
2829
#
2930

31+
def BeforeUpload(target, source, env):
32+
upload_options = {}
33+
if "BOARD" in env:
34+
upload_options = env.BoardConfig().get("upload", {})
35+
36+
if not env.subst("$UPLOAD_PORT"):
37+
env.AutodetectUploadPort()
38+
3039

3140
def _get_board_f_flash(env):
3241
frequency = env.subst("$BOARD_F_FLASH")
@@ -143,6 +152,7 @@ def get_esptoolpy_reset_flags(resetmethod):
143152

144153
env = DefaultEnvironment()
145154
platform = env.PioPlatform()
155+
config = env.GetProjectConfig()
146156
board = env.BoardConfig()
147157
filesystem = board.get("build.filesystem", "littlefs")
148158

@@ -189,6 +199,21 @@ def get_esptoolpy_reset_flags(resetmethod):
189199
PROGSUFFIX=".elf"
190200
)
191201

202+
# Check if lib_archive is set in platformio.ini and set it to False
203+
# if not found. This makes weak defs in framework and libs possible.
204+
def check_lib_archive_exists():
205+
for section in config.sections():
206+
if "lib_archive" in config.options(section):
207+
#print(f"lib_archive in [{section}] found with value: {config.get(section, 'lib_archive')}")
208+
return True
209+
#print("lib_archive was not found in platformio.ini")
210+
return False
211+
212+
if not check_lib_archive_exists():
213+
env_section = "env:" + env["PIOENV"]
214+
config.set(env_section, "lib_archive", "False")
215+
#print(f"lib_archive is set to False in [{env_section}]")
216+
192217
# Allow user to override via pre:script
193218
if env.get("PROGNAME", "program") == "program":
194219
env.Replace(PROGNAME="firmware")
@@ -205,14 +230,23 @@ def get_esptoolpy_reset_flags(resetmethod):
205230
env.Append(
206231
BUILDERS=dict(
207232
DataToBin=Builder(
208-
action=env.VerboseAction(" ".join([
209-
'"$MKFSTOOL"',
210-
"-c", "$SOURCES",
211-
"-p", "$FS_PAGE",
212-
"-b", "$FS_BLOCK",
213-
"-s", "${FS_END - FS_START}",
214-
"$TARGET"
215-
]), "Building file system image from '$SOURCES' directory to $TARGET"),
233+
action=env.VerboseAction(
234+
" ".join(
235+
['"$MKFSTOOL"', "-c", "$SOURCES", "-s", "${FS_END - FS_START}"]
236+
+ (
237+
[
238+
"-p",
239+
"$FS_PAGE",
240+
"-b",
241+
"$FS_BLOCK",
242+
]
243+
if filesystem in ("littlefs", "spiffs")
244+
else []
245+
)
246+
+ ["$TARGET"]
247+
),
248+
"Building FS image from '$SOURCES' directory to $TARGET",
249+
),
216250
emitter=__fetch_fs_size,
217251
source_factory=env.Dir,
218252
suffix=".bin"
@@ -341,8 +375,7 @@ def get_esptoolpy_reset_flags(resetmethod):
341375
)
342376

343377
upload_actions = [
344-
env.VerboseAction(env.AutodetectUploadPort,
345-
"Looking for upload port..."),
378+
env.VerboseAction(BeforeUpload, "Looking for upload port..."),
346379
env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")
347380
]
348381

@@ -366,7 +399,7 @@ def get_esptoolpy_reset_flags(resetmethod):
366399
"erase_upload",
367400
target_firm,
368401
[
369-
env.VerboseAction(env.AutodetectUploadPort, "Looking for serial port..."),
402+
env.VerboseAction(BeforeUpload, "Looking for upload port..."),
370403
env.VerboseAction("$ERASECMD", "Erasing..."),
371404
env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")
372405
],
@@ -381,7 +414,7 @@ def get_esptoolpy_reset_flags(resetmethod):
381414
"erase",
382415
None,
383416
[
384-
env.VerboseAction(env.AutodetectUploadPort, "Looking for serial port..."),
417+
env.VerboseAction(BeforeUpload, "Looking for upload port..."),
385418
env.VerboseAction("$ERASECMD", "Erasing...")
386419
],
387420
"Erase Flash",

platform.json

Lines changed: 55 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,18 @@
66
"license": "Apache-2.0",
77
"keywords": [
88
"dev-platform",
9-
"Wi-Fi",
9+
"WiFi",
1010
"Xtensa",
11-
"106Micro"
11+
"esp8266"
1212
],
1313
"engines": {
14-
"platformio": ">=6.1.14"
14+
"platformio": ">=6.1.18"
1515
},
1616
"repository": {
1717
"type": "git",
1818
"url": "https://github.com/tasmota/platform-espressif8266.git"
1919
},
20-
"version": "2024.09.00",
20+
"version": "2025.05.00",
2121
"frameworks": {
2222
"arduino": {
2323
"package": "framework-arduinoespressif8266",
@@ -27,24 +27,70 @@
2727
"packages": {
2828
"toolchain-xtensa": {
2929
"type": "toolchain",
30-
"owner": "platformio",
31-
"version": "~2.40802.0"
30+
"optional": true,
31+
"owner": "pioarduino",
32+
"package-version": "2.40802.200502",
33+
"version": "https://github.com/pioarduino/registry/releases/download/0.0.1/xtensa-4.8.2.zip"
3234
},
3335
"framework-arduinoespressif8266": {
3436
"type": "framework",
37+
"optional": true,
3538
"owner": "tasmota",
3639
"version": "https://github.com/tasmota/Arduino/releases/download/2.7.8/esp8266-2.7.8.zip"
3740
},
3841
"tool-esptoolpy": {
3942
"type": "uploader",
43+
"optional": true,
4044
"owner": "tasmota",
41-
"version": "https://github.com/tasmota/esptool/releases/download/v4.7.6/esptool.zip"
45+
"version": "https://github.com/pioarduino/esptool/releases/download/v4.8.11/esptool.zip"
46+
},
47+
"tl-install": {
48+
"type": "tool",
49+
"optional": false,
50+
"owner": "pioarduino",
51+
"version": "https://github.com/pioarduino/esp_install/releases/download/v5.0.0/esp_install-v5.0.0.zip"
4252
},
4353
"tool-mklittlefs": {
4454
"type": "uploader",
4555
"optional": true,
46-
"owner": "platformio",
47-
"version": "~1.203.0"
56+
"owner": "pioarduino",
57+
"package-version": "1.203.210628",
58+
"version": "https://github.com/pioarduino/registry/releases/download/0.0.1/mklittlefs-2.3.0.zip"
59+
},
60+
"tool-mkspiffs": {
61+
"type": "uploader",
62+
"optional": true,
63+
"owner": "pioarduino",
64+
"package-version": "1.200.0",
65+
"version": "https://github.com/pioarduino/registry/releases/download/0.0.1/mkspiffs-1.2.0.zip"
66+
},
67+
"tool-cppcheck": {
68+
"type": "tool",
69+
"optional": true,
70+
"owner": "pioarduino",
71+
"package-version": "2.11.0+230717",
72+
"version": "https://github.com/pioarduino/registry/releases/download/0.0.1/cppcheck-v2.11.0-230717.zip"
73+
},
74+
"tool-clangtidy": {
75+
"type": "tool",
76+
"optional": true,
77+
"owner": "pioarduino",
78+
"package-version": "18.1.1",
79+
"version": "https://github.com/pioarduino/registry/releases/download/0.0.1/clangtidy-v18.1.1.zip"
80+
},
81+
"tool-pvs-studio": {
82+
"type": "tool",
83+
"optional": true,
84+
"owner": "pioarduino",
85+
"package-version": "7.36.91321",
86+
"version": "https://github.com/pioarduino/registry/releases/download/0.0.1/pvs-studio-v7.36.91321.zip"
87+
},
88+
"tool-scons": {
89+
"type": "tool",
90+
"optional": true,
91+
"owner": "pioarduino",
92+
"package-version": "4.40801.0",
93+
"version": "https://github.com/pioarduino/registry/releases/download/0.0.1/scons-4.8.1.zip"
4894
}
4995
}
5096
}

platform.py

Lines changed: 117 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,23 +13,134 @@
1313
# limitations under the License.
1414

1515
import os
16+
import json
17+
import subprocess
1618
import sys
19+
import shutil
20+
from os.path import join
21+
22+
from platformio.public import PlatformBase, to_unix_path
23+
from platformio.proc import get_pythonexe_path
24+
from platformio.project.config import ProjectConfig
25+
from platformio.package.manager.tool import ToolPackageManager
1726

18-
from platformio.public import PlatformBase
1927

2028
IS_WINDOWS = sys.platform.startswith("win")
2129
# Set Platformio env var to use windows_amd64 for all windows architectures
2230
# only windows_amd64 native espressif toolchains are available
23-
# needs platformio core >= 6.1.16b2 or pioarduino core 6.1.16+test
31+
# needs platformio/pioarduino core >= 6.1.17
2432
if IS_WINDOWS:
2533
os.environ["PLATFORMIO_SYSTEM_TYPE"] = "windows_amd64"
2634

27-
class Espressif8266Platform(PlatformBase):
35+
python_exe = get_pythonexe_path()
36+
pm = ToolPackageManager()
2837

38+
class Espressif8266Platform(PlatformBase):
2939
def configure_default_packages(self, variables, targets):
30-
framework = variables.get("pioframework", [])
31-
if "buildfs" in targets:
32-
self.packages['tool-mklittlefs']['optional'] = False
40+
if not variables.get("board"):
41+
return super().configure_default_packages(variables, targets)
42+
43+
frameworks = variables.get("pioframework", [])
44+
45+
def install_tool(TOOL, retry_count=0):
46+
self.packages[TOOL]["optional"] = False
47+
TOOL_PATH = os.path.join(ProjectConfig.get_instance().get("platformio", "packages_dir"), TOOL)
48+
TOOL_PACKAGE_PATH = os.path.join(TOOL_PATH, "package.json")
49+
TOOLS_PATH_DEFAULT = os.path.join(os.path.expanduser("~"), ".platformio")
50+
IDF_TOOLS = os.path.join(ProjectConfig.get_instance().get("platformio", "packages_dir"), "tl-install", "tools", "idf_tools.py")
51+
TOOLS_JSON_PATH = os.path.join(TOOL_PATH, "tools.json")
52+
TOOLS_PIO_PATH = os.path.join(TOOL_PATH, ".piopm")
53+
IDF_TOOLS_CMD = (
54+
python_exe,
55+
IDF_TOOLS,
56+
"--quiet",
57+
"--non-interactive",
58+
"--tools-json",
59+
TOOLS_JSON_PATH,
60+
"install"
61+
)
62+
63+
tl_flag = bool(os.path.exists(IDF_TOOLS))
64+
json_flag = bool(os.path.exists(TOOLS_JSON_PATH))
65+
pio_flag = bool(os.path.exists(TOOLS_PIO_PATH))
66+
if tl_flag and json_flag:
67+
rc = subprocess.run(IDF_TOOLS_CMD).returncode
68+
if rc != 0:
69+
sys.stderr.write("Error: Couldn't execute 'idf_tools.py install'\n")
70+
else:
71+
tl_path = "file://" + join(TOOLS_PATH_DEFAULT, "tools", TOOL)
72+
try:
73+
shutil.copyfile(TOOL_PACKAGE_PATH, join(TOOLS_PATH_DEFAULT, "tools", TOOL, "package.json"))
74+
except FileNotFoundError as e:
75+
sys.stderr.write(f"Error copying tool package file: {e}\n")
76+
if os.path.exists(TOOL_PATH) and os.path.isdir(TOOL_PATH):
77+
try:
78+
shutil.rmtree(TOOL_PATH)
79+
except Exception as e:
80+
print(f"Error while removing the tool folder: {e}")
81+
pm.install(tl_path)
82+
# tool is already installed, just activate it
83+
if tl_flag and pio_flag and not json_flag:
84+
with open(TOOL_PACKAGE_PATH, "r") as file:
85+
package_data = json.load(file)
86+
# check installed tool version against listed in platforms.json
87+
if "package-version" in self.packages[TOOL] \
88+
and "version" in package_data \
89+
and self.packages[TOOL]["package-version"] == package_data["version"]:
90+
self.packages[TOOL]["version"] = TOOL_PATH
91+
self.packages[TOOL]["optional"] = False
92+
elif "package-version" not in self.packages[TOOL]:
93+
# No version check needed, just use the installed tool
94+
self.packages[TOOL]["version"] = TOOL_PATH
95+
self.packages[TOOL]["optional"] = False
96+
elif "version" not in package_data:
97+
print(f"Warning: Cannot determine installed version for {TOOL}. Reinstalling...")
98+
else: # Installed version does not match required version, deinstall existing and install needed
99+
if os.path.exists(TOOL_PATH) and os.path.isdir(TOOL_PATH):
100+
try:
101+
shutil.rmtree(TOOL_PATH)
102+
except Exception as e:
103+
print(f"Error while removing the tool folder: {e}")
104+
if retry_count >= 3: # Limit to 3 retries
105+
print(f"Failed to install {TOOL} after multiple attempts. Please check your network connection and try again manually.")
106+
return
107+
print(f"Wrong version for {TOOL}. Installing needed version...")
108+
install_tool(TOOL, retry_count + 1)
109+
110+
return
111+
112+
# Installer only needed for setup, deactivate when installed
113+
if bool(os.path.exists(os.path.join(ProjectConfig.get_instance().get("platformio", "packages_dir"), "tl-install", "tools", "idf_tools.py"))):
114+
self.packages["tl-install"]["optional"] = True
115+
116+
# tool-scons needs to be installed but can be set inactive
117+
if bool(os.path.exists(os.path.join(ProjectConfig.get_instance().get("platformio", "packages_dir"), "tool-scons", "scons.py"))):
118+
self.packages["tool-scons"]["optional"] = True
119+
else:
120+
install_tool("tool-scons")
121+
122+
self.packages["framework-arduinoespressif8266"]["optional"] = False
123+
install_tool("toolchain-xtensa")
124+
125+
CHECK_PACKAGES = [
126+
"tool-cppcheck",
127+
"tool-clangtidy",
128+
"tool-pvs-studio"
129+
]
130+
# Install check tool listed in pio entry "check_tool"
131+
if variables.get("check_tool") is not None:
132+
for package in CHECK_PACKAGES:
133+
for check_tool in variables.get("check_tool", ""):
134+
if check_tool in package:
135+
install_tool(package)
136+
137+
if "buildfs" or "uploadfs" or "downloadfs" in targets:
138+
filesystem = variables.get("board_build.filesystem", "littlefs")
139+
if filesystem == "littlefs":
140+
install_tool("tool-mklittlefs")
141+
elif filesystem == "spiffs":
142+
install_tool("tool-mkspiffs")
143+
33144
return super().configure_default_packages(variables, targets)
34145

35146
def get_boards(self, id_=None):

0 commit comments

Comments
 (0)