From bd48cad645f52f0d783f45a2f55ad4fbced5d421 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Tue, 27 May 2025 07:13:15 +0300 Subject: [PATCH 01/13] Tools - mkbuildoptglobals refactoring & attempt to fix caching Optional include for build-dir header to exclude it from core.a when no build opts are created / used. Preventive reads before writes, too Plus, missing change to makecorever.py doing read before write Prefer to use --output-dir in scripts, tests/device *must* be out-of-tree for caching to work properly (avoid changing core nested dirs) Assume 'aggressive caching' is normal mode of operation, no need for special treatment (can be checked by looking at .d files in the build dir) Even CI uses arduino-cli ~/.cache/arduino/{sketch,core} for building Allow sketches without globals.h to share core.a, rebuild otherwise (same sketch cache is retained, core.a & .o's reused) Drop manual logging in favour of 'import logging'. Arduino-CLI might still need tweaks in debug mode, though. Output flushing does not always happen promptly. Allow multiple entries for the same name, which are then merged. Allow spaces between @ and the signature words. Allow to use line right after signature. Syntax warnings show relevant snippet from globals.h --- .github/workflows/build-host.yml | 2 +- .github/workflows/build-ide.yml | 20 +- package/build_boards_manager_package.sh | 2 +- platform.txt | 14 +- tests/common.sh | 4 +- tests/device/Makefile | 52 +- .../device/test_libc/test_libc.ino.globals.h | 1 - tests/sanity_check.sh | 3 +- tools/makecorever.py | 29 +- tools/mkbuildoptglobals.py | 1555 +++++++++-------- tools/test_mkbuildoptglobals.py | 163 ++ 11 files changed, 1076 insertions(+), 769 deletions(-) mode change 100644 => 100755 tools/mkbuildoptglobals.py create mode 100755 tools/test_mkbuildoptglobals.py diff --git a/.github/workflows/build-host.yml b/.github/workflows/build-host.yml index ae0353068d..bd9172960e 100644 --- a/.github/workflows/build-host.yml +++ b/.github/workflows/build-host.yml @@ -1,5 +1,5 @@ # Run host test suite under valgrind for runtime checking of code. -# Also, a quick test that the mocking builds work at all +# Also, a quick test that the mock builds are actually working name: Build on host OS diff --git a/.github/workflows/build-ide.yml b/.github/workflows/build-ide.yml index edc226b4d9..d5b9b5f0b8 100644 --- a/.github/workflows/build-ide.yml +++ b/.github/workflows/build-ide.yml @@ -9,8 +9,7 @@ permissions: contents: read jobs: - - # Examples are built in parallel to avoid CI total job time limitation + # download toolchain and check whether gcc basic .c compilation works sanity-check: runs-on: ubuntu-latest defaults: @@ -28,6 +27,23 @@ jobs: run: | bash ./tests/sanity_check.sh + # verify that any scripts that will or may be used by the builder are working + tooling-check: + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v4 + with: + submodules: false + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + - run: | + python ./tools/test_mkbuildoptglobals.py --quiet + + # Examples are built in parallel to avoid CI total job time limitation build-linux: name: Linux - LwIP ${{ matrix.lwip }} (${{ matrix.chunk }}) runs-on: ubuntu-latest diff --git a/package/build_boards_manager_package.sh b/package/build_boards_manager_package.sh index 40050f4437..dc15f90fc5 100755 --- a/package/build_boards_manager_package.sh +++ b/package/build_boards_manager_package.sh @@ -117,7 +117,7 @@ $SED -E "s/name=([a-zA-Z0-9\ -]+).*/name=\1(${ver})/g"\ #echo "#define ARDUINO_ESP8266_GIT_DESC `git describe --tags 2>/dev/null`" >>${outdir}/cores/esp8266/core_version.h #echo "#define ARDUINO_ESP8266_RELEASE_${ver_define}" >>${outdir}/cores/esp8266/core_version.h #echo "#define ARDUINO_ESP8266_RELEASE \"${ver_define}\"" >>${outdir}/cores/esp8266/core_version.h -python3 ${srcdir}/tools/makecorever.py -b ${outdir} -i cores/esp8266 -p ${srcdir} -v ${plain_ver} -r +python3 ${srcdir}/tools/makecorever.py --git-root ${srcdir} --version ${plain_ver} --release ${outdir}/cores/esp8266/core_version.h # Zip the package pushd package/versions/${visiblever} diff --git a/platform.txt b/platform.txt index f3392d652d..1a266e63ad 100644 --- a/platform.txt +++ b/platform.txt @@ -66,9 +66,10 @@ build.spiffs_blocksize= build.iramfloat=-DFP_IN_IROM # Fully qualified file names for processing sketch global options -globals.h.source.fqfn={build.source.path}/{build.project_name}.globals.h -commonhfile.fqfn={build.core.path}/CommonHFile.h -build.opt.fqfn={build.path}/core/build.opt +globals.h.build.source.fqfn={build.source.path}/{build.project_name}.globals.h +globals.h.build.fqfn={build.path}/{build.project_name}.globals.h +common.h.fqfn={build.core.path}/CommonHFile.h +build.opt.fqfn={build.path}/sketch/build.opt build.opt.flags="@{build.opt.fqfn}" mkbuildoptglobals.extra_flags= @@ -76,7 +77,8 @@ compiler.path={runtime.tools.xtensa-lx106-elf-gcc.path}/bin/ compiler.sdk.path={runtime.platform.path}/tools/sdk compiler.libc.path={runtime.platform.path}/tools/sdk/libc/xtensa-lx106-elf -compiler.cpreprocessor.flags=-D__ets__ -DICACHE_FLASH -U__STRICT_ANSI__ -D_GNU_SOURCE -DESP8266 {build.debug_optim} {build.opt.flags} "-I{compiler.sdk.path}/include" "-I{compiler.sdk.path}/{build.lwip_include}" "-I{compiler.libc.path}/include" "-I{build.path}/core" +compiler.cpreprocessor.extra_flags= +compiler.cpreprocessor.flags=-D__ets__ -DICACHE_FLASH -U__STRICT_ANSI__ -D_GNU_SOURCE -DESP8266 {build.debug_optim} {build.opt.flags} "-I{compiler.sdk.path}/include" "-I{compiler.sdk.path}/{build.lwip_include}" "-I{compiler.libc.path}/include" "-I{build.path}/core" {compiler.cpreprocessor.extra_flags} # support precompiled libraries in IDE v1.8.6+ compiler.libraries.ldflags= @@ -123,8 +125,8 @@ recipe.hooks.sketch.prebuild.pattern="{runtime.tools.python3.path}/python3" -I " # This is quite a working hack. This form of prebuild hook, while intuitive, is not explicitly documented. recipe.hooks.prebuild.1.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.makecorever}" --git-root "{runtime.platform.path}" --version "{version}" "{build.path}/core/core_version.h" -# Handle processing sketch global options -recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" {runtime.ide.version} "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" {mkbuildoptglobals.extra_flags} +# Handle core & sketch global options +recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.mkbuildoptglobals}" {mkbuildoptglobals.extra_flags} build --build-opt "{build.opt.fqfn}" --source-sketch-header "{globals.h.build.source.fqfn}" --build-sketch-header "{globals.h.build.fqfn}" --common-header "{common.h.fqfn}" ## Build the app.ld linker file diff --git a/tests/common.sh b/tests/common.sh index d26db8f3a6..33daeb5255 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -132,7 +132,7 @@ function build_sketches() build_cmd+=${cli_path} build_cmd+=" compile"\ " --warnings=all"\ -" --build-path $build_dir"\ +" --output-path $build_dir"\ " --fqbn $fqbn"\ " --libraries $library_path"\ " --output-dir $build_out" @@ -311,7 +311,7 @@ function install_core() printf "%s\n" \ "compiler.c.extra_flags=-Wall -Wextra $debug_flags" \ "compiler.cpp.extra_flags=-Wall -Wextra $debug_flags" \ - "mkbuildoptglobals.extra_flags=--ci --cache_core" \ + "recipe.hooks.prebuild.1.pattern=\"{runtime.tools.python3.path}/python3\" -I \"{runtime.tools.makecorever}\" --git-root \"{runtime.platform.path}\" --version \"{version}\" \"{runtime.platform.path}/cores/esp8266/core_version.h\"" \ > ${core_path}/platform.local.txt echo -e "\n----platform.local.txt----" cat platform.local.txt diff --git a/tests/device/Makefile b/tests/device/Makefile index f89c9ea93e..039749b8a6 100644 --- a/tests/device/Makefile +++ b/tests/device/Makefile @@ -2,7 +2,17 @@ SHELL := /bin/bash ESP8266_CORE_PATH ?= $(shell git rev-parse --show-toplevel) +# arduino-cli core/ & sketch/ build and cache directories +# by default, share build location with every other sketch +CACHE_DIR ?= $(HOME)/.cache/arduino +ifneq ("$(findstring $(ESP8266_CORE_PATH),$(CACHE_DIR))", "") +$(warning "CACHE_DIR is located in ESP8266_CORE_PATH, core.a caching will be disabled") +endif + +# binaries (compile --output-dir=...) & test output goes in here. BUILD_DIR ?= $(PWD)/build + +# where to look for BSTest scripts BS_DIR ?= $(PWD)/libraries/BSTest PYTHON ?= python3 @@ -67,7 +77,7 @@ list: showtestlist all: count tests test_report -$(TEST_LIST): | virtualenv $(TEST_CONFIG) $(BUILD_DIR) $(HARDWARE_DIR) +$(TEST_LIST): | virtualenv $(TEST_CONFIG) $(CACHE_DIR) $(BUILD_DIR) $(HARDWARE_DIR) .NOTPARALLEL: $(TEST_LIST) @@ -78,18 +88,21 @@ showtestlist: @printf '%s\n' $(TEST_LIST) @echo "--------------------------------" -$(TEST_LIST): LOCAL_BUILD_DIR=$(BUILD_DIR)/$(notdir $@) -$(TEST_LIST): LOCAL_DATA_IMG=data.img +$(TEST_LIST): LOCAL_BUILD_DIR=$(BUILD_DIR)/$(notdir $@)/ +$(TEST_LIST): LOCAL_TEST_RESULT_XML=$(LOCAL_BUILD_DIR)/$(TEST_RESULT_XML) +$(TEST_LIST): LOCAL_DATA_DIR=$(LOCAL_BUILD_DIR)/data/ +$(TEST_LIST): LOCAL_DATA_IMG=$(LOCAL_BUILD_DIR)/data.img define build-arduino - rm -f $(LOCAL_BUILD_DIR)/build.options.json - $(BUILD_TOOL) compile \ - $(BUILD_FLAGS) \ - --libraries "$(PWD)/libraries" \ - --warnings=all \ - --build-path $(LOCAL_BUILD_DIR) \ - --fqbn=$(FQBN) \ - $@ + export ARDUINO_BUILD_CACHE_PATH="$(CACHE_DIR)"; \ + $(BUILD_TOOL) config dump; \ + $(BUILD_TOOL) compile \ + $(BUILD_FLAGS) \ + --libraries "$(PWD)/libraries" \ + --output-dir "$(LOCAL_BUILD_DIR)" \ + --warnings=all \ + --fqbn=$(FQBN) \ + $@ endef define build-mock @@ -98,7 +111,7 @@ define build-mock $(VENV_PYTHON) $(BS_DIR)/runner.py \ $(BS_FLAGS) \ --name $(basename $(notdir $@)) \ - --output $(LOCAL_BUILD_DIR)/$(TEST_RESULT_XML) \ + --output $(LOCAL_TEST_RESULT_XML) \ --env-file $(TEST_CONFIG) \ $(call mock_script,$@) \ executable "$(ESP8266_CORE_PATH)/tests/host/bin/$(@:%.ino=%)" || echo ""` @@ -111,17 +124,17 @@ define upload-data (cd $(dir $@) && ./make_data.py ) || echo "Filesystem creation skipped" @test -d $(dir $@)/data/ && ( \ $(MKFS) \ - --create $(dir $@)/data/ \ + --create $(LOCAL_DATA_DIR) \ --size 0xFB000 \ --block 8192 \ --page 256 \ - $(LOCAL_BUILD_DIR)/$(LOCAL_DATA_IMG) && \ + $(LOCAL_DATA_IMG) && \ $(ESPTOOL) \ --chip esp8266 \ --port $(UPLOAD_PORT) \ --baud $(UPLOAD_BAUD) \ --after no_reset \ - write_flash 0x300000 $(LOCAL_BUILD_DIR)/$(LOCAL_DATA_IMG) ) \ + write_flash 0x300000 $(LOCAL_DATA_IMG) ) \ && (echo "Uploaded filesystem") \ || (echo "Filesystem upload skipped") endef @@ -149,7 +162,7 @@ define run-test $(VENV_PYTHON) $(BS_DIR)/runner.py \ $(BS_FLAGS) \ --name $(basename $(notdir $@)) \ - --output $(LOCAL_BUILD_DIR)/$(TEST_RESULT_XML) \ + --output $(LOCAL_TEST_RESULT_XML) \ --env-file $(TEST_CONFIG) \ $(call mock_script,$@) \ port $(UPLOAD_PORT) \ @@ -160,6 +173,7 @@ $(TEST_LIST): @echo "--------------------------------" @echo "Running test '$@' of $(words $(TEST_LIST)) tests" mkdir -p $(LOCAL_BUILD_DIR) + mkdir -p $(CACHE_DIR) ifneq ("$(NO_BUILD)","1") @echo Building $(notdir $@) ifeq ("$(MOCK)", "1") @@ -187,13 +201,17 @@ $(TEST_REPORT_HTML): $(TEST_REPORT_XML) | virtualenv test_report: $(TEST_REPORT_HTML) @echo "Test report generated in $(TEST_REPORT_HTML)" +$(CACHE_DIR): + @mkdir -p $@ + $(BUILD_DIR): - @mkdir -p $(BUILD_DIR) + @mkdir -p $@ virtualenv: @make -C $(BS_DIR) PYTHON=$(PYTHON) virtualenv clean: + rm -rf $(CACHE_DIR) rm -rf $(BUILD_DIR) rm -rf $(BS_DIR)/virtualenv rm -f $(TEST_REPORT_HTML) $(TEST_REPORT_XML) diff --git a/tests/device/test_libc/test_libc.ino.globals.h b/tests/device/test_libc/test_libc.ino.globals.h index 1bfe3ba4d5..e271602755 100644 --- a/tests/device/test_libc/test_libc.ino.globals.h +++ b/tests/device/test_libc/test_libc.ino.globals.h @@ -1,4 +1,3 @@ /*@create-file:build.opt@ - -fno-builtin */ diff --git a/tests/sanity_check.sh b/tests/sanity_check.sh index a4754a0ed4..f5f2a6f2e1 100755 --- a/tests/sanity_check.sh +++ b/tests/sanity_check.sh @@ -5,8 +5,9 @@ source "$root/tests/common.sh" pushd "$root"/tools python3 get.py -q - +python3 makecorever.py --git-root "$root" "$root/cores/esp8266/core_version.h" popd + pushd "$cache_dir" gcc="$root/tools/xtensa-lx106-elf/bin/xtensa-lx106-elf-gcc"\ diff --git a/tools/makecorever.py b/tools/makecorever.py index 9062e5b4da..939120fff3 100755 --- a/tools/makecorever.py +++ b/tools/makecorever.py @@ -56,7 +56,6 @@ def check_git(*args: str, cwd: Optional[str]): def generate( - out: TextIO, *, git_root: pathlib.Path, hash_length: int = 8, @@ -118,7 +117,7 @@ def git(*args): #define ARDUINO_ESP8266_DEV 1 // development version """ - out.write(text) + return text if __name__ == "__main__": @@ -158,20 +157,18 @@ def git(*args): args = parser.parse_args() - def select_output(s: str) -> TextIO: - if not s: - return sys.stdout + contents = generate( + git_root=args.git_root, + hash_length=args.hash_length, + release=args.release, + version=args.version, + ) - out = pathlib.Path(s) + if args.output: + out = pathlib.Path(args.output) out.parent.mkdir(parents=True, exist_ok=True) - return out.open("w", encoding="utf-8") - - with select_output(args.output) as out: - generate( - out, - git_root=args.git_root, - hash_length=args.hash_length, - release=args.release, - version=args.version, - ) + if not out.exists() or contents != out.read_text(encoding="utf-8"): + out.write_text(contents, encoding="utf-8") + else: + print(contents, file=sys.stdout) diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py old mode 100644 new mode 100755 index 62a3373aee..6c68f87ab8 --- a/tools/mkbuildoptglobals.py +++ b/tools/mkbuildoptglobals.py @@ -1,14 +1,9 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# This script manages the use of a file with a unique name, like -# `Sketch.ino.globals.h`, in the Sketch source directory to provide compiler -# command-line options (build options) and sketch global macros. The build -# option data is encapsulated in a unique "C" comment block and extracted into -# the build tree during prebuild. -# # Copyright (C) 2022 - M Hightower # +# Updates & fixes for arduino-cli environment by Maxim Prokhorov +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -20,809 +15,925 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# This script manages the use of a file with a unique name, like +# `SKETCH.ino.globals.h`, in the SKETCH source directory to provide compiler +# command-line options (build options), sketch global macros, or inject code. +# The buildo ption data is encapsulated in an unique "C" comment block /* ... */ +# and extracted into the build directory during prebuild. # # A Tip of the hat to: # # This PR continues the effort to get some form of global build support -# presented by brainelectronics' PR https://github.com/esp8266/Arduino/pull/8095 +# presented by brainelectronics' PR +# - https://github.com/esp8266/Arduino/pull/8095 # # Used d-a-v's global name suggestion from arduino PR -# https://github.com/arduino/arduino-cli/pull/1524 -# +# - https://github.com/arduino/arduino-cli/pull/1524 + """ -Operation +Sketch header aka SKETCH.ino.globals.h: -"Sketch.ino.globals.h" - A global h file in the Source Sketch directory. The -string Sketch.ino is the actual name of the sketch program. A matching copy is -kept in the build path/core directory. The file is empty when it does not exist -in the source directory. + SKETCH.ino.globals.h is expected to be created by the user. -Using Sketch.ino.globals.h as a container to hold build.opt, gives implicit -dependency tracking for build.opt by way of Sketch.ino.globals.h's -dependencies. -Example: - gcc ... @{build.path}/core/build.opt -include "{build.path}/core/{build.project_name}.globals.h" ... + It is always located in the root of the SKETCH directory, and must use the + the actual name of the sketch program (SKETCH.ino) in the its name. -In this implementation the '-include "{build.path}/core/{build.project_name}.globals.h"' -component is added to the build.opt file. - gcc ... @{build.path}/core/build.opt ... + Header file format is used because IDE only manages source files it actually + recognizes as valid C / C++ file formats. When building and re-building the + sketch, only valid file formats are taken into an account when building source + code dependencies tree. -At each build cycle, "{build.project_name}.globals.h" is conditoinally copied to -"{build.path}/core/" at prebuild, and build.opt is extraction as needed. The -Sketch.ino.globals.h's dependencies will trigger "rebuild all" as needed. +Command-line options file: -If Sketch.ino.globals.h is not in the source sketch folder, an empty -versions is created in the build tree. The file build.opt always contains a -"-include ..." entry so that file dependencies are generated for -Sketch.ino.globals.h. This allows for change detection when the file is -added. -""" + This file is created by the script. -""" -Arduino `preferences.txt` changes + Contents of the file are then used as gcc options (@file) -"Aggressively cache compiled core" ideally should be turned off; however, -a workaround has been implimented. -In ~/.arduino15/preferences.txt, to disable the feature: - compiler.cache_core=false + In the prebuild stage, options file is generated based on the contents of the + sketch header, and then placed into the build directory. -Reference: -https://forum.arduino.cc/t/no-aggressively-cache-compiled-core-in-ide-1-8-15/878954/2 -""" + Quoting gcc manual + > If file does not exist, or cannot be read, then the option will be treated literally, and not removed. + > + > Options in file are separated by whitespace. A whitespace character may be included + > in an option by surrounding the entire option in either single or double quotes. + > Any character (including a backslash) may be included by prefixing the character + > to be included with a backslash. + > The file may itself contain additional @file options; any such options will be processed recursively. -""" -# Updates or Additions for platform.txt or platform.local.txt + Arduino build system uses timestamps as a method of determining which files should be rebuilt. + Options file is *always* created with at least two of the following lines + > -include "/core/" + > -include "/" -runtime.tools.mkbuildoptglobals={runtime.platform.path}/tools/mkbuildoptglobals.py +Common header: -# Fully qualified file names for processing sketch global options -globals.h.source.fqfn={build.source.path}/{build.project_name}.globals.h -commonhfile.fqfn={build.core.path}/CommonHFile.h -build.opt.fqfn={build.path}/core/build.opt -mkbuildoptglobals.extra_flags= + This file is also created by the script. -recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" {runtime.ide.version} "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" {mkbuildoptglobals.extra_flags} + It is used as a means of triggering core rebuild, because modern Arduino build systems + are agressively caching it and attempt to re-use existing core.a whenever possible. + core.a is also shared between different sketch compilations which use the same board. -compiler.cpreprocessor.flags=-D__ets__ -DICACHE_FLASH -U__STRICT_ANSI__ -D_GNU_SOURCE -DESP8266 @{build.opt.path} "-I{compiler.sdk.path}/include" "-I{compiler.sdk.path}/{build.lwip_include}" "-I{compiler.libc.path}/include" "-I{build.path}/core" -""" + This file would contain path to the currently used command-line options file extracted + from the sketch header. It remains empty otherwise. -""" -A Sketch.ino.globals.h file with embedded build.opt might look like this - -/*@create-file:build.opt@ -// An embedded build.opt file using a "C" block comment. The starting signature -// must be on a line by itself. The closing block comment pattern should be on a -// line by itself. Each line within the block comment will be space trimmed and -// written to build.opt, skipping blank lines and lines starting with '//', '*' -// or '#'. - --DMYDEFINE="\"Chimichangas do not exist\"" --O3 --fanalyzer --DUMM_STATS=2 -*/ - -#ifndef SKETCH_INO_GLOBALS_H -#define SKETCH_INO_GLOBALS_H - -#if defined(__cplusplus) -// Defines kept private to .cpp modules -//#pragma message("__cplusplus has been seen") -#endif - -#if !defined(__cplusplus) && !defined(__ASSEMBLER__) -// Defines kept private to .c modules -#endif - -#if defined(__ASSEMBLER__) -// Defines kept private to assembler modules -#endif - -#endif -""" +Build directory: -""" -Added 2) and 5) to docs + Arduino build process copies every valid source file from the source (sketch) + directory into the build directory. This script is expected to be launched in + the "prebuild" stage. At that point, build directory should already exist, but + it may not yet contain any of the sketch source files. + + Script would always attempt to copy sketch header from the source (sketch) + directory to the build one. If it does not exist, a placeholder would be created. + + Script would always synchronize atime & mtime of every file. When sketch header + exists, stats are taken from it. When it doesn't, stats for the generated common + header are used instead. + +Configuration: + + "platform.txt" is expected to have this script listed as a tool + > runtime.tools.mkbuildoptglobals={runtime.platform.path}/tools/mkbuildoptglobals.py + + Paths are always provided as Fully Qualified File Names (FQFNs). + + For example + > globals.h.source.fqfn={build.source.path}/{build.project_name}.globals.h + > globals.h.common.fqfn={build.core.path}/__common_globals.h + > build.opt.fqfn={build.path}/core/build.opt + > mkbuildoptglobals.extra_flags= + + Both Arduino IDE 1.x and modern 2.x generate prerequisite makefiles (.d files) + at some point in "discovery phase". + ref. https://www.gnu.org/software/make/manual/html_node/Automatic-Prerequisites.html + + "prebuild" hook must be used, allowing this script to run *before* IDE creates them + ref. https://docs.arduino.cc/arduino-cli/platform-specification/#pre-and-post-build-hooks-since-arduino-ide-165 + + > recipe.hooks.prebuild.#.pattern="{runtime.tools.python3.path}/python3" -I \\ + "{runtime.tools.mkbuildoptglobals}" {mkbuildoptglobals.extra_flags} build \\ + --build-path "{build.path}" \\ + --build-opt "{build.opt.fqfn}" \\ + --sketch-header "{globals.h.source.fqfn}" \\ + --common-header "{commonhfile.fqfn}" \\ + + Command-line options file is then shared between other recipes by including it in + the "cpreprocessor" flags. + > compiler.cpreprocessor.flags=... @{build.opt.path} ... + + After that point, prerequisite makefiles should contain either only the common header, + or both the common header and the build sketch header. When any of included headers is + modified, every file in the dependency chain would be rebuilt. This allows us to keep + existing core.a cache when command-line options file is not used by the sketch. + +Example: + + Sketch header file with embedded command-line options file might look like this + + .. code-block:: c++ + + /*@create-file:build.opt@ + // An embedded "build.opt" file using a "C" block comment. The starting signature + // must be on a line by itself. The closing block comment pattern should be on a + // line by itself. Each line within the block comment will be space trimmed and + // written to build.opt, skipping blank lines and lines starting with '//', '*' + // or '#'. + -DMYDEFINE="\"Chimichangas do not exist\"" + -O3 + -fanalyzer + -DUMM_STATS=2 + */ + + #ifndef SKETCH_INO_GLOBALS_H + #define SKETCH_INO_GLOBALS_H + + #if defined(__cplusplus) + // Defines kept private to .cpp modules + //#pragma message("__cplusplus has been seen") + #endif + + #if !defined(__cplusplus) && !defined(__ASSEMBLER__) + // Defines kept private to .c modules + #endif + + #if defined(__ASSEMBLER__) + // Defines kept private to assembler modules + #endif + + #endif + +Caveats, Observations, and Ramblings: + + 1) Edits to "platform.txt" or "platform.local.txt" force a complete rebuild that + removes the core folder. Not a problem, just something to be aware of when + debugging this script. Similarly, changes on the IDE Tools selection cause a + complete rebuild. -Caveats, Observations, and Ramblings + In contrast, the core directory is not deleted when the rebuild occurs from + changing a file with an established dependency. -1) Edits to platform.txt or platform.local.txt force a complete rebuild that -removes the core folder. Not a problem, just something to be aware of when -debugging this script. Similarly, changes on the IDE Tools selection cause a -complete rebuild. + 2) Renaming files does not change the last modified timestamp, possibly causing + issues when replacing files by renaming and rebuilding. -In contrast, the core directory is not deleted when the rebuild occurs from -changing a file with an established dependency. + A good example of this problem is when you correct the spelling of sketch + header file. You need to touch (update time stampt) the file so a + rebuild all is performed. -2) Renaming files does not change the last modified timestamp, possibly causing -issues when replacing files by renaming and rebuilding. + 3) During the build two identical copies of sketch header will exist. + #ifndef fencing will be needed for non comment blocks in SKETCH.ino.globals.h. -A good example of this problem is when you correct the spelling of file -Sketch.ino.globals.h. You need to touch (update time stampt) the file so a -rebuild all is performed. + 4) By using a .h file to encapsulate "build.opt" options, the information is not + lost after a save-as. Before with an individual "build.opt" file, the file was + missing in the saved copy. -3) During the build two identical copies of Sketch.ino.globals.h will exist. -#ifndef fencing will be needed for non comment blocks in Sketch.ino.globals.h. + 5) Previously, when a .h file is renamed, a copy of the old file remains in the build + sketch folder. This created confusion if you missed an edit in updating an + include in one or more of your modules. Module will continue to use the + stale version of the .h, until you restart the IDE or other major changes that + would cause the IDE to delete and recopy the contents from the source sketch. -4) By using a .h file to encapsulate "build.opt" options, the information is not -lost after a save-as. Before with an individual "build.opt" file, the file was -missing in the saved copy. + This may be the culprit for "What! It built fine last night!" -5) When a .h file is renamed, a copy of the old file remains in the build -sketch folder. This can create confusion if you missed an edit in updating an -include in one or more of your modules. That module will continue to use the -stale version of the .h, until you restart the IDE or other major changes that -would cause the IDE to delete and recopy the contents from the source sketch. + 6a) In The case of two Arduino IDE screens up with different programs, they can + share the same core archive file. Defines on one screen will change the core + archive, and a build on the 2nd screen will build with those changes. + The 2nd build will have the core built for the 1st screen. It gets uglier. With + the 2nd program, the newly built modules used headers processed with different + defines than the core. -This may be the culprit for "What! It built fine last night!" + 6b) Problem: Once core has been build, changes to build.opt or globals.h will + not cause the core archive to be rebuild. You either have to change tool + settings or close and reopen the Arduino IDE. This is a variation on 6a) above. + I thought this was working for the single sketch case, but it does not! :( + That is because sometimes it does build properly. What is unknown are the + causes that will make it work and fail? -6a) In The case of two Arduino IDE screens up with different programs, they can -share the same core archive file. Defines on one screen will change the core -archive, and a build on the 2nd screen will build with those changes. -The 2nd build will have the core built for the 1st screen. It gets uglier. With -the 2nd program, the newly built modules used headers processed with different -defines than the core. + 7) Previous IDE versions allowed to disable core.a caching + https://forum.arduino.cc/t/no-aggressively-cache-compiled-core-in-ide-1-8-15/878954/2 -6b) Problem: Once core has been build, changes to build.opt or globals.h will -not cause the core archive to be rebuild. You either have to change tool -settings or close and reopen the Arduino IDE. This is a variation on 6a) above. -I thought this was working for the single sketch case, but it does not! :( -That is because sometimes it does build properly. What is unknown are the -causes that will make it work and fail? - * Fresh single Arduino IDE Window, open with file to build - works + This is not the case for 2.x, where the only way to control it is by calling arduino-cli + directly, or modifying its configuration file / using environment variable. + https://arduino.github.io/arduino-cli/1.2/configuration/ + > "build_cache" configuration options related to the compilation cache -I think these, 6a and 6b, are resolved by setting `compiler.cache_core=false` -in ~/.arduino15/preferences.txt, to disable the aggressive caching feature: - https://forum.arduino.cc/t/no-aggressively-cache-compiled-core-in-ide-1-8-15/878954/2 + "--clean" option would start with a fresh build. -Added workaround for `compiler.cache_core=true` case. -See `if use_aggressive_caching_workaround:` in main(). + 8) Suspected but not confirmed. A quick edit and rebuild don't always work well. + Build does not work as expected. This does not fail often. Maybe PIC NIC. -7) Suspected but not confirmed. A quick edit and rebuild don't always work well. -Build does not work as expected. This does not fail often. Maybe PIC NIC. """ import argparse -import glob import locale +import logging +import io +import re import os -import platform +import dataclasses +import pathlib import sys import textwrap import time -import traceback -from shutil import copyfile +from typing import Optional, TextIO, Union, List, Tuple + +from shutil import copystat # Stay in sync with our bundled version -PYTHON_REQUIRES = (3, 7) +VERSION_MIN = (3, 7) -if sys.version_info < PYTHON_REQUIRES: - raise SystemExit(f"{__file__}\nMinimal supported version of Python is {PYTHON_REQUIRES[0]}.{PYTHON_REQUIRES[1]}") +if sys.version_info < VERSION_MIN: + raise SystemExit( + f"{__file__}\nMinimal supported version of Python is {VERSION_MIN[0]}.{VERSION_MIN[1]}" + ) -# Need to work on signature line used for match to avoid conflicts with -# existing embedded documentation methods. -build_opt_signature = "/*@create-file:build.opt@" +# Like existing documentation methods, signature is embedded in the comment block +# Unlike existing documentation methods, only the first line contains any metadata -docs_url = "https://arduino-esp8266.readthedocs.io/en/latest/faq/a06-global-build-options.html" +# Command-line options file, to be sourced by the compiler +BUILD_OPT_SIGNATURE_RE = re.compile(r"/[\*]@\s*?create-file:(?P\S*?)\s*?@$") +# Script documentation & examples +DOCS_URL = ( + "https://arduino-esp8266.readthedocs.io/en/latest/faq/a06-global-build-options.html" +) +DOCS_EPILOG = f""" +Use platform.local.txt 'mkbuildoptglobals.extra_flags=...' to supply extra command line flags. +See {DOCS_URL} for more information. +""" -err_print_flag = False -msg_print_buf = "" -debug_enabled = False -default_encoding = None -# Issues trying to address through buffered printing -# 1. Arduino IDE 2.0 RC5 does not show stderr text in color. Text printed does -# not stand out from stdout messages. -# 2. Separate pipes, buffering, and multiple threads with output can create -# mixed-up messages. "flush" helped but did not resolve. The Arduino IDE 2.0 -# somehow makes the problem worse. -# 3. With Arduino IDE preferences set for "no verbose output", you only see -# stderr messages. Prior related prints are missing. +# Notify that custom build options exist for the other core, but not this one +OTHER_BUILD_OPTIONS = [ + "build_opt.h", + "file_opt", +] + + +def other_build_options(p: pathlib.Path, sketch_header: pathlib.Path): + return f"""Build options file '{p.name}' is not supported.") +Embed build options and code in '{sketch_header.name}' instead. +Create an empty '{sketch_header.name}' to silence this warning. + + See {DOCS_URL} for more information. +""" + + +def check_other_build_options(sketch_header: pathlib.Path) -> Optional[str]: + if sketch_header.exists(): + return None + + for name in OTHER_BUILD_OPTIONS: + p = sketch_header.parent / name + if p.exists(): + return other_build_options(p, sketch_header) + + return None + + +# Retrieve *system* encoding, not the one used by python internally # -# Locally buffer and merge both stdout and stderr prints. This allows us to -# print a complete context when there is an error. When any buffered prints -# are targeted to stderr, print the whole buffer to stderr. - -def print_msg(*args, **kwargs): - global msg_print_buf - if 'sep' in kwargs: - sep = kwargs['sep'] - else: - sep = ' ' +# Given that GCC will handle lines from an @file as if they were on +# the command line. I assume that the contents of @file need to be encoded +# to match that of the shell running GCC runs. I am not 100% sure this API +# gives me that, but it appears to work. +# +# However, elsewhere when dealing with source code we continue to use 'utf-8', +# ref. https://gcc.gnu.org/onlinedocs/cpp/Character-sets.html +if sys.version_info >= (3, 11): + DEFAULT_ENCODING = locale.getencoding() +else: + DEFAULT_ENCODING = locale.getdefaultlocale()[1] - msg_print_buf += args[0] - for arg in args[1:]: - msg_print_buf += sep - msg_print_buf += arg +FILE_ENCODING = "utf-8" - if 'end' in kwargs: - msg_print_buf += kwargs['end'] - else: - msg_print_buf += '\n' - - -# Bring attention to errors with a blank line and lines starting with "*** ". -def print_err(*args, **kwargs): - global err_print_flag - if (args[0])[0] != ' ': - print_msg("") - print_msg("***", *args, **kwargs) - err_print_flag = True - -def print_dbg(*args, **kwargs): - global debug_enabled - global err_print_flag - if debug_enabled: - print_msg("DEBUG:", *args, **kwargs) - err_print_flag = True - - -def handle_error(err_no): - # on err_no 0, commit print buffer to stderr or stdout - # on err_no != 0, commit print buffer to stderr and sys exist with err_no - global msg_print_buf - global err_print_flag - if len(msg_print_buf): - if err_no or err_print_flag: - fd = sys.stderr - else: - fd = sys.stdout - print(msg_print_buf, file=fd, end='', flush=True) - msg_print_buf = "" - err_print_flag = False - if err_no: - sys.exit(err_no) +# Issues trying to address through logging module & buffered printing +# 1. Arduino IDE 1.x / 2.0 print stderr with color red, allowing any +# messages to stand out in an otherwise white on black console output. +# 2. With Arduino IDE preferences set for "no verbose output", you only see +# stderr messages. Prior related prints are missing. +# 3. logging ensures that stdout & stderr buffers are flushed before exiting. +# While it may not provide consistent output, no messages should be lost. +class LoggingFilter(logging.Filter): + def __init__(self, *filter_only): + self._filter_only = filter_only -def copy_create_build_file(source_fqfn, build_target_fqfn): - """ - Conditionally copy a newer file between the source directory and the build - directory. When source file is missing, create an empty file in the build - directory. - return True when file change detected. - """ - if os.path.exists(source_fqfn): - if os.path.exists(build_target_fqfn) and \ - os.path.getmtime(build_target_fqfn) >= os.path.getmtime(source_fqfn): - # only copy newer files - do nothing, all is good - print_dbg(f"up to date os.path.exists({source_fqfn}) ") - return False - else: - # The new copy gets stamped with the current time, just as other - # files copied by `arduino-builder`. - copyfile(source_fqfn, build_target_fqfn) - print_dbg(f"copyfile({source_fqfn}, {build_target_fqfn})") - else: - if os.path.exists(build_target_fqfn) and \ - os.path.getsize(build_target_fqfn) == 0: - return False - else: - # Place holder - Must have an empty file to satisfy parameter list - # specifications in platform.txt. - with open(build_target_fqfn, 'w', encoding="utf-8"): - pass - return True # file changed - -def add_include_line(build_opt_fqfn, include_fqfn): - global default_encoding - if not os.path.exists(include_fqfn): - # If file is missing, we need an place holder - with open(include_fqfn, 'w', encoding=default_encoding): - pass - print_msg("add_include_line: Created " + include_fqfn) - - with open(build_opt_fqfn, 'a', encoding=default_encoding) as build_opt: - build_opt.write('-include "' + include_fqfn.replace('\\', '\\\\') + '"\n') - -def extract_create_build_opt_file(globals_h_fqfn, file_name, build_opt_fqfn): + def filter(self, rec): + return rec.levelno in self._filter_only + + +# Since default handler is not created, make sure only specific levels go through to stderr +TO_STDERR = logging.StreamHandler(sys.stderr) +TO_STDERR.setFormatter(logging.Formatter("*** %(levelname)s - %(message)s ***")) +TO_STDERR.setLevel(logging.NOTSET) +TO_STDERR.addFilter( + LoggingFilter( + logging.CRITICAL, + logging.DEBUG, + logging.ERROR, + logging.FATAL, + logging.WARNING, + ) +) + +# Generic info messages should be on stdout (but, note the above, these are hidden by IDE defaults) +TO_STDOUT = logging.StreamHandler(sys.stdout) +TO_STDOUT.setFormatter(logging.Formatter("%(message)s")) +TO_STDOUT.setLevel(logging.INFO) +TO_STDOUT.addFilter( + LoggingFilter( + logging.INFO, + logging.NOTSET, + ) +) + +logging.basicConfig(level=logging.INFO, handlers=(TO_STDOUT, TO_STDERR)) + + +class ParsingException(Exception): + def __init__(self, file: Optional[str], lineno: int, line: str): + self.file = file + self.lineno = lineno + self.line = line + + def __str__(self): + out = "" + + if self.file: + out += f"in {self.file}" + + lineno = f" {self.lineno}" + out += f"\n\n{lineno} {self.line}" + + out += f'\n{" " * len(lineno)} ' + out += "^" * len(self.line.strip()) + + return out + + +class InvalidSignature(ParsingException): + pass + + +class InvalidSyntax(ParsingException): + pass + + +def extract_build_opt(name: str, dst: TextIO, src: TextIO): """ - Extract the embedded build.opt from Sketch.ino.globals.h into build - path/core/build.opt. The subdirectory path must already exist as well as the - copy of Sketch.ino.globals.h. + Read src line by line and extract matching 'create-file' directives. + 'name' can match multiple times + + Empty 'src' is always valid. + Never matching anything for 'name' is also valid. + + Incorrectly written signatures always fail with an exception. + + C/C++ syntax validity isn't checked, that's up to the user """ - global build_opt_signature - global default_encoding - - build_opt = open(build_opt_fqfn, 'w', encoding=default_encoding) - if not os.path.exists(globals_h_fqfn) or (0 == os.path.getsize(globals_h_fqfn)): - build_opt.close() - return False - - complete_comment = False - build_opt_error = False - line_no = 0 - # If the source sketch did not have the file Sketch.ino.globals.h, an empty - # file was created in the ./core/ folder. - # By using the copy, open will always succeed. - with open(globals_h_fqfn, 'r', encoding="utf-8") as src: - for line in src: - line = line.strip() - line_no += 1 - if line == build_opt_signature: - if complete_comment: - build_opt_error = True - print_err(" Multiple embedded build.opt blocks in", f'{file_name}:{line_no}') - continue - print_msg("Extracting embedded compiler command-line options from", f'{file_name}:{line_no}') - for line in src: - line = line.strip() - line_no += 1 - if 0 == len(line): - continue - if line.startswith("*/"): - complete_comment = True - break - elif line.startswith("*"): # these are so common - skip these should they occur - continue - elif line.startswith("#"): # allow some embedded comments - continue - elif line.startswith("//"): - continue - # some consistency checking before writing - give some hints about what is wrong - elif line == build_opt_signature: - print_err(" Double begin before end for embedded build.opt block in", f'{file_name}:{line_no}') - build_opt_error = True - elif line.startswith(build_opt_signature): - print_err(" build.opt signature block ignored, trailing character for embedded build.opt block in", f'{file_name}:{line_no}') - build_opt_error = True - elif "/*" in line or "*/" in line : - print_err(" Nesting issue for embedded build.opt block in", f'{file_name}:{line_no}') - build_opt_error = True - else: - print_msg(" ", f'{line_no:2}, Add command-line option: {line}', sep='') - build_opt.write(line + "\n") - elif line.startswith(build_opt_signature): - print_err(" build.opt signature block ignored, trailing character for embedded build.opt block in", f'{file_name}:{line_no}') - build_opt_error = True - if not complete_comment or build_opt_error: - build_opt.truncate(0) - build_opt.close() - if build_opt_error: - # this will help the script start over when the issue is fixed - os.remove(globals_h_fqfn) - print_err(" Extraction failed") - # Don't let the failure get hidden by a spew of nonsensical error - # messages that will follow. Bring things to a halt. - handle_error(1) - return False # not reached - elif complete_comment: - print_msg(" Created compiler command-line options file " + build_opt_fqfn) - build_opt.close() - return complete_comment - - -def enable_override(enable, commonhfile_fqfn): - # Reduce disk IO writes - if os.path.exists(commonhfile_fqfn): - if os.path.getsize(commonhfile_fqfn): # workaround active - if enable: - return - elif not enable: - return - with open(commonhfile_fqfn, 'w', encoding="utf-8") as file: - if enable: - file.write("//Override aggressive caching\n") - # enable workaround when getsize(commonhfile_fqfn) is non-zero, disabled when zero - - -def discover_1st_time_run(build_path): - # Need to know if this is the 1ST compile of the Arduino IDE starting. - # Use empty cache directory as an indicator for 1ST compile. - # Arduino IDE 2.0 RC5 does not cleanup on exist like 1.6.19. Probably for - # debugging like the irregular version number 10607. For RC5 this indicator - # will be true after a reboot instead of a 1ST compile of the IDE starting. - # Another issue for this technique, Windows does not clear the Temp directory. :( - tmp_path, build = os.path.split(build_path) - ide_2_0 = 'arduino-sketch-' - if ide_2_0 == build[:len(ide_2_0)]: - search_path = os.path.join(tmp_path, 'arduino-core-cache/*') # Arduino IDE 2.0 - else: - search_path = os.path.join(tmp_path, 'arduino_cache_*/*') # Arduino IDE 1.6.x and up - - count = 0 - for dirname in glob.glob(search_path): - count += 1 - return 0 == count - - -def get_preferences_txt(file_fqfn, key): - # Get Key Value, key is allowed to be missing. - # We assume file file_fqfn exists - basename = os.path.basename(file_fqfn) - with open(file_fqfn, encoding="utf-8") as file: - for line in file: - name, value = line.partition("=")[::2] - if name.strip().lower() == key: - val = value.strip().lower() - if val != 'true': - val = False - print_msg(f" {basename}: {key}={val}") - return val - print_err(f" Key '{key}' not found in file {basename}. Default to true.") - return True # If we don't find it just assume it is set True - - -def check_preferences_txt(runtime_ide_path, preferences_file): - key = "compiler.cache_core" - # return the state of "compiler.cache_core" found in preferences.txt - if preferences_file != None: - if os.path.exists(preferences_file): - print_msg(f"Using preferences from '{preferences_file}'") - return get_preferences_txt(preferences_file, key) - else: - print_err(f"Override preferences file '{preferences_file}' not found.") - - # Referencing the preferences.txt for an indication of shared "core.a" - # caching is unreliable. There are too many places reference.txt can be - # stored and no hints of which the Arduino build might be using. Unless - # directed otherwise, assume "core.a" caching true. - print_msg(f"Assume aggressive 'core.a' caching enabled.") - return True - -def touch(fname, times=None): - with open(fname, "ab") as file: - file.close(); - os.utime(fname, times) - -def synchronous_touch(globals_h_fqfn, commonhfile_fqfn): - global debug_enabled - # touch both files with the same timestamp - touch(globals_h_fqfn) - with open(globals_h_fqfn, "rb") as file: - file.close() - with open(commonhfile_fqfn, "ab") as file2: - file2.close() - ts = os.stat(globals_h_fqfn) - os.utime(commonhfile_fqfn, ns=(ts.st_atime_ns, ts.st_mtime_ns)) - - if debug_enabled: - print_dbg("After synchronous_touch") - ts = os.stat(globals_h_fqfn) - print_dbg(f" globals_h_fqfn ns_stamp = {ts.st_mtime_ns}") - print_dbg(f" getmtime(globals_h_fqfn) {os.path.getmtime(globals_h_fqfn)}") - ts = os.stat(commonhfile_fqfn) - print_dbg(f" commonhfile_fqfn ns_stamp = {ts.st_mtime_ns}") - print_dbg(f" getmtime(commonhfile_fqfn) {os.path.getmtime(commonhfile_fqfn)}") - -def determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn): - global docs_url - print_dbg(f"runtime_ide_version: {args.runtime_ide_version}") - - if args.cache_core != None: - print_msg(f"Preferences override, this prebuild script assumes the 'compiler.cache_core' parameter is set to {args.cache_core}") - print_msg(f"To change, modify 'mkbuildoptglobals.extra_flags=(--cache_core | --no_cache_core)' in 'platform.local.txt'") - return args.cache_core - else: - ide_path = None - preferences_fqfn = None - if args.preferences_sketch != None: - preferences_fqfn = os.path.join( - os.path.dirname(source_globals_h_fqfn), - os.path.normpath(args.preferences_sketch)) - else: - if args.preferences_file != None: - preferences_fqfn = args.preferences_file - elif args.preferences_env != None: - preferences_fqfn = args.preferences_env + IN_RAW = 1 + IN_BUILD_OPT = 2 + IN_SKIP_OPT = 3 + + state = IN_RAW + + for n, raw_line in enumerate(src, start=1): + line = raw_line.strip().rstrip() + + if state == IN_SKIP_OPT: + if line.startswith("*/"): + state = IN_RAW + continue + + if line.startswith("/*@"): + if not line.endswith("@"): + raise InvalidSyntax(None, n, raw_line) + + result = BUILD_OPT_SIGNATURE_RE.search(line) + if not result or state in (IN_BUILD_OPT, IN_SKIP_OPT): + raise InvalidSignature(None, n, raw_line) + + if name == result.group("name"): + state = IN_BUILD_OPT else: - ide_path = runtime_ide_path - - if preferences_fqfn != None: - preferences_fqfn = os.path.normpath(preferences_fqfn) - root = False - if 'Windows' == platform.system(): - if preferences_fqfn[1:2] == ':\\': - root = True - else: - if preferences_fqfn[0] == '/': - root = True - if not root: - if preferences_fqfn[0] != '~': - preferences_fqfn = os.path.join("~", preferences_fqfn) - preferences_fqfn = os.path.expanduser(preferences_fqfn) - print_dbg(f"determine_cache_state: preferences_fqfn: {preferences_fqfn}") - - return check_preferences_txt(ide_path, preferences_fqfn) + state = IN_SKIP_OPT + continue -""" -TODO + if state == IN_BUILD_OPT: + if line.startswith("*/"): + state = IN_RAW + continue -aggressive caching workaround -========== ======= ========== -The question needs to be asked, is it a good idea? -With all this effort to aid in determining the cache state, it is rendered -usless when arduino command line switches are used that contradict our -settings. + if line.startswith(("#", "//", "*")): + continue -Sort out which of these are imperfect solutions should stay in + if not line: + continue -Possible options for handling problems caused by: - ./arduino --preferences-file other-preferences.txt - ./arduino --pref compiler.cache_core=false + dst.write(f"{line}\n") ---cache_core ---no_cache_core ---preferences_file (relative to IDE or full path) ---preferences_sketch (default looks for preferences.txt or specify path relative to sketch folder) ---preferences_env, python docs say "Availability: most flavors of Unix, Windows." - export ARDUINO15_PREFERENCES_FILE=$(realpath other-name-than-default-preferences.txt ) - ./arduino --preferences-file other-name-than-default-preferences.txt +def extract_build_opt_from_path(dst: TextIO, name: str, p: pathlib.Path): + """ + Same as 'extract_build_opt', but use a file path + """ + try: + with p.open("r", encoding=FILE_ENCODING) as src: + extract_build_opt(name, dst, src) + except ParsingException as e: + e.file = p.name + raise e - platform.local.txt: mkbuildoptglobals.extra_flags=--preferences_env - Tested with: - export ARDUINO15_PREFERENCES_FILE=$(realpath ~/projects/arduino/arduino-1.8.19/portable/preferences.txt) - ~/projects/arduino/arduino-1.8.18/arduino +def is_future_utime(p: pathlib.Path): + return time.time_ns() < p.stat().st_mtime_ns - Future Issues - * "--preferences-file" does not work for Arduino IDE 2.0, they plan to address at a future release - * Arduino IDE 2.0 does not support portable, they plan to address at a future release +def as_stat_result(p: Union[os.stat_result, pathlib.Path]) -> Optional[os.stat_result]: + if not isinstance(p, os.stat_result): + if not p.exists(): + return None + return p.stat() -""" + return p + + +def is_different_utime( + p1: Union[os.stat_result, pathlib.Path], + p2: Union[os.stat_result, pathlib.Path], +) -> bool: + s1 = as_stat_result(p1) + if not s1: + return True + + s2 = as_stat_result(p2) + if not s2: + return True + + for attr in ("st_atime_ns", "st_mtime_ns"): + if getattr(s1, attr) != getattr(s2, attr): + return True + return False -def check_env(env): - system = platform.system() - # From the docs: - # Availability: most flavors of Unix, Windows. - # “Availability: Unix” are supported on macOS - # Because of the soft commitment, I used "help=argparse.SUPPRESS" to keep - # the claim out of the help. The unavailable case is untested. - val = os.getenv(env) - if val == None: - if "Linux" == system or "Windows" == system: - raise argparse.ArgumentTypeError(f'Missing environment variable: {env}') + +# Arduino IDE uses timestamps to determine whether the .o file should be rebuilt +def synchronize_utime(stat: Union[os.stat_result, pathlib.Path], *rest: pathlib.Path): + """ + Retrieve stats from the first 'file' and apply to the 'rest' + """ + if not isinstance(stat, os.stat_result): + logging.debug("using stats from %s", stat.name) + stat = stat.stat() + for p in rest: + if is_different_utime(stat, p.stat()): + os.utime(p, ns=(stat.st_atime_ns, stat.st_mtime_ns)) + logging.debug("synchronized %s", p.name) + + +def as_include_line(p: pathlib.Path) -> str: + out = p.absolute().as_posix() + out = out.replace('"', '\\"') + out = f'-include "{out}"' + return out + + +def as_path_field(help: str): + return dataclasses.field( + default_factory=pathlib.Path, + metadata={ + "help": help, + }, + ) + + +@dataclasses.dataclass +class Context: + build_opt: pathlib.Path = as_path_field( + "resulting options file, used in the gcc command line options" + ) + + source_sketch_header: pathlib.Path = as_path_field( + ".globals.h located in the sketch directory" + ) + build_sketch_header: pathlib.Path = as_path_field( + ".globals.h located in the build directory" + ) + + common_header: pathlib.Path = as_path_field( + "dependency file, copied into the core directory to trigger rebuilds" + ) + + +# build process requires some file to always exist, even when they are empty and thus unused +def ensure_exists_and_empty(*paths: pathlib.Path): + """ + Create empty or replace existing files with an empty ones at 'paths' + """ + for p in paths: + if not p.parent.exists(): + p.parent.mkdir(parents=True, exist_ok=True) + + if not p.exists() or (p.exists() and p.stat().st_size): + p.write_bytes(b"") + logging.debug("placeholder for %s", p.name) else: - # OS/Library limitation - raise argparse.ArgumentTypeError('Not supported') - return val + logging.debug("up-to-date %s", p.name) -def parse_args(): - extra_txt = '''\ - Use platform.local.txt 'mkbuildoptglobals.extra_flags=...' to supply override options: - --cache_core | --no_cache_core | --preferences_file PREFERENCES_FILE | ... +def ensure_normal_time(*paths: pathlib.Path): + for p in paths: + if p.exists() and is_future_utime(p): + logging.debug("fixing timestamp of %s", p.name) + p.touch() - more help at {} - '''.format(docs_url) - parser = argparse.ArgumentParser( - description='Prebuild processing for globals.h and build.opt file', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=textwrap.dedent(extra_txt)) - parser.add_argument('runtime_ide_path', help='Runtime IDE path, {runtime.ide.path}') - parser.add_argument('runtime_ide_version', type=int, help='Runtime IDE Version, {runtime.ide.version}') - parser.add_argument('build_path', help='Build path, {build.path}') - parser.add_argument('build_opt_fqfn', help="Build FQFN to build.opt") - parser.add_argument('source_globals_h_fqfn', help="Source FQFN Sketch.ino.globals.h") - parser.add_argument('commonhfile_fqfn', help="Core Source FQFN CommonHFile.h") - parser.add_argument('--debug', action='store_true', required=False, default=False) - parser.add_argument('-DDEBUG_ESP_PORT', nargs='?', action='store', const="", default="", help='Add mkbuildoptglobals.extra_flags={build.debug_port} to platform.local.txt') - parser.add_argument('--ci', action='store_true', required=False, default=False) - group = parser.add_mutually_exclusive_group(required=False) - group.add_argument('--cache_core', action='store_true', default=None, help='Assume a "compiler.cache_core" value of true') - group.add_argument('--no_cache_core', dest='cache_core', action='store_false', help='Assume a "compiler.cache_core" value of false') - group.add_argument('--preferences_file', help='Full path to preferences file') - group.add_argument('--preferences_sketch', nargs='?', action='store', const="preferences.txt", help='Sketch relative path to preferences file') - # Since the docs say most versions of Windows and Linux support the os.getenv method, suppress the help message. - group.add_argument('--preferences_env', nargs='?', action='store', type=check_env, const="ARDUINO15_PREFERENCES_FILE", help=argparse.SUPPRESS) - # ..., help='Use environment variable for path to preferences file') - return parser.parse_args() - # ref epilog, https://stackoverflow.com/a/50021771 - # ref nargs='*'', https://stackoverflow.com/a/4480202 - # ref no '--n' parameter, https://stackoverflow.com/a/21998252 - - -# retrieve *system* encoding, not the one used by python internally -if sys.version_info >= (3, 11): - def get_encoding(): - return locale.getencoding() -else: - def get_encoding(): - return locale.getdefaultlocale()[1] +# ref. https://gcc.gnu.org/onlinedocs/cpp/Line-Control.html +# arduino builder not just copies sketch files to the build directory, +# but also injects this line at the top to remember the source +def as_arduino_sketch_quoted_header(p: pathlib.Path): + out = str(p.absolute()) + out = out.replace("\\", "\\\\") + out = out.replace('"', '\\"') + return f'#line 1 "{out}"\n' -def show_value(desc, value): - print_dbg(f'{desc:<40} {value}') - return -def locale_dbg(): - show_value("get_encoding()", get_encoding()) - show_value("locale.getdefaultlocale()", locale.getdefaultlocale()) - show_value('sys.getfilesystemencoding()', sys.getfilesystemencoding()) - show_value("sys.getdefaultencoding()", sys.getdefaultencoding()) - show_value("locale.getpreferredencoding(False)", locale.getpreferredencoding(False)) - try: - show_value("locale.getpreferredencoding()", locale.getpreferredencoding()) - except: - pass - show_value("sys.stdout.encoding", sys.stdout.encoding) +def write_or_replace(p: pathlib.Path, contents: str, encoding=FILE_ENCODING) -> bool: + actual = "" - # use current setting - show_value("locale.setlocale(locale.LC_ALL, None)", locale.setlocale(locale.LC_ALL, None)) try: - show_value("locale.getencoding()", locale.getencoding()) - except: - pass - show_value("locale.getlocale()", locale.getlocale()) - - # use user setting - show_value("locale.setlocale(locale.LC_ALL, '')", locale.setlocale(locale.LC_ALL, '')) - # show_value("locale.getencoding()", locale.getencoding()) - show_value("locale.getlocale()", locale.getlocale()) - return - - -def main(): - global build_opt_signature - global docs_url - global debug_enabled - global default_encoding - num_include_lines = 1 - - # Given that GCC will handle lines from an @file as if they were on - # the command line. I assume that the contents of @file need to be encoded - # to match that of the shell running GCC runs. I am not 100% sure this API - # gives me that, but it appears to work. - # - # However, elsewhere when dealing with source code we continue to use 'utf-8', - # ref. https://gcc.gnu.org/onlinedocs/cpp/Character-sets.html - default_encoding = get_encoding() - - args = parse_args() - debug_enabled = args.debug - runtime_ide_path = os.path.normpath(args.runtime_ide_path) - build_path = os.path.normpath(args.build_path) - build_opt_fqfn = os.path.normpath(args.build_opt_fqfn) - source_globals_h_fqfn = os.path.normpath(args.source_globals_h_fqfn) - commonhfile_fqfn = os.path.normpath(args.commonhfile_fqfn) - - globals_name = os.path.basename(source_globals_h_fqfn) - build_path_core, build_opt_name = os.path.split(build_opt_fqfn) - globals_h_fqfn = os.path.join(build_path_core, globals_name) - - if debug_enabled: - locale_dbg() - - print_msg(f'default_encoding: {default_encoding}') - - print_dbg(f"runtime_ide_path: {runtime_ide_path}") - print_dbg(f"runtime_ide_version: {args.runtime_ide_version}") - print_dbg(f"build_path: {build_path}") - print_dbg(f"build_opt_fqfn: {build_opt_fqfn}") - print_dbg(f"source_globals_h_fqfn: {source_globals_h_fqfn}") - print_dbg(f"commonhfile_fqfn: {commonhfile_fqfn}") - print_dbg(f"globals_name: {globals_name}") - print_dbg(f"build_path_core: {build_path_core}") - print_dbg(f"globals_h_fqfn: {globals_h_fqfn}") - print_dbg(f"DDEBUG_ESP_PORT: {args.DDEBUG_ESP_PORT}") - - if len(args.DDEBUG_ESP_PORT): - build_opt_signature = build_opt_signature[:-1] + ":debug@" - - print_dbg(f"build_opt_signature: {build_opt_signature}") - - if args.ci: - # Requires CommonHFile.h to never be checked in. - if os.path.exists(commonhfile_fqfn): - first_time = False - else: - first_time = True + if p.exists(): + actual = p.read_text(encoding=encoding) + except UnicodeDecodeError: + logging.warning("cannot decode %s", p.name) + + if contents != actual: + p.write_text(contents, encoding=encoding) + return True + + return False + + +# arduino builder would copy the file regardless +# prebuild stage has it missing though +def ensure_build_sketch_header_written(ctx: Context): + """ + Sketch header copy or placeholder must always exist in the build directory (even when unused) + """ + if not ctx.source_sketch_header.exists(): + ensure_exists_and_empty(ctx.build_sketch_header) + return + + p = ctx.build_sketch_header + if not p.parent.exists(): + p.parent.mkdir(parents=True, exist_ok=True) + + contents = ctx.source_sketch_header.read_text(encoding=FILE_ENCODING) + contents = "".join( + ( + as_arduino_sketch_quoted_header(ctx.source_sketch_header), + contents, + "\n", + ) + ) + + if write_or_replace(p, contents): + copystat(ctx.source_sketch_header, p) + + +def ensure_common_header_bound(ctx: Context): + """ + Record currently used command-line options file + """ + if write_or_replace( + ctx.common_header, as_arduino_sketch_quoted_header(ctx.build_opt) + ): + logging.debug("wrote to %s", ctx.common_header.name) else: - first_time = discover_1st_time_run(build_path) - if first_time: - print_dbg("First run since Arduino IDE started.") - - use_aggressive_caching_workaround = determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn) - - print_dbg(f"first_time: {first_time}") - print_dbg(f"use_aggressive_caching_workaround: {use_aggressive_caching_workaround}") - - if not os.path.exists(build_path_core): - os.makedirs(build_path_core) - print_msg("Clean build, created dir " + build_path_core) - - if first_time or \ - not use_aggressive_caching_workaround or \ - not os.path.exists(commonhfile_fqfn): - enable_override(False, commonhfile_fqfn) - - # A future timestamp on commonhfile_fqfn will cause everything to - # rebuild. This occurred during development and may happen after - # changing the system time. - if time.time_ns() < os.stat(commonhfile_fqfn).st_mtime_ns: - touch(commonhfile_fqfn) - print_err(f"Neutralized future timestamp on build file: {commonhfile_fqfn}") - - if os.path.exists(source_globals_h_fqfn): - print_msg("Using global include from " + source_globals_h_fqfn) - - copy_create_build_file(source_globals_h_fqfn, globals_h_fqfn) - - # globals_h_fqfn timestamp was only updated if the source changed. This - # controls the rebuild on change. We can always extract a new build.opt - # w/o triggering a needless rebuild. - embedded_options = extract_create_build_opt_file(globals_h_fqfn, globals_name, build_opt_fqfn) - - if use_aggressive_caching_workaround: - # commonhfile_fqfn encodes the following information - # 1. When touched, it causes a rebuild of core.a - # 2. When file size is non-zero, it indicates we are using the - # aggressive cache workaround. The workaround is set to true - # (active) when we discover a non-zero length global .h file in - # any sketch. The aggressive workaround is cleared on the 1ST - # compile by the Arduino IDE after starting. - # 3. When the timestamp matches the build copy of globals.h - # (globals_h_fqfn), we know one two things: - # * The cached core.a matches up to the current build.opt and - # globals.h. The current sketch owns the cached copy of core.a. - # * globals.h has not changed, and no need to rebuild core.a - # 4. When core.a's timestamp does not match the build copy of - # the global .h file, we only know we need to rebuild core.a, and - # that is enough. - # - # When the sketch build has a "Sketch.ino.globals.h" file in the - # build tree that exactly matches the timestamp of "CommonHFile.h" - # in the platform source tree, it owns the core.a cache copy. If - # not, or "Sketch.ino.globals.h" has changed, rebuild core. - # A non-zero file size for commonhfile_fqfn, means we have seen a - # globals.h file before and workaround is active. - if debug_enabled: - print_dbg("Timestamps at start of check aggressive caching workaround") - ts = os.stat(globals_h_fqfn) - print_dbg(f" globals_h_fqfn ns_stamp = {ts.st_mtime_ns}") - print_dbg(f" getmtime(globals_h_fqfn) {os.path.getmtime(globals_h_fqfn)}") - ts = os.stat(commonhfile_fqfn) - print_dbg(f" commonhfile_fqfn ns_stamp = {ts.st_mtime_ns}") - print_dbg(f" getmtime(commonhfile_fqfn) {os.path.getmtime(commonhfile_fqfn)}") - - if os.path.getsize(commonhfile_fqfn): - if (os.path.getmtime(globals_h_fqfn) != os.path.getmtime(commonhfile_fqfn)): - # Need to rebuild core.a - # touching commonhfile_fqfn in the source core tree will cause rebuild. - # Looks like touching or writing unrelated files in the source core tree will cause rebuild. - synchronous_touch(globals_h_fqfn, commonhfile_fqfn) - print_msg("Using 'aggressive caching' workaround, rebuild shared 'core.a' for current globals.") - else: - print_dbg(f"Using old cached 'core.a'") - elif os.path.getsize(globals_h_fqfn): - enable_override(True, commonhfile_fqfn) - synchronous_touch(globals_h_fqfn, commonhfile_fqfn) - print_msg("Using 'aggressive caching' workaround, rebuild shared 'core.a' for current globals.") - else: - print_dbg(f"Workaround not active/needed") - - add_include_line(build_opt_fqfn, commonhfile_fqfn) - add_include_line(build_opt_fqfn, globals_h_fqfn) - - # Provide context help for build option support. - source_build_opt_h_fqfn = os.path.join(os.path.dirname(source_globals_h_fqfn), "build_opt.h") - if os.path.exists(source_build_opt_h_fqfn) and not embedded_options: - print_err("Build options file '" + source_build_opt_h_fqfn + "' not supported.") - print_err(" Add build option content to '" + source_globals_h_fqfn + "'.") - print_err(" Embedd compiler command-line options in a block comment starting with '" + build_opt_signature + "'.") - print_err(" Read more at " + docs_url) - elif os.path.exists(source_globals_h_fqfn): - if not embedded_options: - print_msg("Tip: Embedd compiler command-line options in a block comment starting with '" + build_opt_signature + "'.") - print_msg(" Read more at " + docs_url) + logging.debug("up-to-date %s", ctx.build_opt.name) + + +def make_build_opt_name(ctx: Context, debug: bool) -> str: + name = ctx.build_opt.name + if debug: + name = f"{name}:debug" + + return name + + +def ensure_build_opt_written(ctx: Context, buffer: io.StringIO): + """ + Make sure that 'build_opt' is written to the filesystem. + + '-include ...' lines are always appended at the end of the buffer. + 'build_opt' is not written when its contents remain unchanged. + """ + includes = [ + ctx.common_header, + ] + + if len(buffer.getvalue()): + includes.append(ctx.build_sketch_header) + + for p in includes: + buffer.write(f"{as_include_line(p)}\n") + + value = buffer.getvalue() + + if ( + not ctx.build_opt.exists() + or is_different_utime(ctx.build_sketch_header, ctx.build_opt) + or ctx.build_opt.read_text(encoding=DEFAULT_ENCODING) != value + ): + ctx.build_opt.parent.mkdir(parents=True, exist_ok=True) + ctx.build_opt.write_text(value, encoding=DEFAULT_ENCODING) + logging.debug("wrote to %s", ctx.build_opt.name) else: - print_msg("Note: optional global include file '" + source_globals_h_fqfn + "' does not exist.") - print_msg(" Read more at " + docs_url) + logging.debug("up-to-date %s", ctx.build_opt.name) + + +def maybe_empty_or_missing(p: pathlib.Path): + return not p.exists() or not p.stat().st_size - handle_error(0) # commit print buffer -if __name__ == '__main__': - rc = 1 +def build_with_minimal_build_opt(ctx: Context): + """ + When sketch header is empty or there were no opt files created + """ + logging.debug("building with a minimal %s", ctx.build_opt.name) + + ensure_build_opt_written(ctx, buffer=io.StringIO()) + + ensure_exists_and_empty(ctx.common_header) + ensure_build_sketch_header_written(ctx) + + # sketch directory time ignored, stats are from the only persistent file + synchronize_utime( + ctx.common_header, + ctx.build_opt, + ctx.build_sketch_header, + ) + + +# Before synchronizing targets, find out which file was modified last +# by default, check 'st_mtime_ns' attribute of os.stat_result +def most_recent( + *paths: pathlib.Path, attr="st_mtime_ns" +) -> Tuple[pathlib.Path, os.stat_result]: + def make_pair(p: pathlib.Path): + return (p, p.stat()) + + def key(pair: Tuple[pathlib.Path, os.stat_result]) -> int: + return getattr(pair[1], attr) + + if not paths: + raise ValueError('"paths" cannot be empty') + elif len(paths) == 1: + return make_pair(paths[0]) + + return max((make_pair(p) for p in paths), key=key) + + +def main_build(args: argparse.Namespace): + ctx = Context( + build_opt=args.build_opt, + source_sketch_header=args.source_sketch_header, + build_sketch_header=args.build_sketch_header, + common_header=args.common_header, + ) + + if args.debug: + logging.debug("using the following build context") + for field in dataclasses.fields(ctx): + logging.debug( + " %s %s %s", + field.name, + getattr(ctx, field.name), + field.metadata["help"], + ) + + # notify when other files similar to .globals.h are in the sketch directory + other_build_options = check_other_build_options(ctx.source_sketch_header) + if other_build_options: + logging.warning(other_build_options) + + # future timestamps generally break build order. + # before comparing or synchronizing time, make sure it is current + ensure_normal_time(*dataclasses.astuple(ctx)) + + # when .globals.h is missing, provide placeholder files for the build and exit + if maybe_empty_or_missing(ctx.source_sketch_header): + build_with_minimal_build_opt(ctx) + return + + # when debug port is used, allow for a different set of command line options + build_debug = args.build_debug or "DEBUG_SERIAL_PORT" in ( + args.D or [] + ) # type: bool + name = make_build_opt_name(ctx, build_debug) + + # options file is not written immediately, buffer its contents before commiting + build_opt_buffer = io.StringIO() + + try: + logging.debug("searching for %s", name) + extract_build_opt_from_path(build_opt_buffer, name, ctx.source_sketch_header) + except ParsingException as e: + raise e from None + + # when command-line options were not created / found, it means the same thing as empty or missing .globals.h + if not len(build_opt_buffer.getvalue()): + build_with_minimal_build_opt(ctx) + return + + logging.debug("preparing %s", ctx.build_opt.name) + for line in build_opt_buffer: + logging.debug(" %s", line) + + # at this point, it is necessary to synchronize timestamps of every file + ensure_build_opt_written(ctx, build_opt_buffer) + ensure_build_sketch_header_written(ctx) + ensure_common_header_bound(ctx) + + # stats are now based on the either active sketch or common header + # (thus ensure core.a is rebuilt, even when sketch mtime is earlier) + synchronize_utime( + most_recent(ctx.common_header, ctx.source_sketch_header)[1], + ctx.build_opt, + ctx.build_sketch_header, + ctx.common_header, + ctx.source_sketch_header, + ) + + +def main_inspect(args: argparse.Namespace): + p = args.path # type: pathlib.Path + + buffer = io.StringIO() try: - rc = main() - except: - print_err(traceback.format_exc()) - handle_error(0) - sys.exit(rc) + extract_build_opt_from_path(buffer, args.build_opt_name, p) + except ParsingException as e: + raise e from None + + logging.info(buffer.getvalue()) + + +def main_placeholder(args: argparse.Namespace): + paths = args.path # type: List[pathlib.Path] + ensure_exists_and_empty(*paths) + + +def main_synchronize(args: argparse.Namespace): + first = args.first # type: pathlib.Path + rest = args.rest # type: List[pathlib.Path] + synchronize_utime(first, *rest) + + +def as_path(p: str) -> pathlib.Path: + if p.startswith("{") or p.endswith("}"): + raise ValueError(f'"{p}" was not resolved') from None + + return pathlib.Path(p) + + +def parse_args(args=None, namespace=None): + parser = argparse.ArgumentParser( + description="Handles sketch header containing command-line options, resulting build.opt and the shared core common header", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent(DOCS_EPILOG), + ) + + def main_help(args: argparse.Namespace): + parser.print_help() + + parser.set_defaults(func=main_help) + + parser.add_argument( + "--debug", action="store_true", help=argparse.SUPPRESS + ) # normal debug + parser.add_argument( + "--audit", action="store_true", help=argparse.SUPPRESS + ) # extra noisy debug + + parser.add_argument( + "--build-debug", + action="store_true", + help="Instead of build.opt, use build.opt:debug when searching for the comment signature match", + ) + + parser.add_argument( + "-D", + help="Intended to be used with mkbuildoptglobals.extra_flags={build.debug_port} in platform.local.txt)." + " Only enable --build-debug when debug port is also enabled in the menu / fqbn options.", + ) + + subparsers = parser.add_subparsers() + + # "prebuild" hook recipe command, preparing all of the necessary build files + + build = subparsers.add_parser("build") + + build.add_argument( + "--build-opt", + type=as_path, + required=True, + help="Command-line options file FQFN aka Fully Qualified File Name " + "(%build-path%/core/%build-opt%)", + ) + build.add_argument( + "--source-sketch-header", + type=as_path, + required=True, + help="FQFN of the globals.h header, located in the sketch directory " + "(%sketchname%/%sketchname%.ino.globals.h)", + ) + build.add_argument( + "--build-sketch-header", + type=as_path, + required=True, + help="FQFN of the globals.h header, located in the build directory " + "(%build-path%/sketch/%sketchname%.ino.globals.h)", + ) + build.add_argument( + "--common-header", + type=as_path, + required=True, + help="FQFN of shared dependency header (%core-path%/%common-header%)", + ) + + build.set_defaults(func=main_build) + + # "postbuild" hook recipe command, in case some files have to be cleared + + placeholder = subparsers.add_parser( + "placeholder", + help=ensure_exists_and_empty.__doc__, + ) + placeholder.add_argument( + "path", + action="append", + type=as_path, + ) + placeholder.set_defaults(func=main_placeholder) + + # Parse file path and discover any 'name' options inside + + inspect = subparsers.add_parser( + "inspect", + help=extract_build_opt.__doc__, + ) + inspect.add_argument("--build-opt-name", type=str, default="build.opt") + inspect.add_argument( + "path", + type=as_path, + ) + inspect.set_defaults(func=main_inspect) + + # Retrieve stats from the first file and apply to the rest + + synchronize = subparsers.add_parser( + "synchronize", + help=synchronize_utime.__doc__, + ) + synchronize.add_argument( + "first", + type=as_path, + help="Any file", + ) + synchronize.add_argument( + "rest", + type=as_path, + nargs="+", + help="Any file", + ) + synchronize.set_defaults(func=main_synchronize) + + return parser.parse_args(args, namespace) + + +def main(args: argparse.Namespace): + # mildly verbose logging, intended to notify about the steps taken + if args.debug: + logging.root.setLevel(logging.DEBUG) + + # very verbose logging from the python internals + if args.audit and sys.version_info >= (3, 8): + + def hook(event, args): + # note that logging module itself has audit calls, + # logging.debug(...) here would deadlock output + print(f"{event}:{args}", file=sys.stderr) + + sys.addaudithook(hook) + + return args.func(args) + + +if __name__ == "__main__": + main(parse_args()) diff --git a/tools/test_mkbuildoptglobals.py b/tools/test_mkbuildoptglobals.py new file mode 100755 index 0000000000..5917545199 --- /dev/null +++ b/tools/test_mkbuildoptglobals.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python + +import io +import sys +import unittest + +from typing import TextIO +from mkbuildoptglobals import extract_build_opt, InvalidSignature, InvalidSyntax +import contextlib + +@contextlib.contextmanager +def buffer(init: str): + yield io.StringIO(init), io.StringIO() + +class TestExtractBuildOpt(unittest.TestCase): + def testParseOnce(self): + src = io.StringIO( + """ +/*@create-file:build.opt@ +-fspecial-option +*/ + """ + ) + + dst = io.StringIO() + + extract_build_opt("build.opt", dst, src) + self.assertEqual("-fspecial-option\n", dst.getvalue()) + + def testParseSecond(self): + src = io.StringIO( + r""" +/*@create-file:foo.opt@ +-fno-builtin +# some random comment +-UFOOBAR +// another one +*/ /* extra comment */ +#define FOOBAR=1 +#define SOMETHING=123 +""" + ) + + dst = io.StringIO() + + extract_build_opt("bar.opt", dst, src) + self.assertEqual(dst.getvalue(), "") + + src.seek(0) + + extract_build_opt("foo.opt", dst, src) + self.assertEqual("-fno-builtin\n-UFOOBAR\n", dst.getvalue()) + + def testMultiple(self): + src = io.StringIO( + r""" +/*@ create-file:foo.opt @ +-ffoo +*/ + +/*@create-file:foo.opt:debug@ +-fbaz +*/ + +/*@create-file:bar.opt:debug@ +-DUNUSED +*/ + +/*@create-file:foo.opt@ +-mbar +*/ + +/*@create-file:bar.opt@ +-DALSO_UNUSED +*/ + +""" + ) + + dst = io.StringIO() + extract_build_opt("foo.opt", dst, src) + + self.assertEqual("-ffoo\n-mbar\n", dst.getvalue()) + + def testInvalidSignature(self): + src = io.StringIO( + r""" +#pragma once + +/*@create-file:foo.opt@ +-fanalyzer +*/ +// ordinary c++ code + +const char GlobalVariable[] = /*@ hello world @*/ +const char WriteMkbuildopts[] = /*@create-file:foo.opt@*/ + +/*@make-file:bar.opt@ +-mforce-l32 +*/ + +/* nothing to look at here */ +""" + ) + + dst = io.StringIO() + with self.assertRaises(InvalidSignature) as raises: + extract_build_opt("bar.opt", dst, src) + + self.assertEqual("", dst.getvalue()) + + e = raises.exception + self.assertFalse(e.file) + self.assertEqual(12, e.lineno) + self.assertEqual("/*@make-file:bar.opt@\n", e.line) + + def testPartialDest(self): + src = io.StringIO( + r""" +/*@create-file:foo.opt@ +-DIMPORTANT_FLAG +-DANOTHER_FLAG=123 +*/ +/*@ create-file:foo.opt @ +/*@oops +-mthis-fails +*/ +""" + ) + + dst = io.StringIO() + with self.assertRaises(InvalidSyntax) as raises: + extract_build_opt("foo.opt", dst, src) + + e = raises.exception + self.assertFalse(e.file) + self.assertEqual(7, e.lineno) + self.assertEqual("/*@oops\n", e.line) + self.assertEqual("-DIMPORTANT_FLAG\n-DANOTHER_FLAG=123\n", dst.getvalue()) + + def testParseSignatureSpace(self): + with buffer(r""" +/*@ create-file:test.opt @ +-ftest-test-test +*/ +""") as (src, dst): + extract_build_opt("test.opt", dst, src) + self.assertEqual("-ftest-test-test\n", dst.getvalue()) + + with buffer(r""" +/*@create-file:test.opt +@ +-ftest-test-test +*/ +""") as (src, dst): + with self.assertRaises(InvalidSyntax) as raises: + extract_build_opt("test.opt", dst, src) + + self.assertFalse(dst.getvalue()) + + +if __name__ == "__main__": + unittest.main() From 82f884fc02db6cc3f9bb881fedbe42561f37c795 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Tue, 27 May 2025 07:28:11 +0300 Subject: [PATCH 02/13] compile args typo --- tests/common.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/common.sh b/tests/common.sh index 33daeb5255..3e3d307414 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -132,7 +132,6 @@ function build_sketches() build_cmd+=${cli_path} build_cmd+=" compile"\ " --warnings=all"\ -" --output-path $build_dir"\ " --fqbn $fqbn"\ " --libraries $library_path"\ " --output-dir $build_out" From 26316d3311688c7d075687aea06488302975a8ef Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Tue, 27 May 2025 07:37:06 +0300 Subject: [PATCH 03/13] build_out & no explicit clean --- tests/common.sh | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/tests/common.sh b/tests/common.sh index 3e3d307414..9eb5703e24 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -138,7 +138,6 @@ function build_sketches() print_size_info_header >"$cache_dir"/size.log - local clean_core=1 local testcnt=0 local cnt=0 @@ -163,17 +162,11 @@ function build_sketches() build_cnt=0 fi - # Do we need a clean core build? $build_dir/core/* cannot be shared - # between sketches when global options are present. - clean_core=$(arduino_mkbuildoptglobals_cleanup "$clean_core" "$build_dir" "$sketch") - - # Clear out the last built sketch, map, elf, bin files, but leave the compiled - # objects in the core and libraries available for use so we don't need to rebuild - # them each sketch. - rm -rf "$build_dir"/sketch \ - "$build_dir"/*.bin \ - "$build_dir"/*.map \ - "$build_dir"/*.elf + # Clear out the latest map, elf, bin files + rm -rf \ + "$build_out"/*.bin \ + "$build_out"/*.map \ + "$build_out"/*.elf echo ${ci_group}Building $cnt $sketch echo "$build_cmd $sketch" @@ -193,7 +186,7 @@ function build_sketches() fi print_size_info "$core_path"/tools/xtensa-lx106-elf/bin/xtensa-lx106-elf-size \ - $build_dir/*.elf >>$cache_dir/size.log + $build_out/*.elf >>$cache_dir/size.log echo $ci_end_group done From 75598a7c227d09c31b27a3d3c9bcf481aebcebf3 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Tue, 27 May 2025 20:59:11 +0300 Subject: [PATCH 04/13] drop partial data, handle missing closing comment --- tools/mkbuildoptglobals.py | 12 ++- tools/test_mkbuildoptglobals.py | 153 ++++++++++++++++++++++++++++---- 2 files changed, 149 insertions(+), 16 deletions(-) diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py index 6c68f87ab8..0eb3b6e028 100755 --- a/tools/mkbuildoptglobals.py +++ b/tools/mkbuildoptglobals.py @@ -401,6 +401,7 @@ def extract_build_opt(name: str, dst: TextIO, src: TextIO): IN_SKIP_OPT = 3 state = IN_RAW + block = [] # type: List[str] for n, raw_line in enumerate(src, start=1): line = raw_line.strip().rstrip() @@ -408,6 +409,9 @@ def extract_build_opt(name: str, dst: TextIO, src: TextIO): if state == IN_SKIP_OPT: if line.startswith("*/"): state = IN_RAW + for line in block: + dst.write(line) + block = [] continue if line.startswith("/*@"): @@ -436,7 +440,13 @@ def extract_build_opt(name: str, dst: TextIO, src: TextIO): if not line: continue - dst.write(f"{line}\n") + block.append(f"{line}\n") + + if state != IN_RAW: + raise InvalidSyntax(None, n, raw_line) + + for line in block: + dst.write(line) def extract_build_opt_from_path(dst: TextIO, name: str, p: pathlib.Path): diff --git a/tools/test_mkbuildoptglobals.py b/tools/test_mkbuildoptglobals.py index 5917545199..35f6d038ee 100755 --- a/tools/test_mkbuildoptglobals.py +++ b/tools/test_mkbuildoptglobals.py @@ -8,16 +8,102 @@ from mkbuildoptglobals import extract_build_opt, InvalidSignature, InvalidSyntax import contextlib + @contextlib.contextmanager def buffer(init: str): yield io.StringIO(init), io.StringIO() + class TestExtractBuildOpt(unittest.TestCase): + def testNoSource(self): + src = io.StringIO() + dst = io.StringIO() + + extract_build_opt("anything.opt", dst, src) + self.assertFalse(dst.getvalue()) + + def testSomeSource(self): + src = io.StringIO("12345") + dst = io.StringIO() + + extract_build_opt("numbers.opt", dst, src) + self.assertFalse(dst.getvalue()) + + def testNoOpt(self): + src = io.StringIO( + r""" +#include +int main() { + puts("hello world"); + return 0; +} + """ + ) + + dst = io.StringIO() + + extract_build_opt("something.opt", dst, src) + self.assertFalse(dst.getvalue()) + + def testAfterOpt(self): + src = io.StringIO( + r""" +int main() { + puts("hello world"); + return 0; +} +/*@create-file:after.opt@ +-fhello-world +*/ + """ + ) + + dst = io.StringIO() + + extract_build_opt("after.opt", dst, src) + self.assertEqual("-fhello-world\n", dst.getvalue()) + + def testEmptyBlock(self): + src = io.StringIO( + r""" + + +/*@create-file:empty.opt@ +*/ + +""" + ) + + dst = io.StringIO() + + extract_build_opt("empty.opt", dst, src) + self.assertFalse(dst.getvalue()) + def testParseOnce(self): src = io.StringIO( - """ -/*@create-file:build.opt@ + r""" +/*@create-file:special.opt@ -fspecial-option +*/ +""" + ) + + dst = io.StringIO() + + extract_build_opt("special.opt", dst, src) + self.assertEqual("-fspecial-option\n", dst.getvalue()) + + def testParseOnceLines(self): + src = io.StringIO( + r""" +/*@create-file:build.opt@ +-DFOO="arbitrary definition 1" +#comment + +// comment + +-DBAR="arbitrary definition 2" + // finalize this */ """ ) @@ -25,7 +111,10 @@ def testParseOnce(self): dst = io.StringIO() extract_build_opt("build.opt", dst, src) - self.assertEqual("-fspecial-option\n", dst.getvalue()) + self.assertEqual( + '-DFOO="arbitrary definition 1"\n' '-DBAR="arbitrary definition 2"\n', + dst.getvalue(), + ) def testParseSecond(self): src = io.StringIO( @@ -54,10 +143,13 @@ def testParseSecond(self): def testMultiple(self): src = io.StringIO( r""" +#include +int foo() { return 111; } + /*@ create-file:foo.opt @ -ffoo */ - +#define INTERMIXED_DATA /*@create-file:foo.opt:debug@ -fbaz */ @@ -65,7 +157,6 @@ def testMultiple(self): /*@create-file:bar.opt:debug@ -DUNUSED */ - /*@create-file:foo.opt@ -mbar */ @@ -74,6 +165,7 @@ def testMultiple(self): -DALSO_UNUSED */ + """ ) @@ -107,21 +199,21 @@ def testInvalidSignature(self): with self.assertRaises(InvalidSignature) as raises: extract_build_opt("bar.opt", dst, src) - self.assertEqual("", dst.getvalue()) + self.assertFalse(dst.getvalue()) e = raises.exception self.assertFalse(e.file) self.assertEqual(12, e.lineno) self.assertEqual("/*@make-file:bar.opt@\n", e.line) - def testPartialDest(self): + def testPartialInvalidSyntax(self): src = io.StringIO( r""" -/*@create-file:foo.opt@ +/*@create-file:syntax.opt@ -DIMPORTANT_FLAG -DANOTHER_FLAG=123 */ -/*@ create-file:foo.opt @ +/*@ create-file:syntax.opt @ /*@oops -mthis-fails */ @@ -130,29 +222,60 @@ def testPartialDest(self): dst = io.StringIO() with self.assertRaises(InvalidSyntax) as raises: - extract_build_opt("foo.opt", dst, src) + extract_build_opt("syntax.opt", dst, src) + + self.assertFalse(dst.getvalue()) e = raises.exception self.assertFalse(e.file) self.assertEqual(7, e.lineno) self.assertEqual("/*@oops\n", e.line) - self.assertEqual("-DIMPORTANT_FLAG\n-DANOTHER_FLAG=123\n", dst.getvalue()) + + def testPartialUnclosed(self): + src = io.StringIO( + r""" +/*@create-file:unclosed.opt@ +line 1 +line 2 +""" + ) + dst = io.StringIO() + with self.assertRaises(InvalidSyntax) as raises: + extract_build_opt("unclosed.opt", dst, src) + + self.assertFalse(dst.getvalue()) def testParseSignatureSpace(self): - with buffer(r""" + with buffer( + r""" /*@ create-file:test.opt @ -ftest-test-test */ -""") as (src, dst): +""" + ) as (src, dst): extract_build_opt("test.opt", dst, src) self.assertEqual("-ftest-test-test\n", dst.getvalue()) - with buffer(r""" + with buffer( + r""" /*@create-file:test.opt @ -ftest-test-test */ -""") as (src, dst): +""" + ) as (src, dst): + with self.assertRaises(InvalidSyntax) as raises: + extract_build_opt("test.opt", dst, src) + + self.assertFalse(dst.getvalue()) + + with buffer( + r""" +/*@create-file:test.opt +-ftest-test-test +*/ +""" + ) as (src, dst): with self.assertRaises(InvalidSyntax) as raises: extract_build_opt("test.opt", dst, src) From 64320d1a101bb2fdb2c26d7ed4bc9a50263dba21 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Wed, 28 May 2025 00:24:03 +0300 Subject: [PATCH 05/13] check cache state dir state in ci --- .github/workflows/build-ide.yml | 3 + tests/build.sh | 38 +----- tests/install_arduino.sh | 8 ++ tests/test_mkbuildoptglobals.sh | 112 ++++++++++++++++++ .../buildopt_then_buildopt.sh | 25 ++++ .../buildopt_then_nobuildopt.sh | 25 ++++ tests/test_mkbuildoptglobals/nobuildopt.sh | 25 ++++ .../nobuildopt_then_buildopt.sh | 25 ++++ 8 files changed, 224 insertions(+), 37 deletions(-) create mode 100755 tests/install_arduino.sh create mode 100755 tests/test_mkbuildoptglobals.sh create mode 100644 tests/test_mkbuildoptglobals/buildopt_then_buildopt.sh create mode 100644 tests/test_mkbuildoptglobals/buildopt_then_nobuildopt.sh create mode 100644 tests/test_mkbuildoptglobals/nobuildopt.sh create mode 100644 tests/test_mkbuildoptglobals/nobuildopt_then_buildopt.sh diff --git a/.github/workflows/build-ide.yml b/.github/workflows/build-ide.yml index d5b9b5f0b8..92ba593cd5 100644 --- a/.github/workflows/build-ide.yml +++ b/.github/workflows/build-ide.yml @@ -42,6 +42,9 @@ jobs: python-version: '3.x' - run: | python ./tools/test_mkbuildoptglobals.py --quiet + - run: | + bash ./tests/install_arduino.sh + bash ./tests/test_mkbuildoptglobals.sh run # Examples are built in parallel to avoid CI total job time limitation build-linux: diff --git a/tests/build.sh b/tests/build.sh index af34287a8a..81f756cbac 100755 --- a/tests/build.sh +++ b/tests/build.sh @@ -3,39 +3,7 @@ # expect to have git available root=$(git rev-parse --show-toplevel) -# general configuration related to the builder itself -ESP8266_ARDUINO_BUILD_DIR=${ESP8266_ARDUINO_BUILD_DIR:-$root} -ESP8266_ARDUINO_BUILDER=${ESP8266_ARDUINO_BUILDER:-arduino} -ESP8266_ARDUINO_PRESERVE_CACHE=${ESP8266_ARDUINO_PRESERVE_CACHE:-} - -# sketch build options -ESP8266_ARDUINO_DEBUG=${ESP8266_ARDUINO_DEBUG:-nodebug} -ESP8266_ARDUINO_LWIP=${ESP8266_ARDUINO_LWIP:-default} -ESP8266_ARDUINO_SKETCHES=${ESP8266_ARDUINO_SKETCHES:-} - -ESP8266_ARDUINO_CLI=${ESP8266_ARDUINO_CLI:-$HOME/.local/bin/arduino-cli} - -# ref. https://arduino.github.io/arduino-cli/1.2/configuration/#default-directories -case "${RUNNER_OS:-Linux}" in -"Linux") - ESP8266_ARDUINO_HARDWARE=${ESP8266_ARDUINO_HARDWARE:-$HOME/Arduino/hardware} - ESP8266_ARDUINO_LIBRARIES=${ESP8266_ARDUINO_LIBRARIES:-$HOME/Arduino/libraries} - ;; -"macOS") - ESP8266_ARDUINO_HARDWARE=${ESP8266_ARDUINO_HARDWARE:-$HOME/Documents/Arduino/hardware} - ESP8266_ARDUINO_LIBRARIES=${ESP8266_ARDUINO_LIBRARIES:-$HOME/Documents/Arduino/libraries} - ;; -"Windows") - ESP8266_ARDUINO_HARDWARE=${ESP8266_ARDUINO_HARDWARE:-$HOME/Documents/Arduino/hardware} - ESP8266_ARDUINO_LIBRARIES=${ESP8266_ARDUINO_LIBRARIES:-$HOME/Documents/Arduino/libraries} - ;; -*) - echo 'Unknown ${RUNNER_OS} = "' ${RUNNER_OS} '"' - exit 2 -esac - -source "$root/tests/lib-skip-ino.sh" -source "$root/tests/common.sh" +source "$root/tests/env.sh" cmd=${0##*/} usage=" @@ -56,10 +24,6 @@ USAGE: $cmd - build every .ino file from ESP8266_ARDUINO_SKETCHES " -mod=1 -rem=0 -cnt=0 - if [ "$#" -eq 1 ] ; then case "$1" in "-h") diff --git a/tests/install_arduino.sh b/tests/install_arduino.sh new file mode 100755 index 0000000000..f85895e468 --- /dev/null +++ b/tests/install_arduino.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -x + +root=$(git rev-parse --show-toplevel) +source "$root/tests/env.sh" + +install_arduino "$ESP8266_ARDUINO_DEBUG" diff --git a/tests/test_mkbuildoptglobals.sh b/tests/test_mkbuildoptglobals.sh new file mode 100755 index 0000000000..49a78b8115 --- /dev/null +++ b/tests/test_mkbuildoptglobals.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +root=$(git rev-parse --show-toplevel) +source "$root/tests/env.sh" + +unset -f step_summary +function step_summary() +{ + echo "" +} + +export ARDUINO_BUILD_CACHE_PATH="$cache_dir" + +sketches="${cache_dir}/sketches/" +cores="${cache_dir}/cores/" + +esp8266_dir="${ESP8266_ARDUINO_BUILD_DIR}" +commonfileh="${esp8266_dir}/cores/esp8266/CommonHFile.h" + +tests_dir="$root/tests/test_mkbuildoptglobals" +tests=$(ls -1 "$tests_dir" | sed 's/.sh$//g') + +function name_size_mtime() +{ + stat --printf '%n:%s:%Y' "$1" +} + +function most_recent_file() +{ + local name="$1" + local location="$2" + echo $(readlink -f "${location}/"$(ls -t1 "${location}" | grep "$name" | head -1)) +} + +function most_recent_dir() +{ + local location="$1" + echo $(readlink -f "${location}/"$(ls -t1 "$location" | head -1)) +} + +function assert_build() +{ + local name="$1" + local build_dir="$2" + local size_check="${3:-0}" + + local build_opt="$build_dir"/sketch/build.opt + test -e "$build_opt" + + local globals_h="$build_dir"/${name}.ino.globals.h + test -e "$globals_h" + + if [ "$size_check" = "1" ] ; then + test -s "$build_opt" + test -s "$globals_h" + fi +} + +function assert_core() +{ + local size_check="$1" + + if [ "$size_check" = "1" ] ; then + test -s "$commonfileh" + else + test ! -s "$commonfileh" + fi + +} + +function build_esp8266_example() +{ + local name="$1" + + ESP8266_ARDUINO_SKETCHES="$root/libraries/esp8266/examples/${name}/${name}.ino" + build_sketches_with_arduino "$ESP8266_ARDUINO_LWIP" "$mod" "$rem" "$cnt" +} + +function make_commonh_stat() +{ + local stat=$(name_size_mtime "$commonfileh") + test -n "$stat" + + echo "$stat" +} + +function make_core_stat() +{ + local recent_core=$(most_recent_dir "$cores") + local recent_file=$(most_recent_file "core.a" "$recent_core") + + local stat=$(name_size_mtime "$recent_file") + test -n "$stat" + + echo "$stat" +} + +case "${1:-}" in +"list") + printf "%s\n" $tests + ;; + +"run") + for test in $tests ; do + printf "Checking \"%s\"\n" "$test" + /usr/bin/env bash $root/tests/test_mkbuildoptglobals.sh $test + done + ;; +*) + source "$tests_dir/${1}.sh" + ;; +esac diff --git a/tests/test_mkbuildoptglobals/buildopt_then_buildopt.sh b/tests/test_mkbuildoptglobals/buildopt_then_buildopt.sh new file mode 100644 index 0000000000..3d1ac53830 --- /dev/null +++ b/tests/test_mkbuildoptglobals/buildopt_then_buildopt.sh @@ -0,0 +1,25 @@ +function buildopt_then_buildopt() +{ + build_esp8266_example "GlobalBuildOptions" + + local last_sketch=$(most_recent_dir "$sketches") + assert_build "GlobalBuildOptions" "$last_sketch" 1 + assert_core 1 + + local globalbuildoptions_commonh_stat=$(make_commonh_stat) + local globalbuildoptions_core_stat=$(make_core_stat) + + build_esp8266_example "HwdtStackDump" + + last_sketch=$(most_recent_dir "$sketches") + assert_build "HwdtStackDump" "$last_sketch" 1 + assert_core 1 + + local hwdtstackdump_commonh_stat=$(make_commonh_stat) + local hwdtstackdump_core_stat=$(make_core_stat) + + test "$hwdtstackdump_commonh_stat" != "$globalbuildoptions_commonh_stat" + test "$hwdtstackdump_core_stat" != "$globalbuildoptions_core_stat" +} + +buildopt_then_buildopt diff --git a/tests/test_mkbuildoptglobals/buildopt_then_nobuildopt.sh b/tests/test_mkbuildoptglobals/buildopt_then_nobuildopt.sh new file mode 100644 index 0000000000..4db4927350 --- /dev/null +++ b/tests/test_mkbuildoptglobals/buildopt_then_nobuildopt.sh @@ -0,0 +1,25 @@ +function buildopt_then_nobuildopt() +{ + build_esp8266_example "GlobalBuildOptions" + + local last_sketch=$(most_recent_dir "$sketches") + assert_build "GlobalBuildOptions" "$last_sketch" 1 + assert_core 1 + + local globalbuildoptions_commonh_stat=$(make_commonh_stat) + local globalbuildoptions_core_stat=$(make_core_stat) + + build_esp8266_example "Blink" + + last_sketch=$(most_recent_dir "$sketches") + assert_build "Blink" "$last_sketch" 0 + assert_core 0 + + local blink_commonh_stat=$(make_commonh_stat) + local blink_core_stat=$(make_core_stat) + + test "$globalbuildoptions_commonh_stat" != "$blink_commonh_stat" + test "$globalbuildoptions_core_stat" != "$blink_core_stat" +} + +buildopt_then_nobuildopt diff --git a/tests/test_mkbuildoptglobals/nobuildopt.sh b/tests/test_mkbuildoptglobals/nobuildopt.sh new file mode 100644 index 0000000000..8f5c83709a --- /dev/null +++ b/tests/test_mkbuildoptglobals/nobuildopt.sh @@ -0,0 +1,25 @@ +function nobuildopt() +{ + build_esp8266_example "Blink" + + local last_sketch=$(most_recent_dir "$sketches") + assert_build "Blink.ino" "$last_sketch" 0 + assert_core 0 + + local blink_commonh_stat=$(make_commonh_stat) + local blink_core_stat=$(make_core_stat) + + build_esp8266_example "TestEspApi" + + last_sketch=$(most_recent_dir "$sketches") + assert_build "TestEspApi" "$last_sketch" 0 + assert_core 0 + + local testespapi_commonh_stat=$(make_commonh_stat) + local testespapi_core_stat=$(make_core_stat) + + test "$blink_commonh_stat" = "$testespapi_commonh_stat" + test "$blink_core_stat" = "$testespapi_core_stat" +} + +nobuildopt diff --git a/tests/test_mkbuildoptglobals/nobuildopt_then_buildopt.sh b/tests/test_mkbuildoptglobals/nobuildopt_then_buildopt.sh new file mode 100644 index 0000000000..bb75ec5022 --- /dev/null +++ b/tests/test_mkbuildoptglobals/nobuildopt_then_buildopt.sh @@ -0,0 +1,25 @@ +function nobuildopt_then_buildopt() +{ + build_esp8266_example "Blink" + + local last_sketch=$(most_recent_dir "$sketches") + assert_build "Blink" "$last_sketch" 0 + assert_core 0 + + local blink_commonh_stat=$(make_commonh_stat) + local blink_core_stat=$(make_core_stat) + + build_esp8266_example "HwdtStackDump" + + last_sketch=$(most_recent_dir "$sketches") + assert_build "HwdtStackDump" "$last_sketch" 1 + assert_core 1 + + local hwdtstackdump_commonh_stat=$(make_commonh_stat) + local hwdtstackdump_core_stat=$(make_core_stat) + + test "$hwdtstackdump_commonh_stat" != "$blink_commonh_stat" + test "$hwdtstackdump_core_stat" != "$blink_core_stat" +} + +nobuildopt_then_buildopt From dbd38ff1712157134e2ee8b01fa1ecbab0d3b208 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Wed, 28 May 2025 00:27:18 +0300 Subject: [PATCH 06/13] missing file --- tests/env.sh | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/env.sh diff --git a/tests/env.sh b/tests/env.sh new file mode 100644 index 0000000000..726c9f36ec --- /dev/null +++ b/tests/env.sh @@ -0,0 +1,38 @@ +# general configuration related to the builder itself +ESP8266_ARDUINO_BUILD_DIR=${ESP8266_ARDUINO_BUILD_DIR:-$root} +ESP8266_ARDUINO_BUILDER=${ESP8266_ARDUINO_BUILDER:-arduino} +ESP8266_ARDUINO_PRESERVE_CACHE=${ESP8266_ARDUINO_PRESERVE_CACHE:-} + +# sketch build options +ESP8266_ARDUINO_DEBUG=${ESP8266_ARDUINO_DEBUG:-nodebug} +ESP8266_ARDUINO_LWIP=${ESP8266_ARDUINO_LWIP:-default} +ESP8266_ARDUINO_SKETCHES=${ESP8266_ARDUINO_SKETCHES:-} + +ESP8266_ARDUINO_CLI=${ESP8266_ARDUINO_CLI:-$HOME/.local/bin/arduino-cli} + +# ref. https://arduino.github.io/arduino-cli/1.2/configuration/#default-directories +case "${RUNNER_OS:-Linux}" in +"Linux") + RUNNER_OS="Linux" + ESP8266_ARDUINO_HARDWARE=${ESP8266_ARDUINO_HARDWARE:-$HOME/Arduino/hardware} + ESP8266_ARDUINO_LIBRARIES=${ESP8266_ARDUINO_LIBRARIES:-$HOME/Arduino/libraries} + ;; +"macOS") + ESP8266_ARDUINO_HARDWARE=${ESP8266_ARDUINO_HARDWARE:-$HOME/Documents/Arduino/hardware} + ESP8266_ARDUINO_LIBRARIES=${ESP8266_ARDUINO_LIBRARIES:-$HOME/Documents/Arduino/libraries} + ;; +"Windows") + ESP8266_ARDUINO_HARDWARE=${ESP8266_ARDUINO_HARDWARE:-$HOME/Documents/Arduino/hardware} + ESP8266_ARDUINO_LIBRARIES=${ESP8266_ARDUINO_LIBRARIES:-$HOME/Documents/Arduino/libraries} + ;; +*) + echo 'Unknown ${RUNNER_OS} = "' ${RUNNER_OS} '"' + exit 2 +esac + +source "$root/tests/lib-skip-ino.sh" +source "$root/tests/common.sh" + +mod=1 +rem=0 +cnt=0 From 65c381d8bf1733077f0733bbc38ae66371535ba6 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Wed, 28 May 2025 00:53:33 +0300 Subject: [PATCH 07/13] typo, fix err trap for "test ...something..." --- tests/common.sh | 8 +++++++- tests/install_arduino.sh | 2 -- tests/test_mkbuildoptglobals.sh | 2 ++ tests/test_mkbuildoptglobals/nobuildopt.sh | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/common.sh b/tests/common.sh index 9eb5703e24..32c2564fd5 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -u -e -E -o pipefail +set -u -e -E -o pipefail -o errtrace cache_dir=$(mktemp -d) trap 'trap_exit' EXIT @@ -9,10 +9,16 @@ function trap_exit() { # workaround for macOS shipping with broken bash local exit_code=$? + + # ^ $cache_dir is temporary, prune when exiting if [ -z "${ESP8266_ARDUINO_PRESERVE_CACHE-}" ]; then rm -rf "$cache_dir" fi + if [ "$exit_code" != 0 ] ; then + echo "*** exit_code=$exit_code ***" + fi + exit $exit_code } diff --git a/tests/install_arduino.sh b/tests/install_arduino.sh index f85895e468..152f64e75f 100755 --- a/tests/install_arduino.sh +++ b/tests/install_arduino.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -set -x - root=$(git rev-parse --show-toplevel) source "$root/tests/env.sh" diff --git a/tests/test_mkbuildoptglobals.sh b/tests/test_mkbuildoptglobals.sh index 49a78b8115..be8b63b604 100755 --- a/tests/test_mkbuildoptglobals.sh +++ b/tests/test_mkbuildoptglobals.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +trap 'echo " ${BASH_SOURCE[1]}:$LINENO $BASH_COMMAND"' ERR + root=$(git rev-parse --show-toplevel) source "$root/tests/env.sh" diff --git a/tests/test_mkbuildoptglobals/nobuildopt.sh b/tests/test_mkbuildoptglobals/nobuildopt.sh index 8f5c83709a..93fd688ea2 100644 --- a/tests/test_mkbuildoptglobals/nobuildopt.sh +++ b/tests/test_mkbuildoptglobals/nobuildopt.sh @@ -3,7 +3,7 @@ function nobuildopt() build_esp8266_example "Blink" local last_sketch=$(most_recent_dir "$sketches") - assert_build "Blink.ino" "$last_sketch" 0 + assert_build "Blink" "$last_sketch" 0 assert_core 0 local blink_commonh_stat=$(make_commonh_stat) From 7290d2777348c7c23fa3ff3a226a18befb44960b Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Wed, 28 May 2025 04:18:33 +0300 Subject: [PATCH 08/13] mention indirect and direct dependency on globals .h contents --- .../examples/GlobalBuildOptions/GlobalBuildOptions.ino | 10 ++++++---- .../GlobalBuildOptions.ino.globals.h | 8 +++++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino b/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino index 2bc18751e1..fa4fc5c59d 100644 --- a/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino +++ b/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino @@ -4,11 +4,13 @@ * * Example from https://arduino-esp8266.readthedocs.io/en/latest/faq/a06-global-build-options.html * - * Note, we do not "#include" the special file "GlobalBuildOptions.ino.globals.h". - * The prebuild script will make it available to all modules. + * It is not necessary to have `#include "GlobalBuildOptions.ino.globals.h"`. + * However, any code inside of the special header file would not be available, + * unless there is also a matching `create-file:...` directive inside of it. * - * To track the new sketch name when saving this sketch to a new location and - * name, remember to update the global .h file name. + * Note that when building in debug mode, `build.opt:debug` should be used instead. + * + * Remember to update the global .h file name when saving this sketch to a new location! */ #include // has prototype for umm_free_heap_size_min() diff --git a/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino.globals.h b/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino.globals.h index b51b879f7f..835ec84675 100644 --- a/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino.globals.h +++ b/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino.globals.h @@ -8,11 +8,17 @@ * this line is ignored *@create-file:build.opt@ # this line is ignored +*/ + +/*@create-file:build.opt:debug@ + // Another embedded build.opt, with a different set of flags + -DMYTITLE1="\"Debugging on \"" -Og - // -fanalyzer -DUMM_STATS_FULL=1 + // -fanalyzer */ +// Following *raw* contents are also included #ifndef GLOBALBUILDOPTIONS_INO_GLOBALS_H #define GLOBALBUILDOPTIONS_INO_GLOBALS_H From 47b2d77dc30c14bc3414b6897e50849f5fc74933 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Wed, 28 May 2025 05:52:12 +0300 Subject: [PATCH 09/13] update doc --- doc/faq/a06-global-build-options.rst | 152 +----------------- ...7-mkbuildoptglobals-aggressive-caching.rst | 150 +++++++++++++++++ 2 files changed, 158 insertions(+), 144 deletions(-) create mode 100644 doc/faq/a07-mkbuildoptglobals-aggressive-caching.rst diff --git a/doc/faq/a06-global-build-options.rst b/doc/faq/a06-global-build-options.rst index 3e86b88a58..3f909fc677 100644 --- a/doc/faq/a06-global-build-options.rst +++ b/doc/faq/a06-global-build-options.rst @@ -25,9 +25,9 @@ Actions taken in processing comment block to create ``build.opt`` - for each line, white space is trimmed - blank lines are skipped - lines starting with ``*``, ``//``, or ``#`` are skipped -- the remaining results are written to build tree\ ``/core/build.opt`` -- multiple ``/*@create-file:build.opt@`` ``*/`` comment blocks are not - allowed +- the remaining results are written to arduino-cli build cache directory +- multiple ``/*@create-file:build.opt@`` ``*/`` comment blocks are + allowed and would be merged in order they are written in the file - ``build.opt`` is finished with a ``-include ...`` command, which references the global .h its contents were extracted from. @@ -160,137 +160,6 @@ Updated Global ``.h`` file: ``LowWatermark.ino.globals.h`` #endif -Aggressively cache compiled core -================================ - -This feature appeared with the release of Arduino IDE 1.8.2. The feature -“Aggressively Cache Compiled core” refers to sharing a single copy of -``core.a`` across all Arduino IDE Sketch windows. This feature is on by -default. ``core.a`` is an archive file containing the compiled objects -of ``./core/esp8266/*``. Created after your 1ST successful compilation. -All other open sketch builds use this shared file. When you close all -Arduino IDE windows, the core archive file is deleted. - -This feature is not compatible with using global defines or compiler -command-line options. Without mediation, bad builds could result, when -left enabled. When ``#define`` changes require rebuilding ``core.a`` and -multiple Sketches are open, they can no longer reliably share one cached -``core.a``. In a simple case: The 1st Sketch to be built has its version -of ``core.a`` cached. Other sketches will use this cached version for -their builds. - -There are two solutions to this issue: - -1. Do nothing, and rely on aggressive cache workaround built into the - script. -2. Turn off the “Aggressively Cache Compiled core” feature, by setting - ``compiler.cache_core=false``. - -Using “compiler.cache_core=false” ---------------------------------- - -There are two ways to turn off the “Aggressively Cache Compiled core” -feature: This can be done with the Arduino IDE command-line or a text -editor. - -Using the Arduino IDE command-line from a system command line, enter the -following: - -:: - - arduino --pref compiler.cache_core=false --save-prefs - -For the text editor, you need to find the location of -``preferences.txt``. From the Arduino IDE, go to *File->Preferences*. -Make note of the path to ``prefereces.txt``. You *cannot* edit the file -while the Arduino IDE is running. Close all Arduino IDE windows and edit -the file ``preferences.txt``. Change ``compiler.cache_core=true`` to -``compiler.cache_core=false`` and save. Then each sketch will maintain -its *own* copy of ``core.a`` built with the customization expressed by -their respective ``build.opt`` file. - -The “workaround” ----------------- - -When the “Aggressively Cache Compiled core” feature is enabled and the -global define file is detected, a workaround will turn on and stay on. -When you switch between Sketch windows, core will be recompiled and the -cache updated. The workaround logic is reset when Arduino IDE is -completely shutdown and restarted. - -The workaround is not perfect. These issues may be of concern: - -1. Dirty temp space. Arduino build cache files left over from a previous - run or boot. -2. Arduino command-line options: - - - override default preferences.txt file. - - override a preference, specifically ``compiler.cache_core``. - -3. Multiple versions of the Arduino IDE running - -**Dirty temp space** - -A minor concern, the workaround is always on. Not an issue for build -accuracy, but ``core.a`` maybe rebuild more often than necessary. - -Some operating systems are better at cleaning up their temp space than -others at reboot after a crash. At least for Windows®, you may need to -manually delete the Arduino temp files and directories after a crash. -Otherwise, the workaround logic may be left on. There is no harm in the -workaround being stuck on, the build will be correct; however, the core -files will occasionally be recompiled when not needed. - -For some Windows® systems the temp directory can be found near -``C:\Users\\AppData\Local\Temp\arduino*``. Note ``AppData`` is -a hidden directory. For help with this do an Internet search on -``windows disk cleanup``. Or, type ``disk cleanup`` in the Windows® -taskbar search box. - -With Linux, this problem could occur after an Arduino IDE crash. The -problem would be cleared after a reboot. Or you can manually cleanup the -``/tmp/`` directory before restarting the Arduino IDE. - -**Arduino command-line option overrides** - -If you are building with ``compiler.cache_core=true`` no action is -needed. If ``false`` the script would benefit by knowing that. - -When using either of these two command-line options: - -:: - - ./arduino --preferences-file other-preferences.txt - ./arduino --pref compiler.cache_core=false - -Hints for discovering the value of ``compiler.cache_core``, can be -provided by specifying ``mkbuildoptglobals.extra_flags=...`` in -``platform.local.txt``. - -Examples of hints: - -:: - - mkbuildoptglobals.extra_flags=--preferences_sketch # assume file preferences.txt in the sketch folder - mkbuildoptglobals.extra_flags=--preferences_sketch "pref.txt" # is relative to the sketch folder - mkbuildoptglobals.extra_flags=--no_cache_core - mkbuildoptglobals.extra_flags=--cache_core - mkbuildoptglobals.extra_flags=--preferences_file "other-preferences.txt" # relative to IDE or full path - -If required, remember to quote file or file paths. - -**Multiple versions of the Arduino IDE running** - -You can run multiple Arduino IDE windows as long as you run one version -of the Arduino IDE at a time. When testing different versions, -completely exit one before starting the next version. For example, -Arduino IDE 1.8.19 and Arduino IDE 2.0 work with different temp and -build paths. With this combination, the workaround logic sometimes fails -to enable. - -At the time of this writing, when Arduino IDE 2.0 rc5 exits, it leaves -the temp space dirty. This keeps the workaround active the next time the -IDE is started. If this is an issue, manually delete the temp files. Custom build environments ========================= @@ -301,11 +170,12 @@ require this feature and would like to turn it off, you can add the following lines to the ``platform.local.txt`` used in your build environment: -:: +.. code-block:: ini recipe.hooks.prebuild.2.pattern= build.opt.flags= + Other build confusion ===================== @@ -315,12 +185,6 @@ Other build confusion name ``LowWatermark.ino.globals.h``. You need to touch (update timestamp) the file so a “rebuild all” is performed. -2. When a ``.h`` file is renamed in the sketch folder, a copy of the old - file remains in the build sketch folder. This can create confusion if - you missed an edit in updating an ``#include`` in one or more of your - modules. That module will continue to use the stale version of the - ``.h`` until you restart the IDE or other major changes that would - cause the IDE to delete and recopy the contents from the source - Sketch directory. Changes on the IDE Tools board settings may cause a - complete rebuild, clearing the problem. This may be the culprit for - “What! It built fine last night!” +2. Original script relied on IDE 1.x core & sketch caching behaviour. + Since 3.2.x, we no longer assume aggressive caching can be disabled. + `Read more `__. diff --git a/doc/faq/a07-mkbuildoptglobals-aggressive-caching.rst b/doc/faq/a07-mkbuildoptglobals-aggressive-caching.rst new file mode 100644 index 0000000000..471a38eb63 --- /dev/null +++ b/doc/faq/a07-mkbuildoptglobals-aggressive-caching.rst @@ -0,0 +1,150 @@ +:orphan: + +.. attention:: + + This article applies ONLY to IDE 1.x and original version + of the mkbuildoptglobals.py script shipped with Core 3.1.x + +Aggressively cache compiled core +================================ + +This feature appeared with the release of Arduino IDE 1.8.2. The feature +“Aggressively Cache Compiled core” refers to sharing a single copy of +``core.a`` across all Arduino IDE Sketch windows. This feature is on by +default. ``core.a`` is an archive file containing the compiled objects +of ``./core/esp8266/*``. Created after your 1ST successful compilation. +All other open sketch builds use this shared file. When you close all +Arduino IDE windows, the core archive file is deleted. + +This feature is not compatible with using global defines or compiler +command-line options. Without mediation, bad builds could result, when +left enabled. When ``#define`` changes require rebuilding ``core.a`` and +multiple Sketches are open, they can no longer reliably share one cached +``core.a``. In a simple case: The 1st Sketch to be built has its version +of ``core.a`` cached. Other sketches will use this cached version for +their builds. + +There are two solutions to this issue: + +1. Do nothing, and rely on aggressive cache workaround built into the + script. +2. Turn off the “Aggressively Cache Compiled core” feature, by setting + ``compiler.cache_core=false``. + +Using “compiler.cache_core=false” +--------------------------------- + +There are two ways to turn off the “Aggressively Cache Compiled core” +feature: This can be done with the Arduino IDE command-line or a text +editor. + +Using the Arduino IDE command-line from a system command line, enter the +following: + +.. code-block:: console + + $ arduino --pref compiler.cache_core=false --save-prefs + +For the text editor, you need to find the location of +``preferences.txt``. From the Arduino IDE, go to *File->Preferences*. +Make note of the path to ``prefereces.txt``. You *cannot* edit the file +while the Arduino IDE is running. Close all Arduino IDE windows and edit +the file ``preferences.txt``. Change ``compiler.cache_core=true`` to +``compiler.cache_core=false`` and save. Then each sketch will maintain +its *own* copy of ``core.a`` built with the customization expressed by +their respective ``build.opt`` file. + +The “workaround” +---------------- + +When the “Aggressively Cache Compiled core” feature is enabled and the +global define file is detected, a workaround will turn on and stay on. +When you switch between Sketch windows, core will be recompiled and the +cache updated. The workaround logic is reset when Arduino IDE is +completely shutdown and restarted. + +The workaround is not perfect. These issues may be of concern: + +1. Dirty temp space. Arduino build cache files left over from a previous + run or boot. +2. Arduino command-line options: + + - override default preferences.txt file. + - override a preference, specifically ``compiler.cache_core``. + +3. Multiple versions of the Arduino IDE running + +**Dirty temp space** + +A minor concern, the workaround is always on. Not an issue for build +accuracy, but ``core.a`` maybe rebuild more often than necessary. + +Some operating systems are better at cleaning up their temp space than +others at reboot after a crash. At least for Windows®, you may need to +manually delete the Arduino temp files and directories after a crash. +Otherwise, the workaround logic may be left on. There is no harm in the +workaround being stuck on, the build will be correct; however, the core +files will occasionally be recompiled when not needed. + +For some Windows® systems the temp directory can be found near +``C:\Users\\AppData\Local\Temp\arduino*``. Note ``AppData`` is +a hidden directory. For help with this do an Internet search on +``windows disk cleanup``. Or, type ``disk cleanup`` in the Windows® +taskbar search box. + +With Linux, this problem could occur after an Arduino IDE crash. The +problem would be cleared after a reboot. Or you can manually cleanup the +``/tmp/`` directory before restarting the Arduino IDE. + +**Arduino command-line option overrides** + +If you are building with ``compiler.cache_core=true`` no action is +needed. If ``false`` the script would benefit by knowing that. + +When using either of these two command-line options: + +.. code-block:: console + + $ arduino --preferences-file other-preferences.txt + $ arduino --pref compiler.cache_core=false + +Hints for discovering the value of ``compiler.cache_core``, can be +provided by specifying ``mkbuildoptglobals.extra_flags=...`` in +``platform.local.txt``. + +Examples of hints: + +.. code-block:: ini + + mkbuildoptglobals.extra_flags=--preferences_sketch # assume file preferences.txt in the sketch folder + mkbuildoptglobals.extra_flags=--preferences_sketch "pref.txt" # is relative to the sketch folder + mkbuildoptglobals.extra_flags=--no_cache_core + mkbuildoptglobals.extra_flags=--cache_core + mkbuildoptglobals.extra_flags=--preferences_file "other-preferences.txt" # relative to IDE or full path + +If required, remember to quote file or file paths. + +Multiple versions of the Arduino IDE running +-------------------------------------------- + +You can run multiple Arduino IDE windows as long as you run one version +of the Arduino IDE at a time. When testing different versions, +completely exit one before starting the next version. For example, +Arduino IDE 1.8.19 and Arduino IDE 2.0 work with different temp and +build paths. With this combination, the workaround logic sometimes fails +to enable. + +At the time of this writing, when Arduino IDE 2.0 rc5 exits, it leaves +the temp space dirty. This keeps the workaround active the next time the +IDE is started. If this is an issue, manually delete the temp files. + +Also note when a ``.h`` file is renamed in the sketch folder, a copy of the old +file remains in the build sketch folder. This can create confusion if +you missed an edit in updating an ``#include`` in one or more of your +modules. That module will continue to use the stale version of the +``.h`` until you restart the IDE or other major changes that would +cause the IDE to delete and recopy the contents from the source +Sketch directory. Changes on the IDE Tools board settings may cause a +complete rebuild, clearing the problem. This may be the culprit for +“What! It built fine last night!” + From 5768ba055ec5bbe19b65524634710e86deed2d56 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Wed, 28 May 2025 20:19:55 +0300 Subject: [PATCH 10/13] update docs bump global build options page in the menu hierarchy share pydoc blob as rst instead, reference on the main page note that aggressive caching opts are only for IDE 1.x --- .../aggressive-caching.rst} | 6 +- doc/buildopt/internals.rst | 235 ++++++++++++++++++ doc/faq/readme.rst | 2 +- ...d-options.rst => global_build_options.rst} | 42 ++-- doc/index.rst | 1 + tools/mkbuildoptglobals.py | 194 --------------- 6 files changed, 267 insertions(+), 213 deletions(-) rename doc/{faq/a07-mkbuildoptglobals-aggressive-caching.rst => buildopt/aggressive-caching.rst} (100%) create mode 100644 doc/buildopt/internals.rst rename doc/{faq/a06-global-build-options.rst => global_build_options.rst} (87%) diff --git a/doc/faq/a07-mkbuildoptglobals-aggressive-caching.rst b/doc/buildopt/aggressive-caching.rst similarity index 100% rename from doc/faq/a07-mkbuildoptglobals-aggressive-caching.rst rename to doc/buildopt/aggressive-caching.rst index 471a38eb63..3037b2ca0f 100644 --- a/doc/faq/a07-mkbuildoptglobals-aggressive-caching.rst +++ b/doc/buildopt/aggressive-caching.rst @@ -1,13 +1,13 @@ :orphan: +Aggressively cache compiled core +================================ + .. attention:: This article applies ONLY to IDE 1.x and original version of the mkbuildoptglobals.py script shipped with Core 3.1.x -Aggressively cache compiled core -================================ - This feature appeared with the release of Arduino IDE 1.8.2. The feature “Aggressively Cache Compiled core” refers to sharing a single copy of ``core.a`` across all Arduino IDE Sketch windows. This feature is on by diff --git a/doc/buildopt/internals.rst b/doc/buildopt/internals.rst new file mode 100644 index 0000000000..cbaff9819a --- /dev/null +++ b/doc/buildopt/internals.rst @@ -0,0 +1,235 @@ +:orphan: + +mkbuildoptglobals.py internals +============================== + +Sketch header aka SKETCH.ino.globals.h +-------------------------------------- + +.. hint:: + + ``SKETCH.ino.globals.h`` is expected to be created by the user. + +It is always located in the root of the SKETCH directory, and must use the +the actual name of the sketch program (``SKETCH.ino``) in the its name. + +Header file format is used because IDE only manages source files it actually +recognizes as valid C / C++ file formats. When building and re-building the +sketch, only valid file formats are taken into an account when building source +code dependencies tree. + +Command-line options file +------------------------- + + .. hint:: + + This file is created by the script. + +Options file is generated based on the contents of the sketch header comment block, +and its contents are then used as gcc command-line options (``@file``) + + If file does not exist, or cannot be read, then the option will be treated literally, and not removed. + + Options in file are separated by whitespace. A whitespace character may be included + in an option by surrounding the entire option in either single or double quotes. + Any character (including a backslash) may be included by prefixing the character + to be included with a backslash. + The file may itself contain additional @file options; any such options will be processed recursively. + + --- https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html + + +Arduino build system uses timestamps to determine which files should be rebuilt. +``@file`` cannot be a direct dependency, only other source code files can. + +Thus, command-line options file is *always* created with at least one ``-include`` + +.. code-block:: console + + -include "PLATFORM_PATH / COMMON_HEADER_PATH" + +When matching directive is found in the sketch header, path to its copy in the build directory is also added + +.. code-block:: console + + -include "BUILD_DIRECTORY / SKETCH_HEADER_COPY_PATH" + +Common header +------------- + +.. note:: + + This file is also created by the script. + +It is used as a means of triggering core rebuild, because modern Arduino build systems +are agressively caching it and attempt to re-use existing ``core.a`` whenever possible. + +This file would contain path to the currently used command-line options file extracted +from the sketch header. It remains empty otherwise. + +Build directory +--------------- + +Arduino build process copies every valid source file from the source (sketch) +directory into the build directory. This script is expected to be launched in +the "prebuild" stage. At that point, build directory should already exist, but +it may not yet contain any of the sketch source files. + +Script would always attempt to copy sketch header from the source (sketch) +directory to the build one. If it does not exist, a placeholder would be created. + +Script would always synchronize atime & mtime of every file. When sketch header +exists, stats are taken from it. When it doesn't, stats for the generated common +header are used instead. + +Configuration +------------- + +``platform.txt`` is expected to have this script listed as a tool + +.. code-block:: ini + + runtime.tools.mkbuildoptglobals={runtime.platform.path}/tools/mkbuildoptglobals.py + +Paths are always provided as Fully Qualified File Names (FQFNs): + +.. code-block:: ini + + globals.h.source.fqfn={build.source.path}/{build.project_name}.globals.h + globals.h.common.fqfn={build.core.path}/__common_globals.h + build.opt.fqfn={build.path}/core/build.opt + mkbuildoptglobals.extra_flags= + +`"prebuild" hook `__ must be used, +allowing this script to run *before* build process creates and / or copies them. +Both Arduino IDE 1.x and 2.x generate +`prerequisite makefiles (files with a .d suffix) `__ +at some point in "discovery phase". + +.. code-block:: ini + + recipe.hooks.prebuild.#.pattern= + "{runtime.tools.python3.path}/python3" -I + "{runtime.tools.mkbuildoptglobals}" {mkbuildoptglobals.extra_flags} build + --build-path "{build.path}" + --build-opt "{build.opt.fqfn}" + --sketch-header "{globals.h.source.fqfn}" + --common-header "{commonhfile.fqfn}" + +Command-line options file is then shared between other recipes by including it in +the "cpreprocessor" flags. + +.. code-block:: ini + + compiler.cpreprocessor.flags=... @{build.opt.path} ... + +After that point, prerequisite makefiles should contain either only the common header, +or both the common header and the build sketch header. When any of included headers is +modified, every file in the dependency chain would be rebuilt. This allows us to keep +existing ``core.a`` cache when command-line options file is not used by the sketch. + +Example +------- + +Sketch header file with embedded command-line options file might look like this + +.. code-block:: c++ + :emphasize-lines: 1,2,3,4,5,6,7,8,9,10,11 + + /*@create-file:build.opt@ + // An embedded "build.opt" file using a "C" block comment. The starting signature + // must be on a line by itself. The closing block comment pattern should be on a + // line by itself. Each line within the block comment will be space trimmed and + // written to build.opt, skipping blank lines and lines starting with '//', '*' + // or '#'. + -DMYDEFINE="\"Chimichangas do not exist\"" + -O3 + -fanalyzer + -DUMM_STATS=2 + */ + + #ifndef SKETCH_INO_GLOBALS_H + #define SKETCH_INO_GLOBALS_H + + #if defined(__cplusplus) + // Defines kept private to .cpp modules + //#pragma message("__cplusplus has been seen") + #endif + + #if !defined(__cplusplus) && !defined(__ASSEMBLER__) + // Defines kept private to .c modules + #endif + + #if defined(__ASSEMBLER__) + // Defines kept private to assembler modules + #endif + + #endif + + +Caveats, Observations, and Ramblings +------------------------------------ + +1. Edits to ``platform.txt`` or ``platform.local.txt`` force a complete rebuild that + removes the core folder. Not a problem, just something to be aware of when + debugging this script. Similarly, changes on the IDE Tools selection cause a + complete rebuild. + + In contrast, the core directory is not deleted when the rebuild occurs from + changing a file with an established dependency (inspect .d in the build path) + +2. Renaming files does not change the last modified timestamp, possibly causing + issues when adding or replacing files by renaming and rebuilding. + + A good example of this problem is when you correct the spelling of sketch + header file. You must update mtime (e.g. call touch) of the file. + +3. ``-include ".../Sketch.ino.globals.h"`` is conditionally added to every compilation command, + so it may be reasonable to expect that ``#include "Sketch.ino.globals.h"`` is no longer necessary. + + However, it may not be the case when `create-file:...` directive is missing or does not match. + + When explicit ``#include "Sketch.ino.globals.h"`` is used in the code, it must always be guarded against including it twice: + + .. code-block:: c++ + :emphasize-lines: 1 + + #pragma once + + Or, by using classic header guards: + + .. code-block:: c++ + :emphasize-lines: 1,2,4 + + #infdef SKETCH_GLOBALS_H + #define SKETCH_GLOBALS_H + ... file contents ... + #endif + +4. ``build.opt`` itself is not listed as a dependency in .d, .h files are used + because this is the only obvious way to force arduino-builder / arduino-cli + into tracking it. + +5. When not using ``--build-path``, ``core.a`` is cached and shared. + CI sometimes uses `ARDUINO_BUILD_CACHE_PATH environment variable `__. + This allows to have a private core cache, separate from the system one. + +6. `Referencing upstream arduino-cli code (v1.2.2) `__, ``core.a`` cache key is based on: + + * `ESP8266 Platform Path`, and depends on installation method + + * `Installing ESP8266 Core <../installing.rst>`__ + * `Arduino Platform Installation Directories `__ + + * `FQBN` + + See `Arduino Custom Board Options `__). + + * `Optimization flags` + + .. attention:: + + ``{compiler.optimization_flags}`` is not currently used in the ESP8266 Core + + See `Arduino Optimization Level setting `__). + diff --git a/doc/faq/readme.rst b/doc/faq/readme.rst index cfd65eca90..b4c30bf5f9 100644 --- a/doc/faq/readme.rst +++ b/doc/faq/readme.rst @@ -199,4 +199,4 @@ By using a uniquely named `.h` file, macro definitions can be created and globally used. Additionally, compiler command-line options can be embedded in this file as a unique block comment. -`Read more `__. +`Read more <../global_build_options>`__. diff --git a/doc/faq/a06-global-build-options.rst b/doc/global_build_options.rst similarity index 87% rename from doc/faq/a06-global-build-options.rst rename to doc/global_build_options.rst index 3f909fc677..779a077e9d 100644 --- a/doc/faq/a06-global-build-options.rst +++ b/doc/global_build_options.rst @@ -1,7 +1,8 @@ -:orphan: +Global build defines and options +================================ -How to specify global build defines and options -=============================================== +Basics +------ To create globally usable macro definitions for a Sketch, create a file with a name based on your Sketch’s file name followed by ``.globals.h`` @@ -31,6 +32,9 @@ Actions taken in processing comment block to create ``build.opt`` - ``build.opt`` is finished with a ``-include ...`` command, which references the global .h its contents were extracted from. +Example +------- + Example Sketch: ``LowWatermark.ino`` .. code:: cpp @@ -95,7 +99,7 @@ Global ``.h`` file: ``LowWatermark.ino.globals.h`` #endif Separate production and debug build options -=========================================== +------------------------------------------- If your production and debug build option requirements are different, adding ``mkbuildoptglobals.extra_flags={build.debug_port}`` to @@ -162,7 +166,7 @@ Updated Global ``.h`` file: ``LowWatermark.ino.globals.h`` Custom build environments -========================= +------------------------- Some custom build environments may have already addressed this issue by other means. If you have a custom build environment that does not @@ -175,16 +179,24 @@ environment: recipe.hooks.prebuild.2.pattern= build.opt.flags= +Source Code +----------- + +https://github.com/esp8266/Arduino/blob/master/tools/mkbuildoptglobals.py + + +Internals +--------- + +:doc:`/buildopt/internals` + +IDE 1.x aggressive caching +-------------------------- + +.. attention:: -Other build confusion -===================== + This article applies ONLY to IDE 1.x and original version + of the mkbuildoptglobals.py script shipped with Core 3.1.x -1. Renaming a file does not change the last modified timestamp, possibly - causing issues when adding a file by renaming and rebuilding. A good - example of this problem would be to have then fixed a typo in file - name ``LowWatermark.ino.globals.h``. You need to touch (update - timestamp) the file so a “rebuild all” is performed. +:doc:`/buildopt/aggressive-caching` -2. Original script relied on IDE 1.x core & sketch caching behaviour. - Since 3.2.x, we no longer assume aggressive caching can be disabled. - `Read more `__. diff --git a/doc/index.rst b/doc/index.rst index 5f3ec247c6..bae801a8f8 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -15,6 +15,7 @@ Welcome to ESP8266 Arduino Core's documentation! PROGMEM Using GDB to debug MMU + Global build options Boards FAQ diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py index 0eb3b6e028..e69d32515f 100755 --- a/tools/mkbuildoptglobals.py +++ b/tools/mkbuildoptglobals.py @@ -31,200 +31,6 @@ # Used d-a-v's global name suggestion from arduino PR # - https://github.com/arduino/arduino-cli/pull/1524 -""" -Sketch header aka SKETCH.ino.globals.h: - - SKETCH.ino.globals.h is expected to be created by the user. - - It is always located in the root of the SKETCH directory, and must use the - the actual name of the sketch program (SKETCH.ino) in the its name. - - Header file format is used because IDE only manages source files it actually - recognizes as valid C / C++ file formats. When building and re-building the - sketch, only valid file formats are taken into an account when building source - code dependencies tree. - -Command-line options file: - - This file is created by the script. - - Contents of the file are then used as gcc options (@file) - - In the prebuild stage, options file is generated based on the contents of the - sketch header, and then placed into the build directory. - - Quoting gcc manual - > If file does not exist, or cannot be read, then the option will be treated literally, and not removed. - > - > Options in file are separated by whitespace. A whitespace character may be included - > in an option by surrounding the entire option in either single or double quotes. - > Any character (including a backslash) may be included by prefixing the character - > to be included with a backslash. - > The file may itself contain additional @file options; any such options will be processed recursively. - - Arduino build system uses timestamps as a method of determining which files should be rebuilt. - Options file is *always* created with at least two of the following lines - > -include "/core/" - > -include "/" - -Common header: - - This file is also created by the script. - - It is used as a means of triggering core rebuild, because modern Arduino build systems - are agressively caching it and attempt to re-use existing core.a whenever possible. - core.a is also shared between different sketch compilations which use the same board. - - This file would contain path to the currently used command-line options file extracted - from the sketch header. It remains empty otherwise. - -Build directory: - - Arduino build process copies every valid source file from the source (sketch) - directory into the build directory. This script is expected to be launched in - the "prebuild" stage. At that point, build directory should already exist, but - it may not yet contain any of the sketch source files. - - Script would always attempt to copy sketch header from the source (sketch) - directory to the build one. If it does not exist, a placeholder would be created. - - Script would always synchronize atime & mtime of every file. When sketch header - exists, stats are taken from it. When it doesn't, stats for the generated common - header are used instead. - -Configuration: - - "platform.txt" is expected to have this script listed as a tool - > runtime.tools.mkbuildoptglobals={runtime.platform.path}/tools/mkbuildoptglobals.py - - Paths are always provided as Fully Qualified File Names (FQFNs). - - For example - > globals.h.source.fqfn={build.source.path}/{build.project_name}.globals.h - > globals.h.common.fqfn={build.core.path}/__common_globals.h - > build.opt.fqfn={build.path}/core/build.opt - > mkbuildoptglobals.extra_flags= - - Both Arduino IDE 1.x and modern 2.x generate prerequisite makefiles (.d files) - at some point in "discovery phase". - ref. https://www.gnu.org/software/make/manual/html_node/Automatic-Prerequisites.html - - "prebuild" hook must be used, allowing this script to run *before* IDE creates them - ref. https://docs.arduino.cc/arduino-cli/platform-specification/#pre-and-post-build-hooks-since-arduino-ide-165 - - > recipe.hooks.prebuild.#.pattern="{runtime.tools.python3.path}/python3" -I \\ - "{runtime.tools.mkbuildoptglobals}" {mkbuildoptglobals.extra_flags} build \\ - --build-path "{build.path}" \\ - --build-opt "{build.opt.fqfn}" \\ - --sketch-header "{globals.h.source.fqfn}" \\ - --common-header "{commonhfile.fqfn}" \\ - - Command-line options file is then shared between other recipes by including it in - the "cpreprocessor" flags. - > compiler.cpreprocessor.flags=... @{build.opt.path} ... - - After that point, prerequisite makefiles should contain either only the common header, - or both the common header and the build sketch header. When any of included headers is - modified, every file in the dependency chain would be rebuilt. This allows us to keep - existing core.a cache when command-line options file is not used by the sketch. - -Example: - - Sketch header file with embedded command-line options file might look like this - - .. code-block:: c++ - - /*@create-file:build.opt@ - // An embedded "build.opt" file using a "C" block comment. The starting signature - // must be on a line by itself. The closing block comment pattern should be on a - // line by itself. Each line within the block comment will be space trimmed and - // written to build.opt, skipping blank lines and lines starting with '//', '*' - // or '#'. - -DMYDEFINE="\"Chimichangas do not exist\"" - -O3 - -fanalyzer - -DUMM_STATS=2 - */ - - #ifndef SKETCH_INO_GLOBALS_H - #define SKETCH_INO_GLOBALS_H - - #if defined(__cplusplus) - // Defines kept private to .cpp modules - //#pragma message("__cplusplus has been seen") - #endif - - #if !defined(__cplusplus) && !defined(__ASSEMBLER__) - // Defines kept private to .c modules - #endif - - #if defined(__ASSEMBLER__) - // Defines kept private to assembler modules - #endif - - #endif - -Caveats, Observations, and Ramblings: - - 1) Edits to "platform.txt" or "platform.local.txt" force a complete rebuild that - removes the core folder. Not a problem, just something to be aware of when - debugging this script. Similarly, changes on the IDE Tools selection cause a - complete rebuild. - - In contrast, the core directory is not deleted when the rebuild occurs from - changing a file with an established dependency. - - 2) Renaming files does not change the last modified timestamp, possibly causing - issues when replacing files by renaming and rebuilding. - - A good example of this problem is when you correct the spelling of sketch - header file. You need to touch (update time stampt) the file so a - rebuild all is performed. - - 3) During the build two identical copies of sketch header will exist. - #ifndef fencing will be needed for non comment blocks in SKETCH.ino.globals.h. - - 4) By using a .h file to encapsulate "build.opt" options, the information is not - lost after a save-as. Before with an individual "build.opt" file, the file was - missing in the saved copy. - - 5) Previously, when a .h file is renamed, a copy of the old file remains in the build - sketch folder. This created confusion if you missed an edit in updating an - include in one or more of your modules. Module will continue to use the - stale version of the .h, until you restart the IDE or other major changes that - would cause the IDE to delete and recopy the contents from the source sketch. - - This may be the culprit for "What! It built fine last night!" - - 6a) In The case of two Arduino IDE screens up with different programs, they can - share the same core archive file. Defines on one screen will change the core - archive, and a build on the 2nd screen will build with those changes. - The 2nd build will have the core built for the 1st screen. It gets uglier. With - the 2nd program, the newly built modules used headers processed with different - defines than the core. - - 6b) Problem: Once core has been build, changes to build.opt or globals.h will - not cause the core archive to be rebuild. You either have to change tool - settings or close and reopen the Arduino IDE. This is a variation on 6a) above. - I thought this was working for the single sketch case, but it does not! :( - That is because sometimes it does build properly. What is unknown are the - causes that will make it work and fail? - - 7) Previous IDE versions allowed to disable core.a caching - https://forum.arduino.cc/t/no-aggressively-cache-compiled-core-in-ide-1-8-15/878954/2 - - This is not the case for 2.x, where the only way to control it is by calling arduino-cli - directly, or modifying its configuration file / using environment variable. - https://arduino.github.io/arduino-cli/1.2/configuration/ - > "build_cache" configuration options related to the compilation cache - - "--clean" option would start with a fresh build. - - 8) Suspected but not confirmed. A quick edit and rebuild don't always work well. - Build does not work as expected. This does not fail often. Maybe PIC NIC. - -""" - import argparse import locale import logging From dcb64acd673161f741cb73c7fca74362c41eb8cd Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Wed, 28 May 2025 22:09:19 +0300 Subject: [PATCH 11/13] restore command-line opts info, newlined debugging --- tools/mkbuildoptglobals.py | 62 +++++++++++++++----------------------- 1 file changed, 25 insertions(+), 37 deletions(-) diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py index e69d32515f..9c75c934b6 100755 --- a/tools/mkbuildoptglobals.py +++ b/tools/mkbuildoptglobals.py @@ -135,7 +135,9 @@ def filter(self, rec): # Since default handler is not created, make sure only specific levels go through to stderr TO_STDERR = logging.StreamHandler(sys.stderr) -TO_STDERR.setFormatter(logging.Formatter("*** %(levelname)s - %(message)s ***")) +TO_STDERR.setFormatter( + logging.Formatter("*** %(filename)s %(funcName)s:%(lineno)d ***\n%(message)s\n") +) TO_STDERR.setLevel(logging.NOTSET) TO_STDERR.addFilter( LoggingFilter( @@ -305,12 +307,13 @@ def synchronize_utime(stat: Union[os.stat_result, pathlib.Path], *rest: pathlib. Retrieve stats from the first 'file' and apply to the 'rest' """ if not isinstance(stat, os.stat_result): - logging.debug("using stats from %s", stat.name) stat = stat.stat() + logging.debug( + "setting mtime=%d for:\n%s", stat.st_mtime, "\n".join(f" {p}" for p in rest) + ) for p in rest: if is_different_utime(stat, p.stat()): os.utime(p, ns=(stat.st_atime_ns, stat.st_mtime_ns)) - logging.debug("synchronized %s", p.name) def as_include_line(p: pathlib.Path) -> str: @@ -358,15 +361,15 @@ def ensure_exists_and_empty(*paths: pathlib.Path): if not p.exists() or (p.exists() and p.stat().st_size): p.write_bytes(b"") - logging.debug("placeholder for %s", p.name) + logging.debug("%s is a placeholder", p.name) else: - logging.debug("up-to-date %s", p.name) + logging.debug("%s is up-to-date", p.name) def ensure_normal_time(*paths: pathlib.Path): for p in paths: if p.exists() and is_future_utime(p): - logging.debug("fixing timestamp of %s", p.name) + logging.debug("%s has timestamp in the future, fixing", p.name) p.touch() @@ -391,8 +394,10 @@ def write_or_replace(p: pathlib.Path, contents: str, encoding=FILE_ENCODING) -> if contents != actual: p.write_text(contents, encoding=encoding) + logging.debug("%s contents written", p.name) return True + logging.debug("%s is up-to-date", p.name) return False @@ -427,12 +432,7 @@ def ensure_common_header_bound(ctx: Context): """ Record currently used command-line options file """ - if write_or_replace( - ctx.common_header, as_arduino_sketch_quoted_header(ctx.build_opt) - ): - logging.debug("wrote to %s", ctx.common_header.name) - else: - logging.debug("up-to-date %s", ctx.build_opt.name) + write_or_replace(ctx.common_header, as_arduino_sketch_quoted_header(ctx.build_opt)) def make_build_opt_name(ctx: Context, debug: bool) -> str: @@ -460,18 +460,7 @@ def ensure_build_opt_written(ctx: Context, buffer: io.StringIO): for p in includes: buffer.write(f"{as_include_line(p)}\n") - value = buffer.getvalue() - - if ( - not ctx.build_opt.exists() - or is_different_utime(ctx.build_sketch_header, ctx.build_opt) - or ctx.build_opt.read_text(encoding=DEFAULT_ENCODING) != value - ): - ctx.build_opt.parent.mkdir(parents=True, exist_ok=True) - ctx.build_opt.write_text(value, encoding=DEFAULT_ENCODING) - logging.debug("wrote to %s", ctx.build_opt.name) - else: - logging.debug("up-to-date %s", ctx.build_opt.name) + write_or_replace(ctx.build_opt, buffer.getvalue(), encoding=DEFAULT_ENCODING) def maybe_empty_or_missing(p: pathlib.Path): @@ -525,14 +514,13 @@ def main_build(args: argparse.Namespace): ) if args.debug: - logging.debug("using the following build context") - for field in dataclasses.fields(ctx): - logging.debug( - " %s %s %s", - field.name, - getattr(ctx, field.name), - field.metadata["help"], - ) + logging.debug( + "Build Context:\n%s", + "".join( + f' "{field.name}" at {getattr(ctx, field.name)} - {field.metadata["help"]}\n' + for field in dataclasses.fields(ctx) + ), + ) # notify when other files similar to .globals.h are in the sketch directory other_build_options = check_other_build_options(ctx.source_sketch_header) @@ -558,19 +546,19 @@ def main_build(args: argparse.Namespace): build_opt_buffer = io.StringIO() try: - logging.debug("searching for %s", name) extract_build_opt_from_path(build_opt_buffer, name, ctx.source_sketch_header) except ParsingException as e: - raise e from None + raise # when command-line options were not created / found, it means the same thing as empty or missing .globals.h if not len(build_opt_buffer.getvalue()): build_with_minimal_build_opt(ctx) return - logging.debug("preparing %s", ctx.build_opt.name) - for line in build_opt_buffer: - logging.debug(" %s", line) + logging.info( + "\nExtra command-line options:\n%s", + "\n".join(f" {line}" for line in build_opt_buffer.getvalue().split("\n")), + ) # at this point, it is necessary to synchronize timestamps of every file ensure_build_opt_written(ctx, build_opt_buffer) From bb242d9116600792ca88fa6dcf3eb088b70ed237 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Wed, 28 May 2025 22:09:35 +0300 Subject: [PATCH 12/13] mkdir parent when writing --- tools/mkbuildoptglobals.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py index 9c75c934b6..b96af4812e 100755 --- a/tools/mkbuildoptglobals.py +++ b/tools/mkbuildoptglobals.py @@ -393,6 +393,7 @@ def write_or_replace(p: pathlib.Path, contents: str, encoding=FILE_ENCODING) -> logging.warning("cannot decode %s", p.name) if contents != actual: + p.parent.mkdir(parents=True, exist_ok=True) p.write_text(contents, encoding=encoding) logging.debug("%s contents written", p.name) return True From 22c919b260e15aa519ec754ce5e754a60f69a67a Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Wed, 28 May 2025 22:09:46 +0300 Subject: [PATCH 13/13] useless from None --- tools/mkbuildoptglobals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py index b96af4812e..4b3e3854a3 100755 --- a/tools/mkbuildoptglobals.py +++ b/tools/mkbuildoptglobals.py @@ -584,7 +584,7 @@ def main_inspect(args: argparse.Namespace): try: extract_build_opt_from_path(buffer, args.build_opt_name, p) except ParsingException as e: - raise e from None + raise e logging.info(buffer.getvalue()) @@ -602,7 +602,7 @@ def main_synchronize(args: argparse.Namespace): def as_path(p: str) -> pathlib.Path: if p.startswith("{") or p.endswith("}"): - raise ValueError(f'"{p}" was not resolved') from None + raise ValueError(f'"{p}" was not resolved') return pathlib.Path(p)