diff --git a/.github/workflows/build-host.yml b/.github/workflows/build-host.yml index ae0353068d..bd9172960e 100644 --- a/.github/workflows/build-host.yml +++ b/.github/workflows/build-host.yml @@ -1,5 +1,5 @@ # Run host test suite under valgrind for runtime checking of code. -# Also, a quick test that the mocking builds work at all +# Also, a quick test that the mock builds are actually working name: Build on host OS diff --git a/.github/workflows/build-ide.yml b/.github/workflows/build-ide.yml index edc226b4d9..92ba593cd5 100644 --- a/.github/workflows/build-ide.yml +++ b/.github/workflows/build-ide.yml @@ -9,8 +9,7 @@ permissions: contents: read jobs: - - # Examples are built in parallel to avoid CI total job time limitation + # download toolchain and check whether gcc basic .c compilation works sanity-check: runs-on: ubuntu-latest defaults: @@ -28,6 +27,26 @@ jobs: run: | bash ./tests/sanity_check.sh + # verify that any scripts that will or may be used by the builder are working + tooling-check: + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v4 + with: + submodules: false + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + - run: | + python ./tools/test_mkbuildoptglobals.py --quiet + - run: | + bash ./tests/install_arduino.sh + bash ./tests/test_mkbuildoptglobals.sh run + + # Examples are built in parallel to avoid CI total job time limitation build-linux: name: Linux - LwIP ${{ matrix.lwip }} (${{ matrix.chunk }}) runs-on: ubuntu-latest diff --git a/doc/buildopt/aggressive-caching.rst b/doc/buildopt/aggressive-caching.rst new file mode 100644 index 0000000000..3037b2ca0f --- /dev/null +++ b/doc/buildopt/aggressive-caching.rst @@ -0,0 +1,150 @@ +:orphan: + +Aggressively cache compiled core +================================ + +.. attention:: + + This article applies ONLY to IDE 1.x and original version + of the mkbuildoptglobals.py script shipped with Core 3.1.x + +This feature appeared with the release of Arduino IDE 1.8.2. The feature +“Aggressively Cache Compiled core” refers to sharing a single copy of +``core.a`` across all Arduino IDE Sketch windows. This feature is on by +default. ``core.a`` is an archive file containing the compiled objects +of ``./core/esp8266/*``. Created after your 1ST successful compilation. +All other open sketch builds use this shared file. When you close all +Arduino IDE windows, the core archive file is deleted. + +This feature is not compatible with using global defines or compiler +command-line options. Without mediation, bad builds could result, when +left enabled. When ``#define`` changes require rebuilding ``core.a`` and +multiple Sketches are open, they can no longer reliably share one cached +``core.a``. In a simple case: The 1st Sketch to be built has its version +of ``core.a`` cached. Other sketches will use this cached version for +their builds. + +There are two solutions to this issue: + +1. Do nothing, and rely on aggressive cache workaround built into the + script. +2. Turn off the “Aggressively Cache Compiled core” feature, by setting + ``compiler.cache_core=false``. + +Using “compiler.cache_core=false” +--------------------------------- + +There are two ways to turn off the “Aggressively Cache Compiled core” +feature: This can be done with the Arduino IDE command-line or a text +editor. + +Using the Arduino IDE command-line from a system command line, enter the +following: + +.. code-block:: console + + $ arduino --pref compiler.cache_core=false --save-prefs + +For the text editor, you need to find the location of +``preferences.txt``. From the Arduino IDE, go to *File->Preferences*. +Make note of the path to ``prefereces.txt``. You *cannot* edit the file +while the Arduino IDE is running. Close all Arduino IDE windows and edit +the file ``preferences.txt``. Change ``compiler.cache_core=true`` to +``compiler.cache_core=false`` and save. Then each sketch will maintain +its *own* copy of ``core.a`` built with the customization expressed by +their respective ``build.opt`` file. + +The “workaround” +---------------- + +When the “Aggressively Cache Compiled core” feature is enabled and the +global define file is detected, a workaround will turn on and stay on. +When you switch between Sketch windows, core will be recompiled and the +cache updated. The workaround logic is reset when Arduino IDE is +completely shutdown and restarted. + +The workaround is not perfect. These issues may be of concern: + +1. Dirty temp space. Arduino build cache files left over from a previous + run or boot. +2. Arduino command-line options: + + - override default preferences.txt file. + - override a preference, specifically ``compiler.cache_core``. + +3. Multiple versions of the Arduino IDE running + +**Dirty temp space** + +A minor concern, the workaround is always on. Not an issue for build +accuracy, but ``core.a`` maybe rebuild more often than necessary. + +Some operating systems are better at cleaning up their temp space than +others at reboot after a crash. At least for Windows®, you may need to +manually delete the Arduino temp files and directories after a crash. +Otherwise, the workaround logic may be left on. There is no harm in the +workaround being stuck on, the build will be correct; however, the core +files will occasionally be recompiled when not needed. + +For some Windows® systems the temp directory can be found near +``C:\Users\\AppData\Local\Temp\arduino*``. Note ``AppData`` is +a hidden directory. For help with this do an Internet search on +``windows disk cleanup``. Or, type ``disk cleanup`` in the Windows® +taskbar search box. + +With Linux, this problem could occur after an Arduino IDE crash. The +problem would be cleared after a reboot. Or you can manually cleanup the +``/tmp/`` directory before restarting the Arduino IDE. + +**Arduino command-line option overrides** + +If you are building with ``compiler.cache_core=true`` no action is +needed. If ``false`` the script would benefit by knowing that. + +When using either of these two command-line options: + +.. code-block:: console + + $ arduino --preferences-file other-preferences.txt + $ arduino --pref compiler.cache_core=false + +Hints for discovering the value of ``compiler.cache_core``, can be +provided by specifying ``mkbuildoptglobals.extra_flags=...`` in +``platform.local.txt``. + +Examples of hints: + +.. code-block:: ini + + mkbuildoptglobals.extra_flags=--preferences_sketch # assume file preferences.txt in the sketch folder + mkbuildoptglobals.extra_flags=--preferences_sketch "pref.txt" # is relative to the sketch folder + mkbuildoptglobals.extra_flags=--no_cache_core + mkbuildoptglobals.extra_flags=--cache_core + mkbuildoptglobals.extra_flags=--preferences_file "other-preferences.txt" # relative to IDE or full path + +If required, remember to quote file or file paths. + +Multiple versions of the Arduino IDE running +-------------------------------------------- + +You can run multiple Arduino IDE windows as long as you run one version +of the Arduino IDE at a time. When testing different versions, +completely exit one before starting the next version. For example, +Arduino IDE 1.8.19 and Arduino IDE 2.0 work with different temp and +build paths. With this combination, the workaround logic sometimes fails +to enable. + +At the time of this writing, when Arduino IDE 2.0 rc5 exits, it leaves +the temp space dirty. This keeps the workaround active the next time the +IDE is started. If this is an issue, manually delete the temp files. + +Also note when a ``.h`` file is renamed in the sketch folder, a copy of the old +file remains in the build sketch folder. This can create confusion if +you missed an edit in updating an ``#include`` in one or more of your +modules. That module will continue to use the stale version of the +``.h`` until you restart the IDE or other major changes that would +cause the IDE to delete and recopy the contents from the source +Sketch directory. Changes on the IDE Tools board settings may cause a +complete rebuild, clearing the problem. This may be the culprit for +“What! It built fine last night!” + diff --git a/doc/buildopt/internals.rst b/doc/buildopt/internals.rst new file mode 100644 index 0000000000..cbaff9819a --- /dev/null +++ b/doc/buildopt/internals.rst @@ -0,0 +1,235 @@ +:orphan: + +mkbuildoptglobals.py internals +============================== + +Sketch header aka SKETCH.ino.globals.h +-------------------------------------- + +.. hint:: + + ``SKETCH.ino.globals.h`` is expected to be created by the user. + +It is always located in the root of the SKETCH directory, and must use the +the actual name of the sketch program (``SKETCH.ino``) in the its name. + +Header file format is used because IDE only manages source files it actually +recognizes as valid C / C++ file formats. When building and re-building the +sketch, only valid file formats are taken into an account when building source +code dependencies tree. + +Command-line options file +------------------------- + + .. hint:: + + This file is created by the script. + +Options file is generated based on the contents of the sketch header comment block, +and its contents are then used as gcc command-line options (``@file``) + + If file does not exist, or cannot be read, then the option will be treated literally, and not removed. + + Options in file are separated by whitespace. A whitespace character may be included + in an option by surrounding the entire option in either single or double quotes. + Any character (including a backslash) may be included by prefixing the character + to be included with a backslash. + The file may itself contain additional @file options; any such options will be processed recursively. + + --- https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html + + +Arduino build system uses timestamps to determine which files should be rebuilt. +``@file`` cannot be a direct dependency, only other source code files can. + +Thus, command-line options file is *always* created with at least one ``-include`` + +.. code-block:: console + + -include "PLATFORM_PATH / COMMON_HEADER_PATH" + +When matching directive is found in the sketch header, path to its copy in the build directory is also added + +.. code-block:: console + + -include "BUILD_DIRECTORY / SKETCH_HEADER_COPY_PATH" + +Common header +------------- + +.. note:: + + This file is also created by the script. + +It is used as a means of triggering core rebuild, because modern Arduino build systems +are agressively caching it and attempt to re-use existing ``core.a`` whenever possible. + +This file would contain path to the currently used command-line options file extracted +from the sketch header. It remains empty otherwise. + +Build directory +--------------- + +Arduino build process copies every valid source file from the source (sketch) +directory into the build directory. This script is expected to be launched in +the "prebuild" stage. At that point, build directory should already exist, but +it may not yet contain any of the sketch source files. + +Script would always attempt to copy sketch header from the source (sketch) +directory to the build one. If it does not exist, a placeholder would be created. + +Script would always synchronize atime & mtime of every file. When sketch header +exists, stats are taken from it. When it doesn't, stats for the generated common +header are used instead. + +Configuration +------------- + +``platform.txt`` is expected to have this script listed as a tool + +.. code-block:: ini + + runtime.tools.mkbuildoptglobals={runtime.platform.path}/tools/mkbuildoptglobals.py + +Paths are always provided as Fully Qualified File Names (FQFNs): + +.. code-block:: ini + + globals.h.source.fqfn={build.source.path}/{build.project_name}.globals.h + globals.h.common.fqfn={build.core.path}/__common_globals.h + build.opt.fqfn={build.path}/core/build.opt + mkbuildoptglobals.extra_flags= + +`"prebuild" hook `__ must be used, +allowing this script to run *before* build process creates and / or copies them. +Both Arduino IDE 1.x and 2.x generate +`prerequisite makefiles (files with a .d suffix) `__ +at some point in "discovery phase". + +.. code-block:: ini + + recipe.hooks.prebuild.#.pattern= + "{runtime.tools.python3.path}/python3" -I + "{runtime.tools.mkbuildoptglobals}" {mkbuildoptglobals.extra_flags} build + --build-path "{build.path}" + --build-opt "{build.opt.fqfn}" + --sketch-header "{globals.h.source.fqfn}" + --common-header "{commonhfile.fqfn}" + +Command-line options file is then shared between other recipes by including it in +the "cpreprocessor" flags. + +.. code-block:: ini + + compiler.cpreprocessor.flags=... @{build.opt.path} ... + +After that point, prerequisite makefiles should contain either only the common header, +or both the common header and the build sketch header. When any of included headers is +modified, every file in the dependency chain would be rebuilt. This allows us to keep +existing ``core.a`` cache when command-line options file is not used by the sketch. + +Example +------- + +Sketch header file with embedded command-line options file might look like this + +.. code-block:: c++ + :emphasize-lines: 1,2,3,4,5,6,7,8,9,10,11 + + /*@create-file:build.opt@ + // An embedded "build.opt" file using a "C" block comment. The starting signature + // must be on a line by itself. The closing block comment pattern should be on a + // line by itself. Each line within the block comment will be space trimmed and + // written to build.opt, skipping blank lines and lines starting with '//', '*' + // or '#'. + -DMYDEFINE="\"Chimichangas do not exist\"" + -O3 + -fanalyzer + -DUMM_STATS=2 + */ + + #ifndef SKETCH_INO_GLOBALS_H + #define SKETCH_INO_GLOBALS_H + + #if defined(__cplusplus) + // Defines kept private to .cpp modules + //#pragma message("__cplusplus has been seen") + #endif + + #if !defined(__cplusplus) && !defined(__ASSEMBLER__) + // Defines kept private to .c modules + #endif + + #if defined(__ASSEMBLER__) + // Defines kept private to assembler modules + #endif + + #endif + + +Caveats, Observations, and Ramblings +------------------------------------ + +1. Edits to ``platform.txt`` or ``platform.local.txt`` force a complete rebuild that + removes the core folder. Not a problem, just something to be aware of when + debugging this script. Similarly, changes on the IDE Tools selection cause a + complete rebuild. + + In contrast, the core directory is not deleted when the rebuild occurs from + changing a file with an established dependency (inspect .d in the build path) + +2. Renaming files does not change the last modified timestamp, possibly causing + issues when adding or replacing files by renaming and rebuilding. + + A good example of this problem is when you correct the spelling of sketch + header file. You must update mtime (e.g. call touch) of the file. + +3. ``-include ".../Sketch.ino.globals.h"`` is conditionally added to every compilation command, + so it may be reasonable to expect that ``#include "Sketch.ino.globals.h"`` is no longer necessary. + + However, it may not be the case when `create-file:...` directive is missing or does not match. + + When explicit ``#include "Sketch.ino.globals.h"`` is used in the code, it must always be guarded against including it twice: + + .. code-block:: c++ + :emphasize-lines: 1 + + #pragma once + + Or, by using classic header guards: + + .. code-block:: c++ + :emphasize-lines: 1,2,4 + + #infdef SKETCH_GLOBALS_H + #define SKETCH_GLOBALS_H + ... file contents ... + #endif + +4. ``build.opt`` itself is not listed as a dependency in .d, .h files are used + because this is the only obvious way to force arduino-builder / arduino-cli + into tracking it. + +5. When not using ``--build-path``, ``core.a`` is cached and shared. + CI sometimes uses `ARDUINO_BUILD_CACHE_PATH environment variable `__. + This allows to have a private core cache, separate from the system one. + +6. `Referencing upstream arduino-cli code (v1.2.2) `__, ``core.a`` cache key is based on: + + * `ESP8266 Platform Path`, and depends on installation method + + * `Installing ESP8266 Core <../installing.rst>`__ + * `Arduino Platform Installation Directories `__ + + * `FQBN` + + See `Arduino Custom Board Options `__). + + * `Optimization flags` + + .. attention:: + + ``{compiler.optimization_flags}`` is not currently used in the ESP8266 Core + + See `Arduino Optimization Level setting `__). + diff --git a/doc/faq/a06-global-build-options.rst b/doc/faq/a06-global-build-options.rst deleted file mode 100644 index 3e86b88a58..0000000000 --- a/doc/faq/a06-global-build-options.rst +++ /dev/null @@ -1,326 +0,0 @@ -:orphan: - -How to specify global build defines and options -=============================================== - -To create globally usable macro definitions for a Sketch, create a file -with a name based on your Sketch’s file name followed by ``.globals.h`` -in the Sketch folder. For example, if the main Sketch file is named -``LowWatermark.ino``, its global ``.h`` file would be -``LowWatermark.ino.globals.h``. This file will be implicitly included -with every module built for your Sketch. Do not directly include it in -any of your sketch files or in any other source files. There is no need -to create empty/dummy files, when not used. - -This global ``.h`` also supports embedding compiler command-line options -in a unique “C” block comment. Compiler options are placed in a “C” -block comment starting with ``/*@create-file:build.opt@``. This -signature line must be alone on a single line. The block comment ending -``*/`` should also be alone on a single line. In between, place your -compiler command-line options just as you would have for the GCC @file -command option. - -Actions taken in processing comment block to create ``build.opt`` - -- for each line, white space is trimmed -- blank lines are skipped -- lines starting with ``*``, ``//``, or ``#`` are skipped -- the remaining results are written to build tree\ ``/core/build.opt`` -- multiple ``/*@create-file:build.opt@`` ``*/`` comment blocks are not - allowed -- ``build.opt`` is finished with a ``-include ...`` command, which - references the global .h its contents were extracted from. - -Example Sketch: ``LowWatermark.ino`` - -.. code:: cpp - - #include // has prototype for umm_free_heap_size_min() - - void setup() { - Serial.begin(115200); - delay(200); - #ifdef MYTITLE1 - Serial.printf("\r\n" MYTITLE1 MYTITLE2 "\r\n"); - #else - Serial.println("ERROR: MYTITLE1 not present"); - #endif - Serial.printf("Heap Low Watermark %u\r\n", umm_free_heap_size_min()); - } - - void loop() {} - -Global ``.h`` file: ``LowWatermark.ino.globals.h`` - -.. code:: cpp - - /*@create-file:build.opt@ - // An embedded build.opt file using a "C" block comment. The starting signature - // must be on a line by itself. The closing block comment pattern should be on a - // line by itself. Each line within the block comment will be space trimmed and - // written to build.opt, skipping blank lines and lines starting with '//', '*' - // or '#'. - - * this line is ignored - # this line is ignored - -DMYTITLE1="\"Running on \"" - -O3 - //-fanalyzer - -DUMM_STATS_FULL=1 - */ - - #ifndef LOWWATERMARK_INO_GLOBALS_H - #define LOWWATERMARK_INO_GLOBALS_H - - #if !defined(__ASSEMBLER__) - // Defines kept away from assembler modules - // i.e. Defines for .cpp, .ino, .c ... modules - #endif - - #if defined(__cplusplus) - // Defines kept private to .cpp and .ino modules - //#pragma message("__cplusplus has been seen") - #define MYTITLE2 "Empty" - #endif - - #if !defined(__cplusplus) && !defined(__ASSEMBLER__) - // Defines kept private to .c modules - #define MYTITLE2 "Full" - #endif - - #if defined(__ASSEMBLER__) - // Defines kept private to assembler modules - #endif - - #endif - -Separate production and debug build options -=========================================== - -If your production and debug build option requirements are different, -adding ``mkbuildoptglobals.extra_flags={build.debug_port}`` to -``platform.local.txt`` will create separate build option groups for -debugging and production. For the production build option group, the “C” -block comment starts with ``/*@create-file:build.opt@``, as previously -defined. For the debugging group, the new “C” block comment starts with -``/*@create-file:build.opt:debug@``. You make your group selection -through “Arduino->Tools->Debug port” by selecting or disabling the -“Debug port.” - -Options common to both debug and production builds must be included in -both groups. Neither of the groups is required. You may also omit either -or both. - -Reminder with this change, any old “sketch” with only a “C” block -comment starting with ``/*@create-file:build.opt@`` would not use a -``build.opt`` file for the debug case. Update old sketches as needed. - -Updated Global ``.h`` file: ``LowWatermark.ino.globals.h`` - -.. code:: cpp - - /*@create-file:build.opt:debug@ - // Debug build options - -DMYTITLE1="\"Running on \"" - -DUMM_STATS_FULL=1 - - //-fanalyzer - - // Removing the optimization for "sibling and tail recursive calls" may fill - // in some gaps in the stack decoder report. Preserves the stack frames - // created at each level as you call down to the next. - -fno-optimize-sibling-calls - */ - - /*@create-file:build.opt@ - // Production build options - -DMYTITLE1="\"Running on \"" - -DUMM_STATS_FULL=1 - -O3 - */ - - #ifndef LOWWATERMARK_INO_GLOBALS_H - #define LOWWATERMARK_INO_GLOBALS_H - - #if defined(__cplusplus) - #define MYTITLE2 "Empty" - #endif - - #if !defined(__cplusplus) && !defined(__ASSEMBLER__) - #define MYTITLE2 "Full" - #endif - - #ifdef DEBUG_ESP_PORT - // Global Debug defines - // ... - #else - // Global Production defines - // ... - #endif - - #endif - -Aggressively cache compiled core -================================ - -This feature appeared with the release of Arduino IDE 1.8.2. The feature -“Aggressively Cache Compiled core” refers to sharing a single copy of -``core.a`` across all Arduino IDE Sketch windows. This feature is on by -default. ``core.a`` is an archive file containing the compiled objects -of ``./core/esp8266/*``. Created after your 1ST successful compilation. -All other open sketch builds use this shared file. When you close all -Arduino IDE windows, the core archive file is deleted. - -This feature is not compatible with using global defines or compiler -command-line options. Without mediation, bad builds could result, when -left enabled. When ``#define`` changes require rebuilding ``core.a`` and -multiple Sketches are open, they can no longer reliably share one cached -``core.a``. In a simple case: The 1st Sketch to be built has its version -of ``core.a`` cached. Other sketches will use this cached version for -their builds. - -There are two solutions to this issue: - -1. Do nothing, and rely on aggressive cache workaround built into the - script. -2. Turn off the “Aggressively Cache Compiled core” feature, by setting - ``compiler.cache_core=false``. - -Using “compiler.cache_core=false” ---------------------------------- - -There are two ways to turn off the “Aggressively Cache Compiled core” -feature: This can be done with the Arduino IDE command-line or a text -editor. - -Using the Arduino IDE command-line from a system command line, enter the -following: - -:: - - arduino --pref compiler.cache_core=false --save-prefs - -For the text editor, you need to find the location of -``preferences.txt``. From the Arduino IDE, go to *File->Preferences*. -Make note of the path to ``prefereces.txt``. You *cannot* edit the file -while the Arduino IDE is running. Close all Arduino IDE windows and edit -the file ``preferences.txt``. Change ``compiler.cache_core=true`` to -``compiler.cache_core=false`` and save. Then each sketch will maintain -its *own* copy of ``core.a`` built with the customization expressed by -their respective ``build.opt`` file. - -The “workaround” ----------------- - -When the “Aggressively Cache Compiled core” feature is enabled and the -global define file is detected, a workaround will turn on and stay on. -When you switch between Sketch windows, core will be recompiled and the -cache updated. The workaround logic is reset when Arduino IDE is -completely shutdown and restarted. - -The workaround is not perfect. These issues may be of concern: - -1. Dirty temp space. Arduino build cache files left over from a previous - run or boot. -2. Arduino command-line options: - - - override default preferences.txt file. - - override a preference, specifically ``compiler.cache_core``. - -3. Multiple versions of the Arduino IDE running - -**Dirty temp space** - -A minor concern, the workaround is always on. Not an issue for build -accuracy, but ``core.a`` maybe rebuild more often than necessary. - -Some operating systems are better at cleaning up their temp space than -others at reboot after a crash. At least for Windows®, you may need to -manually delete the Arduino temp files and directories after a crash. -Otherwise, the workaround logic may be left on. There is no harm in the -workaround being stuck on, the build will be correct; however, the core -files will occasionally be recompiled when not needed. - -For some Windows® systems the temp directory can be found near -``C:\Users\\AppData\Local\Temp\arduino*``. Note ``AppData`` is -a hidden directory. For help with this do an Internet search on -``windows disk cleanup``. Or, type ``disk cleanup`` in the Windows® -taskbar search box. - -With Linux, this problem could occur after an Arduino IDE crash. The -problem would be cleared after a reboot. Or you can manually cleanup the -``/tmp/`` directory before restarting the Arduino IDE. - -**Arduino command-line option overrides** - -If you are building with ``compiler.cache_core=true`` no action is -needed. If ``false`` the script would benefit by knowing that. - -When using either of these two command-line options: - -:: - - ./arduino --preferences-file other-preferences.txt - ./arduino --pref compiler.cache_core=false - -Hints for discovering the value of ``compiler.cache_core``, can be -provided by specifying ``mkbuildoptglobals.extra_flags=...`` in -``platform.local.txt``. - -Examples of hints: - -:: - - mkbuildoptglobals.extra_flags=--preferences_sketch # assume file preferences.txt in the sketch folder - mkbuildoptglobals.extra_flags=--preferences_sketch "pref.txt" # is relative to the sketch folder - mkbuildoptglobals.extra_flags=--no_cache_core - mkbuildoptglobals.extra_flags=--cache_core - mkbuildoptglobals.extra_flags=--preferences_file "other-preferences.txt" # relative to IDE or full path - -If required, remember to quote file or file paths. - -**Multiple versions of the Arduino IDE running** - -You can run multiple Arduino IDE windows as long as you run one version -of the Arduino IDE at a time. When testing different versions, -completely exit one before starting the next version. For example, -Arduino IDE 1.8.19 and Arduino IDE 2.0 work with different temp and -build paths. With this combination, the workaround logic sometimes fails -to enable. - -At the time of this writing, when Arduino IDE 2.0 rc5 exits, it leaves -the temp space dirty. This keeps the workaround active the next time the -IDE is started. If this is an issue, manually delete the temp files. - -Custom build environments -========================= - -Some custom build environments may have already addressed this issue by -other means. If you have a custom build environment that does not -require this feature and would like to turn it off, you can add the -following lines to the ``platform.local.txt`` used in your build -environment: - -:: - - recipe.hooks.prebuild.2.pattern= - build.opt.flags= - -Other build confusion -===================== - -1. Renaming a file does not change the last modified timestamp, possibly - causing issues when adding a file by renaming and rebuilding. A good - example of this problem would be to have then fixed a typo in file - name ``LowWatermark.ino.globals.h``. You need to touch (update - timestamp) the file so a “rebuild all” is performed. - -2. When a ``.h`` file is renamed in the sketch folder, a copy of the old - file remains in the build sketch folder. This can create confusion if - you missed an edit in updating an ``#include`` in one or more of your - modules. That module will continue to use the stale version of the - ``.h`` until you restart the IDE or other major changes that would - cause the IDE to delete and recopy the contents from the source - Sketch directory. Changes on the IDE Tools board settings may cause a - complete rebuild, clearing the problem. This may be the culprit for - “What! It built fine last night!” diff --git a/doc/faq/readme.rst b/doc/faq/readme.rst index cfd65eca90..b4c30bf5f9 100644 --- a/doc/faq/readme.rst +++ b/doc/faq/readme.rst @@ -199,4 +199,4 @@ By using a uniquely named `.h` file, macro definitions can be created and globally used. Additionally, compiler command-line options can be embedded in this file as a unique block comment. -`Read more `__. +`Read more <../global_build_options>`__. diff --git a/doc/global_build_options.rst b/doc/global_build_options.rst new file mode 100644 index 0000000000..779a077e9d --- /dev/null +++ b/doc/global_build_options.rst @@ -0,0 +1,202 @@ +Global build defines and options +================================ + +Basics +------ + +To create globally usable macro definitions for a Sketch, create a file +with a name based on your Sketch’s file name followed by ``.globals.h`` +in the Sketch folder. For example, if the main Sketch file is named +``LowWatermark.ino``, its global ``.h`` file would be +``LowWatermark.ino.globals.h``. This file will be implicitly included +with every module built for your Sketch. Do not directly include it in +any of your sketch files or in any other source files. There is no need +to create empty/dummy files, when not used. + +This global ``.h`` also supports embedding compiler command-line options +in a unique “C” block comment. Compiler options are placed in a “C” +block comment starting with ``/*@create-file:build.opt@``. This +signature line must be alone on a single line. The block comment ending +``*/`` should also be alone on a single line. In between, place your +compiler command-line options just as you would have for the GCC @file +command option. + +Actions taken in processing comment block to create ``build.opt`` + +- for each line, white space is trimmed +- blank lines are skipped +- lines starting with ``*``, ``//``, or ``#`` are skipped +- the remaining results are written to arduino-cli build cache directory +- multiple ``/*@create-file:build.opt@`` ``*/`` comment blocks are + allowed and would be merged in order they are written in the file +- ``build.opt`` is finished with a ``-include ...`` command, which + references the global .h its contents were extracted from. + +Example +------- + +Example Sketch: ``LowWatermark.ino`` + +.. code:: cpp + + #include // has prototype for umm_free_heap_size_min() + + void setup() { + Serial.begin(115200); + delay(200); + #ifdef MYTITLE1 + Serial.printf("\r\n" MYTITLE1 MYTITLE2 "\r\n"); + #else + Serial.println("ERROR: MYTITLE1 not present"); + #endif + Serial.printf("Heap Low Watermark %u\r\n", umm_free_heap_size_min()); + } + + void loop() {} + +Global ``.h`` file: ``LowWatermark.ino.globals.h`` + +.. code:: cpp + + /*@create-file:build.opt@ + // An embedded build.opt file using a "C" block comment. The starting signature + // must be on a line by itself. The closing block comment pattern should be on a + // line by itself. Each line within the block comment will be space trimmed and + // written to build.opt, skipping blank lines and lines starting with '//', '*' + // or '#'. + + * this line is ignored + # this line is ignored + -DMYTITLE1="\"Running on \"" + -O3 + //-fanalyzer + -DUMM_STATS_FULL=1 + */ + + #ifndef LOWWATERMARK_INO_GLOBALS_H + #define LOWWATERMARK_INO_GLOBALS_H + + #if !defined(__ASSEMBLER__) + // Defines kept away from assembler modules + // i.e. Defines for .cpp, .ino, .c ... modules + #endif + + #if defined(__cplusplus) + // Defines kept private to .cpp and .ino modules + //#pragma message("__cplusplus has been seen") + #define MYTITLE2 "Empty" + #endif + + #if !defined(__cplusplus) && !defined(__ASSEMBLER__) + // Defines kept private to .c modules + #define MYTITLE2 "Full" + #endif + + #if defined(__ASSEMBLER__) + // Defines kept private to assembler modules + #endif + + #endif + +Separate production and debug build options +------------------------------------------- + +If your production and debug build option requirements are different, +adding ``mkbuildoptglobals.extra_flags={build.debug_port}`` to +``platform.local.txt`` will create separate build option groups for +debugging and production. For the production build option group, the “C” +block comment starts with ``/*@create-file:build.opt@``, as previously +defined. For the debugging group, the new “C” block comment starts with +``/*@create-file:build.opt:debug@``. You make your group selection +through “Arduino->Tools->Debug port” by selecting or disabling the +“Debug port.” + +Options common to both debug and production builds must be included in +both groups. Neither of the groups is required. You may also omit either +or both. + +Reminder with this change, any old “sketch” with only a “C” block +comment starting with ``/*@create-file:build.opt@`` would not use a +``build.opt`` file for the debug case. Update old sketches as needed. + +Updated Global ``.h`` file: ``LowWatermark.ino.globals.h`` + +.. code:: cpp + + /*@create-file:build.opt:debug@ + // Debug build options + -DMYTITLE1="\"Running on \"" + -DUMM_STATS_FULL=1 + + //-fanalyzer + + // Removing the optimization for "sibling and tail recursive calls" may fill + // in some gaps in the stack decoder report. Preserves the stack frames + // created at each level as you call down to the next. + -fno-optimize-sibling-calls + */ + + /*@create-file:build.opt@ + // Production build options + -DMYTITLE1="\"Running on \"" + -DUMM_STATS_FULL=1 + -O3 + */ + + #ifndef LOWWATERMARK_INO_GLOBALS_H + #define LOWWATERMARK_INO_GLOBALS_H + + #if defined(__cplusplus) + #define MYTITLE2 "Empty" + #endif + + #if !defined(__cplusplus) && !defined(__ASSEMBLER__) + #define MYTITLE2 "Full" + #endif + + #ifdef DEBUG_ESP_PORT + // Global Debug defines + // ... + #else + // Global Production defines + // ... + #endif + + #endif + + +Custom build environments +------------------------- + +Some custom build environments may have already addressed this issue by +other means. If you have a custom build environment that does not +require this feature and would like to turn it off, you can add the +following lines to the ``platform.local.txt`` used in your build +environment: + +.. code-block:: ini + + recipe.hooks.prebuild.2.pattern= + build.opt.flags= + +Source Code +----------- + +https://github.com/esp8266/Arduino/blob/master/tools/mkbuildoptglobals.py + + +Internals +--------- + +:doc:`/buildopt/internals` + +IDE 1.x aggressive caching +-------------------------- + +.. attention:: + + This article applies ONLY to IDE 1.x and original version + of the mkbuildoptglobals.py script shipped with Core 3.1.x + +:doc:`/buildopt/aggressive-caching` + diff --git a/doc/index.rst b/doc/index.rst index 5f3ec247c6..bae801a8f8 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -15,6 +15,7 @@ Welcome to ESP8266 Arduino Core's documentation! PROGMEM Using GDB to debug MMU + Global build options Boards FAQ diff --git a/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino b/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino index 2bc18751e1..fa4fc5c59d 100644 --- a/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino +++ b/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino @@ -4,11 +4,13 @@ * * Example from https://arduino-esp8266.readthedocs.io/en/latest/faq/a06-global-build-options.html * - * Note, we do not "#include" the special file "GlobalBuildOptions.ino.globals.h". - * The prebuild script will make it available to all modules. + * It is not necessary to have `#include "GlobalBuildOptions.ino.globals.h"`. + * However, any code inside of the special header file would not be available, + * unless there is also a matching `create-file:...` directive inside of it. * - * To track the new sketch name when saving this sketch to a new location and - * name, remember to update the global .h file name. + * Note that when building in debug mode, `build.opt:debug` should be used instead. + * + * Remember to update the global .h file name when saving this sketch to a new location! */ #include // has prototype for umm_free_heap_size_min() diff --git a/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino.globals.h b/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino.globals.h index b51b879f7f..835ec84675 100644 --- a/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino.globals.h +++ b/libraries/esp8266/examples/GlobalBuildOptions/GlobalBuildOptions.ino.globals.h @@ -8,11 +8,17 @@ * this line is ignored *@create-file:build.opt@ # this line is ignored +*/ + +/*@create-file:build.opt:debug@ + // Another embedded build.opt, with a different set of flags + -DMYTITLE1="\"Debugging on \"" -Og - // -fanalyzer -DUMM_STATS_FULL=1 + // -fanalyzer */ +// Following *raw* contents are also included #ifndef GLOBALBUILDOPTIONS_INO_GLOBALS_H #define GLOBALBUILDOPTIONS_INO_GLOBALS_H diff --git a/platform.txt b/platform.txt index f3392d652d..1a266e63ad 100644 --- a/platform.txt +++ b/platform.txt @@ -66,9 +66,10 @@ build.spiffs_blocksize= build.iramfloat=-DFP_IN_IROM # Fully qualified file names for processing sketch global options -globals.h.source.fqfn={build.source.path}/{build.project_name}.globals.h -commonhfile.fqfn={build.core.path}/CommonHFile.h -build.opt.fqfn={build.path}/core/build.opt +globals.h.build.source.fqfn={build.source.path}/{build.project_name}.globals.h +globals.h.build.fqfn={build.path}/{build.project_name}.globals.h +common.h.fqfn={build.core.path}/CommonHFile.h +build.opt.fqfn={build.path}/sketch/build.opt build.opt.flags="@{build.opt.fqfn}" mkbuildoptglobals.extra_flags= @@ -76,7 +77,8 @@ compiler.path={runtime.tools.xtensa-lx106-elf-gcc.path}/bin/ compiler.sdk.path={runtime.platform.path}/tools/sdk compiler.libc.path={runtime.platform.path}/tools/sdk/libc/xtensa-lx106-elf -compiler.cpreprocessor.flags=-D__ets__ -DICACHE_FLASH -U__STRICT_ANSI__ -D_GNU_SOURCE -DESP8266 {build.debug_optim} {build.opt.flags} "-I{compiler.sdk.path}/include" "-I{compiler.sdk.path}/{build.lwip_include}" "-I{compiler.libc.path}/include" "-I{build.path}/core" +compiler.cpreprocessor.extra_flags= +compiler.cpreprocessor.flags=-D__ets__ -DICACHE_FLASH -U__STRICT_ANSI__ -D_GNU_SOURCE -DESP8266 {build.debug_optim} {build.opt.flags} "-I{compiler.sdk.path}/include" "-I{compiler.sdk.path}/{build.lwip_include}" "-I{compiler.libc.path}/include" "-I{build.path}/core" {compiler.cpreprocessor.extra_flags} # support precompiled libraries in IDE v1.8.6+ compiler.libraries.ldflags= @@ -123,8 +125,8 @@ recipe.hooks.sketch.prebuild.pattern="{runtime.tools.python3.path}/python3" -I " # This is quite a working hack. This form of prebuild hook, while intuitive, is not explicitly documented. recipe.hooks.prebuild.1.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.makecorever}" --git-root "{runtime.platform.path}" --version "{version}" "{build.path}/core/core_version.h" -# Handle processing sketch global options -recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" {runtime.ide.version} "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" {mkbuildoptglobals.extra_flags} +# Handle core & sketch global options +recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.mkbuildoptglobals}" {mkbuildoptglobals.extra_flags} build --build-opt "{build.opt.fqfn}" --source-sketch-header "{globals.h.build.source.fqfn}" --build-sketch-header "{globals.h.build.fqfn}" --common-header "{common.h.fqfn}" ## Build the app.ld linker file diff --git a/tests/build.sh b/tests/build.sh index af34287a8a..81f756cbac 100755 --- a/tests/build.sh +++ b/tests/build.sh @@ -3,39 +3,7 @@ # expect to have git available root=$(git rev-parse --show-toplevel) -# general configuration related to the builder itself -ESP8266_ARDUINO_BUILD_DIR=${ESP8266_ARDUINO_BUILD_DIR:-$root} -ESP8266_ARDUINO_BUILDER=${ESP8266_ARDUINO_BUILDER:-arduino} -ESP8266_ARDUINO_PRESERVE_CACHE=${ESP8266_ARDUINO_PRESERVE_CACHE:-} - -# sketch build options -ESP8266_ARDUINO_DEBUG=${ESP8266_ARDUINO_DEBUG:-nodebug} -ESP8266_ARDUINO_LWIP=${ESP8266_ARDUINO_LWIP:-default} -ESP8266_ARDUINO_SKETCHES=${ESP8266_ARDUINO_SKETCHES:-} - -ESP8266_ARDUINO_CLI=${ESP8266_ARDUINO_CLI:-$HOME/.local/bin/arduino-cli} - -# ref. https://arduino.github.io/arduino-cli/1.2/configuration/#default-directories -case "${RUNNER_OS:-Linux}" in -"Linux") - ESP8266_ARDUINO_HARDWARE=${ESP8266_ARDUINO_HARDWARE:-$HOME/Arduino/hardware} - ESP8266_ARDUINO_LIBRARIES=${ESP8266_ARDUINO_LIBRARIES:-$HOME/Arduino/libraries} - ;; -"macOS") - ESP8266_ARDUINO_HARDWARE=${ESP8266_ARDUINO_HARDWARE:-$HOME/Documents/Arduino/hardware} - ESP8266_ARDUINO_LIBRARIES=${ESP8266_ARDUINO_LIBRARIES:-$HOME/Documents/Arduino/libraries} - ;; -"Windows") - ESP8266_ARDUINO_HARDWARE=${ESP8266_ARDUINO_HARDWARE:-$HOME/Documents/Arduino/hardware} - ESP8266_ARDUINO_LIBRARIES=${ESP8266_ARDUINO_LIBRARIES:-$HOME/Documents/Arduino/libraries} - ;; -*) - echo 'Unknown ${RUNNER_OS} = "' ${RUNNER_OS} '"' - exit 2 -esac - -source "$root/tests/lib-skip-ino.sh" -source "$root/tests/common.sh" +source "$root/tests/env.sh" cmd=${0##*/} usage=" @@ -56,10 +24,6 @@ USAGE: $cmd - build every .ino file from ESP8266_ARDUINO_SKETCHES " -mod=1 -rem=0 -cnt=0 - if [ "$#" -eq 1 ] ; then case "$1" in "-h") diff --git a/tests/common.sh b/tests/common.sh index c52756d177..32c2564fd5 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -u -e -E -o pipefail +set -u -e -E -o pipefail -o errtrace cache_dir=$(mktemp -d) trap 'trap_exit' EXIT @@ -9,10 +9,16 @@ function trap_exit() { # workaround for macOS shipping with broken bash local exit_code=$? + + # ^ $cache_dir is temporary, prune when exiting if [ -z "${ESP8266_ARDUINO_PRESERVE_CACHE-}" ]; then rm -rf "$cache_dir" fi + if [ "$exit_code" != 0 ] ; then + echo "*** exit_code=$exit_code ***" + fi + exit $exit_code } @@ -132,14 +138,12 @@ function build_sketches() build_cmd+=${cli_path} build_cmd+=" compile"\ " --warnings=all"\ -" --build-path $build_dir"\ " --fqbn $fqbn"\ " --libraries $library_path"\ " --output-dir $build_out" print_size_info_header >"$cache_dir"/size.log - local clean_core=1 local testcnt=0 local cnt=0 @@ -164,17 +168,11 @@ function build_sketches() build_cnt=0 fi - # Do we need a clean core build? $build_dir/core/* cannot be shared - # between sketches when global options are present. - clean_core=$(arduino_mkbuildoptglobals_cleanup "$clean_core" "$build_dir" "$sketch") - - # Clear out the last built sketch, map, elf, bin files, but leave the compiled - # objects in the core and libraries available for use so we don't need to rebuild - # them each sketch. - rm -rf "$build_dir"/sketch \ - "$build_dir"/*.bin \ - "$build_dir"/*.map \ - "$build_dir"/*.elf + # Clear out the latest map, elf, bin files + rm -rf \ + "$build_out"/*.bin \ + "$build_out"/*.map \ + "$build_out"/*.elf echo ${ci_group}Building $cnt $sketch echo "$build_cmd $sketch" @@ -194,7 +192,7 @@ function build_sketches() fi print_size_info "$core_path"/tools/xtensa-lx106-elf/bin/xtensa-lx106-elf-size \ - $build_dir/*.elf >>$cache_dir/size.log + $build_out/*.elf >>$cache_dir/size.log echo $ci_end_group done @@ -311,7 +309,6 @@ function install_core() printf "%s\n" \ "compiler.c.extra_flags=-Wall -Wextra $debug_flags" \ "compiler.cpp.extra_flags=-Wall -Wextra $debug_flags" \ - "mkbuildoptglobals.extra_flags=--ci --cache_core" \ "recipe.hooks.prebuild.1.pattern=\"{runtime.tools.python3.path}/python3\" -I \"{runtime.tools.makecorever}\" --git-root \"{runtime.platform.path}\" --version \"{version}\" \"{runtime.platform.path}/cores/esp8266/core_version.h\"" \ > ${core_path}/platform.local.txt echo -e "\n----platform.local.txt----" diff --git a/tests/device/Makefile b/tests/device/Makefile index f89c9ea93e..039749b8a6 100644 --- a/tests/device/Makefile +++ b/tests/device/Makefile @@ -2,7 +2,17 @@ SHELL := /bin/bash ESP8266_CORE_PATH ?= $(shell git rev-parse --show-toplevel) +# arduino-cli core/ & sketch/ build and cache directories +# by default, share build location with every other sketch +CACHE_DIR ?= $(HOME)/.cache/arduino +ifneq ("$(findstring $(ESP8266_CORE_PATH),$(CACHE_DIR))", "") +$(warning "CACHE_DIR is located in ESP8266_CORE_PATH, core.a caching will be disabled") +endif + +# binaries (compile --output-dir=...) & test output goes in here. BUILD_DIR ?= $(PWD)/build + +# where to look for BSTest scripts BS_DIR ?= $(PWD)/libraries/BSTest PYTHON ?= python3 @@ -67,7 +77,7 @@ list: showtestlist all: count tests test_report -$(TEST_LIST): | virtualenv $(TEST_CONFIG) $(BUILD_DIR) $(HARDWARE_DIR) +$(TEST_LIST): | virtualenv $(TEST_CONFIG) $(CACHE_DIR) $(BUILD_DIR) $(HARDWARE_DIR) .NOTPARALLEL: $(TEST_LIST) @@ -78,18 +88,21 @@ showtestlist: @printf '%s\n' $(TEST_LIST) @echo "--------------------------------" -$(TEST_LIST): LOCAL_BUILD_DIR=$(BUILD_DIR)/$(notdir $@) -$(TEST_LIST): LOCAL_DATA_IMG=data.img +$(TEST_LIST): LOCAL_BUILD_DIR=$(BUILD_DIR)/$(notdir $@)/ +$(TEST_LIST): LOCAL_TEST_RESULT_XML=$(LOCAL_BUILD_DIR)/$(TEST_RESULT_XML) +$(TEST_LIST): LOCAL_DATA_DIR=$(LOCAL_BUILD_DIR)/data/ +$(TEST_LIST): LOCAL_DATA_IMG=$(LOCAL_BUILD_DIR)/data.img define build-arduino - rm -f $(LOCAL_BUILD_DIR)/build.options.json - $(BUILD_TOOL) compile \ - $(BUILD_FLAGS) \ - --libraries "$(PWD)/libraries" \ - --warnings=all \ - --build-path $(LOCAL_BUILD_DIR) \ - --fqbn=$(FQBN) \ - $@ + export ARDUINO_BUILD_CACHE_PATH="$(CACHE_DIR)"; \ + $(BUILD_TOOL) config dump; \ + $(BUILD_TOOL) compile \ + $(BUILD_FLAGS) \ + --libraries "$(PWD)/libraries" \ + --output-dir "$(LOCAL_BUILD_DIR)" \ + --warnings=all \ + --fqbn=$(FQBN) \ + $@ endef define build-mock @@ -98,7 +111,7 @@ define build-mock $(VENV_PYTHON) $(BS_DIR)/runner.py \ $(BS_FLAGS) \ --name $(basename $(notdir $@)) \ - --output $(LOCAL_BUILD_DIR)/$(TEST_RESULT_XML) \ + --output $(LOCAL_TEST_RESULT_XML) \ --env-file $(TEST_CONFIG) \ $(call mock_script,$@) \ executable "$(ESP8266_CORE_PATH)/tests/host/bin/$(@:%.ino=%)" || echo ""` @@ -111,17 +124,17 @@ define upload-data (cd $(dir $@) && ./make_data.py ) || echo "Filesystem creation skipped" @test -d $(dir $@)/data/ && ( \ $(MKFS) \ - --create $(dir $@)/data/ \ + --create $(LOCAL_DATA_DIR) \ --size 0xFB000 \ --block 8192 \ --page 256 \ - $(LOCAL_BUILD_DIR)/$(LOCAL_DATA_IMG) && \ + $(LOCAL_DATA_IMG) && \ $(ESPTOOL) \ --chip esp8266 \ --port $(UPLOAD_PORT) \ --baud $(UPLOAD_BAUD) \ --after no_reset \ - write_flash 0x300000 $(LOCAL_BUILD_DIR)/$(LOCAL_DATA_IMG) ) \ + write_flash 0x300000 $(LOCAL_DATA_IMG) ) \ && (echo "Uploaded filesystem") \ || (echo "Filesystem upload skipped") endef @@ -149,7 +162,7 @@ define run-test $(VENV_PYTHON) $(BS_DIR)/runner.py \ $(BS_FLAGS) \ --name $(basename $(notdir $@)) \ - --output $(LOCAL_BUILD_DIR)/$(TEST_RESULT_XML) \ + --output $(LOCAL_TEST_RESULT_XML) \ --env-file $(TEST_CONFIG) \ $(call mock_script,$@) \ port $(UPLOAD_PORT) \ @@ -160,6 +173,7 @@ $(TEST_LIST): @echo "--------------------------------" @echo "Running test '$@' of $(words $(TEST_LIST)) tests" mkdir -p $(LOCAL_BUILD_DIR) + mkdir -p $(CACHE_DIR) ifneq ("$(NO_BUILD)","1") @echo Building $(notdir $@) ifeq ("$(MOCK)", "1") @@ -187,13 +201,17 @@ $(TEST_REPORT_HTML): $(TEST_REPORT_XML) | virtualenv test_report: $(TEST_REPORT_HTML) @echo "Test report generated in $(TEST_REPORT_HTML)" +$(CACHE_DIR): + @mkdir -p $@ + $(BUILD_DIR): - @mkdir -p $(BUILD_DIR) + @mkdir -p $@ virtualenv: @make -C $(BS_DIR) PYTHON=$(PYTHON) virtualenv clean: + rm -rf $(CACHE_DIR) rm -rf $(BUILD_DIR) rm -rf $(BS_DIR)/virtualenv rm -f $(TEST_REPORT_HTML) $(TEST_REPORT_XML) diff --git a/tests/device/test_libc/test_libc.ino.globals.h b/tests/device/test_libc/test_libc.ino.globals.h index 1bfe3ba4d5..e271602755 100644 --- a/tests/device/test_libc/test_libc.ino.globals.h +++ b/tests/device/test_libc/test_libc.ino.globals.h @@ -1,4 +1,3 @@ /*@create-file:build.opt@ - -fno-builtin */ diff --git a/tests/env.sh b/tests/env.sh new file mode 100644 index 0000000000..726c9f36ec --- /dev/null +++ b/tests/env.sh @@ -0,0 +1,38 @@ +# general configuration related to the builder itself +ESP8266_ARDUINO_BUILD_DIR=${ESP8266_ARDUINO_BUILD_DIR:-$root} +ESP8266_ARDUINO_BUILDER=${ESP8266_ARDUINO_BUILDER:-arduino} +ESP8266_ARDUINO_PRESERVE_CACHE=${ESP8266_ARDUINO_PRESERVE_CACHE:-} + +# sketch build options +ESP8266_ARDUINO_DEBUG=${ESP8266_ARDUINO_DEBUG:-nodebug} +ESP8266_ARDUINO_LWIP=${ESP8266_ARDUINO_LWIP:-default} +ESP8266_ARDUINO_SKETCHES=${ESP8266_ARDUINO_SKETCHES:-} + +ESP8266_ARDUINO_CLI=${ESP8266_ARDUINO_CLI:-$HOME/.local/bin/arduino-cli} + +# ref. https://arduino.github.io/arduino-cli/1.2/configuration/#default-directories +case "${RUNNER_OS:-Linux}" in +"Linux") + RUNNER_OS="Linux" + ESP8266_ARDUINO_HARDWARE=${ESP8266_ARDUINO_HARDWARE:-$HOME/Arduino/hardware} + ESP8266_ARDUINO_LIBRARIES=${ESP8266_ARDUINO_LIBRARIES:-$HOME/Arduino/libraries} + ;; +"macOS") + ESP8266_ARDUINO_HARDWARE=${ESP8266_ARDUINO_HARDWARE:-$HOME/Documents/Arduino/hardware} + ESP8266_ARDUINO_LIBRARIES=${ESP8266_ARDUINO_LIBRARIES:-$HOME/Documents/Arduino/libraries} + ;; +"Windows") + ESP8266_ARDUINO_HARDWARE=${ESP8266_ARDUINO_HARDWARE:-$HOME/Documents/Arduino/hardware} + ESP8266_ARDUINO_LIBRARIES=${ESP8266_ARDUINO_LIBRARIES:-$HOME/Documents/Arduino/libraries} + ;; +*) + echo 'Unknown ${RUNNER_OS} = "' ${RUNNER_OS} '"' + exit 2 +esac + +source "$root/tests/lib-skip-ino.sh" +source "$root/tests/common.sh" + +mod=1 +rem=0 +cnt=0 diff --git a/tests/install_arduino.sh b/tests/install_arduino.sh new file mode 100755 index 0000000000..152f64e75f --- /dev/null +++ b/tests/install_arduino.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +root=$(git rev-parse --show-toplevel) +source "$root/tests/env.sh" + +install_arduino "$ESP8266_ARDUINO_DEBUG" diff --git a/tests/test_mkbuildoptglobals.sh b/tests/test_mkbuildoptglobals.sh new file mode 100755 index 0000000000..be8b63b604 --- /dev/null +++ b/tests/test_mkbuildoptglobals.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +trap 'echo " ${BASH_SOURCE[1]}:$LINENO $BASH_COMMAND"' ERR + +root=$(git rev-parse --show-toplevel) +source "$root/tests/env.sh" + +unset -f step_summary +function step_summary() +{ + echo "" +} + +export ARDUINO_BUILD_CACHE_PATH="$cache_dir" + +sketches="${cache_dir}/sketches/" +cores="${cache_dir}/cores/" + +esp8266_dir="${ESP8266_ARDUINO_BUILD_DIR}" +commonfileh="${esp8266_dir}/cores/esp8266/CommonHFile.h" + +tests_dir="$root/tests/test_mkbuildoptglobals" +tests=$(ls -1 "$tests_dir" | sed 's/.sh$//g') + +function name_size_mtime() +{ + stat --printf '%n:%s:%Y' "$1" +} + +function most_recent_file() +{ + local name="$1" + local location="$2" + echo $(readlink -f "${location}/"$(ls -t1 "${location}" | grep "$name" | head -1)) +} + +function most_recent_dir() +{ + local location="$1" + echo $(readlink -f "${location}/"$(ls -t1 "$location" | head -1)) +} + +function assert_build() +{ + local name="$1" + local build_dir="$2" + local size_check="${3:-0}" + + local build_opt="$build_dir"/sketch/build.opt + test -e "$build_opt" + + local globals_h="$build_dir"/${name}.ino.globals.h + test -e "$globals_h" + + if [ "$size_check" = "1" ] ; then + test -s "$build_opt" + test -s "$globals_h" + fi +} + +function assert_core() +{ + local size_check="$1" + + if [ "$size_check" = "1" ] ; then + test -s "$commonfileh" + else + test ! -s "$commonfileh" + fi + +} + +function build_esp8266_example() +{ + local name="$1" + + ESP8266_ARDUINO_SKETCHES="$root/libraries/esp8266/examples/${name}/${name}.ino" + build_sketches_with_arduino "$ESP8266_ARDUINO_LWIP" "$mod" "$rem" "$cnt" +} + +function make_commonh_stat() +{ + local stat=$(name_size_mtime "$commonfileh") + test -n "$stat" + + echo "$stat" +} + +function make_core_stat() +{ + local recent_core=$(most_recent_dir "$cores") + local recent_file=$(most_recent_file "core.a" "$recent_core") + + local stat=$(name_size_mtime "$recent_file") + test -n "$stat" + + echo "$stat" +} + +case "${1:-}" in +"list") + printf "%s\n" $tests + ;; + +"run") + for test in $tests ; do + printf "Checking \"%s\"\n" "$test" + /usr/bin/env bash $root/tests/test_mkbuildoptglobals.sh $test + done + ;; +*) + source "$tests_dir/${1}.sh" + ;; +esac diff --git a/tests/test_mkbuildoptglobals/buildopt_then_buildopt.sh b/tests/test_mkbuildoptglobals/buildopt_then_buildopt.sh new file mode 100644 index 0000000000..3d1ac53830 --- /dev/null +++ b/tests/test_mkbuildoptglobals/buildopt_then_buildopt.sh @@ -0,0 +1,25 @@ +function buildopt_then_buildopt() +{ + build_esp8266_example "GlobalBuildOptions" + + local last_sketch=$(most_recent_dir "$sketches") + assert_build "GlobalBuildOptions" "$last_sketch" 1 + assert_core 1 + + local globalbuildoptions_commonh_stat=$(make_commonh_stat) + local globalbuildoptions_core_stat=$(make_core_stat) + + build_esp8266_example "HwdtStackDump" + + last_sketch=$(most_recent_dir "$sketches") + assert_build "HwdtStackDump" "$last_sketch" 1 + assert_core 1 + + local hwdtstackdump_commonh_stat=$(make_commonh_stat) + local hwdtstackdump_core_stat=$(make_core_stat) + + test "$hwdtstackdump_commonh_stat" != "$globalbuildoptions_commonh_stat" + test "$hwdtstackdump_core_stat" != "$globalbuildoptions_core_stat" +} + +buildopt_then_buildopt diff --git a/tests/test_mkbuildoptglobals/buildopt_then_nobuildopt.sh b/tests/test_mkbuildoptglobals/buildopt_then_nobuildopt.sh new file mode 100644 index 0000000000..4db4927350 --- /dev/null +++ b/tests/test_mkbuildoptglobals/buildopt_then_nobuildopt.sh @@ -0,0 +1,25 @@ +function buildopt_then_nobuildopt() +{ + build_esp8266_example "GlobalBuildOptions" + + local last_sketch=$(most_recent_dir "$sketches") + assert_build "GlobalBuildOptions" "$last_sketch" 1 + assert_core 1 + + local globalbuildoptions_commonh_stat=$(make_commonh_stat) + local globalbuildoptions_core_stat=$(make_core_stat) + + build_esp8266_example "Blink" + + last_sketch=$(most_recent_dir "$sketches") + assert_build "Blink" "$last_sketch" 0 + assert_core 0 + + local blink_commonh_stat=$(make_commonh_stat) + local blink_core_stat=$(make_core_stat) + + test "$globalbuildoptions_commonh_stat" != "$blink_commonh_stat" + test "$globalbuildoptions_core_stat" != "$blink_core_stat" +} + +buildopt_then_nobuildopt diff --git a/tests/test_mkbuildoptglobals/nobuildopt.sh b/tests/test_mkbuildoptglobals/nobuildopt.sh new file mode 100644 index 0000000000..93fd688ea2 --- /dev/null +++ b/tests/test_mkbuildoptglobals/nobuildopt.sh @@ -0,0 +1,25 @@ +function nobuildopt() +{ + build_esp8266_example "Blink" + + local last_sketch=$(most_recent_dir "$sketches") + assert_build "Blink" "$last_sketch" 0 + assert_core 0 + + local blink_commonh_stat=$(make_commonh_stat) + local blink_core_stat=$(make_core_stat) + + build_esp8266_example "TestEspApi" + + last_sketch=$(most_recent_dir "$sketches") + assert_build "TestEspApi" "$last_sketch" 0 + assert_core 0 + + local testespapi_commonh_stat=$(make_commonh_stat) + local testespapi_core_stat=$(make_core_stat) + + test "$blink_commonh_stat" = "$testespapi_commonh_stat" + test "$blink_core_stat" = "$testespapi_core_stat" +} + +nobuildopt diff --git a/tests/test_mkbuildoptglobals/nobuildopt_then_buildopt.sh b/tests/test_mkbuildoptglobals/nobuildopt_then_buildopt.sh new file mode 100644 index 0000000000..bb75ec5022 --- /dev/null +++ b/tests/test_mkbuildoptglobals/nobuildopt_then_buildopt.sh @@ -0,0 +1,25 @@ +function nobuildopt_then_buildopt() +{ + build_esp8266_example "Blink" + + local last_sketch=$(most_recent_dir "$sketches") + assert_build "Blink" "$last_sketch" 0 + assert_core 0 + + local blink_commonh_stat=$(make_commonh_stat) + local blink_core_stat=$(make_core_stat) + + build_esp8266_example "HwdtStackDump" + + last_sketch=$(most_recent_dir "$sketches") + assert_build "HwdtStackDump" "$last_sketch" 1 + assert_core 1 + + local hwdtstackdump_commonh_stat=$(make_commonh_stat) + local hwdtstackdump_core_stat=$(make_core_stat) + + test "$hwdtstackdump_commonh_stat" != "$blink_commonh_stat" + test "$hwdtstackdump_core_stat" != "$blink_core_stat" +} + +nobuildopt_then_buildopt diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py old mode 100644 new mode 100755 index 62a3373aee..4b3e3854a3 --- a/tools/mkbuildoptglobals.py +++ b/tools/mkbuildoptglobals.py @@ -1,14 +1,9 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# This script manages the use of a file with a unique name, like -# `Sketch.ino.globals.h`, in the Sketch source directory to provide compiler -# command-line options (build options) and sketch global macros. The build -# option data is encapsulated in a unique "C" comment block and extracted into -# the build tree during prebuild. -# # Copyright (C) 2022 - M Hightower # +# Updates & fixes for arduino-cli environment by Maxim Prokhorov +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -20,809 +15,730 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# This script manages the use of a file with a unique name, like +# `SKETCH.ino.globals.h`, in the SKETCH source directory to provide compiler +# command-line options (build options), sketch global macros, or inject code. +# The buildo ption data is encapsulated in an unique "C" comment block /* ... */ +# and extracted into the build directory during prebuild. # # A Tip of the hat to: # # This PR continues the effort to get some form of global build support -# presented by brainelectronics' PR https://github.com/esp8266/Arduino/pull/8095 +# presented by brainelectronics' PR +# - https://github.com/esp8266/Arduino/pull/8095 # # Used d-a-v's global name suggestion from arduino PR -# https://github.com/arduino/arduino-cli/pull/1524 -# -""" -Operation - -"Sketch.ino.globals.h" - A global h file in the Source Sketch directory. The -string Sketch.ino is the actual name of the sketch program. A matching copy is -kept in the build path/core directory. The file is empty when it does not exist -in the source directory. - -Using Sketch.ino.globals.h as a container to hold build.opt, gives implicit -dependency tracking for build.opt by way of Sketch.ino.globals.h's -dependencies. -Example: - gcc ... @{build.path}/core/build.opt -include "{build.path}/core/{build.project_name}.globals.h" ... - -In this implementation the '-include "{build.path}/core/{build.project_name}.globals.h"' -component is added to the build.opt file. - gcc ... @{build.path}/core/build.opt ... - -At each build cycle, "{build.project_name}.globals.h" is conditoinally copied to -"{build.path}/core/" at prebuild, and build.opt is extraction as needed. The -Sketch.ino.globals.h's dependencies will trigger "rebuild all" as needed. - -If Sketch.ino.globals.h is not in the source sketch folder, an empty -versions is created in the build tree. The file build.opt always contains a -"-include ..." entry so that file dependencies are generated for -Sketch.ino.globals.h. This allows for change detection when the file is -added. -""" +# - https://github.com/arduino/arduino-cli/pull/1524 -""" -Arduino `preferences.txt` changes +import argparse +import locale +import logging +import io +import re +import os +import dataclasses +import pathlib +import sys +import textwrap +import time -"Aggressively cache compiled core" ideally should be turned off; however, -a workaround has been implimented. -In ~/.arduino15/preferences.txt, to disable the feature: - compiler.cache_core=false +from typing import Optional, TextIO, Union, List, Tuple -Reference: -https://forum.arduino.cc/t/no-aggressively-cache-compiled-core-in-ide-1-8-15/878954/2 -""" +from shutil import copystat -""" -# Updates or Additions for platform.txt or platform.local.txt -runtime.tools.mkbuildoptglobals={runtime.platform.path}/tools/mkbuildoptglobals.py +# Stay in sync with our bundled version +VERSION_MIN = (3, 7) -# Fully qualified file names for processing sketch global options -globals.h.source.fqfn={build.source.path}/{build.project_name}.globals.h -commonhfile.fqfn={build.core.path}/CommonHFile.h -build.opt.fqfn={build.path}/core/build.opt -mkbuildoptglobals.extra_flags= +if sys.version_info < VERSION_MIN: + raise SystemExit( + f"{__file__}\nMinimal supported version of Python is {VERSION_MIN[0]}.{VERSION_MIN[1]}" + ) -recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" {runtime.ide.version} "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" {mkbuildoptglobals.extra_flags} -compiler.cpreprocessor.flags=-D__ets__ -DICACHE_FLASH -U__STRICT_ANSI__ -D_GNU_SOURCE -DESP8266 @{build.opt.path} "-I{compiler.sdk.path}/include" "-I{compiler.sdk.path}/{build.lwip_include}" "-I{compiler.libc.path}/include" "-I{build.path}/core" -""" +# Like existing documentation methods, signature is embedded in the comment block +# Unlike existing documentation methods, only the first line contains any metadata +# Command-line options file, to be sourced by the compiler +BUILD_OPT_SIGNATURE_RE = re.compile(r"/[\*]@\s*?create-file:(?P\S*?)\s*?@$") + +# Script documentation & examples +DOCS_URL = ( + "https://arduino-esp8266.readthedocs.io/en/latest/faq/a06-global-build-options.html" +) +DOCS_EPILOG = f""" +Use platform.local.txt 'mkbuildoptglobals.extra_flags=...' to supply extra command line flags. +See {DOCS_URL} for more information. """ -A Sketch.ino.globals.h file with embedded build.opt might look like this - -/*@create-file:build.opt@ -// An embedded build.opt file using a "C" block comment. The starting signature -// must be on a line by itself. The closing block comment pattern should be on a -// line by itself. Each line within the block comment will be space trimmed and -// written to build.opt, skipping blank lines and lines starting with '//', '*' -// or '#'. - --DMYDEFINE="\"Chimichangas do not exist\"" --O3 --fanalyzer --DUMM_STATS=2 -*/ - -#ifndef SKETCH_INO_GLOBALS_H -#define SKETCH_INO_GLOBALS_H - -#if defined(__cplusplus) -// Defines kept private to .cpp modules -//#pragma message("__cplusplus has been seen") -#endif - -#if !defined(__cplusplus) && !defined(__ASSEMBLER__) -// Defines kept private to .c modules -#endif - -#if defined(__ASSEMBLER__) -// Defines kept private to assembler modules -#endif - -#endif -""" + +# Notify that custom build options exist for the other core, but not this one +OTHER_BUILD_OPTIONS = [ + "build_opt.h", + "file_opt", +] + + +def other_build_options(p: pathlib.Path, sketch_header: pathlib.Path): + return f"""Build options file '{p.name}' is not supported.") +Embed build options and code in '{sketch_header.name}' instead. +Create an empty '{sketch_header.name}' to silence this warning. + + See {DOCS_URL} for more information. """ -Added 2) and 5) to docs -Caveats, Observations, and Ramblings -1) Edits to platform.txt or platform.local.txt force a complete rebuild that -removes the core folder. Not a problem, just something to be aware of when -debugging this script. Similarly, changes on the IDE Tools selection cause a -complete rebuild. +def check_other_build_options(sketch_header: pathlib.Path) -> Optional[str]: + if sketch_header.exists(): + return None -In contrast, the core directory is not deleted when the rebuild occurs from -changing a file with an established dependency. + for name in OTHER_BUILD_OPTIONS: + p = sketch_header.parent / name + if p.exists(): + return other_build_options(p, sketch_header) -2) Renaming files does not change the last modified timestamp, possibly causing -issues when replacing files by renaming and rebuilding. + return None -A good example of this problem is when you correct the spelling of file -Sketch.ino.globals.h. You need to touch (update time stampt) the file so a -rebuild all is performed. -3) During the build two identical copies of Sketch.ino.globals.h will exist. -#ifndef fencing will be needed for non comment blocks in Sketch.ino.globals.h. +# Retrieve *system* encoding, not the one used by python internally +# +# Given that GCC will handle lines from an @file as if they were on +# the command line. I assume that the contents of @file need to be encoded +# to match that of the shell running GCC runs. I am not 100% sure this API +# gives me that, but it appears to work. +# +# However, elsewhere when dealing with source code we continue to use 'utf-8', +# ref. https://gcc.gnu.org/onlinedocs/cpp/Character-sets.html +if sys.version_info >= (3, 11): + DEFAULT_ENCODING = locale.getencoding() +else: + DEFAULT_ENCODING = locale.getdefaultlocale()[1] -4) By using a .h file to encapsulate "build.opt" options, the information is not -lost after a save-as. Before with an individual "build.opt" file, the file was -missing in the saved copy. +FILE_ENCODING = "utf-8" -5) When a .h file is renamed, a copy of the old file remains in the build -sketch folder. This can create confusion if you missed an edit in updating an -include in one or more of your modules. That module will continue to use the -stale version of the .h, until you restart the IDE or other major changes that -would cause the IDE to delete and recopy the contents from the source sketch. -This may be the culprit for "What! It built fine last night!" +# Issues trying to address through logging module & buffered printing +# 1. Arduino IDE 1.x / 2.0 print stderr with color red, allowing any +# messages to stand out in an otherwise white on black console output. +# 2. With Arduino IDE preferences set for "no verbose output", you only see +# stderr messages. Prior related prints are missing. +# 3. logging ensures that stdout & stderr buffers are flushed before exiting. +# While it may not provide consistent output, no messages should be lost. +class LoggingFilter(logging.Filter): + def __init__(self, *filter_only): + self._filter_only = filter_only -6a) In The case of two Arduino IDE screens up with different programs, they can -share the same core archive file. Defines on one screen will change the core -archive, and a build on the 2nd screen will build with those changes. -The 2nd build will have the core built for the 1st screen. It gets uglier. With -the 2nd program, the newly built modules used headers processed with different -defines than the core. + def filter(self, rec): + return rec.levelno in self._filter_only -6b) Problem: Once core has been build, changes to build.opt or globals.h will -not cause the core archive to be rebuild. You either have to change tool -settings or close and reopen the Arduino IDE. This is a variation on 6a) above. -I thought this was working for the single sketch case, but it does not! :( -That is because sometimes it does build properly. What is unknown are the -causes that will make it work and fail? - * Fresh single Arduino IDE Window, open with file to build - works -I think these, 6a and 6b, are resolved by setting `compiler.cache_core=false` -in ~/.arduino15/preferences.txt, to disable the aggressive caching feature: - https://forum.arduino.cc/t/no-aggressively-cache-compiled-core-in-ide-1-8-15/878954/2 +# Since default handler is not created, make sure only specific levels go through to stderr +TO_STDERR = logging.StreamHandler(sys.stderr) +TO_STDERR.setFormatter( + logging.Formatter("*** %(filename)s %(funcName)s:%(lineno)d ***\n%(message)s\n") +) +TO_STDERR.setLevel(logging.NOTSET) +TO_STDERR.addFilter( + LoggingFilter( + logging.CRITICAL, + logging.DEBUG, + logging.ERROR, + logging.FATAL, + logging.WARNING, + ) +) -Added workaround for `compiler.cache_core=true` case. -See `if use_aggressive_caching_workaround:` in main(). +# Generic info messages should be on stdout (but, note the above, these are hidden by IDE defaults) +TO_STDOUT = logging.StreamHandler(sys.stdout) +TO_STDOUT.setFormatter(logging.Formatter("%(message)s")) +TO_STDOUT.setLevel(logging.INFO) +TO_STDOUT.addFilter( + LoggingFilter( + logging.INFO, + logging.NOTSET, + ) +) -7) Suspected but not confirmed. A quick edit and rebuild don't always work well. -Build does not work as expected. This does not fail often. Maybe PIC NIC. -""" +logging.basicConfig(level=logging.INFO, handlers=(TO_STDOUT, TO_STDERR)) -import argparse -import glob -import locale -import os -import platform -import sys -import textwrap -import time -import traceback -from shutil import copyfile +class ParsingException(Exception): + def __init__(self, file: Optional[str], lineno: int, line: str): + self.file = file + self.lineno = lineno + self.line = line + def __str__(self): + out = "" -# Stay in sync with our bundled version -PYTHON_REQUIRES = (3, 7) + if self.file: + out += f"in {self.file}" -if sys.version_info < PYTHON_REQUIRES: - raise SystemExit(f"{__file__}\nMinimal supported version of Python is {PYTHON_REQUIRES[0]}.{PYTHON_REQUIRES[1]}") + lineno = f" {self.lineno}" + out += f"\n\n{lineno} {self.line}" + out += f'\n{" " * len(lineno)} ' + out += "^" * len(self.line.strip()) -# Need to work on signature line used for match to avoid conflicts with -# existing embedded documentation methods. -build_opt_signature = "/*@create-file:build.opt@" + return out -docs_url = "https://arduino-esp8266.readthedocs.io/en/latest/faq/a06-global-build-options.html" +class InvalidSignature(ParsingException): + pass -err_print_flag = False -msg_print_buf = "" -debug_enabled = False -default_encoding = None -# Issues trying to address through buffered printing -# 1. Arduino IDE 2.0 RC5 does not show stderr text in color. Text printed does -# not stand out from stdout messages. -# 2. Separate pipes, buffering, and multiple threads with output can create -# mixed-up messages. "flush" helped but did not resolve. The Arduino IDE 2.0 -# somehow makes the problem worse. -# 3. With Arduino IDE preferences set for "no verbose output", you only see -# stderr messages. Prior related prints are missing. -# -# Locally buffer and merge both stdout and stderr prints. This allows us to -# print a complete context when there is an error. When any buffered prints -# are targeted to stderr, print the whole buffer to stderr. - -def print_msg(*args, **kwargs): - global msg_print_buf - if 'sep' in kwargs: - sep = kwargs['sep'] - else: - sep = ' ' - - msg_print_buf += args[0] - for arg in args[1:]: - msg_print_buf += sep - msg_print_buf += arg - - if 'end' in kwargs: - msg_print_buf += kwargs['end'] - else: - msg_print_buf += '\n' - - -# Bring attention to errors with a blank line and lines starting with "*** ". -def print_err(*args, **kwargs): - global err_print_flag - if (args[0])[0] != ' ': - print_msg("") - print_msg("***", *args, **kwargs) - err_print_flag = True - -def print_dbg(*args, **kwargs): - global debug_enabled - global err_print_flag - if debug_enabled: - print_msg("DEBUG:", *args, **kwargs) - err_print_flag = True - - -def handle_error(err_no): - # on err_no 0, commit print buffer to stderr or stdout - # on err_no != 0, commit print buffer to stderr and sys exist with err_no - global msg_print_buf - global err_print_flag - if len(msg_print_buf): - if err_no or err_print_flag: - fd = sys.stderr - else: - fd = sys.stdout - print(msg_print_buf, file=fd, end='', flush=True) - msg_print_buf = "" - err_print_flag = False - if err_no: - sys.exit(err_no) +class InvalidSyntax(ParsingException): + pass -def copy_create_build_file(source_fqfn, build_target_fqfn): - """ - Conditionally copy a newer file between the source directory and the build - directory. When source file is missing, create an empty file in the build - directory. - return True when file change detected. - """ - if os.path.exists(source_fqfn): - if os.path.exists(build_target_fqfn) and \ - os.path.getmtime(build_target_fqfn) >= os.path.getmtime(source_fqfn): - # only copy newer files - do nothing, all is good - print_dbg(f"up to date os.path.exists({source_fqfn}) ") - return False - else: - # The new copy gets stamped with the current time, just as other - # files copied by `arduino-builder`. - copyfile(source_fqfn, build_target_fqfn) - print_dbg(f"copyfile({source_fqfn}, {build_target_fqfn})") - else: - if os.path.exists(build_target_fqfn) and \ - os.path.getsize(build_target_fqfn) == 0: - return False - else: - # Place holder - Must have an empty file to satisfy parameter list - # specifications in platform.txt. - with open(build_target_fqfn, 'w', encoding="utf-8"): - pass - return True # file changed - -def add_include_line(build_opt_fqfn, include_fqfn): - global default_encoding - if not os.path.exists(include_fqfn): - # If file is missing, we need an place holder - with open(include_fqfn, 'w', encoding=default_encoding): - pass - print_msg("add_include_line: Created " + include_fqfn) - - with open(build_opt_fqfn, 'a', encoding=default_encoding) as build_opt: - build_opt.write('-include "' + include_fqfn.replace('\\', '\\\\') + '"\n') - -def extract_create_build_opt_file(globals_h_fqfn, file_name, build_opt_fqfn): +def extract_build_opt(name: str, dst: TextIO, src: TextIO): """ - Extract the embedded build.opt from Sketch.ino.globals.h into build - path/core/build.opt. The subdirectory path must already exist as well as the - copy of Sketch.ino.globals.h. + Read src line by line and extract matching 'create-file' directives. + 'name' can match multiple times + + Empty 'src' is always valid. + Never matching anything for 'name' is also valid. + + Incorrectly written signatures always fail with an exception. + + C/C++ syntax validity isn't checked, that's up to the user """ - global build_opt_signature - global default_encoding - - build_opt = open(build_opt_fqfn, 'w', encoding=default_encoding) - if not os.path.exists(globals_h_fqfn) or (0 == os.path.getsize(globals_h_fqfn)): - build_opt.close() - return False - - complete_comment = False - build_opt_error = False - line_no = 0 - # If the source sketch did not have the file Sketch.ino.globals.h, an empty - # file was created in the ./core/ folder. - # By using the copy, open will always succeed. - with open(globals_h_fqfn, 'r', encoding="utf-8") as src: - for line in src: - line = line.strip() - line_no += 1 - if line == build_opt_signature: - if complete_comment: - build_opt_error = True - print_err(" Multiple embedded build.opt blocks in", f'{file_name}:{line_no}') - continue - print_msg("Extracting embedded compiler command-line options from", f'{file_name}:{line_no}') - for line in src: - line = line.strip() - line_no += 1 - if 0 == len(line): - continue - if line.startswith("*/"): - complete_comment = True - break - elif line.startswith("*"): # these are so common - skip these should they occur - continue - elif line.startswith("#"): # allow some embedded comments - continue - elif line.startswith("//"): - continue - # some consistency checking before writing - give some hints about what is wrong - elif line == build_opt_signature: - print_err(" Double begin before end for embedded build.opt block in", f'{file_name}:{line_no}') - build_opt_error = True - elif line.startswith(build_opt_signature): - print_err(" build.opt signature block ignored, trailing character for embedded build.opt block in", f'{file_name}:{line_no}') - build_opt_error = True - elif "/*" in line or "*/" in line : - print_err(" Nesting issue for embedded build.opt block in", f'{file_name}:{line_no}') - build_opt_error = True - else: - print_msg(" ", f'{line_no:2}, Add command-line option: {line}', sep='') - build_opt.write(line + "\n") - elif line.startswith(build_opt_signature): - print_err(" build.opt signature block ignored, trailing character for embedded build.opt block in", f'{file_name}:{line_no}') - build_opt_error = True - if not complete_comment or build_opt_error: - build_opt.truncate(0) - build_opt.close() - if build_opt_error: - # this will help the script start over when the issue is fixed - os.remove(globals_h_fqfn) - print_err(" Extraction failed") - # Don't let the failure get hidden by a spew of nonsensical error - # messages that will follow. Bring things to a halt. - handle_error(1) - return False # not reached - elif complete_comment: - print_msg(" Created compiler command-line options file " + build_opt_fqfn) - build_opt.close() - return complete_comment - - -def enable_override(enable, commonhfile_fqfn): - # Reduce disk IO writes - if os.path.exists(commonhfile_fqfn): - if os.path.getsize(commonhfile_fqfn): # workaround active - if enable: - return - elif not enable: - return - with open(commonhfile_fqfn, 'w', encoding="utf-8") as file: - if enable: - file.write("//Override aggressive caching\n") - # enable workaround when getsize(commonhfile_fqfn) is non-zero, disabled when zero - - -def discover_1st_time_run(build_path): - # Need to know if this is the 1ST compile of the Arduino IDE starting. - # Use empty cache directory as an indicator for 1ST compile. - # Arduino IDE 2.0 RC5 does not cleanup on exist like 1.6.19. Probably for - # debugging like the irregular version number 10607. For RC5 this indicator - # will be true after a reboot instead of a 1ST compile of the IDE starting. - # Another issue for this technique, Windows does not clear the Temp directory. :( - tmp_path, build = os.path.split(build_path) - ide_2_0 = 'arduino-sketch-' - if ide_2_0 == build[:len(ide_2_0)]: - search_path = os.path.join(tmp_path, 'arduino-core-cache/*') # Arduino IDE 2.0 - else: - search_path = os.path.join(tmp_path, 'arduino_cache_*/*') # Arduino IDE 1.6.x and up - - count = 0 - for dirname in glob.glob(search_path): - count += 1 - return 0 == count - - -def get_preferences_txt(file_fqfn, key): - # Get Key Value, key is allowed to be missing. - # We assume file file_fqfn exists - basename = os.path.basename(file_fqfn) - with open(file_fqfn, encoding="utf-8") as file: - for line in file: - name, value = line.partition("=")[::2] - if name.strip().lower() == key: - val = value.strip().lower() - if val != 'true': - val = False - print_msg(f" {basename}: {key}={val}") - return val - print_err(f" Key '{key}' not found in file {basename}. Default to true.") - return True # If we don't find it just assume it is set True - - -def check_preferences_txt(runtime_ide_path, preferences_file): - key = "compiler.cache_core" - # return the state of "compiler.cache_core" found in preferences.txt - if preferences_file != None: - if os.path.exists(preferences_file): - print_msg(f"Using preferences from '{preferences_file}'") - return get_preferences_txt(preferences_file, key) - else: - print_err(f"Override preferences file '{preferences_file}' not found.") - - # Referencing the preferences.txt for an indication of shared "core.a" - # caching is unreliable. There are too many places reference.txt can be - # stored and no hints of which the Arduino build might be using. Unless - # directed otherwise, assume "core.a" caching true. - print_msg(f"Assume aggressive 'core.a' caching enabled.") - return True - -def touch(fname, times=None): - with open(fname, "ab") as file: - file.close(); - os.utime(fname, times) - -def synchronous_touch(globals_h_fqfn, commonhfile_fqfn): - global debug_enabled - # touch both files with the same timestamp - touch(globals_h_fqfn) - with open(globals_h_fqfn, "rb") as file: - file.close() - with open(commonhfile_fqfn, "ab") as file2: - file2.close() - ts = os.stat(globals_h_fqfn) - os.utime(commonhfile_fqfn, ns=(ts.st_atime_ns, ts.st_mtime_ns)) - - if debug_enabled: - print_dbg("After synchronous_touch") - ts = os.stat(globals_h_fqfn) - print_dbg(f" globals_h_fqfn ns_stamp = {ts.st_mtime_ns}") - print_dbg(f" getmtime(globals_h_fqfn) {os.path.getmtime(globals_h_fqfn)}") - ts = os.stat(commonhfile_fqfn) - print_dbg(f" commonhfile_fqfn ns_stamp = {ts.st_mtime_ns}") - print_dbg(f" getmtime(commonhfile_fqfn) {os.path.getmtime(commonhfile_fqfn)}") - -def determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn): - global docs_url - print_dbg(f"runtime_ide_version: {args.runtime_ide_version}") - - if args.cache_core != None: - print_msg(f"Preferences override, this prebuild script assumes the 'compiler.cache_core' parameter is set to {args.cache_core}") - print_msg(f"To change, modify 'mkbuildoptglobals.extra_flags=(--cache_core | --no_cache_core)' in 'platform.local.txt'") - return args.cache_core - else: - ide_path = None - preferences_fqfn = None - if args.preferences_sketch != None: - preferences_fqfn = os.path.join( - os.path.dirname(source_globals_h_fqfn), - os.path.normpath(args.preferences_sketch)) - else: - if args.preferences_file != None: - preferences_fqfn = args.preferences_file - elif args.preferences_env != None: - preferences_fqfn = args.preferences_env + IN_RAW = 1 + IN_BUILD_OPT = 2 + IN_SKIP_OPT = 3 + + state = IN_RAW + block = [] # type: List[str] + + for n, raw_line in enumerate(src, start=1): + line = raw_line.strip().rstrip() + + if state == IN_SKIP_OPT: + if line.startswith("*/"): + state = IN_RAW + for line in block: + dst.write(line) + block = [] + continue + + if line.startswith("/*@"): + if not line.endswith("@"): + raise InvalidSyntax(None, n, raw_line) + + result = BUILD_OPT_SIGNATURE_RE.search(line) + if not result or state in (IN_BUILD_OPT, IN_SKIP_OPT): + raise InvalidSignature(None, n, raw_line) + + if name == result.group("name"): + state = IN_BUILD_OPT else: - ide_path = runtime_ide_path - - if preferences_fqfn != None: - preferences_fqfn = os.path.normpath(preferences_fqfn) - root = False - if 'Windows' == platform.system(): - if preferences_fqfn[1:2] == ':\\': - root = True - else: - if preferences_fqfn[0] == '/': - root = True - if not root: - if preferences_fqfn[0] != '~': - preferences_fqfn = os.path.join("~", preferences_fqfn) - preferences_fqfn = os.path.expanduser(preferences_fqfn) - print_dbg(f"determine_cache_state: preferences_fqfn: {preferences_fqfn}") - - return check_preferences_txt(ide_path, preferences_fqfn) + state = IN_SKIP_OPT + continue -""" -TODO + if state == IN_BUILD_OPT: + if line.startswith("*/"): + state = IN_RAW + continue + + if line.startswith(("#", "//", "*")): + continue -aggressive caching workaround -========== ======= ========== -The question needs to be asked, is it a good idea? -With all this effort to aid in determining the cache state, it is rendered -usless when arduino command line switches are used that contradict our -settings. + if not line: + continue -Sort out which of these are imperfect solutions should stay in + block.append(f"{line}\n") -Possible options for handling problems caused by: - ./arduino --preferences-file other-preferences.txt - ./arduino --pref compiler.cache_core=false + if state != IN_RAW: + raise InvalidSyntax(None, n, raw_line) ---cache_core ---no_cache_core ---preferences_file (relative to IDE or full path) ---preferences_sketch (default looks for preferences.txt or specify path relative to sketch folder) ---preferences_env, python docs say "Availability: most flavors of Unix, Windows." + for line in block: + dst.write(line) - export ARDUINO15_PREFERENCES_FILE=$(realpath other-name-than-default-preferences.txt ) - ./arduino --preferences-file other-name-than-default-preferences.txt - platform.local.txt: mkbuildoptglobals.extra_flags=--preferences_env +def extract_build_opt_from_path(dst: TextIO, name: str, p: pathlib.Path): + """ + Same as 'extract_build_opt', but use a file path + """ + try: + with p.open("r", encoding=FILE_ENCODING) as src: + extract_build_opt(name, dst, src) + except ParsingException as e: + e.file = p.name + raise e - Tested with: - export ARDUINO15_PREFERENCES_FILE=$(realpath ~/projects/arduino/arduino-1.8.19/portable/preferences.txt) - ~/projects/arduino/arduino-1.8.18/arduino +def is_future_utime(p: pathlib.Path): + return time.time_ns() < p.stat().st_mtime_ns - Future Issues - * "--preferences-file" does not work for Arduino IDE 2.0, they plan to address at a future release - * Arduino IDE 2.0 does not support portable, they plan to address at a future release -""" +def as_stat_result(p: Union[os.stat_result, pathlib.Path]) -> Optional[os.stat_result]: + if not isinstance(p, os.stat_result): + if not p.exists(): + return None + return p.stat() + + return p + + +def is_different_utime( + p1: Union[os.stat_result, pathlib.Path], + p2: Union[os.stat_result, pathlib.Path], +) -> bool: + s1 = as_stat_result(p1) + if not s1: + return True + s2 = as_stat_result(p2) + if not s2: + return True -def check_env(env): - system = platform.system() - # From the docs: - # Availability: most flavors of Unix, Windows. - # “Availability: Unix” are supported on macOS - # Because of the soft commitment, I used "help=argparse.SUPPRESS" to keep - # the claim out of the help. The unavailable case is untested. - val = os.getenv(env) - if val == None: - if "Linux" == system or "Windows" == system: - raise argparse.ArgumentTypeError(f'Missing environment variable: {env}') + for attr in ("st_atime_ns", "st_mtime_ns"): + if getattr(s1, attr) != getattr(s2, attr): + return True + + return False + + +# Arduino IDE uses timestamps to determine whether the .o file should be rebuilt +def synchronize_utime(stat: Union[os.stat_result, pathlib.Path], *rest: pathlib.Path): + """ + Retrieve stats from the first 'file' and apply to the 'rest' + """ + if not isinstance(stat, os.stat_result): + stat = stat.stat() + logging.debug( + "setting mtime=%d for:\n%s", stat.st_mtime, "\n".join(f" {p}" for p in rest) + ) + for p in rest: + if is_different_utime(stat, p.stat()): + os.utime(p, ns=(stat.st_atime_ns, stat.st_mtime_ns)) + + +def as_include_line(p: pathlib.Path) -> str: + out = p.absolute().as_posix() + out = out.replace('"', '\\"') + out = f'-include "{out}"' + return out + + +def as_path_field(help: str): + return dataclasses.field( + default_factory=pathlib.Path, + metadata={ + "help": help, + }, + ) + + +@dataclasses.dataclass +class Context: + build_opt: pathlib.Path = as_path_field( + "resulting options file, used in the gcc command line options" + ) + + source_sketch_header: pathlib.Path = as_path_field( + ".globals.h located in the sketch directory" + ) + build_sketch_header: pathlib.Path = as_path_field( + ".globals.h located in the build directory" + ) + + common_header: pathlib.Path = as_path_field( + "dependency file, copied into the core directory to trigger rebuilds" + ) + + +# build process requires some file to always exist, even when they are empty and thus unused +def ensure_exists_and_empty(*paths: pathlib.Path): + """ + Create empty or replace existing files with an empty ones at 'paths' + """ + for p in paths: + if not p.parent.exists(): + p.parent.mkdir(parents=True, exist_ok=True) + + if not p.exists() or (p.exists() and p.stat().st_size): + p.write_bytes(b"") + logging.debug("%s is a placeholder", p.name) else: - # OS/Library limitation - raise argparse.ArgumentTypeError('Not supported') - return val + logging.debug("%s is up-to-date", p.name) -def parse_args(): - extra_txt = '''\ - Use platform.local.txt 'mkbuildoptglobals.extra_flags=...' to supply override options: - --cache_core | --no_cache_core | --preferences_file PREFERENCES_FILE | ... +def ensure_normal_time(*paths: pathlib.Path): + for p in paths: + if p.exists() and is_future_utime(p): + logging.debug("%s has timestamp in the future, fixing", p.name) + p.touch() - more help at {} - '''.format(docs_url) - parser = argparse.ArgumentParser( - description='Prebuild processing for globals.h and build.opt file', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=textwrap.dedent(extra_txt)) - parser.add_argument('runtime_ide_path', help='Runtime IDE path, {runtime.ide.path}') - parser.add_argument('runtime_ide_version', type=int, help='Runtime IDE Version, {runtime.ide.version}') - parser.add_argument('build_path', help='Build path, {build.path}') - parser.add_argument('build_opt_fqfn', help="Build FQFN to build.opt") - parser.add_argument('source_globals_h_fqfn', help="Source FQFN Sketch.ino.globals.h") - parser.add_argument('commonhfile_fqfn', help="Core Source FQFN CommonHFile.h") - parser.add_argument('--debug', action='store_true', required=False, default=False) - parser.add_argument('-DDEBUG_ESP_PORT', nargs='?', action='store', const="", default="", help='Add mkbuildoptglobals.extra_flags={build.debug_port} to platform.local.txt') - parser.add_argument('--ci', action='store_true', required=False, default=False) - group = parser.add_mutually_exclusive_group(required=False) - group.add_argument('--cache_core', action='store_true', default=None, help='Assume a "compiler.cache_core" value of true') - group.add_argument('--no_cache_core', dest='cache_core', action='store_false', help='Assume a "compiler.cache_core" value of false') - group.add_argument('--preferences_file', help='Full path to preferences file') - group.add_argument('--preferences_sketch', nargs='?', action='store', const="preferences.txt", help='Sketch relative path to preferences file') - # Since the docs say most versions of Windows and Linux support the os.getenv method, suppress the help message. - group.add_argument('--preferences_env', nargs='?', action='store', type=check_env, const="ARDUINO15_PREFERENCES_FILE", help=argparse.SUPPRESS) - # ..., help='Use environment variable for path to preferences file') - return parser.parse_args() - # ref epilog, https://stackoverflow.com/a/50021771 - # ref nargs='*'', https://stackoverflow.com/a/4480202 - # ref no '--n' parameter, https://stackoverflow.com/a/21998252 - - -# retrieve *system* encoding, not the one used by python internally -if sys.version_info >= (3, 11): - def get_encoding(): - return locale.getencoding() -else: - def get_encoding(): - return locale.getdefaultlocale()[1] +# ref. https://gcc.gnu.org/onlinedocs/cpp/Line-Control.html +# arduino builder not just copies sketch files to the build directory, +# but also injects this line at the top to remember the source +def as_arduino_sketch_quoted_header(p: pathlib.Path): + out = str(p.absolute()) + out = out.replace("\\", "\\\\") + out = out.replace('"', '\\"') + return f'#line 1 "{out}"\n' -def show_value(desc, value): - print_dbg(f'{desc:<40} {value}') - return -def locale_dbg(): - show_value("get_encoding()", get_encoding()) - show_value("locale.getdefaultlocale()", locale.getdefaultlocale()) - show_value('sys.getfilesystemencoding()', sys.getfilesystemencoding()) - show_value("sys.getdefaultencoding()", sys.getdefaultencoding()) - show_value("locale.getpreferredencoding(False)", locale.getpreferredencoding(False)) +def write_or_replace(p: pathlib.Path, contents: str, encoding=FILE_ENCODING) -> bool: + actual = "" + try: - show_value("locale.getpreferredencoding()", locale.getpreferredencoding()) - except: - pass - show_value("sys.stdout.encoding", sys.stdout.encoding) + if p.exists(): + actual = p.read_text(encoding=encoding) + except UnicodeDecodeError: + logging.warning("cannot decode %s", p.name) + + if contents != actual: + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(contents, encoding=encoding) + logging.debug("%s contents written", p.name) + return True + + logging.debug("%s is up-to-date", p.name) + return False + + +# arduino builder would copy the file regardless +# prebuild stage has it missing though +def ensure_build_sketch_header_written(ctx: Context): + """ + Sketch header copy or placeholder must always exist in the build directory (even when unused) + """ + if not ctx.source_sketch_header.exists(): + ensure_exists_and_empty(ctx.build_sketch_header) + return + + p = ctx.build_sketch_header + if not p.parent.exists(): + p.parent.mkdir(parents=True, exist_ok=True) + + contents = ctx.source_sketch_header.read_text(encoding=FILE_ENCODING) + contents = "".join( + ( + as_arduino_sketch_quoted_header(ctx.source_sketch_header), + contents, + "\n", + ) + ) + + if write_or_replace(p, contents): + copystat(ctx.source_sketch_header, p) + + +def ensure_common_header_bound(ctx: Context): + """ + Record currently used command-line options file + """ + write_or_replace(ctx.common_header, as_arduino_sketch_quoted_header(ctx.build_opt)) + + +def make_build_opt_name(ctx: Context, debug: bool) -> str: + name = ctx.build_opt.name + if debug: + name = f"{name}:debug" + + return name + + +def ensure_build_opt_written(ctx: Context, buffer: io.StringIO): + """ + Make sure that 'build_opt' is written to the filesystem. + + '-include ...' lines are always appended at the end of the buffer. + 'build_opt' is not written when its contents remain unchanged. + """ + includes = [ + ctx.common_header, + ] + + if len(buffer.getvalue()): + includes.append(ctx.build_sketch_header) + + for p in includes: + buffer.write(f"{as_include_line(p)}\n") + + write_or_replace(ctx.build_opt, buffer.getvalue(), encoding=DEFAULT_ENCODING) + + +def maybe_empty_or_missing(p: pathlib.Path): + return not p.exists() or not p.stat().st_size + + +def build_with_minimal_build_opt(ctx: Context): + """ + When sketch header is empty or there were no opt files created + """ + logging.debug("building with a minimal %s", ctx.build_opt.name) + + ensure_build_opt_written(ctx, buffer=io.StringIO()) + + ensure_exists_and_empty(ctx.common_header) + ensure_build_sketch_header_written(ctx) + + # sketch directory time ignored, stats are from the only persistent file + synchronize_utime( + ctx.common_header, + ctx.build_opt, + ctx.build_sketch_header, + ) + + +# Before synchronizing targets, find out which file was modified last +# by default, check 'st_mtime_ns' attribute of os.stat_result +def most_recent( + *paths: pathlib.Path, attr="st_mtime_ns" +) -> Tuple[pathlib.Path, os.stat_result]: + def make_pair(p: pathlib.Path): + return (p, p.stat()) + + def key(pair: Tuple[pathlib.Path, os.stat_result]) -> int: + return getattr(pair[1], attr) + + if not paths: + raise ValueError('"paths" cannot be empty') + elif len(paths) == 1: + return make_pair(paths[0]) + + return max((make_pair(p) for p in paths), key=key) + + +def main_build(args: argparse.Namespace): + ctx = Context( + build_opt=args.build_opt, + source_sketch_header=args.source_sketch_header, + build_sketch_header=args.build_sketch_header, + common_header=args.common_header, + ) + + if args.debug: + logging.debug( + "Build Context:\n%s", + "".join( + f' "{field.name}" at {getattr(ctx, field.name)} - {field.metadata["help"]}\n' + for field in dataclasses.fields(ctx) + ), + ) + + # notify when other files similar to .globals.h are in the sketch directory + other_build_options = check_other_build_options(ctx.source_sketch_header) + if other_build_options: + logging.warning(other_build_options) + + # future timestamps generally break build order. + # before comparing or synchronizing time, make sure it is current + ensure_normal_time(*dataclasses.astuple(ctx)) + + # when .globals.h is missing, provide placeholder files for the build and exit + if maybe_empty_or_missing(ctx.source_sketch_header): + build_with_minimal_build_opt(ctx) + return + + # when debug port is used, allow for a different set of command line options + build_debug = args.build_debug or "DEBUG_SERIAL_PORT" in ( + args.D or [] + ) # type: bool + name = make_build_opt_name(ctx, build_debug) + + # options file is not written immediately, buffer its contents before commiting + build_opt_buffer = io.StringIO() - # use current setting - show_value("locale.setlocale(locale.LC_ALL, None)", locale.setlocale(locale.LC_ALL, None)) try: - show_value("locale.getencoding()", locale.getencoding()) - except: - pass - show_value("locale.getlocale()", locale.getlocale()) - - # use user setting - show_value("locale.setlocale(locale.LC_ALL, '')", locale.setlocale(locale.LC_ALL, '')) - # show_value("locale.getencoding()", locale.getencoding()) - show_value("locale.getlocale()", locale.getlocale()) - return - - -def main(): - global build_opt_signature - global docs_url - global debug_enabled - global default_encoding - num_include_lines = 1 - - # Given that GCC will handle lines from an @file as if they were on - # the command line. I assume that the contents of @file need to be encoded - # to match that of the shell running GCC runs. I am not 100% sure this API - # gives me that, but it appears to work. - # - # However, elsewhere when dealing with source code we continue to use 'utf-8', - # ref. https://gcc.gnu.org/onlinedocs/cpp/Character-sets.html - default_encoding = get_encoding() - - args = parse_args() - debug_enabled = args.debug - runtime_ide_path = os.path.normpath(args.runtime_ide_path) - build_path = os.path.normpath(args.build_path) - build_opt_fqfn = os.path.normpath(args.build_opt_fqfn) - source_globals_h_fqfn = os.path.normpath(args.source_globals_h_fqfn) - commonhfile_fqfn = os.path.normpath(args.commonhfile_fqfn) - - globals_name = os.path.basename(source_globals_h_fqfn) - build_path_core, build_opt_name = os.path.split(build_opt_fqfn) - globals_h_fqfn = os.path.join(build_path_core, globals_name) - - if debug_enabled: - locale_dbg() - - print_msg(f'default_encoding: {default_encoding}') - - print_dbg(f"runtime_ide_path: {runtime_ide_path}") - print_dbg(f"runtime_ide_version: {args.runtime_ide_version}") - print_dbg(f"build_path: {build_path}") - print_dbg(f"build_opt_fqfn: {build_opt_fqfn}") - print_dbg(f"source_globals_h_fqfn: {source_globals_h_fqfn}") - print_dbg(f"commonhfile_fqfn: {commonhfile_fqfn}") - print_dbg(f"globals_name: {globals_name}") - print_dbg(f"build_path_core: {build_path_core}") - print_dbg(f"globals_h_fqfn: {globals_h_fqfn}") - print_dbg(f"DDEBUG_ESP_PORT: {args.DDEBUG_ESP_PORT}") - - if len(args.DDEBUG_ESP_PORT): - build_opt_signature = build_opt_signature[:-1] + ":debug@" - - print_dbg(f"build_opt_signature: {build_opt_signature}") - - if args.ci: - # Requires CommonHFile.h to never be checked in. - if os.path.exists(commonhfile_fqfn): - first_time = False - else: - first_time = True - else: - first_time = discover_1st_time_run(build_path) - if first_time: - print_dbg("First run since Arduino IDE started.") - - use_aggressive_caching_workaround = determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn) - - print_dbg(f"first_time: {first_time}") - print_dbg(f"use_aggressive_caching_workaround: {use_aggressive_caching_workaround}") - - if not os.path.exists(build_path_core): - os.makedirs(build_path_core) - print_msg("Clean build, created dir " + build_path_core) - - if first_time or \ - not use_aggressive_caching_workaround or \ - not os.path.exists(commonhfile_fqfn): - enable_override(False, commonhfile_fqfn) - - # A future timestamp on commonhfile_fqfn will cause everything to - # rebuild. This occurred during development and may happen after - # changing the system time. - if time.time_ns() < os.stat(commonhfile_fqfn).st_mtime_ns: - touch(commonhfile_fqfn) - print_err(f"Neutralized future timestamp on build file: {commonhfile_fqfn}") - - if os.path.exists(source_globals_h_fqfn): - print_msg("Using global include from " + source_globals_h_fqfn) - - copy_create_build_file(source_globals_h_fqfn, globals_h_fqfn) - - # globals_h_fqfn timestamp was only updated if the source changed. This - # controls the rebuild on change. We can always extract a new build.opt - # w/o triggering a needless rebuild. - embedded_options = extract_create_build_opt_file(globals_h_fqfn, globals_name, build_opt_fqfn) - - if use_aggressive_caching_workaround: - # commonhfile_fqfn encodes the following information - # 1. When touched, it causes a rebuild of core.a - # 2. When file size is non-zero, it indicates we are using the - # aggressive cache workaround. The workaround is set to true - # (active) when we discover a non-zero length global .h file in - # any sketch. The aggressive workaround is cleared on the 1ST - # compile by the Arduino IDE after starting. - # 3. When the timestamp matches the build copy of globals.h - # (globals_h_fqfn), we know one two things: - # * The cached core.a matches up to the current build.opt and - # globals.h. The current sketch owns the cached copy of core.a. - # * globals.h has not changed, and no need to rebuild core.a - # 4. When core.a's timestamp does not match the build copy of - # the global .h file, we only know we need to rebuild core.a, and - # that is enough. - # - # When the sketch build has a "Sketch.ino.globals.h" file in the - # build tree that exactly matches the timestamp of "CommonHFile.h" - # in the platform source tree, it owns the core.a cache copy. If - # not, or "Sketch.ino.globals.h" has changed, rebuild core. - # A non-zero file size for commonhfile_fqfn, means we have seen a - # globals.h file before and workaround is active. - if debug_enabled: - print_dbg("Timestamps at start of check aggressive caching workaround") - ts = os.stat(globals_h_fqfn) - print_dbg(f" globals_h_fqfn ns_stamp = {ts.st_mtime_ns}") - print_dbg(f" getmtime(globals_h_fqfn) {os.path.getmtime(globals_h_fqfn)}") - ts = os.stat(commonhfile_fqfn) - print_dbg(f" commonhfile_fqfn ns_stamp = {ts.st_mtime_ns}") - print_dbg(f" getmtime(commonhfile_fqfn) {os.path.getmtime(commonhfile_fqfn)}") - - if os.path.getsize(commonhfile_fqfn): - if (os.path.getmtime(globals_h_fqfn) != os.path.getmtime(commonhfile_fqfn)): - # Need to rebuild core.a - # touching commonhfile_fqfn in the source core tree will cause rebuild. - # Looks like touching or writing unrelated files in the source core tree will cause rebuild. - synchronous_touch(globals_h_fqfn, commonhfile_fqfn) - print_msg("Using 'aggressive caching' workaround, rebuild shared 'core.a' for current globals.") - else: - print_dbg(f"Using old cached 'core.a'") - elif os.path.getsize(globals_h_fqfn): - enable_override(True, commonhfile_fqfn) - synchronous_touch(globals_h_fqfn, commonhfile_fqfn) - print_msg("Using 'aggressive caching' workaround, rebuild shared 'core.a' for current globals.") - else: - print_dbg(f"Workaround not active/needed") - - add_include_line(build_opt_fqfn, commonhfile_fqfn) - add_include_line(build_opt_fqfn, globals_h_fqfn) - - # Provide context help for build option support. - source_build_opt_h_fqfn = os.path.join(os.path.dirname(source_globals_h_fqfn), "build_opt.h") - if os.path.exists(source_build_opt_h_fqfn) and not embedded_options: - print_err("Build options file '" + source_build_opt_h_fqfn + "' not supported.") - print_err(" Add build option content to '" + source_globals_h_fqfn + "'.") - print_err(" Embedd compiler command-line options in a block comment starting with '" + build_opt_signature + "'.") - print_err(" Read more at " + docs_url) - elif os.path.exists(source_globals_h_fqfn): - if not embedded_options: - print_msg("Tip: Embedd compiler command-line options in a block comment starting with '" + build_opt_signature + "'.") - print_msg(" Read more at " + docs_url) - else: - print_msg("Note: optional global include file '" + source_globals_h_fqfn + "' does not exist.") - print_msg(" Read more at " + docs_url) - - handle_error(0) # commit print buffer - -if __name__ == '__main__': - rc = 1 + extract_build_opt_from_path(build_opt_buffer, name, ctx.source_sketch_header) + except ParsingException as e: + raise + + # when command-line options were not created / found, it means the same thing as empty or missing .globals.h + if not len(build_opt_buffer.getvalue()): + build_with_minimal_build_opt(ctx) + return + + logging.info( + "\nExtra command-line options:\n%s", + "\n".join(f" {line}" for line in build_opt_buffer.getvalue().split("\n")), + ) + + # at this point, it is necessary to synchronize timestamps of every file + ensure_build_opt_written(ctx, build_opt_buffer) + ensure_build_sketch_header_written(ctx) + ensure_common_header_bound(ctx) + + # stats are now based on the either active sketch or common header + # (thus ensure core.a is rebuilt, even when sketch mtime is earlier) + synchronize_utime( + most_recent(ctx.common_header, ctx.source_sketch_header)[1], + ctx.build_opt, + ctx.build_sketch_header, + ctx.common_header, + ctx.source_sketch_header, + ) + + +def main_inspect(args: argparse.Namespace): + p = args.path # type: pathlib.Path + + buffer = io.StringIO() try: - rc = main() - except: - print_err(traceback.format_exc()) - handle_error(0) - sys.exit(rc) + extract_build_opt_from_path(buffer, args.build_opt_name, p) + except ParsingException as e: + raise e + + logging.info(buffer.getvalue()) + + +def main_placeholder(args: argparse.Namespace): + paths = args.path # type: List[pathlib.Path] + ensure_exists_and_empty(*paths) + + +def main_synchronize(args: argparse.Namespace): + first = args.first # type: pathlib.Path + rest = args.rest # type: List[pathlib.Path] + synchronize_utime(first, *rest) + + +def as_path(p: str) -> pathlib.Path: + if p.startswith("{") or p.endswith("}"): + raise ValueError(f'"{p}" was not resolved') + + return pathlib.Path(p) + + +def parse_args(args=None, namespace=None): + parser = argparse.ArgumentParser( + description="Handles sketch header containing command-line options, resulting build.opt and the shared core common header", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent(DOCS_EPILOG), + ) + + def main_help(args: argparse.Namespace): + parser.print_help() + + parser.set_defaults(func=main_help) + + parser.add_argument( + "--debug", action="store_true", help=argparse.SUPPRESS + ) # normal debug + parser.add_argument( + "--audit", action="store_true", help=argparse.SUPPRESS + ) # extra noisy debug + + parser.add_argument( + "--build-debug", + action="store_true", + help="Instead of build.opt, use build.opt:debug when searching for the comment signature match", + ) + + parser.add_argument( + "-D", + help="Intended to be used with mkbuildoptglobals.extra_flags={build.debug_port} in platform.local.txt)." + " Only enable --build-debug when debug port is also enabled in the menu / fqbn options.", + ) + + subparsers = parser.add_subparsers() + + # "prebuild" hook recipe command, preparing all of the necessary build files + + build = subparsers.add_parser("build") + + build.add_argument( + "--build-opt", + type=as_path, + required=True, + help="Command-line options file FQFN aka Fully Qualified File Name " + "(%build-path%/core/%build-opt%)", + ) + build.add_argument( + "--source-sketch-header", + type=as_path, + required=True, + help="FQFN of the globals.h header, located in the sketch directory " + "(%sketchname%/%sketchname%.ino.globals.h)", + ) + build.add_argument( + "--build-sketch-header", + type=as_path, + required=True, + help="FQFN of the globals.h header, located in the build directory " + "(%build-path%/sketch/%sketchname%.ino.globals.h)", + ) + build.add_argument( + "--common-header", + type=as_path, + required=True, + help="FQFN of shared dependency header (%core-path%/%common-header%)", + ) + + build.set_defaults(func=main_build) + + # "postbuild" hook recipe command, in case some files have to be cleared + + placeholder = subparsers.add_parser( + "placeholder", + help=ensure_exists_and_empty.__doc__, + ) + placeholder.add_argument( + "path", + action="append", + type=as_path, + ) + placeholder.set_defaults(func=main_placeholder) + + # Parse file path and discover any 'name' options inside + + inspect = subparsers.add_parser( + "inspect", + help=extract_build_opt.__doc__, + ) + inspect.add_argument("--build-opt-name", type=str, default="build.opt") + inspect.add_argument( + "path", + type=as_path, + ) + inspect.set_defaults(func=main_inspect) + + # Retrieve stats from the first file and apply to the rest + + synchronize = subparsers.add_parser( + "synchronize", + help=synchronize_utime.__doc__, + ) + synchronize.add_argument( + "first", + type=as_path, + help="Any file", + ) + synchronize.add_argument( + "rest", + type=as_path, + nargs="+", + help="Any file", + ) + synchronize.set_defaults(func=main_synchronize) + + return parser.parse_args(args, namespace) + + +def main(args: argparse.Namespace): + # mildly verbose logging, intended to notify about the steps taken + if args.debug: + logging.root.setLevel(logging.DEBUG) + + # very verbose logging from the python internals + if args.audit and sys.version_info >= (3, 8): + + def hook(event, args): + # note that logging module itself has audit calls, + # logging.debug(...) here would deadlock output + print(f"{event}:{args}", file=sys.stderr) + + sys.addaudithook(hook) + + return args.func(args) + + +if __name__ == "__main__": + main(parse_args()) diff --git a/tools/test_mkbuildoptglobals.py b/tools/test_mkbuildoptglobals.py new file mode 100755 index 0000000000..35f6d038ee --- /dev/null +++ b/tools/test_mkbuildoptglobals.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python + +import io +import sys +import unittest + +from typing import TextIO +from mkbuildoptglobals import extract_build_opt, InvalidSignature, InvalidSyntax +import contextlib + + +@contextlib.contextmanager +def buffer(init: str): + yield io.StringIO(init), io.StringIO() + + +class TestExtractBuildOpt(unittest.TestCase): + def testNoSource(self): + src = io.StringIO() + dst = io.StringIO() + + extract_build_opt("anything.opt", dst, src) + self.assertFalse(dst.getvalue()) + + def testSomeSource(self): + src = io.StringIO("12345") + dst = io.StringIO() + + extract_build_opt("numbers.opt", dst, src) + self.assertFalse(dst.getvalue()) + + def testNoOpt(self): + src = io.StringIO( + r""" +#include +int main() { + puts("hello world"); + return 0; +} + """ + ) + + dst = io.StringIO() + + extract_build_opt("something.opt", dst, src) + self.assertFalse(dst.getvalue()) + + def testAfterOpt(self): + src = io.StringIO( + r""" +int main() { + puts("hello world"); + return 0; +} +/*@create-file:after.opt@ +-fhello-world +*/ + """ + ) + + dst = io.StringIO() + + extract_build_opt("after.opt", dst, src) + self.assertEqual("-fhello-world\n", dst.getvalue()) + + def testEmptyBlock(self): + src = io.StringIO( + r""" + + +/*@create-file:empty.opt@ +*/ + +""" + ) + + dst = io.StringIO() + + extract_build_opt("empty.opt", dst, src) + self.assertFalse(dst.getvalue()) + + def testParseOnce(self): + src = io.StringIO( + r""" +/*@create-file:special.opt@ +-fspecial-option +*/ +""" + ) + + dst = io.StringIO() + + extract_build_opt("special.opt", dst, src) + self.assertEqual("-fspecial-option\n", dst.getvalue()) + + def testParseOnceLines(self): + src = io.StringIO( + r""" +/*@create-file:build.opt@ +-DFOO="arbitrary definition 1" +#comment + +// comment + +-DBAR="arbitrary definition 2" + // finalize this +*/ + """ + ) + + dst = io.StringIO() + + extract_build_opt("build.opt", dst, src) + self.assertEqual( + '-DFOO="arbitrary definition 1"\n' '-DBAR="arbitrary definition 2"\n', + dst.getvalue(), + ) + + def testParseSecond(self): + src = io.StringIO( + r""" +/*@create-file:foo.opt@ +-fno-builtin +# some random comment +-UFOOBAR +// another one +*/ /* extra comment */ +#define FOOBAR=1 +#define SOMETHING=123 +""" + ) + + dst = io.StringIO() + + extract_build_opt("bar.opt", dst, src) + self.assertEqual(dst.getvalue(), "") + + src.seek(0) + + extract_build_opt("foo.opt", dst, src) + self.assertEqual("-fno-builtin\n-UFOOBAR\n", dst.getvalue()) + + def testMultiple(self): + src = io.StringIO( + r""" +#include +int foo() { return 111; } + +/*@ create-file:foo.opt @ +-ffoo +*/ +#define INTERMIXED_DATA +/*@create-file:foo.opt:debug@ +-fbaz +*/ + +/*@create-file:bar.opt:debug@ +-DUNUSED +*/ +/*@create-file:foo.opt@ +-mbar +*/ + +/*@create-file:bar.opt@ +-DALSO_UNUSED +*/ + + +""" + ) + + dst = io.StringIO() + extract_build_opt("foo.opt", dst, src) + + self.assertEqual("-ffoo\n-mbar\n", dst.getvalue()) + + def testInvalidSignature(self): + src = io.StringIO( + r""" +#pragma once + +/*@create-file:foo.opt@ +-fanalyzer +*/ +// ordinary c++ code + +const char GlobalVariable[] = /*@ hello world @*/ +const char WriteMkbuildopts[] = /*@create-file:foo.opt@*/ + +/*@make-file:bar.opt@ +-mforce-l32 +*/ + +/* nothing to look at here */ +""" + ) + + dst = io.StringIO() + with self.assertRaises(InvalidSignature) as raises: + extract_build_opt("bar.opt", dst, src) + + self.assertFalse(dst.getvalue()) + + e = raises.exception + self.assertFalse(e.file) + self.assertEqual(12, e.lineno) + self.assertEqual("/*@make-file:bar.opt@\n", e.line) + + def testPartialInvalidSyntax(self): + src = io.StringIO( + r""" +/*@create-file:syntax.opt@ +-DIMPORTANT_FLAG +-DANOTHER_FLAG=123 +*/ +/*@ create-file:syntax.opt @ +/*@oops +-mthis-fails +*/ +""" + ) + + dst = io.StringIO() + with self.assertRaises(InvalidSyntax) as raises: + extract_build_opt("syntax.opt", dst, src) + + self.assertFalse(dst.getvalue()) + + e = raises.exception + self.assertFalse(e.file) + self.assertEqual(7, e.lineno) + self.assertEqual("/*@oops\n", e.line) + + def testPartialUnclosed(self): + src = io.StringIO( + r""" +/*@create-file:unclosed.opt@ +line 1 +line 2 +""" + ) + dst = io.StringIO() + with self.assertRaises(InvalidSyntax) as raises: + extract_build_opt("unclosed.opt", dst, src) + + self.assertFalse(dst.getvalue()) + + def testParseSignatureSpace(self): + with buffer( + r""" +/*@ create-file:test.opt @ +-ftest-test-test +*/ +""" + ) as (src, dst): + extract_build_opt("test.opt", dst, src) + self.assertEqual("-ftest-test-test\n", dst.getvalue()) + + with buffer( + r""" +/*@create-file:test.opt +@ +-ftest-test-test +*/ +""" + ) as (src, dst): + with self.assertRaises(InvalidSyntax) as raises: + extract_build_opt("test.opt", dst, src) + + self.assertFalse(dst.getvalue()) + + with buffer( + r""" +/*@create-file:test.opt +-ftest-test-test +*/ +""" + ) as (src, dst): + with self.assertRaises(InvalidSyntax) as raises: + extract_build_opt("test.opt", dst, src) + + self.assertFalse(dst.getvalue()) + + +if __name__ == "__main__": + unittest.main()