From c26c48372f929d12491f8ffb4480d4e8a7760cde Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Fri, 28 Feb 2025 11:45:50 +0100 Subject: [PATCH 1/2] test: Set INVALID_UTF8_VAR for pilot wrapper tests --- .github/workflows/pilotWrapper.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pilotWrapper.yml b/.github/workflows/pilotWrapper.yml index 3c7bc9fe9de..3010db7c0b6 100644 --- a/.github/workflows/pilotWrapper.yml +++ b/.github/workflows/pilotWrapper.yml @@ -35,6 +35,7 @@ jobs: conda create -c conda-forge -c free -n python_${{ matrix.python }} python=${{ matrix.python }} - name: run pilot wrapper test run: | + export INVALID_UTF8_VAR=$'\xff' cp tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py . eval "$(conda shell.bash hook)" && conda activate python_${{ matrix.python }} # use github APIs to get the artifacts URLS from https://github.com/DIRACGrid/Pilot/, for those named Pilot_${{ matrix.pilot_branch }} From 4b7af9548abd1ee36ddb4158009d1eb682d9a53f Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Fri, 28 Feb 2025 11:24:19 +0100 Subject: [PATCH 2/2] fix: Supress non-UTF8 variables from pilot environment --- .../WorkloadManagementSystem/Utilities/PilotWrapper.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py b/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py index 50b8bf5ed99..2936b3d52e8 100644 --- a/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py @@ -68,7 +68,15 @@ # just logging the environment as first thing logger.debug('===========================================================') logger.debug('Environment of execution host\\n') -for key, val in os.environ.items(): +for key, val in getattr(os, "environb", os.environ).items(): + # Clean the environment of non-utf-8 characters + try: + key = key.decode("utf-8") + val = val.decode("utf-8") + except UnicodeDecodeError as e: + logger.error("Dropping %%s=%%s due to: %%s", key, val, e) + del os.environ[key] + continue logger.debug(key + '=' + val) logger.debug('===========================================================\\n')