From 090f3447baa954acd619cb83f0e5a704bf33e79c Mon Sep 17 00:00:00 2001 From: David Vogt Date: Tue, 4 Mar 2025 21:03:29 +0100 Subject: [PATCH 1/7] chore(ci): deprecate python and postgres versions Python version 3.9 and PostgreSQL 12 are being deprecated. In their place, start testing with Python 3.13 and PostgreSQL 17 --- .github/workflows/compatibility-tests.yml | 2 +- .github/workflows/tests.yml | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/compatibility-tests.yml b/.github/workflows/compatibility-tests.yml index 2aa90472d..3b823eede 100644 --- a/.github/workflows/compatibility-tests.yml +++ b/.github/workflows/compatibility-tests.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python: ["3.9", "3.10", "3.11", "3.12"] + python: ["3.10", "3.11", "3.12", "3.13"] services: postgres: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c455664b3..da841cf54 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -125,13 +125,6 @@ jobs: - name: Run tests run: poetry run pytest --no-cov-on-fail --cov --create-db -vv - compatibility-tests-postgres-12: - name: "Compatibility tests" - uses: ./.github/workflows/compatibility-tests.yml - needs: [lint] - with: - postgres: "12" - compatibility-tests-postgres-13: name: "Compatibility tests" uses: ./.github/workflows/compatibility-tests.yml @@ -159,3 +152,10 @@ jobs: needs: [lint] with: postgres: "16" + + compatibility-tests-postgres-17: + name: "Compatibility tests" + uses: ./.github/workflows/compatibility-tests.yml + needs: [lint] + with: + postgres: "17" From 059606a610edb9d457c3c0dadaefbf223ec3b0c8 Mon Sep 17 00:00:00 2001 From: Jonas Metzener Date: Wed, 5 Mar 2025 13:10:04 +0100 Subject: [PATCH 2/7] chore: remove duplicate string casting --- caluma/caluma_form/structure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caluma/caluma_form/structure.py b/caluma/caluma_form/structure.py index 85e0777fe..8412ab82f 100644 --- a/caluma/caluma_form/structure.py +++ b/caluma/caluma_form/structure.py @@ -332,7 +332,7 @@ def form_by_id(self, form_id: str) -> Form: return self._forms[form_id] def rows_for_table_answer(self, answer_id: str) -> list[Document]: - document_ids = self._table_rows_by_answer[str(str(answer_id))] + document_ids = self._table_rows_by_answer[str(answer_id)] return [self._documents[doc_id] for doc_id in document_ids] def options_for_question(self, question_id): From 1282dbb3af1b9d3df8b8456864e7381bf3c11eea Mon Sep 17 00:00:00 2001 From: David Vogt Date: Wed, 5 Mar 2025 14:43:29 +0100 Subject: [PATCH 3/7] chore: add python version deprecation note to 11.0.0 release notes --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 612f50eb3..a1109e164 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,10 @@ * Code that uses the form jexl and / or structure code most likely will need to be rewritten. The changes are small-ish, but still semantically not exactly equal. ([`7c997b7`](https://github.com/projectcaluma/caluma/commit/7c997b70850a4c4e7f046714d8a1ab56a89ef950)) See https://github.com/projectcaluma/caluma/pull/2356 for further details. - +* While not yet (technically) breaking, we do not support Python versions + of 3.9 and earlier. Update your Python to 3.10 or ideally, 3.13. Support + for Python versions will be according to Python's own version schedule: + https://devguide.python.org/versions/ # v10.7.0 (09 January 2025) From 6b698ef99cfc4f32ad62b4154a7bc61a5c973db1 Mon Sep 17 00:00:00 2001 From: Jonas Metzener Date: Thu, 6 Mar 2025 10:09:59 +0100 Subject: [PATCH 4/7] chore(build): fix docker syntax deprecations --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4dbe70c6b..ca41dd2f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ENV HOME=/home/caluma ENV PYTHONUNBUFFERED=1 ENV APP_HOME=/app -ENV DJANGO_SETTINGS_MODULE caluma.settings.django +ENV DJANGO_SETTINGS_MODULE=caluma.settings.django RUN mkdir -p $APP_HOME \ && useradd -u 901 -r caluma --create-home \ @@ -33,4 +33,4 @@ COPY . $APP_HOME EXPOSE 8000 -CMD /bin/sh -c "wait-for-it $DATABASE_HOST:${DATABASE_PORT:-5432} -- poetry run python manage.py migrate && poetry run gunicorn --workers 10 --access-logfile - --limit-request-line 16384 --bind :8000 caluma.wsgi" +CMD ["/bin/sh", "-c", "wait-for-it $DATABASE_HOST:${DATABASE_PORT:-5432} -- poetry run python manage.py migrate && poetry run gunicorn --workers 10 --access-logfile - --limit-request-line 16384 --bind :8000 caluma.wsgi"] From 581d2d056daeee0394c94f20dd64c36ff12a7be8 Mon Sep 17 00:00:00 2001 From: David Vogt Date: Wed, 5 Mar 2025 18:13:35 +0100 Subject: [PATCH 5/7] fix(perf): don't load full document in memory if not needed for options Question options can have a JEXL to decide whether they're hidden or not. Most of them however do not use this feature, and for those, loading the full document into memory just to evaluate their (non-existing) `is_hidden` expression is just a waste of resources/time. --- caluma/caluma_form/tests/test_validators.py | 40 +++++++++++++++++++++ caluma/caluma_form/validators.py | 8 +++++ 2 files changed, 48 insertions(+) diff --git a/caluma/caluma_form/tests/test_validators.py b/caluma/caluma_form/tests/test_validators.py index 3bec140fe..b29e20a72 100644 --- a/caluma/caluma_form/tests/test_validators.py +++ b/caluma/caluma_form/tests/test_validators.py @@ -799,3 +799,43 @@ def test_validate_integer_0( with pytest.raises(ValidationError) as excinfo: DocumentValidator().validate(document, admin_user) assert excinfo.match("Invalid value 0") + + +@pytest.mark.parametrize( + "option_jexl, expect_queries, expect_jexl_evaluations", + [ + # pre-recognized "visible" jexl, no JEXL and few queries + ("", 9, 0), + # pre-recognized "visible" jexl, no JEXL and few queries + ("false", 9, 0), + # not pre-recognized - needs to be evaluated in full doc context + # therefore more queries needed, and JEXL expressions are evaluated + ("!true", 18, 5), + ], +) +def test_validate_options_without_jexl( + db, + django_assert_num_queries, + form_question_factory, + question_option_factory, + document, + option_jexl, + expect_queries, + expect_jexl_evaluations, + mocker, +): + """Ensure that validating options only evaluates JEXL if actually needed.""" + form_question_factory(form=document.form) + form_question_factory(form=document.form) + form_question_factory(form=document.form) + fq = form_question_factory(question__type=Question.TYPE_CHOICE, form=document.form) + options = question_option_factory.create_batch( + 5, question=fq.question, option__is_hidden=option_jexl + ) + + spy = mocker.spy(structure.BaseField, "evaluate_jexl") + + with django_assert_num_queries(expect_queries): + api.save_answer(fq.question, document, value=options[3].option_id) + + assert spy.call_count == expect_jexl_evaluations diff --git a/caluma/caluma_form/validators.py b/caluma/caluma_form/validators.py index 2895727ec..e444257b1 100644 --- a/caluma/caluma_form/validators.py +++ b/caluma/caluma_form/validators.py @@ -130,6 +130,14 @@ def _evaluate_options_jexl( # and use the structure code) return [o.slug for o in question.options.all()] + elif not validation_context: + # First see if we really need to build up the validation context. If + # not, we can speed things up significantly + all_options = list(question.options.all()) + if all(opt.is_hidden in ("false", "") for opt in all_options): + # no JEXL evaluation neccessary + return [o.slug for o in all_options] + field, _root = self._structure_field(document, question, validation_context) if not field: # pragma: no cover # This only happens if *programmer* made an error, therefore we're From 85138939f60e411a5a1f47f83ebe3262481ce193 Mon Sep 17 00:00:00 2001 From: Niels Verbeek Date: Thu, 6 Mar 2025 16:54:34 +0100 Subject: [PATCH 6/7] chore(ci): update minimal python version in github actions --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index da841cf54..7e1780b1e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: - name: Setup python uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" cache: "poetry" - name: Set UID @@ -108,7 +108,7 @@ jobs: - name: Setup python uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" cache: "poetry" - name: Set UID From afd3a06443e9305457ffd9cd1cf8cda570f3671c Mon Sep 17 00:00:00 2001 From: David Vogt Date: Fri, 7 Mar 2025 14:46:01 +0100 Subject: [PATCH 7/7] chore: release 11.1.2 Note this is on a maintenance branch that won't be merged to `main`. All the fixes / chores in here have already been merged. --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1109e164..be7cccb36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# v11.1.2 (07 March 2025) + +### Fix + +* **perf:** Don't load full document in memory if not needed for options ([`581d2d0`](https://github.com/projectcaluma/caluma/commit/581d2d056daeee0394c94f20dd64c36ff12a7be8)) + + # v11.1.1 (04 March 2025) ### Fix diff --git a/pyproject.toml b/pyproject.toml index 51ee54367..a4bd5ba94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "caluma" -version = "11.1.1" +version = "11.1.2" description = "Caluma Service providing GraphQL API" homepage = "https://caluma.io" repository = "https://github.com/projectcaluma/caluma"