From 70f083b0ba25312effd77d67a2cb862ba2e6a039 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 25 Apr 2024 19:37:35 -0400 Subject: [PATCH 1/3] drop python 3.5, 3.6, 3.7 support --- CHANGES.rst | 6 ++++++ requirements-dev.txt | 32 +++++++++----------------------- requirements-doc.txt | 15 +++++---------- requirements-sys.txt | 3 +-- requirements.txt | 40 +++++++++++----------------------------- setup.py | 6 ++---- 6 files changed, 34 insertions(+), 68 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index d3799bea5..df52b6481 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -9,6 +9,12 @@ Changes `Unreleased `_ (latest) ------------------------------------------------------------------------------------ +Features / Changes +~~~~~~~~~~~~~~~~~~~~~ + +* Drop support of Python 3.5, 3.6 and 3.7. Minimum version is Python 3.8. +* Pin ``gunicorn>=22`` to address CVE-2024-1135. + Bug Fixes ~~~~~~~~~ diff --git a/requirements-dev.txt b/requirements-dev.txt index 9c9672878..bcca6abfb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,35 +1,21 @@ -r requirements-sys.txt -r requirements-doc.txt -autopep8; python_version < "3" -autopep8>=1.5.4; python_version >= "3.6" -backports.tempfile; python_version < "3" -bandit==1.7.1; python_version < "3.7" # pyup: ignore -bandit==1.7.5; python_version == "3.7" # pyup: ignore -bandit==1.7.7; python_version >= "3.8" +autopep8>=1.5.4 +bandit==1.7.7 bump2version==1.0.1 codacy-coverage>=1.3.11 -coverage==5.5; python_version < "3" # pyup: ignore -coverage>=5.5; python_version >= "3" -doc8; python_version < "3.6" -doc8>=0.8; python_version >= "3.6" -docformatter==1.4; python_version < "3.6" # pyup: ignore -docformatter; python_version >= "3.6" +coverage>=5.5 +doc8>=0.8 +docformatter flake8 -# Fix for https://github.com/PyCQA/flake8/issues/1701 -importlib-metadata>=1.1.0,<4.3; python_version < "3.8" -isort; python_version < "3.6" -isort>5.5,!=5.11.0; python_version >= "3.6" -mock; python_version < "3.6" -mock>4; python_version >= "3.6" -pylint<2.7; python_version < "3.6" # pyup: ignore +isort>5.5,!=5.11.0 +mock>4 # ignore 2.12 bad docstring asterisks args handling (https://github.com/PyCQA/pylint/issues/5406) -pylint>=2.11,!=2.12,!=2.15; python_version >= "3.6" +pylint>=2.11,!=2.12,!=2.15 pylint-quotes # bird-house/twticher, must match version in Dockerfile.adapater -pyramid-twitcher>=0.5.3; python_version < "3.6" # pyup: ignore -pyramid-twitcher>=0.9.0; python_version >= "3.6" +pyramid-twitcher>=0.9.0 pytest -python2-secrets; python_version <= "3.5" safety tox>=3.0 webtest diff --git a/requirements-doc.txt b/requirements-doc.txt index 9afc953fa..8f20d6fb1 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -3,16 +3,11 @@ # we actually need to install all requirements during docs build because of OpenAPI generation # (see 'docs/conf.py') astroid<2.12 # pin to resolve sphinx-autoapi (see https://github.com/readthedocs/sphinx-autoapi/issues/349) -pycodestyle>=2.6.0,<2.9.0; python_version <= "3.5" # pyup: ignore -pycodestyle>=2.6.0,<3; python_version >= "3.6" +pycodestyle>=2.6.0,<3 # sphinx-autoapi dropped 3.5 support at 1.3.0 # latest to fullfil requirements, but that is not the main doc builder version -sphinx-autoapi; python_version < "3.6" -sphinx-autoapi>=1.7.0; python_version >= "3.6" -sphinx-paramlinks>=0.4.1,<0.5; python_version < "3" -sphinx-paramlinks>=0.4.1; python_version >= "3" -sphinx; python_version < "3" -sphinx>=3.1,<8; python_version >= "3" +sphinx-autoapi>=1.7.0 +sphinx-paramlinks>=0.4.1 +sphinx>=5.3,<8 sphinxcontrib-redoc>=1.6.0 -sphinx_autodoc_typehints[type_comments]; python_version < "3.6" -sphinx_autodoc_typehints[type_comments]>=1.11.0; python_version >= "3.6" +sphinx_autodoc_typehints[type_comments]>=1.11.0 diff --git a/requirements-sys.txt b/requirements-sys.txt index 539640e1a..540f0dd53 100644 --- a/requirements-sys.txt +++ b/requirements-sys.txt @@ -1,3 +1,2 @@ pip -setuptools<60; python_version <= "3.6" # pyup: ignore -setuptools>=65.5.1; python_version >= "3.7" +setuptools>=65.5.1 diff --git a/requirements.txt b/requirements.txt index 5b672056c..c0df0da6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,5 @@ -r requirements-sys.txt -aenum; python_version < "3.6" -alembic>=1.3.0,<1.5; python_version <= "3.6" -alembic>=1.10; python_version >= "3.7" +alembic>=1.10 # TODO: remove when merged # until fix merged and deployed (https://github.com/authomatic/authomatic/pull/195) # old variants: @@ -14,19 +12,12 @@ bcrypt>=3.1.6 # FIXME: integrate when implemnted by official package (see https://github.com/bbangert/beaker/issues/201) beaker @ https://github.com/crim-ca/beaker/archive/0ac88bcd8cca063a571fc385ffbe9bcc8acaa690.zip colander -cornice<5; python_version < "3" -cornice; python_version >= "3.6" -cornice<6.1; python_version == "3.5" +cornice cornice_swagger>=0.7.0 dicttoxml -# futures is required for gunicorn threads -futures; python_version < "3" -# flag --paste breaks for >20, must use pserve instead -gunicorn<20; python_version < "3.5" # pyup: ignore -gunicorn>=20; python_version >= "3" +gunicorn>=22 humanize -jsonschema<4; python_version < "3.6" -jsonschema>=4; python_version >= "3.6" +jsonschema>=4 lxml>=3.7 mako # controlled by pyramid_mako paste @@ -40,29 +31,20 @@ pyramid_mako>=1.0.2 pyramid_retry==2.1.1 pyramid_tm>=2.2.1 python-dotenv -python2-secrets; python_version <= "3.5" pyyaml>=5.1 requests requests_file simplejson six>=1.12.0 -# see also: safety 51668, https://github.com/sqlalchemy/sqlalchemy/pull/8563, still beta + major version change 2.0.0b1 -sqlalchemy==1.3.*; python_version <= "3.6" # pyup: ignore -sqlalchemy>=1.4.44,<2; python_version >= "3.7" # pyup: ignore +# see also: safety 51668, https://github.com/sqlalchemy/sqlalchemy/pull/8563 +sqlalchemy>=1.4.44,<2 # pyup: ignore # see also: safety 42194, https://github.com/kvesteri/sqlalchemy-utils/issues/166, not fixed since 2015 -sqlalchemy-utils<0.36.4; python_version < "3" # pyup: ignore -sqlalchemy-utils==0.37.9; python_version >= "3" and python_version <= "3.5" # pyup: ignore -sqlalchemy-utils==0.39.0; python_version >= "3.6" # pyup: ignore -threddsclient==0.4.2; python_version < "3" # pyup: ignore -threddsclient>=0.4.2; python_version >= "3" +sqlalchemy-utils==0.39.0 # pyup: ignore +threddsclient>=0.4.2 transaction -typing; python_version < "3" -# typing extension required for TypedDict -typing_extensions; python_version < "3.8" -wheel; python_version <= "3.6" -wheel>=0.38; python_version >= "3.7" +typing_extensions +wheel>=0.38 webob -ziggurat_foundations==0.8.4; python_version <= "3.6" # pyup: ignore -ziggurat_foundations==0.9.1; python_version >= "3.7" +ziggurat_foundations==0.9.1 zope.interface>=4.7.2,<5 zope.sqlalchemy==1.6 diff --git a/setup.py b/setup.py index ec8fdd753..8c31e1a77 100644 --- a/setup.py +++ b/setup.py @@ -211,15 +211,13 @@ def _extra_requirements(base_requirements, other_requirements): "License :: OSI Approved :: Apache Software License", "Natural Language :: English", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ], - python_requires=">=3.5, <4", + python_requires=">=3.8, <4", # -- Package structure ------------------------------------------------- packages=[__meta__.__package__], From 1094d9d0d9a6cf0096e44e64d423170471c77817 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 25 Apr 2024 19:50:23 -0400 Subject: [PATCH 2/3] add Python 3.12 support --- .github/workflows/tests.yml | 13 ++----------- CHANGES.rst | 8 ++++++++ magpie/__init__.py | 10 ++++++++++ requirements.txt | 7 +++++-- setup.py | 2 +- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 124e948f6..038a6b67f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,7 +48,7 @@ jobs: matrix: # os: [ubuntu-latest, windows-latest] os: [ubuntu-latest] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] allow-failure: [false] test-case: [test-local] # can use below option to set environment variables or makefile settings applied during test execution @@ -79,18 +79,9 @@ jobs: test-case: test-docker # test allowed failing for legacy versions - os: ubuntu-20.04 - python-version: "3.5" + python-version: "3.7" allow-failure: true test-case: test-local - - os: ubuntu-20.04 - python-version: "3.6" - allow-failure: true - test-case: test-local - # test allowed failing for recent versions - # - os: ubuntu-latest - # python-version: 3.12 - # allow-failure: true - # test-case: test-local steps: - uses: actions/checkout@v2 with: diff --git a/CHANGES.rst b/CHANGES.rst index df52b6481..7b127b1bc 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,6 +12,14 @@ Changes Features / Changes ~~~~~~~~~~~~~~~~~~~~~ +* | Add support of Python 3.12. + | + | **NOTE**: + | Requires patch of ``authomatic.six.moves`` in ``magpie.__init__.py`` to work around vendor-specific definitions. + Fix inspiration + from `dpkp/kafka-python#2401 `_. + Pending official fix (see `authomatic/authomatic#233 `_). + * Drop support of Python 3.5, 3.6 and 3.7. Minimum version is Python 3.8. * Pin ``gunicorn>=22`` to address CVE-2024-1135. diff --git a/magpie/__init__.py b/magpie/__init__.py index 6db442527..7b94770ac 100644 --- a/magpie/__init__.py +++ b/magpie/__init__.py @@ -7,6 +7,16 @@ import sys +# FIXME: patch workaround for Python 3.12 issue with vendor-specific 'six.moves' +# Inspired from https://github.com/dpkp/kafka-python/issues/2401#issuecomment-1760208950 +# Applied here such that anything importing 'magpie' does it first, to avoid patch everywhere importing 'authomatic'. +# Official fix required (see https://github.com/authomatic/authomatic/issues/233). +if sys.version_info >= (3, 12, 0): + import six + sys.modules["authomatic.six.moves"] = six.moves + sys.modules["authomatic.six.moves.urllib"] = six.moves.urllib + sys.modules["authomatic.six.moves.urllib.parse"] = six.moves.urllib.parse + def includeme(config): # import needs to be here, otherwise ImportError happens during setup.py install (modules not yet installed) diff --git a/requirements.txt b/requirements.txt index c0df0da6e..a267244c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,8 +5,11 @@ alembic>=1.10 # old variants: # -e git+https://github.com/fmigneault/authomatic.git@httplib-port#egg=Authomatic # https://github.com/fmigneault/authomatic/archive/httplib-port.zip#egg=Authomatic-1.0.1 -# new authomatic handles openid install correctly -# leave http port until merged: +# - new authomatic handles openid install correctly +# - https://github.com/authomatic/authomatic/tree/master includes #195 (http-po#rt) +# via https://github.com/authomatic/authomatic/pull/227 +# However, this is not the latest tag https://github.com/authomatic/authomatic/tree/1.2.1 (done before #227). +# - Requires https://github.com/authomatic/authomatic/issues/233 for Python 3.12 (temporary patch in magpie.__init__) authomatic[OpenID] @ https://github.com/fmigneault/authomatic/archive/httplib-port.zip bcrypt>=3.1.6 # FIXME: integrate when implemnted by official package (see https://github.com/bbangert/beaker/issues/201) diff --git a/setup.py b/setup.py index 8c31e1a77..cc5f73eeb 100644 --- a/setup.py +++ b/setup.py @@ -217,7 +217,7 @@ def _extra_requirements(base_requirements, other_requirements): "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ], - python_requires=">=3.8, <4", + python_requires=">=3.5, <4", # -- Package structure ------------------------------------------------- packages=[__meta__.__package__], From c59b2c9ecad342703293db7c8458ce0f882c5fe1 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 25 Apr 2024 19:55:10 -0400 Subject: [PATCH 3/3] remove python 3.7 from CI --- .github/workflows/tests.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 038a6b67f..0c3b1c64f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -77,11 +77,6 @@ jobs: python-version: none allow-failure: true test-case: test-docker - # test allowed failing for legacy versions - - os: ubuntu-20.04 - python-version: "3.7" - allow-failure: true - test-case: test-local steps: - uses: actions/checkout@v2 with: