diff --git a/CHANGELOG.md b/CHANGELOG.md index 06b7063240e..e77526adbbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ Note: Breaking changes between versions are indicated by "💥". ## Unreleased + +- [Improvement] Improved the output of `tutor plugins list`. + ## v13.1.8 (2022-03-18) - [Bugfix] Fix "evalsymlink failure" during `k8s quickstart` (#611). diff --git a/Makefile b/Makefile index 2eb30e8dd13..8255ddf4d9f 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,8 @@ push-pythonpackage: ## Push python package to pypi test: test-lint test-unit test-types test-format test-pythonpackage ## Run all tests by decreasing order of priority +test-static: test-lint test-types test-format ## Run only static tests + test-format: ## Run code formatting tests black --check --diff $(BLACK_OPTS) @@ -38,7 +40,7 @@ test-unit: ## Run unit tests python -m unittest discover tests test-types: ## Check type definitions - mypy --exclude=templates --ignore-missing-imports --strict tutor/ tests/ + mypy --exclude=templates --ignore-missing-imports --strict ${SRC_DIRS} test-pythonpackage: build-pythonpackage ## Test that package can be uploaded to pypi twine check dist/tutor-$(shell make version).tar.gz @@ -49,6 +51,9 @@ test-k8s: ## Validate the k8s format with kubectl. Not part of the standard test format: ## Format code automatically black $(BLACK_OPTS) +isort: ## Sort imports. This target is not mandatory because the output may be incompatible with black formatting. Provided for convenience purposes. + isort --skip=templates ${SRC_DIRS} + bootstrap-dev: ## Install dev requirements pip install . pip install -r requirements/dev.txt diff --git a/bin/main.py b/bin/main.py index 0055f90bb65..8335a212776 100755 --- a/bin/main.py +++ b/bin/main.py @@ -1,25 +1,16 @@ #!/usr/bin/env python3 -from tutor.plugins import OfficialPlugin +from tutor import hooks from tutor.commands.cli import main +from tutor.plugins.v0 import OfficialPlugin + + +@hooks.actions.on(hooks.Actions.INSTALL_PLUGINS) +def _install_official_plugins() -> None: + # Manually install plugins: that's because entrypoint plugins are not properly + # detected within the binary bundle. + OfficialPlugin.install_all() -# Manually install plugins (this is for creating the bundle) -for plugin_name in [ - "android", - "discovery", - "ecommerce", - "forum", - "license", - "mfe", - "minio", - "notes", - "richie", - "webui", - "xqueue", -]: - try: - OfficialPlugin.load(plugin_name) - except ImportError: - pass if __name__ == "__main__": + # Call the regular main function, which will not detect any entrypoint plugin main() diff --git a/docs/Makefile b/docs/Makefile index 8026e4c87e4..a8149769ec5 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -11,4 +11,7 @@ browse: sensible-browser _build/html/index.html watch: build browse - while true; do inotifywait -e modify *.rst */*.rst */*/*.rst ../*.rst conf.py; $(MAKE) build || true; done + while true; do $(MAKE) wait-for-change build || true; done + +wait-for-change: + inotifywait -e modify $(shell find . -name "*.rst") ../*.rst ../tutor/hooks/*.py conf.py diff --git a/docs/conf.py b/docs/conf.py index f1334a34cda..c256f7a6904 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,6 +27,10 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] pygments_style = None +# Autodocumentation of modules +extensions.append("sphinx.ext.autodoc") +autodoc_typehints = "description" + # -- Sphinx-Click configuration # https://sphinx-click.readthedocs.io/ extensions.append("sphinx_click") @@ -108,5 +112,5 @@ def youtube( ] -youtube.content = True -docutils.parsers.rst.directives.register_directive("youtube", youtube) +setattr(youtube, "content", True) +docutils.parsers.rst.directives.register_directive("youtube", youtube) # type: ignore diff --git a/docs/index.rst b/docs/index.rst index 6fd06b13ec0..eed95370195 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -59,6 +59,6 @@ This work is licensed under the terms of the `GNU Affero General Public License The AGPL license covers the Tutor code, including the Dockerfiles, but not the content of the Docker images which can be downloaded from https://hub.docker.com. Software other than Tutor provided with the docker images retain their original license. -The Tutor plugin system is licensed under the terms of the `Apache License, Version 2.0 `__. +The Tutor plugin and hooks system is licensed under the terms of the `Apache License, Version 2.0 `__. © 2021 Tutor is a registered trademark of SASU NULI NULI. All Rights Reserved. diff --git a/docs/intro.rst b/docs/intro.rst index 82c737911e0..9157a594d1b 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -70,6 +70,8 @@ Urls: The platform is reset every day at 9:00 AM, `Paris (France) time `__, so feel free to try and break things as much as you want. +.. _how_does_tutor_work: + How does Tutor work? -------------------- diff --git a/docs/plugins.rst b/docs/plugins.rst index 2f58115e4d9..f2d9dedd89d 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -12,10 +12,12 @@ Tutor comes with a plugin system that allows anyone to customise the deployment # 3) Reconfigure and restart the platform tutor local quickstart -For simple changes, it may be extremely easy to create a Tutor plugin: even non-technical users may get started with :ref:`simple yaml plugins `. +For simple changes, it may be extremely easy to create a Tutor plugin: even non-technical users may get started with our :ref:`plugin_development_tutorial`. In the following we learn how to use and create Tutor plugins. +.. TODO move this part of the docs to the tutorial? + Commands -------- @@ -39,12 +41,14 @@ Existing plugins Officially-supported plugins are listed on the `Overhang.IO `__ website. -Plugin development ------------------- +Legacy plugin v0 development +---------------------------- + +.. TODO put more words here .. toctree:: :maxdepth: 2 - plugins/api - plugins/gettingstarted - plugins/examples + plugins/v0/api + plugins/v0/gettingstarted + plugins/v0/examples diff --git a/docs/plugins/api.rst b/docs/plugins/v0/api.rst similarity index 100% rename from docs/plugins/api.rst rename to docs/plugins/v0/api.rst diff --git a/docs/plugins/examples.rst b/docs/plugins/v0/examples.rst similarity index 100% rename from docs/plugins/examples.rst rename to docs/plugins/v0/examples.rst diff --git a/docs/plugins/gettingstarted.rst b/docs/plugins/v0/gettingstarted.rst similarity index 100% rename from docs/plugins/gettingstarted.rst rename to docs/plugins/v0/gettingstarted.rst diff --git a/docs/reference.rst b/docs/reference.rst index 006044b1f59..f2ade14381b 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -1,13 +1,9 @@ -CLI Reference -============= +Reference +========= .. toctree:: :maxdepth: 2 - reference/cli/tutor - reference/cli/config - reference/cli/dev - reference/cli/images - reference/cli/k8s - reference/cli/local - reference/cli/plugins + reference/api + reference/cli + reference/patches diff --git a/docs/reference/api.rst b/docs/reference/api.rst new file mode 100644 index 00000000000..ef90b56f272 --- /dev/null +++ b/docs/reference/api.rst @@ -0,0 +1,14 @@ +=== +API +=== + +Hooks +===== + +.. toctree:: + :maxdepth: 2 + + api/hooks/actions + api/hooks/filters + api/hooks/contexts + api/hooks/consts diff --git a/docs/reference/api/hooks/actions.rst b/docs/reference/api/hooks/actions.rst new file mode 100644 index 00000000000..2ffb6be6dec --- /dev/null +++ b/docs/reference/api/hooks/actions.rst @@ -0,0 +1,8 @@ +======= +Actions +======= + +.. autofunction:: tutor.hooks.actions::on +.. autofunction:: tutor.hooks.actions::do +.. autofunction:: tutor.hooks.actions::clear +.. autofunction:: tutor.hooks.actions::clear_all diff --git a/docs/reference/api/hooks/consts.rst b/docs/reference/api/hooks/consts.rst new file mode 100644 index 00000000000..d9064408cb0 --- /dev/null +++ b/docs/reference/api/hooks/consts.rst @@ -0,0 +1,21 @@ +========= +Constants +========= + +Actions +======= + +.. autoclass:: tutor.hooks.Actions + :members: + +Filters +======= + +.. autoclass:: tutor.hooks.Filters + :members: + +Contexts +======== + +.. autoclass:: tutor.hooks.Contexts + :members: diff --git a/docs/reference/api/hooks/contexts.rst b/docs/reference/api/hooks/contexts.rst new file mode 100644 index 00000000000..91ecda96645 --- /dev/null +++ b/docs/reference/api/hooks/contexts.rst @@ -0,0 +1,5 @@ +======== +Contexts +======== + +.. autofunction:: tutor.hooks.contexts::enter diff --git a/docs/reference/api/hooks/filters.rst b/docs/reference/api/hooks/filters.rst new file mode 100644 index 00000000000..0b03b56cf52 --- /dev/null +++ b/docs/reference/api/hooks/filters.rst @@ -0,0 +1,13 @@ +.. _filters: + +======= +Filters +======= + +.. autofunction:: tutor.hooks.filters::add +.. autofunction:: tutor.hooks.filters::add_item +.. autofunction:: tutor.hooks.filters::add_items +.. autofunction:: tutor.hooks.filters::apply +.. autofunction:: tutor.hooks.filters::iterate +.. autofunction:: tutor.hooks.filters::clear +.. autofunction:: tutor.hooks.filters::clear_all diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst new file mode 100644 index 00000000000..f356ddca6f7 --- /dev/null +++ b/docs/reference/cli.rst @@ -0,0 +1,13 @@ +Command line interface (CLI) +============================ + +.. toctree:: + :maxdepth: 2 + + cli/tutor + cli/config + cli/dev + cli/images + cli/k8s + cli/local + cli/plugins diff --git a/docs/reference/patches.rst b/docs/reference/patches.rst new file mode 100644 index 00000000000..faa5188c56d --- /dev/null +++ b/docs/reference/patches.rst @@ -0,0 +1,13 @@ +.. _patches: + +================ +Template patches +================ + +.. TODO fill me + +This is a work-in-progress. For the moment, the list of patches in Tutor templates can be obtained by grepping the source code:: + + git clone https://github.com/overhangio/tutor + cd tutor + git grep "{{ patch" -- tutor/templates diff --git a/docs/tutorials.rst b/docs/tutorials.rst index b8a9f2d2138..642ac90948e 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -7,6 +7,7 @@ Open edX customization .. toctree:: :maxdepth: 2 + tutorials/plugin tutorials/theming tutorials/edx-platform-settings tutorials/google-smtp diff --git a/docs/tutorials/plugin.rst b/docs/tutorials/plugin.rst new file mode 100644 index 00000000000..4e96462c1cf --- /dev/null +++ b/docs/tutorials/plugin.rst @@ -0,0 +1,93 @@ +.. _plugin_development_tutorial: + +======================= +Creating a Tutor plugin +======================= + +Tutor plugins are the offically recommended way of customizing the behaviour of Tutor. If Tutor does not do things the way you want, then your first reaction should *not* be to fork Tutor, but instead to figure out whether you can create a plugin that will allow you to achieve what you want. + +You may be thinking that creating a plugin might be overkill for your use case. It's almost certainly not! The stable plugin API guarantees that your changes will keep working even after you upgrade from one major release to the next, with little to no extra work. Also, it allows you to distribute your changes to other users. + +.. TODO reformat everything below this line + +:: + + touch "$(tutor plugins printroot)/myplugin.py" + +:: + + tutor plugins list + +:: + + myplugin (disabled) /home/yourusername/.local/share/tutor-plugins/myplugin.py + +Enable your plugin:: + + tutor plugins enable myplugin + +:: + + Plugin myplugin enabled + Configuration saved to /home/yourusername/.local/share/tutor/config.yml + You should now re-generate your environment with `tutor config save`. + +At this point you could re-generate your environment with ``tutor config save``, but there would not be any change to your environment... because the plugin does not do anything. So let's get started and make some changes. + +Defining patches +================ + +We'll start by modifying some of our Open edX settings. It's a frequent requirement to modify the ``FEATURES`` setting from the LMS or the CMS in edx-platform. In the legacy native installation, this was done by modifying the ``lms.env.yml`` and ``cms.env.yml`` files. Here we'll modify the Python setting files that define the edx-platform configuration. To achieve that we'll make use of two concepts from the Tutor API: :ref:`patches` and :ref:`filters`. + +If you have not already read :ref:`how_does_tutor_work` now would be a good time :-) Tutor uses templates to generate various files, such as settings, Dockerfiles, etc. These templates include ``{{ patch("patch-name") }}`` statements that allow plugins to insert arbitrary content in there. These patches are located at strategic locations. See :ref:`patches` for more information. + +Let's say that we would like to limit access to our brandh new Open edX platform. It is not ready for prime-time yet, so we want to prevent users from registering new accounts. There is a feature flag for that in the LMS: `FEATURES['ALLOW_PUBLIC_ACCOUNT_CREATION'] `__. By default this flag is set to a true value, enabling anyone to create an account. In the following we'll set it to false. + +Add the following content to the ```myplugin.py`` file that you created earlier:: + + from tutor import hooks + + hooks.filters.add_item( + hooks.Filters.ENV_PATCHES % "openedx-lms-common-settings", + "FEATURES['ALLOW_PUBLIC_ACCOUNT_CREATION'] = False" + ) + +Let's go over these changes one by one:: + + from tutor import hooks + +This imports the ``hooks`` module from Tutor, which grants us access to ``hooks.filters`` and ``hooks.Filters`` (among other things). + +:: + + hooks.filters.add_item( + , + + ) + +This means "add to the ". In our case, we want to modify the LMS settings, both in production and development. The right patch for that is ``"openedx-lms-common-settings"`` and the corresponding filter is named ``hooks.Filters.ENV_PATCHES % "openedx-lms-common-settings"``. We add one item, which is a single Python-formatted line of code:: + + "FEATURES['ALLOW_PUBLIC_ACCOUNT_CREATION'] = False" + +Notice how "False" starts with a capital "F"? That's how booleans are created in Python. + +Now, re-render your environment with:: + + tutor config save + +You can check that the feature was added to your environment by running:: + + grep -r ALLOW_PUBLIC_ACCOUNT_CREATION "$(tutor config printroot)/env" + +This should print:: + + /home/yourusername/.local/share/tutor/env/apps/openedx/settings/lms/production.py:FEATURES['ALLOW_PUBLIC_ACCOUNT_CREATION'] = False + /home/yourusername/.local/share/tutor/env/apps/openedx/settings/lms/development.py:FEATURES['ALLOW_PUBLIC_ACCOUNT_CREATION'] = False + +Your new settings will be taken into account by restarting your platform:: + + tutor local restart + +Congratulations! You've made your first working plugins. As you can guess, you can add changes to other files by adding other similar patch statements to your plugin. + +.. TODO to be continued diff --git a/requirements/base.txt b/requirements/base.txt index 3a8401b1b57..81ef2d8d607 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -26,7 +26,7 @@ kubernetes==18.20.0 # via -r requirements/base.in markupsafe==2.0.1 # via jinja2 -mypy==0.931 +mypy==0.941 # via -r requirements/base.in mypy-extensions==0.4.3 # via mypy diff --git a/requirements/dev.in b/requirements/dev.in index b126909716a..a8fb2a8509f 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -7,5 +7,6 @@ twine coverage # Types packages +types-docutils types-PyYAML types-setuptools diff --git a/requirements/dev.txt b/requirements/dev.txt index 6cd8fd11885..eee2c57f1fc 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -10,7 +10,7 @@ appdirs==1.4.4 # via -r requirements/base.txt astroid==2.8.3 # via pylint -black==21.9b0 +black==22.1.0 # via -r requirements/dev.in bleach==4.1.0 # via readme-renderer @@ -74,7 +74,7 @@ markupsafe==2.0.1 # jinja2 mccabe==0.6.1 # via pylint -mypy==0.910 +mypy==0.941 # via -r requirements/base.txt mypy-extensions==0.4.3 # via @@ -132,8 +132,6 @@ pyyaml==6.0 # kubernetes readme-renderer==30.0 # via twine -regex==2021.10.23 - # via black requests==2.26.0 # via # -r requirements/base.txt @@ -163,18 +161,19 @@ six==1.16.0 # kubernetes # python-dateutil toml==0.10.2 + # via pylint +tomli==2.0.1 # via # -r requirements/base.txt - # mypy - # pylint -tomli==1.2.2 - # via # black + # mypy # pep517 tqdm==4.62.3 # via twine twine==3.4.2 # via -r requirements/dev.in +types-docutils==0.18.0 + # via -r requirements/dev.in types-pyyaml==6.0.0 # via -r requirements/dev.in types-setuptools==57.4.2 diff --git a/requirements/docs.txt b/requirements/docs.txt index c402f38ceba..4540307de64 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -52,7 +52,7 @@ markupsafe==2.0.1 # via # -r requirements/base.txt # jinja2 -mypy==0.910 +mypy==0.941 # via -r requirements/base.txt mypy-extensions==0.4.3 # via @@ -132,7 +132,7 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -toml==0.10.2 +tomli==2.0.1 # via # -r requirements/base.txt # mypy diff --git a/tests/commands/base.py b/tests/commands/base.py new file mode 100644 index 00000000000..a39d934b09c --- /dev/null +++ b/tests/commands/base.py @@ -0,0 +1,35 @@ +import typing as t + +import click.testing + +from tests.helpers import TestContext, temporary_root +from tutor.commands.cli import cli + + +class TestCommandMixin: + """ + Run CLI tests in an isolated test root. + """ + + @staticmethod + def invoke(args: t.List[str]) -> click.testing.Result: + with temporary_root() as root: + return TestCommandMixin.invoke_in_root(root, args) + + @staticmethod + def invoke_in_root(root: str, args: t.List[str]) -> click.testing.Result: + """ + Use this method for commands that all need to run in the same root: + + with temporary_root() as root: + result1 = self.invoke_in_root(root, ...) + result2 = self.invoke_in_root(root, ...) + """ + runner = click.testing.CliRunner( + env={ + "TUTOR_ROOT": root, + "TUTOR_IGNORE_ENTRYPOINT_PLUGINS": "1", + "TUTOR_IGNORE_DICT_PLUGINS": "1", + } + ) + return runner.invoke(cli, args, obj=TestContext(root)) diff --git a/tests/commands/test_cli.py b/tests/commands/test_cli.py index 4c2f7d41b8d..65e65254558 100644 --- a/tests/commands/test_cli.py +++ b/tests/commands/test_cli.py @@ -1,27 +1,23 @@ import unittest -from click.testing import CliRunner - from tutor.__about__ import __version__ -from tutor.commands.cli import cli, print_help + +from .base import TestCommandMixin -class CliTests(unittest.TestCase): +class CliTests(unittest.TestCase, TestCommandMixin): def test_help(self) -> None: - runner = CliRunner() - result = runner.invoke(print_help) + result = self.invoke(["help"]) self.assertEqual(0, result.exit_code) self.assertIsNone(result.exception) def test_cli_help(self) -> None: - runner = CliRunner() - result = runner.invoke(cli, ["--help"]) + result = self.invoke(["--help"]) self.assertEqual(0, result.exit_code) self.assertIsNone(result.exception) def test_cli_version(self) -> None: - runner = CliRunner() - result = runner.invoke(cli, ["--version"]) + result = self.invoke(["--version"]) self.assertEqual(0, result.exit_code) self.assertIsNone(result.exception) - self.assertRegex(result.output, r"cli, version {}\n".format(__version__)) + self.assertRegex(result.output, rf"cli, version {__version__}\n") diff --git a/tests/commands/test_config.py b/tests/commands/test_config.py index 319f5c89af3..f044e116f9a 100644 --- a/tests/commands/test_config.py +++ b/tests/commands/test_config.py @@ -2,115 +2,88 @@ import tempfile import unittest -from click.testing import CliRunner - -from tests.helpers import TestContext, temporary_root +from tests.helpers import temporary_root from tutor import config as tutor_config -from tutor.commands.config import config_command + +from .base import TestCommandMixin -class ConfigTests(unittest.TestCase): +class ConfigTests(unittest.TestCase, TestCommandMixin): def test_config_help(self) -> None: - runner = CliRunner() - result = runner.invoke(config_command, ["--help"]) + result = self.invoke(["config", "--help"]) self.assertEqual(0, result.exit_code) self.assertFalse(result.exception) def test_config_save(self) -> None: - with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke(config_command, ["save"], obj=context) - self.assertEqual(0, result.exit_code) - self.assertFalse(result.exception) + result = self.invoke(["config", "save"]) + self.assertFalse(result.exception) + self.assertEqual(0, result.exit_code) def test_config_save_interactive(self) -> None: - with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke(config_command, ["save", "-i"], obj=context) - self.assertEqual(0, result.exit_code) - self.assertFalse(result.exception) + result = self.invoke(["config", "save", "-i"]) + self.assertFalse(result.exception) + self.assertEqual(0, result.exit_code) def test_config_save_skip_update(self) -> None: - with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke(config_command, ["save", "-e"], obj=context) - self.assertEqual(0, result.exit_code) - self.assertFalse(result.exception) + result = self.invoke(["config", "save", "-e"]) + self.assertFalse(result.exception) + self.assertEqual(0, result.exit_code) def test_config_save_set_value(self) -> None: with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke( - config_command, ["save", "-s", "key=value"], obj=context - ) - self.assertEqual(0, result.exit_code) - self.assertFalse(result.exception) - result = runner.invoke(config_command, ["printvalue", "key"], obj=context) - self.assertIn("value", result.output) + result1 = self.invoke_in_root(root, ["config", "save", "-s", "key=value"]) + result2 = self.invoke_in_root(root, ["config", "printvalue", "key"]) + self.assertFalse(result1.exception) + self.assertEqual(0, result1.exit_code) + self.assertIn("value", result2.output) def test_config_save_unset_value(self) -> None: with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke(config_command, ["save", "-U", "key"], obj=context) - self.assertEqual(0, result.exit_code) - self.assertFalse(result.exception) - result = runner.invoke(config_command, ["printvalue", "key"], obj=context) - self.assertEqual(1, result.exit_code) + result1 = self.invoke_in_root(root, ["config", "save", "-U", "key"]) + result2 = self.invoke_in_root(root, ["config", "printvalue", "key"]) + self.assertFalse(result1.exception) + self.assertEqual(0, result1.exit_code) + self.assertEqual(1, result2.exit_code) def test_config_printroot(self) -> None: with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke(config_command, ["printroot"], obj=context) - self.assertEqual(0, result.exit_code) - self.assertFalse(result.exception) - self.assertIn(context.root, result.output) + result = self.invoke_in_root(root, ["config", "printroot"]) + self.assertFalse(result.exception) + self.assertEqual(0, result.exit_code) + self.assertIn(root, result.output) def test_config_printvalue(self) -> None: with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - runner.invoke(config_command, ["save"], obj=context) - result = runner.invoke( - config_command, ["printvalue", "MYSQL_ROOT_PASSWORD"], obj=context + self.invoke_in_root(root, ["config", "save"]) + result = self.invoke_in_root( + root, ["config", "printvalue", "MYSQL_ROOT_PASSWORD"] ) - self.assertEqual(0, result.exit_code) - self.assertFalse(result.exception) - self.assertTrue(result.output) + self.assertFalse(result.exception) + self.assertEqual(0, result.exit_code) + self.assertTrue(result.output) def test_config_render(self) -> None: with tempfile.TemporaryDirectory() as dest: with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - runner.invoke(config_command, ["save"], obj=context) - result = runner.invoke( - config_command, ["render", context.root, dest], obj=context - ) - self.assertEqual(0, result.exit_code) - self.assertFalse(result.exception) + self.invoke_in_root(root, ["config", "save"]) + result = self.invoke_in_root(root, ["config", "render", root, dest]) + self.assertEqual(0, result.exit_code) + self.assertFalse(result.exception) def test_config_render_with_extra_configs(self) -> None: with tempfile.TemporaryDirectory() as dest: with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - runner.invoke(config_command, ["save"], obj=context) - result = runner.invoke( - config_command, + self.invoke_in_root(root, ["config", "save"]) + result = self.invoke_in_root( + root, [ + "config", "render", "-x", - os.path.join(context.root, tutor_config.CONFIG_FILENAME), - context.root, + os.path.join(root, tutor_config.CONFIG_FILENAME), + root, dest, ], - obj=context, ) - self.assertEqual(0, result.exit_code) - self.assertFalse(result.exception) + self.assertEqual(0, result.exit_code) + self.assertFalse(result.exception) diff --git a/tests/commands/test_dev.py b/tests/commands/test_dev.py index 7a7205a1c42..2f911f584f6 100644 --- a/tests/commands/test_dev.py +++ b/tests/commands/test_dev.py @@ -1,20 +1,15 @@ import unittest -from click.testing import CliRunner +from .base import TestCommandMixin -from tutor.commands.compose import bindmount_command -from tutor.commands.dev import dev - -class DevTests(unittest.TestCase): +class DevTests(unittest.TestCase, TestCommandMixin): def test_dev_help(self) -> None: - runner = CliRunner() - result = runner.invoke(dev, ["--help"]) + result = self.invoke(["dev", "--help"]) self.assertEqual(0, result.exit_code) self.assertIsNone(result.exception) def test_dev_bindmount(self) -> None: - runner = CliRunner() - result = runner.invoke(bindmount_command, ["--help"]) + result = self.invoke(["dev", "bindmount", "--help"]) self.assertEqual(0, result.exit_code) self.assertIsNone(result.exception) diff --git a/tests/commands/test_images.py b/tests/commands/test_images.py index 7dc9fdca4fb..1013174dd7c 100644 --- a/tests/commands/test_images.py +++ b/tests/commands/test_images.py @@ -1,177 +1,145 @@ -import unittest from unittest.mock import Mock, patch -from click.testing import CliRunner - -from tests.helpers import TestContext, temporary_root -from tutor.__about__ import __version__ +from tests.helpers import PluginsTestCase, temporary_root from tutor import images, plugins -from tutor.commands.config import config_command -from tutor.commands.images import ImageNotFoundError, images_command +from tutor.__about__ import __version__ +from tutor.commands.images import ImageNotFoundError + +from .base import TestCommandMixin -class ImagesTests(unittest.TestCase): +class ImagesTests(PluginsTestCase, TestCommandMixin): def test_images_help(self) -> None: - runner = CliRunner() - result = runner.invoke(images_command, ["--help"]) - self.assertEqual(0, result.exit_code) + result = self.invoke(["images", "--help"]) self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) def test_images_pull_image(self) -> None: - with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke(images_command, ["pull"], obj=context) - self.assertEqual(0, result.exit_code) - self.assertIsNone(result.exception) + result = self.invoke(["images", "pull"]) + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) def test_images_pull_plugin_invalid_plugin_should_throw_error(self) -> None: - with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke(images_command, ["pull", "plugin"], obj=context) - self.assertEqual(1, result.exit_code) - self.assertEqual(ImageNotFoundError, type(result.exception)) + result = self.invoke(["images", "pull", "plugin"]) + self.assertEqual(1, result.exit_code) + self.assertEqual(ImageNotFoundError, type(result.exception)) - @patch.object(plugins.BasePlugin, "iter_installed", return_value=[]) - @patch.object( - plugins.Plugins, - "iter_hooks", - return_value=[ - ( - "dev-plugins", - {"plugin": "plugin:dev-1.0.0", "plugin2": "plugin2:dev-1.0.0"}, - ) - ], - ) @patch.object(images, "pull", return_value=None) - def test_images_pull_plugin( - self, _image_pull: Mock, iter_hooks: Mock, iter_installed: Mock - ) -> None: - with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke(images_command, ["pull", "plugin"], obj=context) - self.assertEqual(0, result.exit_code) - self.assertIsNone(result.exception) - iter_hooks.assert_called_once_with("remote-image") - _image_pull.assert_called_once_with("plugin:dev-1.0.0") - iter_installed.assert_called() + def test_images_pull_plugin(self, image_pull: Mock) -> None: + plugins.v0.DictPlugin( + { + "name": "plugin1", + "hooks": { + "remote-image": { + "service1": "service1:1.0.0", + "service2": "service2:2.0.0", + } + }, + } + ) + plugins.enable("plugin1") + result = self.invoke(["images", "pull", "service1"]) + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + image_pull.assert_called_once_with("service1:1.0.0") + + @patch.object(images, "pull", return_value=None) + def test_images_pull_all_vendor_images(self, image_pull: Mock) -> None: + result = self.invoke(["images", "pull", "mysql"]) + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + # Note: we should update this tag whenever the mysql image is updated + image_pull.assert_called_once_with("docker.io/mysql:5.7.35") def test_images_printtag_image(self) -> None: - with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke(images_command, ["printtag", "openedx"], obj=context) - self.assertEqual(0, result.exit_code) - self.assertIsNone(result.exception) - self.assertRegex( - result.output, r"docker.io/overhangio/openedx:{}\n".format(__version__) - ) + result = self.invoke(["images", "printtag", "openedx"]) + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + self.assertRegex( + result.output, rf"docker.io/overhangio/openedx:{__version__}\n" + ) - @patch.object(plugins.BasePlugin, "iter_installed", return_value=[]) - @patch.object( - plugins.Plugins, - "iter_hooks", - return_value=[ - ( - "dev-plugins", - {"plugin": "plugin:dev-1.0.0", "plugin2": "plugin2:dev-1.0.0"}, - ) - ], - ) - def test_images_printtag_plugin( - self, iter_hooks: Mock, iter_installed: Mock - ) -> None: - with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke(images_command, ["printtag", "plugin"], obj=context) - self.assertEqual(0, result.exit_code) - self.assertIsNone(result.exception) - iter_hooks.assert_called_once_with("build-image") - iter_installed.assert_called() - self.assertEqual(result.output, "plugin:dev-1.0.0\n") + def test_images_printtag_plugin(self) -> None: + plugins.v0.DictPlugin( + { + "name": "plugin1", + "hooks": { + "build-image": { + "service1": "service1:1.0.0", + "service2": "service2:2.0.0", + } + }, + } + ) + plugins.enable("plugin1") + result = self.invoke(["images", "printtag", "service1"]) + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code, result) + self.assertEqual(result.output, "service1:1.0.0\n") - @patch.object(plugins.BasePlugin, "iter_installed", return_value=[]) - @patch.object( - plugins.Plugins, - "iter_hooks", - return_value=[ - ( - "dev-plugins", - {"plugin": "plugin:dev-1.0.0", "plugin2": "plugin2:dev-1.0.0"}, - ) - ], - ) @patch.object(images, "build", return_value=None) - def test_images_build_plugin( - self, image_build: Mock, iter_hooks: Mock, iter_installed: Mock - ) -> None: + def test_images_build_plugin(self, mock_image_build: Mock) -> None: + plugins.v0.DictPlugin( + { + "name": "plugin1", + "hooks": { + "build-image": { + "service1": "service1:1.0.0", + "service2": "service2:2.0.0", + } + }, + } + ) + plugins.enable("plugin1") with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - runner.invoke(config_command, ["save"], obj=context) - result = runner.invoke(images_command, ["build", "plugin"], obj=context) - self.assertIsNone(result.exception) - self.assertEqual(0, result.exit_code) - image_build.assert_called() - iter_hooks.assert_called_once_with("build-image") - iter_installed.assert_called() - self.assertIn("plugin:dev-1.0.0", image_build.call_args[0]) + self.invoke_in_root(root, ["config", "save"]) + result = self.invoke_in_root(root, ["images", "build", "service1"]) + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + mock_image_build.assert_called() + self.assertIn("service1:1.0.0", mock_image_build.call_args[0]) - @patch.object(plugins.BasePlugin, "iter_installed", return_value=[]) - @patch.object( - plugins.Plugins, - "iter_hooks", - return_value=[ - ( - "dev-plugins", - {"plugin": "plugin:dev-1.0.0", "plugin2": "plugin2:dev-1.0.0"}, - ) - ], - ) @patch.object(images, "build", return_value=None) - def test_images_build_plugin_with_args( - self, image_build: Mock, iter_hooks: Mock, iter_installed: Mock - ) -> None: + def test_images_build_plugin_with_args(self, image_build: Mock) -> None: + plugins.v0.DictPlugin( + { + "name": "plugin1", + "hooks": { + "build-image": { + "service1": "service1:1.0.0", + "service2": "service2:2.0.0", + } + }, + } + ) + plugins.enable("plugin1") + build_args = [ + "images", + "build", + "--no-cache", + "-a", + "myarg=value", + "--add-host", + "host", + "--target", + "target", + "-d", + "docker_args", + "service1", + ] with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - runner.invoke(config_command, ["save"], obj=context) - args = [ - "build", - "--no-cache", - "-a", - "myarg=value", - "--add-host", - "host", - "--target", - "target", - "-d", - "docker_args", - "plugin", - ] - result = runner.invoke( - images_command, - args, - obj=context, - ) - self.assertEqual(0, result.exit_code) - self.assertIsNone(result.exception) - iter_hooks.assert_called_once_with("build-image") - iter_installed.assert_called() - image_build.assert_called() - self.assertIn("plugin:dev-1.0.0", image_build.call_args[0]) - for arg in image_build.call_args[0][2:]: - if arg == "--build-arg": - continue - self.assertIn(arg, args) + self.invoke_in_root(root, ["config", "save"]) + result = self.invoke_in_root(root, build_args) + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + image_build.assert_called() + self.assertIn("service1:1.0.0", image_build.call_args[0]) + for arg in image_build.call_args[0][2:]: + # The only extra args are `--build-arg` + if arg != "--build-arg": + self.assertIn(arg, build_args) def test_images_push(self) -> None: - with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke(images_command, ["push"], obj=context) - self.assertEqual(0, result.exit_code) - self.assertIsNone(result.exception) + result = self.invoke(["images", "push"]) + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) diff --git a/tests/commands/test_k8s.py b/tests/commands/test_k8s.py index 4a9a730f67a..f8513fd49eb 100644 --- a/tests/commands/test_k8s.py +++ b/tests/commands/test_k8s.py @@ -1,13 +1,10 @@ import unittest -from click.testing import CliRunner +from .base import TestCommandMixin -from tutor.commands.k8s import k8s - -class K8sTests(unittest.TestCase): +class K8sTests(unittest.TestCase, TestCommandMixin): def test_k8s_help(self) -> None: - runner = CliRunner() - result = runner.invoke(k8s, ["--help"]) - self.assertEqual(0, result.exit_code) + result = self.invoke(["k8s", "--help"]) self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) diff --git a/tests/commands/test_local.py b/tests/commands/test_local.py index 88d930d2fac..bae0edfaccf 100644 --- a/tests/commands/test_local.py +++ b/tests/commands/test_local.py @@ -1,25 +1,20 @@ import unittest -from click.testing import CliRunner +from .base import TestCommandMixin -from tutor.commands.local import local, quickstart, upgrade - -class LocalTests(unittest.TestCase): +class LocalTests(unittest.TestCase, TestCommandMixin): def test_local_help(self) -> None: - runner = CliRunner() - result = runner.invoke(local, ["--help"]) - self.assertEqual(0, result.exit_code) + result = self.invoke(["local", "--help"]) self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) def test_local_quickstart_help(self) -> None: - runner = CliRunner() - result = runner.invoke(quickstart, ["--help"]) - self.assertEqual(0, result.exit_code) + result = self.invoke(["local", "quickstart", "--help"]) self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) def test_local_upgrade_help(self) -> None: - runner = CliRunner() - result = runner.invoke(upgrade, ["--help"]) - self.assertEqual(0, result.exit_code) + result = self.invoke(["local", "upgrade", "--help"]) self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) diff --git a/tests/commands/test_plugins.py b/tests/commands/test_plugins.py index f6a27b0cb46..81b9d3a5b29 100644 --- a/tests/commands/test_plugins.py +++ b/tests/commands/test_plugins.py @@ -1,64 +1,42 @@ import unittest from unittest.mock import Mock, patch -from click.testing import CliRunner - -from tests.helpers import TestContext, temporary_root from tutor import plugins -from tutor.commands.plugins import plugins_command + +from .base import TestCommandMixin -class PluginsTests(unittest.TestCase): +class PluginsTests(unittest.TestCase, TestCommandMixin): def test_plugins_help(self) -> None: - runner = CliRunner() - result = runner.invoke(plugins_command, ["--help"]) - self.assertEqual(0, result.exit_code) + result = self.invoke(["plugins", "--help"]) self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) def test_plugins_printroot(self) -> None: - with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke(plugins_command, ["printroot"], obj=context) - self.assertEqual(0, result.exit_code) - self.assertIsNone(result.exception) - self.assertTrue(result.output) + result = self.invoke(["plugins", "printroot"]) + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + self.assertTrue(result.output) - @patch.object(plugins.BasePlugin, "iter_installed", return_value=[]) - def test_plugins_list(self, _iter_installed: Mock) -> None: - with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke(plugins_command, ["list"], obj=context) - self.assertEqual(0, result.exit_code) - self.assertIsNone(result.exception) - self.assertFalse(result.output) - _iter_installed.assert_called() + @patch.object(plugins, "iter_info", return_value=[]) + def test_plugins_list(self, _iter_info: Mock) -> None: + result = self.invoke(["plugins", "list"]) + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + self.assertFalse(result.output) + _iter_info.assert_called() def test_plugins_install_not_found_plugin(self) -> None: - with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke( - plugins_command, ["install", "notFound"], obj=context - ) - self.assertEqual(1, result.exit_code) - self.assertTrue(result.exception) + result = self.invoke(["plugins", "install", "notFound"]) + self.assertEqual(1, result.exit_code) + self.assertTrue(result.exception) def test_plugins_enable_not_installed_plugin(self) -> None: - with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke(plugins_command, ["enable", "notFound"], obj=context) - self.assertEqual(1, result.exit_code) - self.assertTrue(result.exception) + result = self.invoke(["plugins", "enable", "notFound"]) + self.assertEqual(1, result.exit_code) + self.assertTrue(result.exception) def test_plugins_disable_not_installed_plugin(self) -> None: - with temporary_root() as root: - context = TestContext(root) - runner = CliRunner() - result = runner.invoke( - plugins_command, ["disable", "notFound"], obj=context - ) - self.assertEqual(0, result.exit_code) - self.assertFalse(result.exception) + result = self.invoke(["plugins", "disable", "notFound"]) + self.assertEqual(0, result.exit_code) + self.assertFalse(result.exception) diff --git a/tests/helpers.py b/tests/helpers.py index 3762b8bf892..d6a2da8ad1a 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,25 +1,25 @@ import os import tempfile +import typing as t +import unittest +import unittest.result +from tutor import hooks from tutor.commands.context import BaseJobContext from tutor.jobs import BaseJobRunner from tutor.types import Config class TestJobRunner(BaseJobRunner): - def __init__(self, root: str, config: Config): - """ - Mock job runner for unit testing. + """ + Mock job runner for unit testing. - This runner does nothing except print the service name and command, - separated by dashes. - """ - super().__init__(root, config) + This runner does nothing except print the service name and command, + separated by dashes. + """ def run_job(self, service: str, command: str) -> int: - print( - os.linesep.join(["Service: {}".format(service), "-----", command, "----- "]) - ) + print(os.linesep.join([f"Service: {service}", "-----", command, "----- "])) return 0 @@ -43,3 +43,36 @@ class TestContext(BaseJobContext): def job_runner(self, config: Config) -> TestJobRunner: return TestJobRunner(self.root, config) + + +class PluginsTestCase(unittest.TestCase): + """ + This test case class clears the hooks created during tests. It also makes sure that + we don't accidentally load entrypoint/dict plugins from the user. + """ + + def setUp(self) -> None: + self.clean() + self.addCleanup(self.clean) + super().setUp() + + def clean(self) -> None: + # We clear hooks created in some contexts, such that user plugins are never loaded. + for context in [ + hooks.Contexts.PLUGINS, + hooks.Contexts.PLUGINS_V0_ENTRYPOINT, + hooks.Contexts.PLUGINS_V0_YAML, + "unittests", + ]: + hooks.filters.clear_all(context=context) + hooks.actions.clear_all(context=context) + + def run( + self, result: t.Optional[unittest.result.TestResult] = None + ) -> t.Optional[unittest.result.TestResult]: + """ + Run all actions and filters with a test context, such that they can be cleared + from one run to the next. + """ + with hooks.contexts.enter("unittests"): + return super().run(result=result) diff --git a/tests/hooks/__init__.py b/tests/hooks/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/hooks/test_actions.py b/tests/hooks/test_actions.py new file mode 100644 index 00000000000..e02760c8e64 --- /dev/null +++ b/tests/hooks/test_actions.py @@ -0,0 +1,29 @@ +import typing as t +import unittest + +from tutor import hooks + + +class PluginActionsTests(unittest.TestCase): + def setUp(self) -> None: + self.side_effect_int = 0 + + def tearDown(self) -> None: + super().tearDown() + hooks.actions.clear_all(context="tests") + + def run(self, result: t.Any = None) -> t.Any: + with hooks.contexts.enter("tests"): + return super().run(result=result) + + def test_on(self) -> None: + @hooks.actions.on("test-action") + def _test_action_1(increment: int) -> None: + self.side_effect_int += increment + + @hooks.actions.on("test-action") + def _test_action_2(increment: int) -> None: + self.side_effect_int += increment * 2 + + hooks.actions.do("test-action", 1) + self.assertEqual(3, self.side_effect_int) diff --git a/tests/hooks/test_filters.py b/tests/hooks/test_filters.py new file mode 100644 index 00000000000..4dafcc90987 --- /dev/null +++ b/tests/hooks/test_filters.py @@ -0,0 +1,60 @@ +import typing as t +import unittest + +from tutor import hooks + + +class PluginFiltersTests(unittest.TestCase): + def tearDown(self) -> None: + super().tearDown() + hooks.filters.clear_all(context="tests") + + def run(self, result: t.Any = None) -> t.Any: + with hooks.contexts.enter("tests"): + return super().run(result=result) + + def test_add(self) -> None: + @hooks.filters.add("tests:count-sheeps") + def filter1(value: int) -> int: + return value + 1 + + value = hooks.filters.apply("tests:count-sheeps", 0) + self.assertEqual(1, value) + + def test_add_items(self) -> None: + @hooks.filters.add("tests:add-sheeps") + def filter1(sheeps: t.List[int]) -> t.List[int]: + return sheeps + [0] + + hooks.filters.add_item("tests:add-sheeps", 1) + hooks.filters.add_item("tests:add-sheeps", 2) + hooks.filters.add_items("tests:add-sheeps", [3, 4]) + + sheeps: t.List[int] = hooks.filters.apply("tests:add-sheeps", []) + self.assertEqual([0, 1, 2, 3, 4], sheeps) + + def test_filter_class(self) -> None: + filtre = hooks.filters.Filter(lambda _: 1) + self.assertTrue(filtre.is_in_context(None)) + self.assertFalse(filtre.is_in_context("customcontext")) + self.assertEqual(1, filtre.apply(0)) + self.assertEqual(0, filtre.apply(0, context="customcontext")) + + def test_filter_context(self) -> None: + with hooks.contexts.enter("testcontext"): + hooks.filters.add_item("test:sheeps", 1) + hooks.filters.add_item("test:sheeps", 2) + + self.assertEqual([1, 2], hooks.filters.apply("test:sheeps", [])) + self.assertEqual( + [1], hooks.filters.apply("test:sheeps", [], context="testcontext") + ) + + def test_clear_context(self) -> None: + with hooks.contexts.enter("testcontext"): + hooks.filters.add_item("test:sheeps", 1) + hooks.filters.add_item("test:sheeps", 2) + + self.assertEqual([1, 2], hooks.filters.apply("test:sheeps", [])) + hooks.filters.clear("test:sheeps", context="testcontext") + self.assertEqual([2], hooks.filters.apply("test:sheeps", [])) diff --git a/tests/test_config.py b/tests/test_config.py index b3d5b1823ac..38a3ea8799a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,15 +5,15 @@ import click -from tests.helpers import temporary_root +from tests.helpers import PluginsTestCase, temporary_root from tutor import config as tutor_config -from tutor import interactive +from tutor import hooks, interactive from tutor.types import Config, get_typed class ConfigTests(unittest.TestCase): def test_version(self) -> None: - defaults = tutor_config.get_defaults({}) + defaults = tutor_config.get_defaults() self.assertNotIn("TUTOR_VERSION", defaults) def test_merge(self) -> None: @@ -24,7 +24,7 @@ def test_merge(self) -> None: def test_merge_not_render(self) -> None: config: Config = {} - base = tutor_config.get_base({}) + base = tutor_config.get_base() with patch.object(tutor_config.utils, "random_string", return_value="abcd"): tutor_config.merge(config, base) @@ -40,26 +40,6 @@ def test_update_twice_should_return_same_config(self, _: Mock) -> None: self.assertEqual(config1, config2) - @patch.object(tutor_config.fmt, "echo") - def test_removed_entry_is_added_on_save(self, _: Mock) -> None: - with temporary_root() as root: - with patch.object( - tutor_config.utils, "random_string" - ) as mock_random_string: - mock_random_string.return_value = "abcd" - config1 = tutor_config.load_full(root) - password1 = config1["MYSQL_ROOT_PASSWORD"] - - config1.pop("MYSQL_ROOT_PASSWORD") - tutor_config.save_config_file(root, config1) - - mock_random_string.return_value = "efgh" - config2 = tutor_config.load_full(root) - password2 = config2["MYSQL_ROOT_PASSWORD"] - - self.assertEqual("abcd", password1) - self.assertEqual("efgh", password2) - def test_interactive(self) -> None: def mock_prompt(*_args: None, **kwargs: str) -> str: return kwargs["default"] @@ -100,3 +80,27 @@ def test_json_config_is_overwritten_by_yaml(self, _: Mock) -> None: self.assertTrue(os.path.exists(config_yml_path)) self.assertFalse(os.path.exists(config_json_path)) self.assertEqual(config, current) + + +class ConfigPluginTestCase(PluginsTestCase): + @patch.object(tutor_config.fmt, "echo") + def test_removed_entry_is_added_on_save(self, _: Mock) -> None: + with temporary_root() as root: + mock_random_string = Mock() + + hooks.filters.add_item( + hooks.Filters.ENV_TEMPLATE_FILTERS, + ("random_string", mock_random_string), + ) + mock_random_string.return_value = "abcd" + config1 = tutor_config.load_full(root) + password1 = config1.pop("MYSQL_ROOT_PASSWORD") + + tutor_config.save_config_file(root, config1) + + mock_random_string.return_value = "efgh" + config2 = tutor_config.load_full(root) + password2 = config2["MYSQL_ROOT_PASSWORD"] + + self.assertEqual("abcd", password1) + self.assertEqual("efgh", password2) diff --git a/tests/test_env.py b/tests/test_env.py index 52e5065ecc5..cf21cdd07db 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -3,14 +3,15 @@ import unittest from unittest.mock import Mock, patch -from tutor.__about__ import __version__ +from tests.helpers import PluginsTestCase, temporary_root from tutor import config as tutor_config -from tutor import env, exceptions, fmt +from tutor import env, exceptions, fmt, plugins +from tutor.__about__ import __version__ +from tutor.plugins.v0 import DictPlugin from tutor.types import Config -from tests.helpers import temporary_root -class EnvTests(unittest.TestCase): +class EnvTests(PluginsTestCase): def test_walk_templates(self) -> None: renderer = env.Renderer({}, [env.TEMPLATES_ROOT]) templates = list(renderer.walk_templates("local")) @@ -103,15 +104,15 @@ def test_save_full_with_https(self) -> None: def test_patch(self) -> None: patches = {"plugin1": "abcd", "plugin2": "efgh"} with patch.object( - env.plugins, "iter_patches", return_value=patches.items() + env.plugins, "iter_patches", return_value=patches.values() ) as mock_iter_patches: rendered = env.render_str({}, '{{ patch("location") }}') - mock_iter_patches.assert_called_once_with({}, "location") + mock_iter_patches.assert_called_once_with("location") self.assertEqual("abcd\nefgh", rendered) def test_patch_separator_suffix(self) -> None: patches = {"plugin1": "abcd", "plugin2": "efgh"} - with patch.object(env.plugins, "iter_patches", return_value=patches.items()): + with patch.object(env.plugins, "iter_patches", return_value=patches.values()): rendered = env.render_str( {}, '{{ patch("location", separator=",\n", suffix=",") }}' ) @@ -119,11 +120,9 @@ def test_patch_separator_suffix(self) -> None: def test_plugin_templates(self) -> None: with tempfile.TemporaryDirectory() as plugin_templates: - # Create plugin - plugin1 = env.plugins.DictPlugin( + DictPlugin( {"name": "plugin1", "version": "0", "templates": plugin_templates} ) - # Create two templates os.makedirs(os.path.join(plugin_templates, "plugin1", "apps")) with open( @@ -139,36 +138,37 @@ def test_plugin_templates(self) -> None: ) as f: f.write("Hello my ID is {{ ID }}") - # Create configuration - config: Config = {"ID": "abcd"} - # Render templates - with patch.object( - env.plugins, - "iter_enabled", - return_value=[plugin1], - ): - with temporary_root() as root: - # Render plugin templates - env.save_plugin_templates(plugin1, root, config) - - # Check that plugin template was rendered - dst_unrendered = os.path.join( - root, "env", "plugins", "plugin1", "unrendered.txt" - ) - dst_rendered = os.path.join( - root, "env", "plugins", "plugin1", "apps", "rendered.txt" - ) - self.assertFalse(os.path.exists(dst_unrendered)) - self.assertTrue(os.path.exists(dst_rendered)) - with open(dst_rendered, encoding="utf-8") as f: - self.assertEqual("Hello my ID is abcd", f.read()) + with temporary_root() as root: + # Create configuration + config: Config = tutor_config.load_full(root) + config["ID"] = "Hector Rumblethorpe" + plugins.enable("plugin1") + tutor_config.save_enabled_plugins(config) + + # Render environment + with patch.object(fmt, "STDOUT"): + env.save(root, config) + + # Check that plugin template was rendered + root_env = os.path.join(root, "env") + dst_unrendered = os.path.join( + root_env, "plugins", "plugin1", "unrendered.txt" + ) + dst_rendered = os.path.join( + root_env, "plugins", "plugin1", "apps", "rendered.txt" + ) + self.assertFalse(os.path.exists(dst_unrendered)) + self.assertTrue(os.path.exists(dst_rendered)) + with open(dst_rendered, encoding="utf-8") as f: + self.assertEqual("Hello my ID is Hector Rumblethorpe", f.read()) def test_renderer_is_reset_on_config_change(self) -> None: with tempfile.TemporaryDirectory() as plugin_templates: - plugin1 = env.plugins.DictPlugin( + plugin1 = DictPlugin( {"name": "plugin1", "version": "0", "templates": plugin_templates} ) + # Create one template os.makedirs(os.path.join(plugin_templates, plugin1.name)) with open( @@ -182,14 +182,12 @@ def test_renderer_is_reset_on_config_change(self) -> None: config: Config = {"PLUGINS": []} env1 = env.Renderer.instance(config).environment - with patch.object( - env.plugins, - "iter_enabled", - return_value=[plugin1], - ): - # Load env a second time - config["PLUGINS"] = ["myplugin"] - env2 = env.Renderer.instance(config).environment + # Enable plugins + plugins.enable("plugin1") + + # Load env a second time + config["PLUGINS"] = ["myplugin"] + env2 = env.Renderer.instance(config).environment self.assertNotIn("plugin1/myplugin.txt", env1.loader.list_templates()) self.assertIn("plugin1/myplugin.txt", env2.loader.list_templates()) diff --git a/tests/test_plugins.py b/tests/test_plugins.py deleted file mode 100644 index 8bb9ba3f3c5..00000000000 --- a/tests/test_plugins.py +++ /dev/null @@ -1,244 +0,0 @@ -import unittest -from unittest.mock import Mock, patch - -from tutor import config as tutor_config -from tutor import exceptions, fmt, plugins -from tutor.types import Config, get_typed - - -class PluginsTests(unittest.TestCase): - def setUp(self) -> None: - plugins.Plugins.clear_cache() - - @patch.object(plugins.DictPlugin, "iter_installed", return_value=[]) - def test_iter_installed(self, dict_plugin_iter_installed: Mock) -> None: - with patch.object(plugins.pkg_resources, "iter_entry_points", return_value=[]): # type: ignore - self.assertEqual([], list(plugins.iter_installed())) - dict_plugin_iter_installed.assert_called_once() - - def test_is_installed(self) -> None: - self.assertFalse(plugins.is_installed("dummy")) - - @patch.object(plugins.DictPlugin, "iter_installed", return_value=[]) - def test_official_plugins(self, dict_plugin_iter_installed: Mock) -> None: - with patch.object(plugins.importlib, "import_module", return_value=42): # type: ignore - plugin1 = plugins.OfficialPlugin.load("plugin1") - with patch.object(plugins.importlib, "import_module", return_value=43): # type: ignore - plugin2 = plugins.OfficialPlugin.load("plugin2") - with patch.object( - plugins.EntrypointPlugin, - "iter_installed", - return_value=[plugin1], - ): - self.assertEqual( - [plugin1, plugin2], - list(plugins.iter_installed()), - ) - dict_plugin_iter_installed.assert_called_once() - - def test_enable(self) -> None: - config: Config = {plugins.CONFIG_KEY: []} - with patch.object(plugins, "is_installed", return_value=True): - plugins.enable(config, "plugin2") - plugins.enable(config, "plugin1") - self.assertEqual(["plugin1", "plugin2"], config[plugins.CONFIG_KEY]) - - def test_enable_twice(self) -> None: - config: Config = {plugins.CONFIG_KEY: []} - with patch.object(plugins, "is_installed", return_value=True): - plugins.enable(config, "plugin1") - plugins.enable(config, "plugin1") - self.assertEqual(["plugin1"], config[plugins.CONFIG_KEY]) - - def test_enable_not_installed_plugin(self) -> None: - config: Config = {"PLUGINS": []} - with patch.object(plugins, "is_installed", return_value=False): - self.assertRaises(exceptions.TutorError, plugins.enable, config, "plugin1") - - @patch.object( - plugins.Plugins, - "iter_installed", - return_value=[ - plugins.DictPlugin( - { - "name": "plugin1", - "version": "1.0.0", - "config": {"set": {"KEY": "value"}}, - } - ), - plugins.DictPlugin( - { - "name": "plugin2", - "version": "1.0.0", - } - ), - ], - ) - def test_disable(self, _iter_installed_mock: Mock) -> None: - config: Config = {"PLUGINS": ["plugin1", "plugin2"]} - with patch.object(fmt, "STDOUT"): - plugin = plugins.get_enabled(config, "plugin1") - plugins.disable(config, plugin) - self.assertEqual(["plugin2"], config["PLUGINS"]) - - @patch.object( - plugins.Plugins, - "iter_installed", - return_value=[ - plugins.DictPlugin( - { - "name": "plugin1", - "version": "1.0.0", - "config": {"set": {"KEY": "value"}}, - } - ), - ], - ) - def test_disable_removes_set_config(self, _iter_installed_mock: Mock) -> None: - config: Config = {"PLUGINS": ["plugin1"], "KEY": "value"} - plugin = plugins.get_enabled(config, "plugin1") - with patch.object(fmt, "STDOUT"): - plugins.disable(config, plugin) - self.assertEqual([], config["PLUGINS"]) - self.assertNotIn("KEY", config) - - def test_none_plugins(self) -> None: - config: Config = {plugins.CONFIG_KEY: None} - self.assertFalse(plugins.is_enabled(config, "myplugin")) - - def test_patches(self) -> None: - class plugin1: - patches = {"patch1": "Hello {{ ID }}"} - - with patch.object( - plugins.Plugins, - "iter_enabled", - return_value=[plugins.BasePlugin("plugin1", plugin1)], - ): - patches = list(plugins.iter_patches({}, "patch1")) - self.assertEqual([("plugin1", "Hello {{ ID }}")], patches) - - def test_plugin_without_patches(self) -> None: - with patch.object( - plugins.Plugins, - "iter_enabled", - return_value=[plugins.BasePlugin("plugin1", None)], - ): - patches = list(plugins.iter_patches({}, "patch1")) - self.assertEqual([], patches) - - def test_configure(self) -> None: - class plugin1: - config: Config = { - "add": {"PARAM1": "value1", "PARAM2": "value2"}, - "set": {"PARAM3": "value3"}, - "defaults": {"PARAM4": "value4"}, - } - - with patch.object( - plugins.Plugins, - "iter_enabled", - return_value=[plugins.BasePlugin("plugin1", plugin1)], - ): - base = tutor_config.get_base({}) - defaults = tutor_config.get_defaults({}) - - self.assertEqual(base["PARAM3"], "value3") - self.assertEqual(base["PLUGIN1_PARAM1"], "value1") - self.assertEqual(base["PLUGIN1_PARAM2"], "value2") - self.assertEqual(defaults["PLUGIN1_PARAM4"], "value4") - - def test_configure_set_does_not_override(self) -> None: - config: Config = {"ID1": "oldid"} - - class plugin1: - config: Config = {"set": {"ID1": "newid", "ID2": "id2"}} - - with patch.object( - plugins.Plugins, - "iter_enabled", - return_value=[plugins.BasePlugin("plugin1", plugin1)], - ): - tutor_config.update_with_base(config) - - self.assertEqual("oldid", config["ID1"]) - self.assertEqual("id2", config["ID2"]) - - def test_configure_set_random_string(self) -> None: - class plugin1: - config: Config = {"set": {"PARAM1": "{{ 128|random_string }}"}} - - with patch.object( - plugins.Plugins, - "iter_enabled", - return_value=[plugins.BasePlugin("plugin1", plugin1)], - ): - config = tutor_config.get_base({}) - tutor_config.render_full(config) - - self.assertEqual(128, len(get_typed(config, "PARAM1", str))) - - def test_configure_default_value_with_previous_definition(self) -> None: - config: Config = {"PARAM1": "value"} - - class plugin1: - config: Config = {"defaults": {"PARAM2": "{{ PARAM1 }}"}} - - with patch.object( - plugins.Plugins, - "iter_enabled", - return_value=[plugins.BasePlugin("plugin1", plugin1)], - ): - tutor_config.update_with_defaults(config) - self.assertEqual("{{ PARAM1 }}", config["PLUGIN1_PARAM2"]) - - def test_config_load_from_plugins(self) -> None: - config: Config = {} - - class plugin1: - config: Config = {"add": {"PARAM1": "{{ 10|random_string }}"}} - - with patch.object( - plugins.Plugins, - "iter_enabled", - return_value=[plugins.BasePlugin("plugin1", plugin1)], - ): - tutor_config.update_with_base(config) - tutor_config.update_with_defaults(config) - tutor_config.render_full(config) - value1 = get_typed(config, "PLUGIN1_PARAM1", str) - - self.assertEqual(10, len(value1)) - - def test_hooks(self) -> None: - class plugin1: - hooks = {"init": ["myclient"]} - - with patch.object( - plugins.Plugins, - "iter_enabled", - return_value=[plugins.BasePlugin("plugin1", plugin1)], - ): - self.assertEqual( - [("plugin1", ["myclient"])], list(plugins.iter_hooks({}, "init")) - ) - - def test_plugins_are_updated_on_config_change(self) -> None: - config: Config = {"PLUGINS": []} - plugins1 = plugins.Plugins(config) - self.assertEqual(0, len(list(plugins1.iter_enabled()))) - config["PLUGINS"] = ["plugin1"] - with patch.object( - plugins.Plugins, - "iter_installed", - return_value=[plugins.BasePlugin("plugin1", None)], - ): - plugins2 = plugins.Plugins(config) - self.assertEqual(1, len(list(plugins2.iter_enabled()))) - - def test_dict_plugin(self) -> None: - plugin = plugins.DictPlugin( - {"name": "myplugin", "config": {"set": {"KEY": "value"}}, "version": "0.1"} - ) - self.assertEqual("myplugin", plugin.name) - self.assertEqual({"KEY": "value"}, plugin.config_set) diff --git a/tests/test_plugins_v0.py b/tests/test_plugins_v0.py new file mode 100644 index 00000000000..a21aa042825 --- /dev/null +++ b/tests/test_plugins_v0.py @@ -0,0 +1,222 @@ +import typing as t +from unittest.mock import patch + +from tests.helpers import PluginsTestCase, temporary_root +from tutor import config as tutor_config +from tutor import exceptions, fmt, hooks, plugins +from tutor.plugins import v0 as plugins_v0 +from tutor.types import Config, get_typed + + +class PluginsTests(PluginsTestCase): + def test_iter_installed(self) -> None: + self.assertEqual([], list(plugins.iter_installed())) + + def test_is_installed(self) -> None: + self.assertFalse(plugins.is_installed("dummy")) + + def test_official_plugins(self) -> None: + # Create 2 official plugins + plugins_v0.OfficialPlugin("plugin1") + plugins_v0.OfficialPlugin("plugin2") + self.assertEqual( + ["plugin1", "plugin2"], + list(plugins.iter_installed()), + ) + + def test_enable(self) -> None: + config: Config = {tutor_config.PLUGINS_CONFIG_KEY: []} + plugins_v0.DictPlugin({"name": "plugin1"}) + plugins_v0.DictPlugin({"name": "plugin2"}) + plugins.enable("plugin2") + plugins.enable("plugin1") + tutor_config.save_enabled_plugins(config) + self.assertEqual( + ["plugin1", "plugin2"], config[tutor_config.PLUGINS_CONFIG_KEY] + ) + + def test_enable_twice(self) -> None: + plugins_v0.DictPlugin({"name": "plugin1"}) + plugins.enable("plugin1") + plugins.enable("plugin1") + config: Config = {tutor_config.PLUGINS_CONFIG_KEY: []} + tutor_config.save_enabled_plugins(config) + self.assertEqual(["plugin1"], config[tutor_config.PLUGINS_CONFIG_KEY]) + + def test_enable_not_installed_plugin(self) -> None: + self.assertRaises(exceptions.TutorError, plugins.enable, "plugin1") + + def test_disable(self) -> None: + plugins_v0.DictPlugin( + { + "name": "plugin1", + "version": "1.0.0", + "config": {"set": {"KEY": "value"}}, + } + ) + plugins_v0.DictPlugin( + { + "name": "plugin2", + "version": "1.0.0", + } + ) + config: Config = {"PLUGINS": ["plugin1", "plugin2"]} + tutor_config.enable_plugins(config) + with patch.object(fmt, "STDOUT"): + tutor_config.disable_plugin(config, "plugin1") + self.assertEqual(["plugin2"], config["PLUGINS"]) + + def test_disable_removes_set_config(self) -> None: + plugins_v0.DictPlugin( + { + "name": "plugin1", + "version": "1.0.0", + "config": {"set": {"KEY": "value"}}, + } + ) + config: Config = {"PLUGINS": ["plugin1"], "KEY": "value"} + tutor_config.enable_plugins(config) + with patch.object(fmt, "STDOUT"): + tutor_config.disable_plugin(config, "plugin1") + self.assertEqual([], config["PLUGINS"]) + self.assertNotIn("KEY", config) + + def test_patches(self) -> None: + plugins_v0.DictPlugin( + {"name": "plugin1", "patches": {"patch1": "Hello {{ ID }}"}} + ) + plugins.enable("plugin1") + patches = list(plugins.iter_patches("patch1")) + self.assertEqual(["Hello {{ ID }}"], patches) + + def test_plugin_without_patches(self) -> None: + plugins_v0.DictPlugin({"name": "plugin1"}) + plugins.enable("plugin1") + patches = list(plugins.iter_patches("patch1")) + self.assertEqual([], patches) + + def test_configure(self) -> None: + plugins_v0.DictPlugin( + { + "name": "plugin1", + "config": { + "add": {"PARAM1": "value1", "PARAM2": "value2"}, + "set": {"PARAM3": "value3"}, + "defaults": {"PARAM4": "value4"}, + }, + } + ) + plugins.enable("plugin1") + + base = tutor_config.get_base() + defaults = tutor_config.get_defaults() + + self.assertEqual(base["PARAM3"], "value3") + self.assertEqual(base["PLUGIN1_PARAM1"], "value1") + self.assertEqual(base["PLUGIN1_PARAM2"], "value2") + self.assertEqual(defaults["PLUGIN1_PARAM4"], "value4") + + def test_configure_set_does_not_override(self) -> None: + config: Config = {"ID1": "oldid"} + + plugins_v0.DictPlugin( + {"name": "plugin1", "config": {"set": {"ID1": "newid", "ID2": "id2"}}} + ) + plugins.enable("plugin1") + tutor_config.update_with_base(config) + + self.assertEqual("oldid", config["ID1"]) + self.assertEqual("id2", config["ID2"]) + + def test_configure_set_random_string(self) -> None: + plugins_v0.DictPlugin( + { + "name": "plugin1", + "config": {"set": {"PARAM1": "{{ 128|random_string }}"}}, + } + ) + plugins.enable("plugin1") + config = tutor_config.get_base() + tutor_config.render_full(config) + + self.assertEqual(128, len(get_typed(config, "PARAM1", str))) + + def test_configure_default_value_with_previous_definition(self) -> None: + config: Config = {"PARAM1": "value"} + plugins_v0.DictPlugin( + {"name": "plugin1", "config": {"defaults": {"PARAM2": "{{ PARAM1 }}"}}} + ) + plugins.enable("plugin1") + tutor_config.update_with_defaults(config) + self.assertEqual("{{ PARAM1 }}", config["PLUGIN1_PARAM2"]) + + def test_config_load_from_plugins(self) -> None: + config: Config = {} + + plugins_v0.DictPlugin( + {"name": "plugin1", "config": {"add": {"PARAM1": "{{ 10|random_string }}"}}} + ) + plugins.enable("plugin1") + + tutor_config.update_with_base(config) + tutor_config.update_with_defaults(config) + tutor_config.render_full(config) + value1 = get_typed(config, "PLUGIN1_PARAM1", str) + + self.assertEqual(10, len(value1)) + + def test_init_tasks(self) -> None: + plugins_v0.DictPlugin({"name": "plugin1", "hooks": {"init": ["myclient"]}}) + plugins.enable("plugin1") + self.assertIn( + ("myclient", ("plugin1", "hooks", "myclient", "init")), + list(hooks.filters.iterate(hooks.Filters.APP_TASK_INIT)), + ) + + def test_plugins_are_updated_on_config_change(self) -> None: + config: Config = {} + plugins_v0.DictPlugin({"name": "plugin1"}) + tutor_config.enable_plugins(config) + plugins1 = list(plugins.iter_enabled()) + config["PLUGINS"] = ["plugin1"] + tutor_config.enable_plugins(config) + plugins2 = list(plugins.iter_enabled()) + + self.assertEqual([], plugins1) + self.assertEqual(1, len(plugins2)) + + def test_dict_plugin(self) -> None: + plugin = plugins_v0.DictPlugin( + {"name": "myplugin", "config": {"set": {"KEY": "value"}}, "version": "0.1"} + ) + plugins.enable("myplugin") + overriden_items: t.List[t.Tuple[str, t.Any]] = hooks.filters.apply( + hooks.Filters.CONFIG_OVERRIDES, [] + ) + versions = list(plugins.iter_info()) + self.assertEqual("myplugin", plugin.name) + self.assertEqual([("myplugin", "0.1")], versions) + self.assertEqual([("KEY", "value")], overriden_items) + + def test_config_disable_plugin(self) -> None: + plugins_v0.DictPlugin( + {"name": "plugin1", "config": {"set": {"KEY1": "value1"}}} + ) + plugins_v0.DictPlugin( + {"name": "plugin2", "config": {"set": {"KEY2": "value2"}}} + ) + plugins.enable("plugin1") + plugins.enable("plugin2") + + with temporary_root() as root: + config = tutor_config.load_minimal(root) + config_pre = config.copy() + with patch.object(fmt, "STDOUT"): + tutor_config.disable_plugin(config, "plugin1") + config_post = tutor_config.load_minimal(root) + + self.assertEqual("value1", config_pre["KEY1"]) + self.assertEqual("value2", config_pre["KEY2"]) + self.assertNotIn("KEY1", config) + self.assertNotIn("KEY1", config_post) + self.assertEqual("value2", config["KEY2"]) diff --git a/tutor.spec b/tutor.spec index bfd63aaa752..8b0e40f6964 100644 --- a/tutor.spec +++ b/tutor.spec @@ -11,7 +11,11 @@ hidden_imports = [] # Auto-discover plugins and include patches & templates folders for entrypoint in pkg_resources.iter_entry_points("tutor.plugin.v0"): plugin_name = entrypoint.name - plugin = entrypoint.load() + try: + plugin = entrypoint.load() + except Exception as e: + print(f"ERROR Failed to load plugin {plugin_name}: {e}") + continue plugin_root = os.path.dirname(plugin.__file__) plugin_root_module_name = os.path.basename(plugin_root) hidden_imports.append(entrypoint.module_name) diff --git a/tutor/commands/cli.py b/tutor/commands/cli.py index 07f2757eded..21c74ccf153 100755 --- a/tutor/commands/cli.py +++ b/tutor/commands/cli.py @@ -1,39 +1,91 @@ import sys +from typing import List, Optional import appdirs import click -from .. import exceptions, fmt, utils -from ..__about__ import __app__, __version__ -from .config import config_command -from .context import Context -from .dev import dev -from .images import images_command -from .k8s import k8s -from .local import local -from .plugins import add_plugin_commands, plugins_command +from tutor import exceptions, fmt, hooks, utils +from tutor.__about__ import __app__, __version__ +from tutor.commands.config import config_command +from tutor.commands.context import Context +from tutor.commands.dev import dev +from tutor.commands.images import images_command +from tutor.commands.k8s import k8s +from tutor.commands.local import local +from tutor.commands.plugins import plugins_command + +# Everyone on board +hooks.actions.do(hooks.Actions.CORE_READY) def main() -> None: try: - cli.add_command(images_command) - cli.add_command(config_command) - cli.add_command(local) - cli.add_command(dev) - cli.add_command(k8s) - cli.add_command(print_help) - cli.add_command(plugins_command) - add_plugin_commands(cli) cli() # pylint: disable=no-value-for-parameter except KeyboardInterrupt: pass except exceptions.TutorError as e: - fmt.echo_error("Error: {}".format(e.args[0])) + fmt.echo_error(f"Error: {e.args[0]}") sys.exit(1) +class TutorCli(click.MultiCommand): + """ + Dynamically load subcommands at runtime. + + This is necessary to load plugin subcommands, based on the list of enabled + plugins (and thus of config.yml). + Docs: https://click.palletsprojects.com/en/latest/commands/#custom-multi-commands + """ + + IS_ROOT_READY = False + + @classmethod + def get_commands(cls, ctx: click.Context) -> List[click.Command]: + """ + Return the list of subcommands (click.Command). + """ + cls.ensure_plugins_enabled(ctx) + return hooks.filters.apply(hooks.Filters.CLI_COMMANDS, []) + + @classmethod + def ensure_plugins_enabled(cls, ctx: click.Context) -> None: + """ + We enable plugins as soon as possible to have access to commands. + """ + if not isinstance(ctx, click.Context): + # When generating docs, this function is incorrectly called with a + # multicommand object instead of a Context. That's ok, we just + # ignore it. + # https://github.com/click-contrib/sphinx-click/issues/70 + return + if not cls.IS_ROOT_READY: + hooks.actions.do(hooks.Actions.CORE_ROOT_READY, ctx.params["root"]) + cls.IS_ROOT_READY = True + + def list_commands(self, ctx: click.Context) -> List[str]: + """ + This is run in the following cases: + - shell autocompletion: tutor + - print help: tutor, tutor -h + """ + return sorted( + [command.name or "" for command in self.get_commands(ctx)] + ) + + def get_command(self, ctx: click.Context, cmd_name: str) -> Optional[click.Command]: + """ + This is run when passing a command from the CLI. E.g: tutor config ... + """ + for command in self.get_commands(ctx): + if cmd_name == command.name: + return command + return None + + @click.group( - context_settings={"help_option_names": ["-h", "--help", "help"]}, + cls=TutorCli, + invoke_without_command=True, + add_help_option=False, # Context is incorrectly loaded when help option is automatically added help="Tutor is the Docker-based Open edX distribution designed for peace of mind.", ) @click.version_option(version=__version__) @@ -46,8 +98,15 @@ def main() -> None: type=click.Path(resolve_path=True), help="Root project directory (environment variable: TUTOR_ROOT)", ) +@click.option( + "-h", + "--help", + "show_help", + is_flag=True, + help="Print this help", +) @click.pass_context -def cli(context: click.Context, root: str) -> None: +def cli(context: click.Context, root: str, show_help: bool) -> None: if utils.is_root(): fmt.echo_alert( "You are running Tutor as root. This is strongly not recommended. If you are doing this in order to access" @@ -55,12 +114,28 @@ def cli(context: click.Context, root: str) -> None: "/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user)" ) context.obj = Context(root) + if context.invoked_subcommand is None or show_help: + click.echo(context.get_help()) @click.command(help="Print this help", name="help") -def print_help() -> None: - context = click.Context(cli) - click.echo(cli.get_help(context)) +@click.pass_context +def help_command(context: click.Context) -> None: + context.invoke(cli, show_help=True) + + +hooks.filters.add_items( + "cli:commands", + [ + images_command, + config_command, + local, + dev, + k8s, + help_command, + plugins_command, + ], +) if __name__ == "__main__": diff --git a/tutor/commands/config.py b/tutor/commands/config.py index a776dc3218d..644a5952e02 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -80,10 +80,9 @@ def render(context: Context, extra_configs: List[str], src: str, dst: str) -> No config.update( env.render_unknown(config, tutor_config.get_yaml_file(extra_config)) ) - renderer = env.Renderer(config, [src]) renderer.render_all_to(dst) - fmt.echo_info("Templates rendered to {}".format(dst)) + fmt.echo_info(f"Templates rendered to {dst}") @click.command(help="Print the project root") @@ -101,9 +100,7 @@ def printvalue(context: Context, key: str) -> None: # Note that this will incorrectly print None values fmt.echo(str(config[key])) except KeyError as e: - raise exceptions.TutorError( - "Missing configuration value: {}".format(key) - ) from e + raise exceptions.TutorError(f"Missing configuration value: {key}") from e config_command.add_command(save) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 29a839270c5..3e3c1a18c67 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -1,12 +1,12 @@ -from typing import Iterator, List, Tuple +import typing as t import click -from .. import config as tutor_config -from .. import env as tutor_env -from .. import exceptions, images, plugins -from ..types import Config -from .context import Context +from tutor import config as tutor_config +from tutor import env as tutor_env +from tutor import exceptions, hooks, images +from tutor.commands.context import Context +from tutor.types import Config BASE_IMAGE_NAMES = ["openedx", "permissions"] VENDOR_IMAGES = [ @@ -19,6 +19,47 @@ ] +@hooks.filters.add(hooks.Filters.APP_TASK_IMAGES_BUILD) +def _add_core_images_to_build( + build_images: t.List[t.Tuple[str, t.Tuple[str, str], str, t.List[str]]], + config: Config, +) -> t.List[t.Tuple[str, t.Tuple[str, str], str, t.List[str]]]: + """ + Add base images to the list of Docker images to build on `tutor build all`. + """ + for image in BASE_IMAGE_NAMES: + tag = images.get_tag(config, image) + build_images.append((image, ("build", image), tag, [])) + return build_images + + +@hooks.filters.add(hooks.Filters.APP_TASK_IMAGES_PULL) +def _add_images_to_pull( + remote_images: t.List[t.Tuple[str, str]], config: Config +) -> t.List[t.Tuple[str, str]]: + """ + Add base and vendor images to the list of Docker images to pull on `tutor pull all`. + """ + for image in VENDOR_IMAGES: + if config.get(f"RUN_{image.upper()}", True): + remote_images.append((image, images.get_tag(config, image))) + for image in BASE_IMAGE_NAMES: + remote_images.append((image, images.get_tag(config, image))) + return remote_images + + +@hooks.filters.add(hooks.Filters.APP_TASK_IMAGES_PULL) +def _add_core_images_to_push( + remote_images: t.List[t.Tuple[str, str]], config: Config +) -> t.List[t.Tuple[str, str]]: + """ + Add base images to the list of Docker images to push on `tutor push all`. + """ + for image in BASE_IMAGE_NAMES: + remote_images.append((image, images.get_tag(config, image))) + return remote_images + + @click.group(name="images", short_help="Manage docker images") def images_command() -> None: pass @@ -59,12 +100,12 @@ def images_command() -> None: @click.pass_obj def build( context: Context, - image_names: List[str], + image_names: t.List[str], no_cache: bool, - build_args: List[str], - add_hosts: List[str], + build_args: t.List[str], + add_hosts: t.List[str], target: str, - docker_args: List[str], + docker_args: t.List[str], ) -> None: config = tutor_config.load(context.root) command_args = [] @@ -79,134 +120,97 @@ def build( if docker_args: command_args += docker_args for image in image_names: - build_image(context.root, config, image, *command_args) + for _name, path, tag, custom_args in find_images_to_build(config, image): + images.build( + tutor_env.pathjoin(context.root, *path), + tag, + *command_args, + *custom_args, + ) @click.command(short_help="Pull images from the Docker registry") @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj -def pull(context: Context, image_names: List[str]) -> None: +def pull(context: Context, image_names: t.List[str]) -> None: config = tutor_config.load_full(context.root) for image in image_names: - pull_image(config, image) + for tag in find_remote_image_tags( + config, hooks.Filters.APP_TASK_IMAGES_PULL, image + ): + images.pull(tag) @click.command(short_help="Push images to the Docker registry") @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj -def push(context: Context, image_names: List[str]) -> None: +def push(context: Context, image_names: t.List[str]) -> None: config = tutor_config.load_full(context.root) for image in image_names: - push_image(config, image) + for tag in find_remote_image_tags( + config, hooks.Filters.APP_TASK_IMAGES_PUSH, image + ): + images.push(tag) @click.command(short_help="Print tag associated to a Docker image") @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj -def printtag(context: Context, image_names: List[str]) -> None: +def printtag(context: Context, image_names: t.List[str]) -> None: config = tutor_config.load_full(context.root) for image in image_names: - to_print = [] - for _img, tag in iter_images(config, image, BASE_IMAGE_NAMES): - to_print.append(tag) - for _plugin, _img, tag in iter_plugin_images(config, image, "build-image"): - to_print.append(tag) - - if not to_print: - raise ImageNotFoundError(image) - - for tag in to_print: + for _name, _path, tag, _args in find_images_to_build(config, image): print(tag) -def build_image(root: str, config: Config, image: str, *args: str) -> None: - to_build = [] - - # Build base images - for img, tag in iter_images(config, image, BASE_IMAGE_NAMES): - to_build.append((tutor_env.pathjoin(root, "build", img), tag, args)) - - # Build plugin images - for plugin, img, tag in iter_plugin_images(config, image, "build-image"): - to_build.append( - (tutor_env.pathjoin(root, "plugins", plugin, "build", img), tag, args) - ) - - if not to_build: - raise ImageNotFoundError(image) - - for path, tag, build_args in to_build: - images.build(path, tag, *args) +def find_images_to_build( + config: Config, image: str +) -> t.Iterator[t.Tuple[str, t.Tuple[str], str, t.List[str]]]: + """ + Iterate over all images to build. + If no corresponding image is found, raise exception. -def pull_image(config: Config, image: str) -> None: - to_pull = [] - for _img, tag in iter_images(config, image, all_image_names(config)): - to_pull.append(tag) - for _plugin, _img, tag in iter_plugin_images(config, image, "remote-image"): - to_pull.append(tag) + Yield: (name, path, tag, build args) + """ + all_images_to_build: t.Iterator[ + t.Tuple[str, t.Tuple[str], str, t.List[str]] + ] = hooks.filters.iterate(hooks.Filters.APP_TASK_IMAGES_BUILD, config) + found = False + for name, path, tag, args in all_images_to_build: + if name == image or image == "all": + found = True + tag = tutor_env.render_str(config, tag) + yield (name, path, tag, args) - if not to_pull: + if not found: raise ImageNotFoundError(image) - for tag in to_pull: - images.pull(tag) - -def push_image(config: Config, image: str) -> None: - to_push = [] - for _img, tag in iter_images(config, image, BASE_IMAGE_NAMES): - to_push.append(tag) - for _plugin, _img, tag in iter_plugin_images(config, image, "remote-image"): - to_push.append(tag) - - if not to_push: +def find_remote_image_tags( + config: Config, filter_name: str, image: str +) -> t.Iterator[str]: + """ + Iterate over all images to push or pull. + + If no corresponding image is found, raise exception. + + Yield: tag + """ + all_remote_images: t.List[t.Tuple[str, str]] = [] + all_remote_images = hooks.filters.apply(filter_name, all_remote_images, config) + found = False + for name, tag in all_remote_images: + if name == image or image == "all": + found = True + yield tutor_env.render_str(config, tag) + if not found: raise ImageNotFoundError(image) - for tag in to_push: - images.push(tag) - - -def iter_images( - config: Config, image: str, image_list: List[str] -) -> Iterator[Tuple[str, str]]: - for img in image_list: - if image in [img, "all"]: - tag = images.get_tag(config, img) - yield img, tag - - -def iter_plugin_images( - config: Config, image: str, hook_name: str -) -> Iterator[Tuple[str, str, str]]: - for plugin, hook in plugins.iter_hooks(config, hook_name): - if not isinstance(hook, dict): - raise exceptions.TutorError( - "Invalid hook '{}': expected dict, got {}".format( - hook_name, hook.__class__ - ) - ) - for img, tag in hook.items(): - if image in [img, "all"]: - tag = tutor_env.render_str(config, tag) - yield plugin, img, tag - - -def all_image_names(config: Config) -> List[str]: - return BASE_IMAGE_NAMES + vendor_image_names(config) - - -def vendor_image_names(config: Config) -> List[str]: - vendor_images = VENDOR_IMAGES[:] - for image in VENDOR_IMAGES: - if not config.get("RUN_" + image.upper(), True): - vendor_images.remove(image) - return vendor_images - class ImageNotFoundError(exceptions.TutorError): def __init__(self, image_name: str): - super().__init__("Image '{}' could not be found".format(image_name)) + super().__init__(f"Image '{image_name}' could not be found") images_command.add_command(build) diff --git a/tutor/commands/plugins.py b/tutor/commands/plugins.py index 332378c1ab5..77502b5ca33 100644 --- a/tutor/commands/plugins.py +++ b/tutor/commands/plugins.py @@ -5,9 +5,11 @@ import click -from .. import config as tutor_config -from .. import env as tutor_env -from .. import exceptions, fmt, plugins +from tutor import config as tutor_config +from tutor import env as tutor_env +from tutor import exceptions, fmt, plugins +from tutor.plugins.base import PLUGINS_ROOT, PLUGINS_ROOT_ENV_VAR_NAME + from .context import Context @@ -24,16 +26,18 @@ def plugins_command() -> None: @click.command(name="list", help="List installed plugins") -@click.pass_obj -def list_command(context: Context) -> None: - config = tutor_config.load_full(context.root) - for plugin in plugins.iter_installed(): - status = "" if plugins.is_enabled(config, plugin.name) else " (disabled)" - print( - "{plugin}=={version}{status}".format( - plugin=plugin.name, status=status, version=plugin.version - ) - ) +def list_command() -> None: + lines = [] + first_column_width = 1 + for plugin, plugin_info in plugins.iter_info(): + plugin_info = plugin_info or "" + plugin_info.replace("\n", " ") + status = "" if plugins.is_enabled(plugin) else "(disabled)" + lines.append((plugin, status, plugin_info)) + first_column_width = max([first_column_width, len(plugin) + 2]) + + for line in lines: + print("{:{width}}\t{:10}\t{}".format(*line, width=first_column_width)) @click.command(help="Enable a plugin") @@ -42,8 +46,9 @@ def list_command(context: Context) -> None: def enable(context: Context, plugin_names: List[str]) -> None: config = tutor_config.load_minimal(context.root) for plugin in plugin_names: - plugins.enable(config, plugin) - fmt.echo_info("Plugin {} enabled".format(plugin)) + plugins.enable(plugin) + fmt.echo_info(f"Plugin {plugin} enabled") + tutor_config.save_enabled_plugins(config) tutor_config.save_config_file(context.root, config) fmt.echo_info( "You should now re-generate your environment with `tutor config save`." @@ -59,14 +64,11 @@ def enable(context: Context, plugin_names: List[str]) -> None: def disable(context: Context, plugin_names: List[str]) -> None: config = tutor_config.load_minimal(context.root) disable_all = "all" in plugin_names - for plugin in plugins.iter_enabled(config): - if disable_all or plugin.name in plugin_names: - fmt.echo_info("Disabling plugin {}...".format(plugin.name)) - for key, value in plugin.config_set.items(): - value = tutor_env.render_unknown(config, value) - fmt.echo_info(" Removing config entry {}={}".format(key, value)) - plugins.disable(config, plugin) - delete_plugin(context.root, plugin.name) + for plugin in plugins.iter_enabled(): + if disable_all or plugin in plugin_names: + fmt.echo_info(f"Disabling plugin {plugin}...") + tutor_config.disable_plugin(config, plugin) + delete_plugin(context.root, plugin) fmt.echo_info(" Plugin disabled") tutor_config.save_config_file(context.root, config) fmt.echo_info( @@ -81,36 +83,31 @@ def delete_plugin(root: str, name: str) -> None: shutil.rmtree(plugin_dir) except PermissionError as e: raise exceptions.TutorError( - "Could not delete file {} from plugin {} in folder {}".format( - e.filename, name, plugin_dir - ) + f"Could not delete file {e.filename} from plugin {name} in folder {plugin_dir}" ) @click.command( short_help="Print the location of yaml-based plugins", - help="""Print the location of yaml-based plugins. This location can be manually -defined by setting the {} environment variable""".format( - plugins.DictPlugin.ROOT_ENV_VAR_NAME - ), + help=f"""Print the location of yaml-based plugins. This location can be manually +defined by setting the {PLUGINS_ROOT_ENV_VAR_NAME} environment variable""", ) def printroot() -> None: - fmt.echo(plugins.DictPlugin.ROOT) + fmt.echo(PLUGINS_ROOT) @click.command( short_help="Install a plugin", - help="""Install a plugin, either from a local YAML file or a remote, web-hosted -location. The plugin will be installed to {}.""".format( - plugins.DictPlugin.ROOT_ENV_VAR_NAME - ), + help=f"""Install a plugin, either from a local YAML file or a remote, web-hosted +location. The plugin will be installed to {PLUGINS_ROOT_ENV_VAR_NAME}.""", ) @click.argument("location") def install(location: str) -> None: basename = os.path.basename(location) + # TODO install py files as well if not basename.endswith(".yml"): basename += ".yml" - plugin_path = os.path.join(plugins.DictPlugin.ROOT, basename) + plugin_path = os.path.join(PLUGINS_ROOT, basename) if location.startswith("http"): # Download file @@ -118,28 +115,17 @@ def install(location: str) -> None: content = response.read().decode() elif os.path.isfile(location): # Read file - with open(location) as f: + with open(location, encoding="utf-8") as f: content = f.read() else: - raise exceptions.TutorError("No plugin found at {}".format(location)) + raise exceptions.TutorError(f"No plugin found at {location}") # Save file - if not os.path.exists(plugins.DictPlugin.ROOT): - os.makedirs(plugins.DictPlugin.ROOT) - with open(plugin_path, "w", newline="\n") as f: + if not os.path.exists(PLUGINS_ROOT): + os.makedirs(PLUGINS_ROOT) + with open(plugin_path, "w", newline="\n", encoding="utf-8") as f: f.write(content) - fmt.echo_info("Plugin installed at {}".format(plugin_path)) - - -def add_plugin_commands(command_group: click.Group) -> None: - """ - Add commands provided by all plugins to the given command group. Each command is - added with a name that is equal to the plugin name. - """ - for plugin in plugins.iter_installed(): - if isinstance(plugin.command, click.Command): - plugin.command.name = plugin.name - command_group.add_command(plugin.command) + fmt.echo_info(f"Plugin installed at {plugin_path}") plugins_command.add_command(list_command) diff --git a/tutor/commands/upgrade/common.py b/tutor/commands/upgrade/common.py index e186af7776b..56f40f0c450 100644 --- a/tutor/commands/upgrade/common.py +++ b/tutor/commands/upgrade/common.py @@ -1,5 +1,5 @@ -from tutor import fmt -from tutor import plugins +from tutor import config as tutor_config +from tutor import fmt, plugins from tutor.types import Config @@ -9,23 +9,25 @@ def upgrade_from_lilac(config: Config) -> None: "The Open edX forum feature was moved to a separate plugin in Maple. To keep using this feature, " "you must install and enable the tutor-forum plugin: https://github.com/overhangio/tutor-forum" ) - elif not plugins.is_enabled(config, "forum"): + elif not plugins.is_enabled("forum"): fmt.echo_info( "The Open edX forum feature was moved to a separate plugin in Maple. To keep using this feature, " "we will now enable the 'forum' plugin. If you do not want to use this feature, you should disable the " "plugin with: `tutor plugins disable forum`." ) - plugins.enable(config, "forum") + plugins.enable("forum") + tutor_config.save_enabled_plugins(config) if not plugins.is_installed("mfe"): fmt.echo_alert( "In Maple the legacy courseware is no longer supported. You need to install and enable the 'mfe' plugin " "to make use of the new learning microfrontend: https://github.com/overhangio/tutor-mfe" ) - elif not plugins.is_enabled(config, "mfe"): + elif not plugins.is_enabled("mfe"): fmt.echo_info( "In Maple the legacy courseware is no longer supported. To start using the new learning microfrontend, " "we will now enable the 'mfe' plugin. If you do not want to use this feature, you should disable the " "plugin with: `tutor plugins disable mfe`." ) - plugins.enable(config, "mfe") + plugins.enable("mfe") + tutor_config.save_enabled_plugins(config) diff --git a/tutor/commands/upgrade/k8s.py b/tutor/commands/upgrade/k8s.py index 29d5b5b42c3..02d086049ba 100644 --- a/tutor/commands/upgrade/k8s.py +++ b/tutor/commands/upgrade/k8s.py @@ -3,6 +3,7 @@ from tutor.commands import k8s from tutor.commands.context import Context from tutor.types import Config + from . import common as common_upgrade diff --git a/tutor/commands/upgrade/local.py b/tutor/commands/upgrade/local.py index 353fbc50e95..527447d5633 100644 --- a/tutor/commands/upgrade/local.py +++ b/tutor/commands/upgrade/local.py @@ -7,6 +7,7 @@ from tutor import fmt from tutor.commands import compose from tutor.types import Config + from . import common as common_upgrade diff --git a/tutor/config.py b/tutor/config.py index 2185de37c9a..b77fe0889dc 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -1,7 +1,8 @@ import os +import typing as t -from . import env, exceptions, fmt, plugins, serialize, utils -from .types import Config, cast_config +from tutor import env, exceptions, fmt, hooks, plugins, serialize, utils +from tutor.types import Config, ConfigValue, cast_config, get_typed CONFIG_FILENAME = "config.yml" @@ -58,7 +59,7 @@ def update_with_base(config: Config) -> None: Note that configuration entries are unrendered at this point. """ - base = get_base(config) + base = get_base() merge(config, base) @@ -68,7 +69,7 @@ def update_with_defaults(config: Config) -> None: Note that configuration entries are unrendered at this point. """ - defaults = get_defaults(config) + defaults = get_defaults() merge(config, defaults) @@ -100,41 +101,37 @@ def get_user(root: str) -> Config: return config -def get_base(config: Config) -> Config: +def get_base() -> Config: """ Load the base configuration. Entries in this configuration are unrendered. """ base = get_template("base.yml") - - # Load base values from plugins - for plugin in plugins.iter_enabled(config): - # Add new config key/values - for key, value in plugin.config_add.items(): - new_key = plugin.config_key(key) - base[new_key] = value - - # Set existing config key/values - for key, value in plugin.config_set.items(): - base[key] = value - + extra_base: t.List[t.Tuple[str, ConfigValue]] = [] + extra_base = hooks.filters.apply(hooks.Filters.CONFIG_BASE, extra_base) + extra_base = hooks.filters.apply(hooks.Filters.CONFIG_OVERRIDES, extra_base) + for name, value in extra_base: + if name in base: + fmt.echo_alert( + f"Found conflicting values for setting '{name}': '{value}' or '{base[name]}'" + ) + base[name] = value return base -def get_defaults(config: Config) -> Config: +def get_defaults() -> Config: """ Get default configuration, including from plugins. Entries in this configuration are unrendered. """ defaults = get_template("defaults.yml") - - for plugin in plugins.iter_enabled(config): - # Create new defaults - for key, value in plugin.config_defaults.items(): - defaults[plugin.config_key(key)] = value - + extra_defaults: t.Iterator[t.Tuple[str, ConfigValue]] = hooks.filters.iterate( + hooks.Filters.CONFIG_DEFAULTS + ) + for name, value in extra_defaults: + defaults[name] = value update_with_env(defaults) return defaults @@ -153,7 +150,7 @@ def get_yaml_file(path: str) -> Config: """ Load config from yaml file. """ - with open(path) as f: + with open(path, encoding="utf-8") as f: config = serialize.load(f.read()) return cast_config(config) @@ -198,11 +195,13 @@ def upgrade_obsolete(config: Config) -> None: config["OPENEDX_MYSQL_USERNAME"] = config.pop("MYSQL_USERNAME") if "RUN_NOTES" in config: if config["RUN_NOTES"]: - plugins.enable(config, "notes") + plugins.enable("notes") + save_enabled_plugins(config) config.pop("RUN_NOTES") if "RUN_XQUEUE" in config: if config["RUN_XQUEUE"]: - plugins.enable(config, "xqueue") + plugins.enable("xqueue") + save_enabled_plugins(config) config.pop("RUN_XQUEUE") if "SECRET_KEY" in config: config["OPENEDX_SECRET_KEY"] = config.pop("SECRET_KEY") @@ -254,10 +253,61 @@ def convert_json2yml(root: str) -> None: def save_config_file(root: str, config: Config) -> None: path = config_path(root) utils.ensure_file_directory_exists(path) - with open(path, "w") as of: + with open(path, "w", encoding="utf-8") as of: serialize.dump(config, of) fmt.echo_info(f"Configuration saved to {path}") def config_path(root: str) -> str: return os.path.join(root, CONFIG_FILENAME) + + +# Key name under which plugins are listed +PLUGINS_CONFIG_KEY = "PLUGINS" + + +def enable_plugins(config: Config) -> None: + """ + Enable all plugins listed in the configuration. + """ + names: t.List[str] = get_typed(config, PLUGINS_CONFIG_KEY, list, []) + names = sorted(set(names)) + installed = set(plugins.iter_installed()) + for name in names: + if name in installed: + try: + plugins.enable(name) + except exceptions.TutorError as e: + fmt.echo_alert(f"Failed to enable plugin '{name}' : {e.args[0]}") + + +def save_enabled_plugins(config: Config) -> None: + """ + Save the list of enabled plugins. + + Plugins are deduplicated by name. + """ + config[PLUGINS_CONFIG_KEY] = list(plugins.iter_enabled()) + + +def disable_plugin(config: Config, plugin: str) -> None: + # Find the configuration entries that were overridden by the plugin and + # remove them from the current config + plugin_context = hooks.Contexts.APP % plugin + overriden_config_items: t.Iterator[ + t.Tuple[str, ConfigValue] + ] = hooks.filters.iterate(hooks.Filters.CONFIG_OVERRIDES, context=plugin_context) + for key, _value in overriden_config_items: + value = config.pop(key, None) + value = env.render_unknown(config, value) + fmt.echo_info(f"Disabling {plugin}: removing config entry {key}={value}") + + # Disable plugin and remove it from the list of enabled plugins + plugins.disable(plugin) + save_enabled_plugins(config) + + +@hooks.actions.on(hooks.Actions.CORE_ROOT_READY) +def _enable_plugins(root: str) -> None: + config = load_minimal(root) + enable_plugins(config) diff --git a/tutor/env.py b/tutor/env.py index 5a89a49803e..e280007e06d 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -1,91 +1,140 @@ import os +import typing as t from copy import deepcopy -from typing import Any, Iterable, List, Optional, Type, Union import jinja2 import pkg_resources -from . import exceptions, fmt, plugins, utils -from .__about__ import __app__, __version__ -from .types import Config, ConfigValue +from tutor import exceptions, fmt, hooks, plugins, utils +from tutor.__about__ import __app__, __version__ +from tutor.types import Config, ConfigValue TEMPLATES_ROOT = pkg_resources.resource_filename("tutor", "templates") VERSION_FILENAME = "version" BIN_FILE_EXTENSIONS = [".ico", ".jpg", ".patch", ".png", ".ttf", ".woff", ".woff2"] +JinjaFilter = t.Callable[..., t.Any] + + +def _prepare_environment() -> None: + """ + Prepare environment by adding core data to filters. + """ + # Core template targets + hooks.filters.add_items( + hooks.Filters.ENV_TEMPLATE_TARGETS, + [ + ("apps/", ""), + ("build/", ""), + ("dev/", ""), + ("k8s/", ""), + ("local/", ""), + (VERSION_FILENAME, ""), + ("kustomization.yml", ""), + ], + ) + # Template filters + hooks.filters.add_items( + hooks.Filters.ENV_TEMPLATE_FILTERS, + [ + ("common_domain", utils.common_domain), + ("encrypt", utils.encrypt), + ("list_if", utils.list_if), + ("long_to_base64", utils.long_to_base64), + ("random_string", utils.random_string), + ("reverse_host", utils.reverse_host), + ("rsa_private_key", utils.rsa_private_key), + ], + ) + # Template variables + hooks.filters.add_items( + hooks.Filters.ENV_TEMPLATE_VARIABLES, + [ + ("rsa_import_key", utils.rsa_import_key), + ("HOST_USER_ID", utils.get_user_id()), + ("TUTOR_APP", __app__.replace("-", "_")), + ("TUTOR_VERSION", __version__), + ], + ) + + +_prepare_environment() class JinjaEnvironment(jinja2.Environment): loader: jinja2.BaseLoader - def __init__(self, template_roots: List[str]) -> None: + def __init__(self, template_roots: t.List[str]) -> None: loader = jinja2.FileSystemLoader(template_roots) super().__init__(loader=loader, undefined=jinja2.StrictUndefined) class Renderer: @classmethod - def instance(cls: Type["Renderer"], config: Config) -> "Renderer": + def instance(cls: t.Type["Renderer"], config: Config) -> "Renderer": # Load template roots: these are required to be able to use # {% include .. %} directives - template_roots = [TEMPLATES_ROOT] - for plugin in plugins.iter_enabled(config): - if plugin.templates_root: - template_roots.append(plugin.templates_root) - + template_roots = hooks.filters.apply( + hooks.Filters.ENV_TEMPLATE_ROOTS, [TEMPLATES_ROOT] + ) return cls(config, template_roots, ignore_folders=["partials"]) def __init__( self, config: Config, - template_roots: List[str], - ignore_folders: Optional[List[str]] = None, + template_roots: t.List[str], + ignore_folders: t.Optional[t.List[str]] = None, ): self.config = deepcopy(config) self.template_roots = template_roots self.ignore_folders = ignore_folders or [] self.ignore_folders.append(".git") - # Create environment - environment = JinjaEnvironment(template_roots) - environment.filters["common_domain"] = utils.common_domain - environment.filters["encrypt"] = utils.encrypt - environment.filters["list_if"] = utils.list_if - environment.filters["long_to_base64"] = utils.long_to_base64 - environment.globals["iter_values_named"] = self.iter_values_named - environment.globals["patch"] = self.patch - environment.filters["random_string"] = utils.random_string - environment.filters["reverse_host"] = utils.reverse_host - environment.globals["rsa_import_key"] = utils.rsa_import_key - environment.filters["rsa_private_key"] = utils.rsa_private_key - environment.filters["walk_templates"] = self.walk_templates - environment.globals["HOST_USER_ID"] = utils.get_user_id() - environment.globals["TUTOR_APP"] = __app__.replace("-", "_") - environment.globals["TUTOR_VERSION"] = __version__ - self.environment = environment - - def iter_templates_in(self, *prefix: str) -> Iterable[str]: + # Create environment with extra filters and globals + self.environment = JinjaEnvironment(template_roots) + + # Filters + plugin_filters: t.Iterator[t.Tuple[str, JinjaFilter]] = hooks.filters.iterate( + hooks.Filters.ENV_TEMPLATE_FILTERS + ) + for name, func in plugin_filters: + if name in self.environment.filters: + fmt.echo_alert(f"Found conflicting template filters named '{name}'") + self.environment.filters[name] = func + self.environment.filters["walk_templates"] = self.walk_templates + + # Globals + plugin_globals: t.Iterator[t.Tuple[str, JinjaFilter]] = hooks.filters.iterate( + hooks.Filters.ENV_TEMPLATE_VARIABLES + ) + for name, value in plugin_globals: + if name in self.environment.globals: + fmt.echo_alert(f"Found conflicting template variables named '{name}'") + self.environment.globals[name] = value + self.environment.globals["iter_values_named"] = self.iter_values_named + self.environment.globals["patch"] = self.patch + + def iter_templates_in(self, *prefix: str) -> t.Iterable[str]: """ The elements of `prefix` must contain only "/", and not os.sep. """ full_prefix = "/".join(prefix) - env_templates: List[str] = self.environment.loader.list_templates() + env_templates: t.List[str] = self.environment.loader.list_templates() for template in env_templates: if template.startswith(full_prefix) and self.is_part_of_env(template): yield template def iter_values_named( self, - prefix: Optional[str] = None, - suffix: Optional[str] = None, + prefix: t.Optional[str] = None, + suffix: t.Optional[str] = None, allow_empty: bool = False, - ) -> Iterable[ConfigValue]: + ) -> t.Iterable[ConfigValue]: """ Iterate on all config values for which the name match the given pattern. Note that here we only iterate on the values, not the key names. Empty values (those that evaluate to boolean `false`) will not be yielded, unless `allow_empty` is True. - TODO document this in the plugins API """ for var_name, value in self.config.items(): if prefix is not None and not var_name.startswith(prefix): @@ -96,7 +145,7 @@ def iter_values_named( continue yield value - def walk_templates(self, subdir: str) -> Iterable[str]: + def walk_templates(self, subdir: str) -> t.Iterable[str]: """ Iterate on the template files from `templates/`. @@ -134,11 +183,11 @@ def patch(self, name: str, separator: str = "\n", suffix: str = "") -> str: Render calls to {{ patch("...") }} in environment templates from plugin patches. """ patches = [] - for plugin, patch in plugins.iter_patches(self.config, name): + for patch in plugins.iter_patches(name): try: patches.append(self.render_str(patch)) except exceptions.TutorError: - fmt.echo_error(f"Error rendering patch '{name}' from plugin {plugin}") + fmt.echo_error(f"Error rendering patch '{name}': {patch}") raise rendered = separator.join(patches) if rendered: @@ -149,7 +198,7 @@ def render_str(self, text: str) -> str: template = self.environment.from_string(text) return self.__render(template) - def render_template(self, template_name: str) -> Union[str, bytes]: + def render_template(self, template_name: str) -> t.Union[str, bytes]: """ Render a template file. Return the corresponding string. If it's a binary file (as indicated by its path), return bytes. @@ -177,14 +226,14 @@ def render_template(self, template_name: str) -> Union[str, bytes]: fmt.echo_error("Unknown error rendering template " + template_name) raise - def render_all_to(self, root: str, *prefix: str) -> None: + def render_all_to(self, dst: str, *prefix: str) -> None: """ `prefix` can be used to limit the templates to render. """ for template_name in self.iter_templates_in(*prefix): rendered = self.render_template(template_name) - dst = os.path.join(root, template_name.replace("/", os.sep)) - write_to(rendered, dst) + template_dst = os.path.join(dst, template_name.replace("/", os.sep)) + write_to(rendered, template_dst) def __render(self, template: jinja2.Template) -> str: try: @@ -198,20 +247,11 @@ def save(root: str, config: Config) -> None: Save the full environment, including version information. """ root_env = pathjoin(root) - for prefix in [ - "apps/", - "build/", - "dev/", - "k8s/", - "local/", - VERSION_FILENAME, - "kustomization.yml", - ]: - save_all_from(prefix, root_env, config) - - for plugin in plugins.iter_enabled(config): - if plugin.templates_root: - save_plugin_templates(plugin, root, config) + targets: t.Iterator[t.Tuple[str, str]] = hooks.filters.iterate( + hooks.Filters.ENV_TEMPLATE_TARGETS + ) + for src, dst in targets: + save_all_from(src, os.path.join(root_env, dst), config) upgrade_obsolete(root) fmt.echo_info(f"Environment generated in {base_dir(root)}") @@ -223,29 +263,16 @@ def upgrade_obsolete(_root: str) -> None: """ -def save_plugin_templates( - plugin: plugins.BasePlugin, root: str, config: Config -) -> None: - """ - Save plugin templates to plugins//*. - Only the "apps" and "build" subfolders are rendered. - """ - plugins_root = pathjoin(root, "plugins") - for subdir in ["apps", "build"]: - subdir_path = os.path.join(plugin.name, subdir) - save_all_from(subdir_path, plugins_root, config) - - -def save_all_from(prefix: str, root: str, config: Config) -> None: +def save_all_from(prefix: str, dst: str, config: Config) -> None: """ Render the templates that start with `prefix` and store them with the same - hierarchy at `root`. Here, `prefix` can be the result of os.path.join(...). + hierarchy at `dst`. Here, `prefix` can be the result of os.path.join(...). """ renderer = Renderer.instance(config) - renderer.render_all_to(root, prefix.replace(os.sep, "/")) + renderer.render_all_to(dst, prefix.replace(os.sep, "/")) -def write_to(content: Union[str, bytes], path: str) -> None: +def write_to(content: t.Union[str, bytes], path: str) -> None: """ Write some content to a path. Content can be either str or bytes. """ @@ -258,7 +285,7 @@ def write_to(content: Union[str, bytes], path: str) -> None: of_text.write(content) -def render_file(config: Config, *path: str) -> Union[str, bytes]: +def render_file(config: Config, *path: str) -> t.Union[str, bytes]: """ Return the rendered contents of a template. """ @@ -267,7 +294,7 @@ def render_file(config: Config, *path: str) -> Union[str, bytes]: return renderer.render_template(template_name) -def render_unknown(config: Config, value: Any) -> Any: +def render_unknown(config: Config, value: t.Any) -> t.Any: """ Render an unknown `value` object with the selected config. @@ -311,7 +338,7 @@ def is_up_to_date(root: str) -> bool: return current is None or current == __version__ -def should_upgrade_from_release(root: str) -> Optional[str]: +def should_upgrade_from_release(root: str) -> t.Optional[str]: """ Return the name of the currently installed release that we should upgrade from. Return None If we already run the latest release. @@ -326,7 +353,7 @@ def should_upgrade_from_release(root: str) -> Optional[str]: return get_release(current) -def get_env_release(root: str) -> Optional[str]: +def get_env_release(root: str) -> t.Optional[str]: """ Return the Open edX release name from the current environment. @@ -356,7 +383,7 @@ def get_release(version: str) -> str: }[version.split(".", maxsplit=1)[0]] -def current_version(root: str) -> Optional[str]: +def current_version(root: str) -> t.Optional[str]: """ Return the current environment version. If the current environment has no version, return None. diff --git a/tutor/hooks/__init__.py b/tutor/hooks/__init__.py new file mode 100644 index 00000000000..156d6e69010 --- /dev/null +++ b/tutor/hooks/__init__.py @@ -0,0 +1,16 @@ +# The Tutor plugin system is licensed under the terms of the Apache 2.0 license. +__license__ = "Apache 2.0" + +import typing as t + +# These imports are the hooks API +from . import actions, contexts, filters +from .consts import * + + +def clear_all(context: t.Optional[str] = None) -> None: + """ + Clear both actions and filters. + """ + filters.clear_all(context=context) + actions.clear_all(context=context) diff --git a/tutor/hooks/actions.py b/tutor/hooks/actions.py new file mode 100644 index 00000000000..6caa6915992 --- /dev/null +++ b/tutor/hooks/actions.py @@ -0,0 +1,132 @@ +# The Tutor plugin system is licensed under the terms of the Apache 2.0 license. +__license__ = "Apache 2.0" + +import sys +import typing as t + +from . import contexts + +# Similarly to CallableFilter, it should be possible to refine the definition of +# CallableAction in the future. +CallableAction = t.Callable[..., None] + + +class Action(contexts.Contextualized): + def __init__(self, func: CallableAction): + super().__init__() + self.func = func + + def do( + self, *args: t.Any, context: t.Optional[str] = None, **kwargs: t.Any + ) -> None: + if self.is_in_context(context): + self.func(*args, **kwargs) + + +class Actions: + """ + Singleton set of named actions. + """ + + INSTANCE = None + + @classmethod + def instance(cls) -> "Actions": + if cls.INSTANCE is None: + cls.INSTANCE = cls() + return cls.INSTANCE + + def __init__(self) -> None: + self.actions: t.Dict[str, t.List[Action]] = {} + + def on(self, name: str, func: CallableAction) -> None: + self.actions.setdefault(name, []).append(Action(func)) + + def do( + self, name: str, *args: t.Any, context: t.Optional[str] = None, **kwargs: t.Any + ) -> None: + """ + Run callback actions associated to a name. + + :param name: name of the action for which callbacks will be run. + :param context: limit the set of callback actions to those that + were declared within a certain context (see + :py:func:`tutor.hooks.contexts.enter`). + + Extra ``*args`` and ``*kwargs`` arguments will be passed as-is to + callback functions. + + Callbacks are executed in the order they were added. There is no error + management here: a single exception will cause all following callbacks + not to be run and the exception to be bubbled up. + """ + for action in self.actions.get(name, []): + try: + action.do(*args, context=context, **kwargs) + except: + sys.stderr.write( + f"Error applying action '{name}': func={action.func} contexts={action.contexts}'\n" + ) + raise + + def clear_all(self, context: t.Optional[str] = None) -> None: + """ + Clear any previously defined filter with the given context. + + This will call :py:func:`clear` with all action names. + """ + for name in self.actions: + self.clear(name, context=context) + + def clear(self, name: str, context: t.Optional[str] = None) -> None: + """ + Clear any previously defined action with the given name and context. + + :param name: name of the action callbacks to remove. + :param context: when defined, will clear only the actions that were + created within that context. + + Actions will be removed from the list of callbacks and will no longer be + run in :py:func:`do` calls. + + This function should almost certainly never be called by plugins. It is + mostly useful to disable some plugins at runtime or in unit tests. + """ + if name not in self.actions: + return + self.actions[name] = [ + action for action in self.actions[name] if not action.is_in_context(context) + ] + + +def on(name: str) -> t.Callable[[CallableAction], CallableAction]: + """ + Decorator to add a callback action associated to a name. + + :param name: name of the action. For forward compatibility, it is + recommended not to hardcode any string here, but to pick a value from + :py:class:`tutor.hooks.Actions` instead. + + Use as follows:: + + from tutor import hooks + + @hooks.actions.on("my-action") + def do_stuff(): + ... + + The ``do_stuff`` callback function will be called on ``hooks.actions.do("my-action")``. (see :py:func:`do`) + + The signature of each callback action function must match the signature of the corresponding ``hooks.actions.do`` call. Callback action functions are not supposed to return any value. Returned values will be ignored. + """ + + def inner(action_func: CallableAction) -> CallableAction: + Actions.instance().on(name, action_func) + return action_func + + return inner + + +do = Actions.instance().do +clear = Actions.instance().clear +clear_all = Actions.instance().clear_all diff --git a/tutor/hooks/consts.py b/tutor/hooks/consts.py new file mode 100644 index 00000000000..ae5d2abf564 --- /dev/null +++ b/tutor/hooks/consts.py @@ -0,0 +1,236 @@ +# The Tutor plugin system is licensed under the terms of the Apache 2.0 license. +__license__ = "Apache 2.0" + +""" +List of all the action, filter and context names used across Tutor. This module is used +to generate part of the reference documentation. +""" + +__all__ = ["Actions", "Filters", "Contexts"] + + +class Actions: + """ + This class is a container for the names of all actions used across Tutor + (see :py:mod:`tutor.hooks.actions.do`). For each action, we describe the + arguments that are passed to the callback functions. + + To create a new callback for an existing action, write the following:: + + @hooks.actions.on(hooks.Actions.YOUR_ACTION_NAME) + def your_action(): + # Do stuff here + """ + + #: Called whenever the core project is ready to run. This is the right time to install plugins, for instance. + #: + #: This action does not have any parameter. + CORE_READY = "core:ready" + + #: Called as soon as we have access to the Tutor project root. + #: + #: :parameter str root: absolute path to the project root. + CORE_ROOT_READY = "core:root:ready" + + #: Enable a specific plugin. Only plugins that have previously been installed can be enabled. (see :py:data:`INSTALL_PLUGINS`) + #: + #: Most plugin developers will not have to implement this action themselves, unless + #: they want to perform a specific action at the moment the plugin is enabled. + #: + #: This action does not have any parameter. + ENABLE_PLUGIN = "plugins:enable:%s" + + #: This action is done to auto-detect plugins. In particular, we load the following plugins: + #: + #: - Python packages that declare a "tutor.plugin.v0" entrypoint. + #: - YAML plugins stored in ~/.local/share/tutor-plugins (as indicated by ``tutor plugins printroot``) + #: - When running the binary version of Tutor, official plugins that ship with the binary are automatically installed. + #: + #: Installing a plugin is typically done by the Tutor plugin mechanism. Thus, plugin + #: developers don't have to implement this action themselves. + #: + #: This action does not have any parameter. + INSTALL_PLUGINS = "plugins:install" + + +class Filters: + """ + Here are the names of all filters used across Tutor. For each filter, the + type of the first argument also indicates the type of the expected returned value. + + Filter names are all namespaced with domains separated by colons (":"). + + To add custom data to any filter, write the following in your plugin:: + + import typing as t from tutor import hooks + + @hooks.filters.add(hooks.Filters.YOUR_FILTER_NAME) + def your_filter(items): + # do stuff with items + ... + # return the modified list of items + return items + """ + + #: List of images to be built when we run ``tutor images build ...``. + #: + #: :parameter list[tuple[str, tuple[str, ...], str, tuple[str, ...]]] tasks: list of ``(name, path, tag, args)`` tuples. + #: + #: - ``name`` is the name of the image, as in ``tutor images build myimage``. + #: - ``path`` is the relative path to the folder that contains the Dockerfile. + #: For instance ``("myplugin", "build", "myservice")`` indicates that the template will be read from + #: ``myplugin/build/myservice/Dockerfile`` + #: - ``tag`` is the Docker tag that will be applied to the image. It will + #: rendered at runtime with the user configuration. Thus, the image tag could be ``"{{ + #: DOCKER_REGISTRY }}/myimage:{{ TUTOR_VERSION }}"``. + #: - ``args`` is a list of arguments that will be passed to ``docker build ...``. + #: :parameter dict config: user configuration. + APP_TASK_IMAGES_BUILD = "app:tasks:images:build" + + #: List of images to be pulled when we run ``tutor images pull ...``. + #: + #: :parameter list[tuple[str, str]] tasks: list of ``(name, tag)`` tuples. + #: + #: - ``name`` is the name of the image, as in ``tutor images pull myimage``. + #: - ``tag`` is the Docker tag that will be applied to the image. (see :py:data:`APP_TASK_IMAGES_BUILD`). + #: :parameter dict config: user configuration. + APP_TASK_IMAGES_PULL = "app:tasks:images:pull" + APP_TASK_IMAGES_PUSH = "app:tasks:images:push" + + #: List of tasks to be performed during initialization. These tasks typically + #: include database migrations, setting feature flags, etc. + #: + #: :parameter list[tuple[str, tuple[str, ...]]] tasks: list of ``(service, path)`` tasks. + #: + #: - ``service`` is the name of the container in which the task will be executed. + #: - ``path`` is a tuple that corresponds to a template relative path. Example: + #: ``("myplugin", "hooks", "myservice", "pre-init")`` (see :py:data:`APP_TASK_IMAGES_BUILD`). + APP_TASK_INIT = "app:tasks:init" + + #: List of tasks to be performed prior to initialization. These tasks are run even + #: before the mysql databases are created and the migrations are applied. + #: + #: :parameter list[tuple[str, tuple[str, ...]]] tasks: list of ``(service, path)`` tasks. (see :py:data:`APP_TASK_INIT`). + APP_TASK_PRE_INIT = "app:tasks:pre-init" + + #: List of command line interface (CLI) commands. + #: + #: :parameter list commands: commands are instances of ``click.Command``. They will + #: all be added as subcommands of the main ``tutor`` command. + CLI_COMMANDS = "cli:commands" + + #: Declare new configuration settings that must be saved in the user ``config.yml`` file. This is where + #: you should declare passwords and randomly-generated values. + #: + #: :parameter list[tuple[str, ...]] items: list of (name, value) new settings. All + #: names must be prefixed with the plugin name in all-caps. + CONFIG_BASE = "config:base" + + #: Declare new default configuration settings that don't necessarily have to be saved in the user + #: ``config.yml`` file. Default settings may be overridden with ``tutor config save --set=...``, in which + #: case they will automatically be added to ``config.yml``. + #: + #: :parameter list[tuple[str, ...]] items: list of (name, value) new settings. All + # new entries must be prefixed with the plugin name in all-caps. + CONFIG_DEFAULTS = "config:defaults" + + #: Modify existing settings, either from Tutor core or from other plugins. Beware not to override any + #: important setting, such as passwords! Overridden setting values will be printed to stdout when the plugin + #: is disabled, such that users have a chance to back them up. + #: + #: :parameter list[tuple[str, ...]] items: list of (name, value) settings. + CONFIG_OVERRIDES = "config:overrides" + + #: List of patches that should be inserted in a given location of the templates. The + #: filter name will be formatted with the patch name at runtime. + #: + #: :parameter list[str] patches: each item is the unrendered patch content. Use this + #: filter to modify the Tutor templates. + ENV_PATCHES = "env:patches:%s" + + #: List of all template root folders. + #: + #: :parameter list[str] templates_root: absolute paths to folders which contain templates. + #: The templates in these folders will then be accessible by the environment + #: renderer using paths that are relative to their template root. + ENV_TEMPLATE_ROOTS = "env:templates:roots" + + #: List of template source/destination targets. + #: + #: :parameter list[tuple[str, str]] targets: list of (source, destination) pairs. + #: Each source is a path relative to one of the template roots, and each destination + #: is a path relative to the environment root. For instance: adding ``("c/d", + #: "a/b")`` to the filter will cause all files from "c/d" to be rendered to the ``a/b/c/d`` + #: subfolder. + ENV_TEMPLATE_TARGETS = "env:templates:targets" + + #: List of `Jinja2 filters `__ that will be + #: available in templates. Jinja2 filters are basically functions that can be used + #: as follows within templates:: + #: + #: {{ "somevalue"|my_filter }} + #: + #: :parameter filters: list of (name, function) tuples. The function signature + #: should correspond to its usage in templates. + ENV_TEMPLATE_FILTERS = "env:templates:filters" + + #: List of extra variables to be included in all templates. + #: + #: :parameter filters: list of (name, value) tuples. + ENV_TEMPLATE_VARIABLES = "env:templates:variables" + + #: List of installed plugins. A plugin is first installed, then enabled. + #: + #: :param list[str] plugins: plugin developers probably don't have to modify this + #: filter themselves, but they can apply it to check for the presence of other + #: plugins. + PLUGINS_INSTALLED = "plugins:installed" + + #: Information about each installed plugin, including its version. + #: Keep this information to a single line for easier parsing by 3rd-party scripts. + #: + #: :param list[tuple[str, str]] versions: each pair is a ``(plugin, info)`` tuple. + PLUGINS_INFO = "plugins:installed:versions" + + #: List of enabled plugins. + #: + #: :param list[str] plugins: plugin developers probably don't have to modify this + #: filter themselves, but they can apply it to check whether other plugins are enabled. + PLUGINS_ENABLED = "plugins:enabled" + + +class Contexts: + """ + Contexts are used to track in which parts of the code filters and actions have been + declared. Let's look at an example:: + + from tutor import hooks + + with hooks.contexts.enter("c1"): + @hooks.filters.add("f1") def add_stuff_to_filter(...): + ... + + The fact that our custom filter was added in a certain context allows us to later + remove it. To do so, we write:: + + from tutor import hooks + hooks.filters.clear("f1", context="c1") + + This makes it easy to disable side-effects by plugins, provided they were created with appropriate contexts. + + Here we list all the contexts that are used across Tutor. + """ + + #: We enter this context whenever we create hooks for a specific application or : + #: plugin. For instance, plugin "myplugin" will be enabled within the "app:myplugin" + #: context. + APP = "app:%s" + + #: Plugins will be installed and enabled within this context. + PLUGINS = "plugins" + + #: YAML-formatted v0 plugins will be installed within that context. + PLUGINS_V0_YAML = "plugins:v0:yaml" + + #: Python entrypoint plugins will be installed within that context. + PLUGINS_V0_ENTRYPOINT = "plugins:v0:entrypoint" diff --git a/tutor/hooks/contexts.py b/tutor/hooks/contexts.py new file mode 100644 index 00000000000..e007a855b9b --- /dev/null +++ b/tutor/hooks/contexts.py @@ -0,0 +1,65 @@ +# The Tutor plugin system is licensed under the terms of the Apache 2.0 license. +__license__ = "Apache 2.0" + +import typing as t +from contextlib import contextmanager + + +class Contextualized: + """ + This is a simple class to store the current context in hooks. + + The current context is stored as a static variable. + """ + + CURRENT: t.List[str] = [] + + def __init__(self) -> None: + self.contexts = self.CURRENT[:] + + def is_in_context(self, context: t.Optional[str]) -> bool: + return context is None or context in self.contexts + + @classmethod + @contextmanager + def enter(cls, *names: str) -> t.Iterator[None]: + """ + Identify created hooks with one or multiple context strings. + + :param names: names of the contexts that will be attached to hooks. Multiple + context names may be defined. + + Usage:: + + from tutor import hooks + + with hooks.contexts.enter("my-context"): + # declare new actions and filters + ... + + # Later on, actions and filters can be disabled with: + hooks.actions.clear_all(context="my-context") + hooks.filters.clear_all(context="my-context") + + This is a context manager that will attach a context name to all hooks + created within its scope. The purpose of contexts is to solve an issue that + is inherent to pluggable hooks: it is difficult to track in which part of the + code each hook was created. This makes things hard to debug when a specific + hook goes wrong. It also makes it impossible to disable some hooks after + they have been created. + + We resolve this issue by storing the current contexts in a static list. + Whenever a hook is created, the list of current contexts is copied as a + ``contexts`` attribute. This attribute can be later examined, either for + removal or for limiting the set of hooks that should be applied. + """ + try: + for name in names: + cls.CURRENT.append(name) + yield + finally: + for _ in range(len(names)): + cls.CURRENT.pop() + + +enter = Contextualized.enter diff --git a/tutor/hooks/filters.py b/tutor/hooks/filters.py new file mode 100644 index 00000000000..fec8b77dd95 --- /dev/null +++ b/tutor/hooks/filters.py @@ -0,0 +1,198 @@ +# The Tutor plugin system is licensed under the terms of the Apache 2.0 license. +__license__ = "Apache 2.0" + +import sys +import typing as t + +from . import contexts + +# For now, this signature is not very restrictive. In the future, we could improve it by writing: +# +# P = ParamSpec("P") +# CallableFilter = t.Callable[Concatenate[T, P], T] +# +# See PEP-612: https://www.python.org/dev/peps/pep-0612/ +# Unfortunately, this piece of code fails because of a bug in mypy: +# https://github.com/python/mypy/issues/11833 +# https://github.com/python/mypy/issues/8645 +# https://github.com/python/mypy/issues/5876 +# https://github.com/python/typing/issues/696 +T = t.TypeVar("T") +CallableFilter = t.Callable[..., t.Any] + + +class Filter(contexts.Contextualized): + """ + A filter is simply a function associated to a context. + """ + + def __init__(self, func: CallableFilter): + super().__init__() + self.func = func + + def apply( + self, value: T, *args: t.Any, context: t.Optional[str] = None, **kwargs: t.Any + ) -> T: + if self.is_in_context(context): + value = self.func(value, *args, **kwargs) + return value + + +class Filters: + """ + Singleton set of named filters. + """ + + INSTANCE = None + + @classmethod + def instance(cls) -> "Filters": + if cls.INSTANCE is None: + cls.INSTANCE = cls() + return cls.INSTANCE + + def __init__(self) -> None: + self.filters: t.Dict[str, t.List[Filter]] = {} + + def add(self, name: str, func: CallableFilter) -> None: + self.filters.setdefault(name, []).append(Filter(func)) + + def iterate( + self, name: str, *args: t.Any, context: t.Optional[str] = None, **kwargs: t.Any + ) -> t.Iterator[T]: + """ + Convenient function to iterate over the results of a filter result list. + + This pieces of code are equivalent:: + + for value in filters.apply("my-filter", [], *args, **kwargs): + ... + + for value in filters.iterate("my-filter", *args, **kwargs): + ... + + :rtype iterator[T]: iterator over the list items from the filter with the same name. + """ + yield from self.apply(name, [], *args, context=context, **kwargs) + + def apply( + self, + name: str, + value: T, + *args: t.Any, + context: t.Optional[str] = None, + **kwargs: t.Any, + ) -> T: + """ + Apply all declared filters to a single value, passing along the additional arguments. + + The return value of every filter is passed as the first argument to the next callback. + + Usage:: + + results = filters.apply("my-filter", ["item0"]) + + :type value: object + :rtype: same as the type of ``value``. + """ + for filtre in self.filters.get(name, []): + try: + value = filtre.apply(value, *args, context=context, **kwargs) + except: + sys.stderr.write( + f"Error applying filter '{name}': func={filtre.func} contexts={filtre.contexts}'\n" + ) + raise + return value + + def clear_all(self, context: t.Optional[str] = None) -> None: + """ + Clear any previously defined filter with the given context. + """ + for name in self.filters: + self.clear(name, context=context) + + def clear(self, name: str, context: t.Optional[str] = None) -> None: + """ + Clear any previously defined filter with the given name and context. + """ + if name not in self.filters: + return + self.filters[name] = [ + filtre for filtre in self.filters[name] if not filtre.is_in_context(context) + ] + + +def add(name: str) -> t.Callable[[CallableFilter], CallableFilter]: + """ + Decorator for functions that will be applied to a single named filter. + + :param name: name of the filter to which the decorated function should be added. + + The return value of each filter function will be passed as the first argument to the next one. + + Usage:: + + from tutor import hooks + + @hooks.filters.add("my-filter") + def my_func(value, some_other_arg): + # Do something with `value` + ... + return value + + # After filters have been created, the result of calling all filter callbacks is obtained by running: + hooks.filters.apply("my-filter", initial_value, some_other_argument_value) + """ + + def inner(func: CallableFilter) -> CallableFilter: + Filters.instance().add(name, func) + return func + + return inner + + +def add_item(name: str, item: T) -> None: + """ + Convenience function to add a single item to a filter that returns a list of items. + + :param name: filter name. + :param object item: item that will be appended to the resulting list. + + Usage:: + + from tutor import hooks + + hooks.filters.add_item("my-filter", "item1") + hooks.filters.add_item("my-filter", "item2") + + assert ["item1", "item2"] == hooks.filters.apply("my-filter", []) + """ + add_items(name, [item]) + + +def add_items(name: str, items: t.List[T]) -> None: + """ + Convenience function to add multiple item to a filter that returns a list of items. + + :param name: filter name. + :param list[object] items: items that will be appended to the resulting list. + + Usage:: + + from tutor import hooks + + hooks.filters.add_items("my-filter", ["item1", "item2"]) + + assert ["item1", "item2"] == hooks.filters.apply("my-filter", []) + """ + + @add(name) + def callback(value: t.List[T], *_args: t.Any, **_kwargs: t.Any) -> t.List[T]: + return value + items + + +iterate = Filters.instance().iterate +apply = Filters.instance().apply +clear = Filters.instance().clear +clear_all = Filters.instance().clear_all diff --git a/tutor/images.py b/tutor/images.py index 1939d3ab8a0..b87b55010c5 100644 --- a/tutor/images.py +++ b/tutor/images.py @@ -8,15 +8,15 @@ def get_tag(config: Config, name: str) -> str: def build(path: str, tag: str, *args: str) -> None: - fmt.echo_info("Building image {}".format(tag)) + fmt.echo_info(f"Building image {tag}") utils.docker("build", "-t", tag, *args, path) def pull(tag: str) -> None: - fmt.echo_info("Pulling image {}".format(tag)) + fmt.echo_info(f"Pulling image {tag}") utils.docker("pull", tag) def push(tag: str) -> None: - fmt.echo_info("Pushing image {}".format(tag)) + fmt.echo_info(f"Pushing image {tag}") utils.docker("push", tag) diff --git a/tutor/interactive.py b/tutor/interactive.py index 25ba3d9f750..ce33756458b 100644 --- a/tutor/interactive.py +++ b/tutor/interactive.py @@ -18,7 +18,8 @@ def load_user_config(root: str, interactive: bool = True) -> Config: def ask_questions(config: Config) -> None: - defaults = tutor_config.get_defaults(config) + # TODO we should add interactive questions as hooks here. + defaults = tutor_config.get_defaults() run_for_prod = config.get("LMS_HOST") != "local.overhang.io" run_for_prod = click.confirm( fmt.question( @@ -38,7 +39,7 @@ def ask_questions(config: Config) -> None: ) for k, v in dev_values.items(): config[k] = v - fmt.echo_info(" {} = {}".format(k, v)) + fmt.echo_info(f" {k} = {v}") if run_for_prod: ask("Your website domain name for students (LMS)", "LMS_HOST", config, defaults) diff --git a/tutor/jobs.py b/tutor/jobs.py index 56624c81d21..1896f0ff988 100644 --- a/tutor/jobs.py +++ b/tutor/jobs.py @@ -1,7 +1,7 @@ -from typing import Dict, Iterator, List, Optional, Tuple, Union +import typing as t -from . import env, fmt, plugins -from .types import Config, get_typed +from tutor import env, fmt, hooks +from tutor.types import Config, get_typed BASE_OPENEDX_COMMAND = """ export DJANGO_SETTINGS_MODULE=$SERVICE_VARIANT.envs.$SETTINGS @@ -36,44 +36,54 @@ def run_job(self, service: str, command: str) -> int: """ raise NotImplementedError - def iter_plugin_hooks( - self, hook: str - ) -> Iterator[Tuple[str, Union[Dict[str, str], List[str]]]]: - yield from plugins.iter_hooks(self.config, hook) - class BaseComposeJobRunner(BaseJobRunner): def docker_compose(self, *command: str) -> int: raise NotImplementedError -def initialise(runner: BaseJobRunner, limit_to: Optional[str] = None) -> None: +@hooks.actions.on(hooks.Actions.CORE_READY) +def _bootstrap_init_scripts() -> None: + """ + Declare core init scripts at runtime. + + The context is important, because it allows us to select the init scripts based on + the --limit argument. + """ + with hooks.contexts.enter(hooks.Contexts.APP % "mysql"): + hooks.filters.add_item( + hooks.Filters.APP_TASK_INIT, ("mysql", ("hooks", "mysql", "init")) + ) + with hooks.contexts.enter(hooks.Contexts.APP % "lms"): + hooks.filters.add_item( + hooks.Filters.APP_TASK_INIT, ("lms", ("hooks", "lms", "init")) + ) + with hooks.contexts.enter(hooks.Contexts.APP % "cms"): + hooks.filters.add_item( + hooks.Filters.APP_TASK_INIT, ("cms", ("hooks", "cms", "init")) + ) + + +def initialise(runner: BaseJobRunner, limit_to: t.Optional[str] = None) -> None: fmt.echo_info("Initialising all services...") - if limit_to is None or limit_to == "mysql": - fmt.echo_info("Initialising mysql...") - runner.run_job_from_template("mysql", "hooks", "mysql", "init") - for plugin_name, hook in runner.iter_plugin_hooks("pre-init"): - if limit_to is None or limit_to == plugin_name: - for service in hook: - fmt.echo_info( - f"Plugin {plugin_name}: running pre-init for service {service}..." - ) - runner.run_job_from_template( - service, plugin_name, "hooks", service, "pre-init" - ) - for service in ["lms", "cms"]: - if limit_to is None or limit_to == service: - fmt.echo_info(f"Initialising {service}...") - runner.run_job_from_template(service, "hooks", service, "init") - for plugin_name, hook in runner.iter_plugin_hooks("init"): - if limit_to is None or limit_to == plugin_name: - for service in hook: - fmt.echo_info( - f"Plugin {plugin_name}: running init for service {service}..." - ) - runner.run_job_from_template( - service, plugin_name, "hooks", service, "init" - ) + filter_context = hooks.Contexts.APP % limit_to if limit_to else None + + # Pre-init tasks + iter_pre_init_tasks: t.Iterator[ + t.Tuple[str, t.Iterable[str]] + ] = hooks.filters.iterate(hooks.Filters.APP_TASK_PRE_INIT, context=filter_context) + for service, path in iter_pre_init_tasks: + fmt.echo_info(f"Running pre-init task: {'/'.join(path)}") + runner.run_job_from_template(service, *path) + + # Init tasks + iter_init_tasks: t.Iterator[t.Tuple[str, t.Iterable[str]]] = hooks.filters.iterate( + hooks.Filters.APP_TASK_INIT, context=filter_context + ) + for service, path in iter_init_tasks: + fmt.echo_info(f"Running init task: {'/'.join(path)}") + runner.run_job_from_template(service, *path) + fmt.echo_info("All services initialised.") @@ -82,7 +92,7 @@ def create_user_command( staff: bool, username: str, email: str, - password: Optional[str] = None, + password: t.Optional[str] = None, ) -> str: command = BASE_OPENEDX_COMMAND @@ -113,7 +123,9 @@ def import_demo_course(runner: BaseJobRunner) -> None: runner.run_job_from_template("cms", "hooks", "cms", "importdemocourse") -def set_theme(theme_name: str, domain_names: List[str], runner: BaseJobRunner) -> None: +def set_theme( + theme_name: str, domain_names: t.List[str], runner: BaseJobRunner +) -> None: """ For each domain, get or create a Site object and assign the selected theme. """ @@ -136,7 +148,7 @@ def set_theme(theme_name: str, domain_names: List[str], runner: BaseJobRunner) - runner.run_job("lms", command) -def get_all_openedx_domains(config: Config) -> List[str]: +def get_all_openedx_domains(config: Config) -> t.List[str]: return [ get_typed(config, "LMS_HOST", str), get_typed(config, "LMS_HOST", str) + ":8000", diff --git a/tutor/plugins.py b/tutor/plugins.py deleted file mode 100644 index fef582aa602..00000000000 --- a/tutor/plugins.py +++ /dev/null @@ -1,439 +0,0 @@ -import importlib -import os -from copy import deepcopy -from glob import glob -from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, Union - -import appdirs -import click -import pkg_resources - -from . import exceptions, fmt, serialize -from .__about__ import __app__ -from .types import Config, get_typed - -CONFIG_KEY = "PLUGINS" - - -class BasePlugin: - """ - Tutor plugins are defined by a name and an object that implements one or more of the - following properties: - - `config` (dict str->dict(str->str)): contains "add", "defaults", "set" keys. Entries - in these dicts will be added or override the global configuration. Keys in "add" and - "defaults" will be prefixed by the plugin name in uppercase. - - `patches` (dict str->str): entries in this dict will be used to patch the rendered - Tutor templates. For instance, to add "somecontent" to a template that includes '{{ - patch("mypatch") }}', set: `patches["mypatch"] = "somecontent"`. It is recommended - to store all patches in separate files, and to dynamically list patches by listing - the contents of a "patches" subdirectory. - - `templates` (str): path to a directory that includes new template files for the - plugin. It is recommended that all files in the template directory are stored in a - `myplugin` folder to avoid conflicts with other plugins. Plugin templates are useful - for content re-use, e.g: "{% include 'myplugin/mytemplate.html'}". - - `hooks` (dict str->list[str]): hooks are commands that will be run at various points - during the lifetime of the platform. For instance, to run `service1` and `service2` - in sequence during initialization, you should define: - - hooks["init"] = ["service1", "service2"] - - It is then assumed that there are `myplugin/hooks/service1/init` and - `myplugin/hooks/service2/init` templates in the plugin `templates` directory. - - `command` (click.Command): if a plugin exposes a `command` attribute, users will be able to run it from the command line as `tutor pluginname`. - """ - - INSTALLED: List["BasePlugin"] = [] - _IS_LOADED = False - - def __init__(self, name: str, obj: Any) -> None: - self.name = name - self.config = self.load_config(obj, self.name) - self.patches = self.load_patches(obj, self.name) - self.hooks = self.load_hooks(obj, self.name) - - templates_root = get_callable_attr(obj, "templates", default=None) - if templates_root is not None: - assert isinstance(templates_root, str) - self.templates_root = templates_root - - command = getattr(obj, "command", None) - if command is not None: - assert isinstance(command, click.Command) - self.command: Optional[click.Command] = command - - @staticmethod - def load_config(obj: Any, plugin_name: str) -> Dict[str, Config]: - """ - Load config and check types. - """ - config = get_callable_attr(obj, "config", {}) - if not isinstance(config, dict): - raise exceptions.TutorError( - f"Invalid config in plugin {plugin_name}. Expected dict, got {config.__class__}." - ) - for name, subconfig in config.items(): - if not isinstance(name, str): - raise exceptions.TutorError( - f"Invalid config entry '{name}' in plugin {plugin_name}. Expected str, got {config.__class__}." - ) - if not isinstance(subconfig, dict): - raise exceptions.TutorError( - f"Invalid config entry '{name}' in plugin {plugin_name}. Expected str keys, got {config.__class__}." - ) - for key in subconfig.keys(): - if not isinstance(key, str): - raise exceptions.TutorError( - f"Invalid config entry '{name}.{key}' in plugin {plugin_name}. Expected str, got {key.__class__}." - ) - return config - - @staticmethod - def load_patches(obj: Any, plugin_name: str) -> Dict[str, str]: - """ - Load patches and check the types are right. - """ - patches = get_callable_attr(obj, "patches", {}) - if not isinstance(patches, dict): - raise exceptions.TutorError( - f"Invalid patches in plugin {plugin_name}. Expected dict, got {patches.__class__}." - ) - for patch_name, content in patches.items(): - if not isinstance(patch_name, str): - raise exceptions.TutorError( - f"Invalid patch name '{patch_name}' in plugin {plugin_name}. Expected str, got {patch_name.__class__}." - ) - if not isinstance(content, str): - raise exceptions.TutorError( - f"Invalid patch '{patch_name}' in plugin {plugin_name}. Expected str, got {content.__class__}." - ) - return patches - - @staticmethod - def load_hooks( - obj: Any, plugin_name: str - ) -> Dict[str, Union[Dict[str, str], List[str]]]: - """ - Load hooks and check types. - """ - hooks = get_callable_attr(obj, "hooks", default={}) - if not isinstance(hooks, dict): - raise exceptions.TutorError( - f"Invalid hooks in plugin {plugin_name}. Expected dict, got {hooks.__class__}." - ) - for hook_name, hook in hooks.items(): - if not isinstance(hook_name, str): - raise exceptions.TutorError( - f"Invalid hook name '{hook_name}' in plugin {plugin_name}. Expected str, got {hook_name.__class__}." - ) - if isinstance(hook, list): - for service in hook: - if not isinstance(service, str): - raise exceptions.TutorError( - f"Invalid service in hook '{hook_name}' from plugin {plugin_name}. Expected str, got {service.__class__}." - ) - elif isinstance(hook, dict): - for name, value in hook.items(): - if not isinstance(name, str) or not isinstance(value, str): - raise exceptions.TutorError( - f"Invalid hook '{hook_name}' in plugin {plugin_name}. Only str -> str entries are supported." - ) - else: - raise exceptions.TutorError( - f"Invalid hook '{hook_name}' in plugin {plugin_name}. Expected dict or list, got {hook.__class__}." - ) - return hooks - - def config_key(self, key: str) -> str: - """ - Config keys in the "add" and "defaults" dicts should be prefixed by the plugin name, in uppercase. - """ - return self.name.upper() + "_" + key - - @property - def config_add(self) -> Config: - return self.config.get("add", {}) - - @property - def config_set(self) -> Config: - return self.config.get("set", {}) - - @property - def config_defaults(self) -> Config: - return self.config.get("defaults", {}) - - @property - def version(self) -> str: - raise NotImplementedError - - @classmethod - def iter_installed(cls) -> Iterator["BasePlugin"]: - if not cls._IS_LOADED: - for plugin in cls.iter_load(): - cls.INSTALLED.append(plugin) - cls._IS_LOADED = True - yield from cls.INSTALLED - - @classmethod - def iter_load(cls) -> Iterator["BasePlugin"]: - raise NotImplementedError - - @classmethod - def clear_cache(cls) -> None: - cls._IS_LOADED = False - cls.INSTALLED.clear() - - -class EntrypointPlugin(BasePlugin): - """ - Entrypoint plugins are regular python packages that have a 'tutor.plugin.v0' entrypoint. - - The API for Tutor plugins is currently in development. The entrypoint will switch to - 'tutor.plugin.v1' once it is stabilised. - """ - - ENTRYPOINT = "tutor.plugin.v0" - - def __init__(self, entrypoint: pkg_resources.EntryPoint) -> None: - super().__init__(entrypoint.name, entrypoint.load()) - self.entrypoint = entrypoint - - @property - def version(self) -> str: - if not self.entrypoint.dist: - return "0.0.0" - return self.entrypoint.dist.version - - @classmethod - def iter_load(cls) -> Iterator["EntrypointPlugin"]: - for entrypoint in pkg_resources.iter_entry_points(cls.ENTRYPOINT): - try: - error: Optional[str] = None - yield cls(entrypoint) - except pkg_resources.VersionConflict as e: - error = e.report() - except Exception as e: # pylint: disable=broad-except - error = str(e) - if error: - fmt.echo_error( - f"Failed to load entrypoint '{entrypoint.name} = {entrypoint.module_name}' from distribution {entrypoint.dist}: {error}" - ) - - -class OfficialPlugin(BasePlugin): - """ - Official plugins have a "plugin" module which exposes a __version__ attribute. - Official plugins should be manually added by calling `OfficialPlugin.load()`. - """ - - @classmethod - def load(cls, name: str) -> BasePlugin: - plugin = cls(name) - cls.INSTALLED.append(plugin) - return plugin - - def __init__(self, name: str): - self.module = importlib.import_module(f"tutor{name}.plugin") - super().__init__(name, self.module) - - @property - def version(self) -> str: - version = getattr(self.module, "__version__") - if not isinstance(version, str): - raise TypeError("OfficialPlugin __version__ must be 'str'") - return version - - @classmethod - def iter_load(cls) -> Iterator[BasePlugin]: - yield from [] - - -class DictPlugin(BasePlugin): - ROOT_ENV_VAR_NAME = "TUTOR_PLUGINS_ROOT" - ROOT = os.path.expanduser( - os.environ.get(ROOT_ENV_VAR_NAME, "") - ) or appdirs.user_data_dir(appname=__app__ + "-plugins") - - def __init__(self, data: Config): - name = data["name"] - if not isinstance(name, str): - raise exceptions.TutorError( - f"Invalid plugin name: '{name}'. Expected str, got {name.__class__}" - ) - - # Create a generic object (sort of a named tuple) which will contain all key/values from data - class Module: - pass - - obj = Module() - for key, value in data.items(): - setattr(obj, key, value) - - super().__init__(name, obj) - - version = data["version"] - if not isinstance(version, str): - raise TypeError("DictPlugin.__version__ must be str") - self._version: str = version - - @property - def version(self) -> str: - return self._version - - @classmethod - def iter_load(cls) -> Iterator[BasePlugin]: - for path in glob(os.path.join(cls.ROOT, "*.yml")): - with open(path, encoding="utf-8") as f: - data = serialize.load(f) - if not isinstance(data, dict): - raise exceptions.TutorError( - f"Invalid plugin: {path}. Expected dict." - ) - try: - yield cls(data) - except KeyError as e: - raise exceptions.TutorError( - f"Invalid plugin: {path}. Missing key: {e.args[0]}" - ) - - -class Plugins: - PLUGIN_CLASSES: List[Type[BasePlugin]] = [ - OfficialPlugin, - EntrypointPlugin, - DictPlugin, - ] - - def __init__(self, config: Config): - self.config = deepcopy(config) - # patches has the following structure: - # {patch_name -> {plugin_name -> "content"}} - self.patches: Dict[str, Dict[str, str]] = {} - # some hooks have a dict-like structure, like "build", others are list of services. - self.hooks: Dict[str, Dict[str, Union[Dict[str, str], List[str]]]] = {} - self.template_roots: Dict[str, str] = {} - - for plugin in self.iter_enabled(): - for patch_name, content in plugin.patches.items(): - if patch_name not in self.patches: - self.patches[patch_name] = {} - self.patches[patch_name][plugin.name] = content - - for hook_name, services in plugin.hooks.items(): - if hook_name not in self.hooks: - self.hooks[hook_name] = {} - self.hooks[hook_name][plugin.name] = services - - @classmethod - def clear_cache(cls) -> None: - for PluginClass in cls.PLUGIN_CLASSES: - PluginClass.clear_cache() - - @classmethod - def iter_installed(cls) -> Iterator[BasePlugin]: - """ - Iterate on all installed plugins. Plugins are deduplicated by name. The list of installed plugins is cached to - prevent too many re-computations, which happens a lot. - """ - installed_plugin_names = set() - plugins = [] - for PluginClass in cls.PLUGIN_CLASSES: - for plugin in PluginClass.iter_installed(): - if plugin.name not in installed_plugin_names: - installed_plugin_names.add(plugin.name) - plugins.append(plugin) - plugins = sorted(plugins, key=lambda plugin: plugin.name) - yield from plugins - - def iter_enabled(self) -> Iterator[BasePlugin]: - for plugin in self.iter_installed(): - if is_enabled(self.config, plugin.name): - yield plugin - - def iter_patches(self, name: str) -> Iterator[Tuple[str, str]]: - plugin_patches = self.patches.get(name, {}) - plugins = sorted(plugin_patches.keys()) - for plugin in plugins: - yield plugin, plugin_patches[plugin] - - def iter_hooks( - self, hook_name: str - ) -> Iterator[Tuple[str, Union[Dict[str, str], List[str]]]]: - yield from self.hooks.get(hook_name, {}).items() - - -def get_callable_attr( - plugin: Any, attr_name: str, default: Optional[Any] = None -) -> Optional[Any]: - attr = getattr(plugin, attr_name, default) - if callable(attr): - attr = attr() - return attr - - -def is_installed(name: str) -> bool: - for plugin in iter_installed(): - if name == plugin.name: - return True - return False - - -def iter_installed() -> Iterator[BasePlugin]: - yield from Plugins.iter_installed() - - -def enable(config: Config, name: str) -> None: - if not is_installed(name): - raise exceptions.TutorError(f"plugin '{name}' is not installed.") - if is_enabled(config, name): - return - enabled = enabled_plugins(config) - enabled.append(name) - enabled.sort() - - -def disable(config: Config, plugin: BasePlugin) -> None: - # Remove plugin-specific set config - for key in plugin.config_set.keys(): - config.pop(key, None) - - # Remove plugin from list of enabled plugins - enabled = enabled_plugins(config) - while plugin.name in enabled: - enabled.remove(plugin.name) - - -def get_enabled(config: Config, name: str) -> BasePlugin: - for plugin in iter_enabled(config): - if plugin.name == name: - return plugin - raise ValueError(f"Enabled plugin {name} could not be found.") - - -def iter_enabled(config: Config) -> Iterator[BasePlugin]: - yield from Plugins(config).iter_enabled() - - -def is_enabled(config: Config, name: str) -> bool: - return name in enabled_plugins(config) - - -def enabled_plugins(config: Config) -> List[str]: - if not config.get(CONFIG_KEY): - config[CONFIG_KEY] = [] - plugins = get_typed(config, CONFIG_KEY, list) - return plugins - - -def iter_patches(config: Config, name: str) -> Iterator[Tuple[str, str]]: - yield from Plugins(config).iter_patches(name) - - -def iter_hooks( - config: Config, hook_name: str -) -> Iterator[Tuple[str, Union[Dict[str, str], List[str]]]]: - yield from Plugins(config).iter_hooks(hook_name) diff --git a/tutor/plugins/__init__.py b/tutor/plugins/__init__.py new file mode 100644 index 00000000000..7b81ffaf4f1 --- /dev/null +++ b/tutor/plugins/__init__.py @@ -0,0 +1,102 @@ +""" +Provide API for plugin features. +""" +import typing as t +from copy import deepcopy + +from tutor import exceptions, hooks +from tutor.types import Config, get_typed + +# Import modules to trigger hook creation +from . import v0 +from . import v1 + + +@hooks.actions.on(hooks.Actions.CORE_READY) +def install() -> None: + """ + Find all installed plugins. + + This method must be called once prior to loading enabled plugins. Plugins are + installed within a context, such that they can easily be disabled later, for + instance in tests. + """ + with hooks.contexts.enter(hooks.Contexts.PLUGINS): + hooks.actions.do(hooks.Actions.INSTALL_PLUGINS) + + +def is_installed(name: str) -> bool: + """ + Return true if the plugin is installed. + + The install() method must have been called prior to calling this one, + otherwise no installed plugin will be detected. + """ + return name in iter_installed() + + +def iter_installed() -> t.Iterator[str]: + """ + Iterate on all installed plugins, sorted by name. + + This will yield all plugins, including those that have the same name. + """ + plugins: t.Iterator[str] = hooks.filters.iterate(hooks.Filters.PLUGINS_INSTALLED) + yield from sorted(plugins) + + +def iter_info() -> t.Iterator[t.Tuple[str, t.Optional[str]]]: + """ + Iterate on the information of all installed plugins. + + Yields (, ) tuples. + """ + versions: t.Iterator[t.Tuple[str, t.Optional[str]]] = hooks.filters.iterate( + hooks.Filters.PLUGINS_INFO + ) + yield from sorted(versions, key=lambda v: v[0]) + + +def is_enabled(name: str) -> bool: + for plugin in iter_enabled(): + if plugin == name: + return True + return False + + +def enable(name: str) -> None: + """ + Enable a given plugin. + + Enabling a plugin is done within a context, such that we can remove all hooks when a + plugin is disabled, or during unit tests. + """ + if not is_installed(name): + raise exceptions.TutorError(f"plugin '{name}' is not installed.") + with hooks.contexts.enter(hooks.Contexts.PLUGINS, hooks.Contexts.APP % name): + hooks.actions.do(hooks.Actions.ENABLE_PLUGIN % name) + + +def iter_enabled() -> t.Iterator[str]: + """ + Iterate on the list of enabled plugin names, sorted in alphabetical order. + + Note that enabled plugin names are deduplicated. Thus, if two plugins have + the same name, just one name will be displayed. + """ + plugins: t.Iterable[str] = hooks.filters.iterate(hooks.Filters.PLUGINS_ENABLED) + yield from sorted(set(plugins)) + + +def iter_patches(name: str) -> t.Iterator[str]: + """ + Yields: patch (str) + """ + yield from hooks.filters.apply(hooks.Filters.ENV_PATCHES % name, []) + + +def disable(plugin: str) -> None: + """ + Remove all filters and actions associated to a given plugin. + """ + hooks.clear_all(context=hooks.Contexts.APP % plugin) diff --git a/tutor/plugins/base.py b/tutor/plugins/base.py new file mode 100644 index 00000000000..307e1f3d298 --- /dev/null +++ b/tutor/plugins/base.py @@ -0,0 +1,17 @@ +import os + +import appdirs + +from tutor.__about__ import __app__ + + +PLUGINS_ROOT_ENV_VAR_NAME = "TUTOR_PLUGINS_ROOT" + +# Folder path which contains *.yml and *.py file plugins. +# On linux this is typically ``~/.local/share/tutor-plugins``. On the nightly branch +# this will be ``~/.local/share/tutor-plugins-nightly``. +# The path can be overridden by defining the ``TUTOR_PLUGINS_ROOT`` environment +# variable. +PLUGINS_ROOT = os.path.expanduser( + os.environ.get(PLUGINS_ROOT_ENV_VAR_NAME, "") +) or appdirs.user_data_dir(appname=__app__ + "-plugins") diff --git a/tutor/plugins/v0.py b/tutor/plugins/v0.py new file mode 100644 index 00000000000..f3e7ba5d5e6 --- /dev/null +++ b/tutor/plugins/v0.py @@ -0,0 +1,402 @@ +import importlib +import importlib.util +import os +import typing as t +from glob import glob + +import click +import pkg_resources + +from tutor import exceptions, fmt, hooks, serialize +from tutor.__about__ import __app__ +from tutor.types import Config + +from .base import PLUGINS_ROOT + + +class BasePlugin: + """ + Tutor plugins are defined by a name and an object that implements one or more of the + following properties: + + `config` (dict str->dict(str->str)): contains "add", "defaults", "set" keys. Entries + in these dicts will be added or override the global configuration. Keys in "add" and + "defaults" will be prefixed by the plugin name in uppercase. + + `patches` (dict str->str): entries in this dict will be used to patch the rendered + Tutor templates. For instance, to add "somecontent" to a template that includes '{{ + patch("mypatch") }}', set: `patches["mypatch"] = "somecontent"`. It is recommended + to store all patches in separate files, and to dynamically list patches by listing + the contents of a "patches" subdirectory. + + `templates` (str): path to a directory that includes new template files for the + plugin. It is recommended that all files in the template directory are stored in a + `myplugin` folder to avoid conflicts with other plugins. Plugin templates are useful + for content re-use, e.g: "{% include 'myplugin/mytemplate.html'}". + + `hooks` (dict str->list[str]): hooks are commands that will be run at various points + during the lifetime of the platform. For instance, to run `service1` and `service2` + in sequence during initialization, you should define: + + hooks["init"] = ["service1", "service2"] + + It is then assumed that there are `myplugin/hooks/service1/init` and + `myplugin/hooks/service2/init` templates in the plugin `templates` directory. + + `command` (click.Command): if a plugin exposes a `command` attribute, users will be able to run it from the command line as `tutor pluginname`. + """ + + def __init__(self, name: str, loader: t.Optional[t.Any] = None) -> None: + self.name = name + self.loader = loader + self.obj: t.Optional[t.Any] = None + self._install() + + def _install(self) -> None: + # Add itself to the list of installed plugins + hooks.filters.add_item(hooks.Filters.PLUGINS_INSTALLED, self.name) + + # Add plugin version + hooks.filters.add_item(hooks.Filters.PLUGINS_INFO, (self.name, self._version())) + + # Create actions and filters on enable + hooks.actions.on(hooks.Actions.ENABLE_PLUGIN % self.name)(self.__enable) + + def __enable(self) -> None: + """ + On enabling a plugin, we create all the required actions and filters. + + Note that this method is quite costly. Thus it is important that as little is + done as part of installing the plugin. For instance, we should not import + modules during installation, but only when the plugin is enabled. + """ + # Add all actions/filters + self._load_obj() + self._load_config() + self._load_patches() + self._load_tasks() + self._load_templates_root() + self._load_command() + # Add self to enabled plugins + hooks.filters.add_item(hooks.Filters.PLUGINS_ENABLED, self.name) + + def _load_obj(self) -> None: + """ + Override this method to write to the `obj` attribute based on the `loader`. + """ + raise NotImplementedError + + def _load_config(self) -> None: + """ + Load config and check types. + """ + config = get_callable_attr(self.obj, "config", {}) + if not isinstance(config, dict): + raise exceptions.TutorError( + f"Invalid config in plugin {self.name}. Expected dict, got {config.__class__}." + ) + for name, subconfig in config.items(): + if not isinstance(name, str): + raise exceptions.TutorError( + f"Invalid config entry '{name}' in plugin {self.name}. Expected str, got {config.__class__}." + ) + if not isinstance(subconfig, dict): + raise exceptions.TutorError( + f"Invalid config entry '{name}' in plugin {self.name}. Expected str keys, got {config.__class__}." + ) + for key in subconfig.keys(): + if not isinstance(key, str): + raise exceptions.TutorError( + f"Invalid config entry '{name}.{key}' in plugin {self.name}. Expected str, got {key.__class__}." + ) + + # Config keys in the "add" and "defaults" dicts must be prefixed by + # the plugin name, in uppercase. + key_prefix = self.name.upper() + "_" + + hooks.filters.add_items( + hooks.Filters.CONFIG_BASE, + [ + (f"{key_prefix}{key}", value) + for key, value in config.get("add", {}).items() + ], + ) + hooks.filters.add_items( + hooks.Filters.CONFIG_DEFAULTS, + [ + (f"{key_prefix}{key}", value) + for key, value in config.get("defaults", {}).items() + ], + ) + hooks.filters.add_items( + hooks.Filters.CONFIG_OVERRIDES, + [(key, value) for key, value in config.get("set", {}).items()], + ) + + def _load_patches(self) -> None: + """ + Load patches and check the types are right. + """ + patches = get_callable_attr(self.obj, "patches", {}) + if not isinstance(patches, dict): + raise exceptions.TutorError( + f"Invalid patches in plugin {self.name}. Expected dict, got {patches.__class__}." + ) + for patch_name, content in patches.items(): + if not isinstance(patch_name, str): + raise exceptions.TutorError( + f"Invalid patch name '{patch_name}' in plugin {self.name}. Expected str, got {patch_name.__class__}." + ) + if not isinstance(content, str): + raise exceptions.TutorError( + f"Invalid patch '{patch_name}' in plugin {self.name}. Expected str, got {content.__class__}." + ) + hooks.filters.add_item(hooks.Filters.ENV_PATCHES % patch_name, content) + + def _load_tasks(self) -> None: + """ + Load hooks and check types. + """ + tasks = get_callable_attr(self.obj, "hooks", default={}) + if not isinstance(tasks, dict): + raise exceptions.TutorError( + f"Invalid hooks in plugin {self.name}. Expected dict, got {tasks.__class__}." + ) + + build_image_tasks = tasks.get("build-image", {}) + remote_image_tasks = tasks.get("remote-image", {}) + pre_init_tasks = tasks.get("pre-init", []) + init_tasks = tasks.get("init", []) + + # Build images: hooks = {"build-image": {"myimage": "myimage:latest"}} + # We assume that the dockerfile is in the build/myimage folder. + for img, tag in build_image_tasks.items(): + hooks.filters.add_item( + hooks.Filters.APP_TASK_IMAGES_BUILD, + (img, ("plugins", self.name, "build", img), tag, []), + ) + # Remote images: hooks = {"remote-image": {"myimage": "myimage:latest"}} + for img, tag in remote_image_tasks.items(): + hooks.filters.add_item( + hooks.Filters.APP_TASK_IMAGES_PULL, + (img, tag), + ) + hooks.filters.add_item( + hooks.Filters.APP_TASK_IMAGES_PUSH, + (img, tag), + ) + # Pre-init scripts: hooks = {"pre-init": ["myservice1", "myservice2"]} + for service in pre_init_tasks: + path = (self.name, "hooks", service, "pre-init") + hooks.filters.add_item(hooks.Filters.APP_TASK_PRE_INIT, (service, path)) + # Init scripts: hooks = {"init": ["myservice1", "myservice2"]} + for service in init_tasks: + path = (self.name, "hooks", service, "init") + hooks.filters.add_item(hooks.Filters.APP_TASK_INIT, (service, path)) + + def _load_templates_root(self) -> None: + templates_root = get_callable_attr(self.obj, "templates", default=None) + if templates_root is None: + return + if not isinstance(templates_root, str): + raise exceptions.TutorError( + f"Invalid templates in plugin {self.name}. Expected str, got {templates_root.__class__}." + ) + + hooks.filters.add_item(hooks.Filters.ENV_TEMPLATE_ROOTS, templates_root) + # We only add the "apps" and "build" folders and we render them in the + # "plugins/" folder. + hooks.filters.add_items( + "env:templates:targets", + [ + ( + os.path.join(self.name, "apps"), + "plugins", + ), + ( + os.path.join(self.name, "build"), + "plugins", + ), + ], + ) + + def _load_command(self) -> None: + command = getattr(self.obj, "command", None) + if command is None: + return + if not isinstance(command, click.Command): + raise exceptions.TutorError( + f"Invalid command in plugin {self.name}. Expected click.Command, got {command.__class__}." + ) + # We force the command name to the plugin name + command.name = self.name + hooks.filters.add_item(hooks.Filters.CLI_COMMANDS, command) + + def _version(self) -> t.Optional[str]: + return None + + +class EntrypointPlugin(BasePlugin): + """ + Entrypoint plugins are regular python packages that have a 'tutor.plugin.v0' entrypoint. + + The API for Tutor plugins is currently in development. The entrypoint will switch to + 'tutor.plugin.v1' once it is stabilised. + """ + + ENTRYPOINT = "tutor.plugin.v0" + + def __init__(self, entrypoint: pkg_resources.EntryPoint) -> None: + self.loader: pkg_resources.EntryPoint + super().__init__(entrypoint.name, entrypoint) + + def _load_obj(self) -> None: + self.obj = self.loader.load() + + def _version(self) -> t.Optional[str]: + if not self.loader.dist: + raise exceptions.TutorError(f"Entrypoint plugin '{self.name}' has no dist.") + return self.loader.dist.version + + @classmethod + def install_all(cls) -> None: + for entrypoint in pkg_resources.iter_entry_points(cls.ENTRYPOINT): + try: + error: t.Optional[str] = None + cls(entrypoint) + except pkg_resources.VersionConflict as e: + error = e.report() + except Exception as e: # pylint: disable=broad-except + error = str(e) + if error: + fmt.echo_error( + f"Failed to load entrypoint '{entrypoint.name} = {entrypoint.module_name}' from distribution {entrypoint.dist}: {error}" + ) + + +class OfficialPlugin(BasePlugin): + """ + Official plugins have a "plugin" module which exposes a __version__ attribute. + Official plugins should be manually added by instantiating them with: `OfficialPlugin('name')`. + """ + + NAMES = [ + "android", + "discovery", + "ecommerce", + "forum", + "license", + "mfe", + "minio", + "notes", + "richie", + "webui", + "xqueue", + ] + + def _load_obj(self) -> None: + self.obj = importlib.import_module(f"tutor{self.name}.plugin") + + def _version(self) -> t.Optional[str]: + try: + module = importlib.import_module(f"tutor{self.name}.__about__") + except ModuleNotFoundError: + return None + version = getattr(module, "__version__") + if version is None: + return None + if not isinstance(version, str): + raise TypeError("OfficialPlugin __version__ must be 'str'") + return version + + @classmethod + def install_all(cls) -> None: + """ + This function must be called explicitely from the main. This is to handle + detection of official plugins from within the compiled binary. When not running + the binary, official plugins are treated as regular entrypoint plugins. + """ + for plugin_name in cls.NAMES: + if importlib.util.find_spec(f"tutor{plugin_name}") is not None: + OfficialPlugin(plugin_name) + + +class DictPlugin(BasePlugin): + def __init__(self, data: Config): + self.loader: Config + name = data["name"] + if not isinstance(name, str): + raise exceptions.TutorError( + f"Invalid plugin name: '{name}'. Expected str, got {name.__class__}" + ) + super().__init__(name, data) + + def _load_obj(self) -> None: + # Create a generic object (sort of a named tuple) which will contain all + # key/values from data + class Module: + pass + + self.obj = Module() + for key, value in self.loader.items(): + setattr(self.obj, key, value) + + def _version(self) -> t.Optional[str]: + version = self.loader.get("version", None) + if version is None: + return None + if not isinstance(version, str): + raise TypeError("DictPlugin.version must be str") + return version + + @classmethod + def install_all(cls) -> None: + for path in glob(os.path.join(PLUGINS_ROOT, "*.yml")): + with open(path, encoding="utf-8") as f: + data = serialize.load(f) + if not isinstance(data, dict): + raise exceptions.TutorError( + f"Invalid plugin: {path}. Expected dict." + ) + try: + cls(data) + except KeyError as e: + raise exceptions.TutorError( + f"Invalid plugin: {path}. Missing key: {e.args[0]}" + ) + + +@hooks.actions.on(hooks.Actions.INSTALL_PLUGINS) +def _install_v0_plugins() -> None: + """ + Install all entrypoint and dict plugins. + + Plugins from both classes are installed in a contexts, to make it easier to disable + them in tests. + + Note that official plugins are not installed here. That's because they are expected + to be installed manually from within the tutor binary. + + Installing entrypoint or dict plugins can be disabled by defining the + ``TUTOR_IGNORE_DICT_PLUGINS`` and ``TUTOR_IGNORE_ENTRYPOINT_PLUGINS`` + environment variables. + """ + if "TUTOR_IGNORE_ENTRYPOINT_PLUGINS" not in os.environ: + with hooks.contexts.enter(hooks.Contexts.PLUGINS_V0_ENTRYPOINT): + EntrypointPlugin.install_all() + if "TUTOR_IGNORE_DICT_PLUGINS" not in os.environ: + with hooks.contexts.enter(hooks.Contexts.PLUGINS_V0_YAML): + DictPlugin.install_all() + + +def get_callable_attr( + plugin: t.Any, attr_name: str, default: t.Optional[t.Any] = None +) -> t.Optional[t.Any]: + """ + Return the attribute of a plugin. If this attribute is a callable, return + the return value instead. + """ + attr = getattr(plugin, attr_name, default) + if callable(attr): + attr = attr() # pylint: disable=not-callable + return attr diff --git a/tutor/plugins/v1.py b/tutor/plugins/v1.py new file mode 100644 index 00000000000..f5b5258953b --- /dev/null +++ b/tutor/plugins/v1.py @@ -0,0 +1,33 @@ +from glob import glob +import importlib.util +import os + +from tutor import hooks + +from .base import PLUGINS_ROOT + + +@hooks.actions.on(hooks.Actions.INSTALL_PLUGINS) +def _install_module_plugins() -> None: + for path in glob(os.path.join(PLUGINS_ROOT, "*.py")): + install(path) + + +def install(path: str) -> None: + name = os.path.splitext(os.path.basename(path))[0] + + # Add plugin to list of installed plugins + hooks.filters.add_item(hooks.Filters.PLUGINS_INSTALLED, name) + # Add plugin information + hooks.filters.add_item(hooks.Filters.PLUGINS_INFO, (name, path)) + # Import module on enable + @hooks.actions.on(hooks.Actions.ENABLE_PLUGIN % name) + def enable() -> None: + # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly + spec = importlib.util.spec_from_file_location("tutor.plugin.v1.{name}", path) + if spec is None or spec.loader is None: + raise ValueError("Plugin could not be found: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + # Add to enabled plugins + hooks.filters.add_item(hooks.Filters.PLUGINS_ENABLED, name) diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 4b148c30502..0986d25b7e1 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -18,7 +18,6 @@ DOCKER_IMAGE_MYSQL: "docker.io/mysql:5.7.35" DOCKER_IMAGE_PERMISSIONS: "{{ DOCKER_REGISTRY }}overhangio/openedx-permissions:{{ TUTOR_VERSION }}" DOCKER_IMAGE_REDIS: "docker.io/redis:6.2.6" DOCKER_IMAGE_SMTP: "docker.io/devture/exim-relay:4.95-r0-2" -LOCAL_PROJECT_NAME: "{{ TUTOR_APP }}_local" ELASTICSEARCH_HOST: "elasticsearch" ELASTICSEARCH_PORT: 9200 ELASTICSEARCH_SCHEME: "http" diff --git a/tutor/types.py b/tutor/types.py index 70193d893c8..973f580c316 100644 --- a/tutor/types.py +++ b/tutor/types.py @@ -1,39 +1,45 @@ -from typing import Any, Dict, List, Optional, Type, TypeVar, Union +import typing as t + +# https://mypy.readthedocs.io/en/latest/kinds_of_types.html#type-aliases +from typing_extensions import TypeAlias from . import exceptions -ConfigValue = Union[ - str, float, None, bool, List[str], List[Any], Dict[str, Any], Dict[Any, Any] +ConfigValue: TypeAlias = t.Union[ + str, + float, + None, + bool, + t.List[str], + t.List[t.Any], + t.Dict[str, t.Any], + t.Dict[t.Any, t.Any], ] -Config = Dict[str, ConfigValue] +Config: TypeAlias = t.Dict[str, ConfigValue] -def cast_config(config: Any) -> Config: +def cast_config(config: t.Any) -> Config: if not isinstance(config, dict): raise exceptions.TutorError( - "Invalid configuration: expected dict, got {}".format(config.__class__) + f"Invalid configuration: expected dict, got {config.__class__}" ) for key in config.keys(): if not isinstance(key, str): raise exceptions.TutorError( - "Invalid configuration: expected str, got {} for key '{}'".format( - key.__class__, key - ) + f"Invalid configuration: expected str, got {key.__class__} for key '{key}'" ) return config -T = TypeVar("T") +T = t.TypeVar("T") def get_typed( - config: Config, key: str, expected_type: Type[T], default: Optional[T] = None + config: Config, key: str, expected_type: t.Type[T], default: t.Optional[T] = None ) -> T: value = config.get(key, default) if not isinstance(value, expected_type): raise exceptions.TutorError( - "Invalid config entry: expected {}, got {} for key '{}'".format( - expected_type.__name__, value.__class__, key - ) + "Invalid config entry: expected {expected_type.__name__}, got {value.__class__} for key '{key}'" ) return value