From 0a5913b512df8004da14c8abf7dcc867813ec886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 17 Oct 2024 16:34:30 +0200 Subject: [PATCH] feat: migrate from Ruby to forum v2 Python app Obviously, this is a very important change. But it drastically simplifies this plugin, in the sense that we no longer have to run a separate Ruby container. --- .gitlab-ci.yml | 1 - changelog.d/20241017_163150_regis_forumv2.md | 1 + tutorforum/hooks.py | 11 - tutorforum/patches/k8s-deployments | 26 -- tutorforum/patches/k8s-jobs | 16 -- tutorforum/patches/k8s-services | 12 - tutorforum/patches/lms-env | 2 - .../patches/local-docker-compose-dev-services | 3 - .../local-docker-compose-jobs-services | 12 - .../local-docker-compose-lms-dependencies | 1 - .../patches/local-docker-compose-services | 13 - tutorforum/patches/openedx-common-settings | 1 - .../patches/openedx-lms-development-settings | 1 - tutorforum/plugin.py | 232 +++--------------- .../templates/forum/build/forum/Dockerfile | 53 ---- tutorforum/templates/forum/tasks/forum/init | 8 - 16 files changed, 30 insertions(+), 363 deletions(-) create mode 100644 changelog.d/20241017_163150_regis_forumv2.md delete mode 100644 tutorforum/hooks.py delete mode 100644 tutorforum/patches/k8s-deployments delete mode 100644 tutorforum/patches/k8s-jobs delete mode 100644 tutorforum/patches/k8s-services delete mode 100644 tutorforum/patches/lms-env delete mode 100644 tutorforum/patches/local-docker-compose-dev-services delete mode 100644 tutorforum/patches/local-docker-compose-jobs-services delete mode 100644 tutorforum/patches/local-docker-compose-lms-dependencies delete mode 100644 tutorforum/patches/local-docker-compose-services delete mode 100644 tutorforum/patches/openedx-common-settings delete mode 100644 tutorforum/patches/openedx-lms-development-settings delete mode 100644 tutorforum/templates/forum/build/forum/Dockerfile delete mode 100644 tutorforum/templates/forum/tasks/forum/init diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 86c7c00..523e27e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,5 @@ variables: TUTOR_PLUGIN: forum - TUTOR_IMAGES: forum TUTOR_PYPI_PACKAGE: tutor-forum GITHUB_REPO: overhangio/tutor-forum diff --git a/changelog.d/20241017_163150_regis_forumv2.md b/changelog.d/20241017_163150_regis_forumv2.md new file mode 100644 index 0000000..c0510a8 --- /dev/null +++ b/changelog.d/20241017_163150_regis_forumv2.md @@ -0,0 +1 @@ +- 💥[Feature] Switch from the legacy `cs_comments_service` Ruby app to the new forum v2 Python app. In addition, forum data is now stored in MySQL, and not in MongoDb. This considerably simplifies this plugin. Change should be transparent for most users, unless the forum backend has been customised in some way. (by @regisb) diff --git a/tutorforum/hooks.py b/tutorforum/hooks.py deleted file mode 100644 index 21dfb3b..0000000 --- a/tutorforum/hooks.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -These hooks are stored in a separate module. If they were included in plugin.py, then -the tutor-forum hooks would be created in the context of some other plugin that imports -them. -""" - -from __future__ import annotations - -from tutor.core.hooks import Filter - -FORUM_ENV: Filter[dict[str, str], []] = Filter() diff --git a/tutorforum/patches/k8s-deployments b/tutorforum/patches/k8s-deployments deleted file mode 100644 index 8b68cb1..0000000 --- a/tutorforum/patches/k8s-deployments +++ /dev/null @@ -1,26 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: forum - labels: - app.kubernetes.io/name: forum -spec: - selector: - matchLabels: - app.kubernetes.io/name: forum - template: - metadata: - labels: - app.kubernetes.io/name: forum - spec: - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - containers: - - name: forum - image: {{ FORUM_DOCKER_IMAGE }} - ports: - - containerPort: {{ FORUM_PORT }} - env: - {{ patch("forum-k8s-env")|indent(12) }} diff --git a/tutorforum/patches/k8s-jobs b/tutorforum/patches/k8s-jobs deleted file mode 100644 index 993d4e7..0000000 --- a/tutorforum/patches/k8s-jobs +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: forum-job - labels: - app.kubernetes.io/component: job -spec: - template: - spec: - restartPolicy: Never - containers: - - name: forum - image: {{ FORUM_DOCKER_IMAGE }} - env: - {{ patch("forum-k8s-env")|indent(10) }} diff --git a/tutorforum/patches/k8s-services b/tutorforum/patches/k8s-services deleted file mode 100644 index 0131b65..0000000 --- a/tutorforum/patches/k8s-services +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: forum -spec: - type: NodePort - ports: - - port: {{ FORUM_PORT }} - protocol: TCP - selector: - app.kubernetes.io/name: forum diff --git a/tutorforum/patches/lms-env b/tutorforum/patches/lms-env deleted file mode 100644 index 46f9bd2..0000000 --- a/tutorforum/patches/lms-env +++ /dev/null @@ -1,2 +0,0 @@ -COMMENTS_SERVICE_URL: "http://forum:{{FORUM_PORT}}" -COMMENTS_SERVICE_KEY: "{{ FORUM_API_KEY }}" \ No newline at end of file diff --git a/tutorforum/patches/local-docker-compose-dev-services b/tutorforum/patches/local-docker-compose-dev-services deleted file mode 100644 index 9fbb086..0000000 --- a/tutorforum/patches/local-docker-compose-dev-services +++ /dev/null @@ -1,3 +0,0 @@ -forum: - stdin_open: true - tty: true diff --git a/tutorforum/patches/local-docker-compose-jobs-services b/tutorforum/patches/local-docker-compose-jobs-services deleted file mode 100644 index 488e107..0000000 --- a/tutorforum/patches/local-docker-compose-jobs-services +++ /dev/null @@ -1,12 +0,0 @@ -forum-job: - image: {{ FORUM_DOCKER_IMAGE }} - environment: - {{ patch("forum-local-env")|indent(4) }} -{%- set mounts = iter_mounts(MOUNTS, "forum")|list %} -{%- if mounts %} - volumes: - {%- for mount in iter_mounts(MOUNTS, "forum") %} - - {{ mount }} - {%- endfor %} -{%- endif %} - depends_on: {{ [("elasticsearch", RUN_ELASTICSEARCH), ("mongodb", RUN_MONGODB)]|list_if }} diff --git a/tutorforum/patches/local-docker-compose-lms-dependencies b/tutorforum/patches/local-docker-compose-lms-dependencies deleted file mode 100644 index 39a2b1a..0000000 --- a/tutorforum/patches/local-docker-compose-lms-dependencies +++ /dev/null @@ -1 +0,0 @@ -- forum diff --git a/tutorforum/patches/local-docker-compose-services b/tutorforum/patches/local-docker-compose-services deleted file mode 100644 index 090790a..0000000 --- a/tutorforum/patches/local-docker-compose-services +++ /dev/null @@ -1,13 +0,0 @@ -forum: - image: {{ FORUM_DOCKER_IMAGE }} - environment: - {{ patch("forum-local-env")|indent(4) }} -{%- set mounts = iter_mounts(MOUNTS, "forum")|list %} -{%- if mounts %} - volumes: - {%- for mount in iter_mounts(MOUNTS, "forum") %} - - {{ mount }} - {%- endfor %} -{%- endif %} - restart: unless-stopped - depends_on: {{ [("elasticsearch", RUN_ELASTICSEARCH), ("mongodb", RUN_MONGODB)]|list_if }} diff --git a/tutorforum/patches/openedx-common-settings b/tutorforum/patches/openedx-common-settings deleted file mode 100644 index 5d00a9a..0000000 --- a/tutorforum/patches/openedx-common-settings +++ /dev/null @@ -1 +0,0 @@ -FEATURES["ENABLE_DISCUSSION_SERVICE"] = True diff --git a/tutorforum/patches/openedx-lms-development-settings b/tutorforum/patches/openedx-lms-development-settings deleted file mode 100644 index bc02072..0000000 --- a/tutorforum/patches/openedx-lms-development-settings +++ /dev/null @@ -1 +0,0 @@ -COMMENTS_SERVICE_URL = "http://forum:{{ FORUM_PORT }}" diff --git a/tutorforum/plugin.py b/tutorforum/plugin.py index d8aa3a0..1c68661 100644 --- a/tutorforum/plugin.py +++ b/tutorforum/plugin.py @@ -1,15 +1,9 @@ from __future__ import annotations -import os -import urllib.parse -from glob import glob - -import importlib_resources from tutor import hooks as tutor_hooks from tutor.__about__ import __version_suffix__ from .__about__ import __version__ -from .hooks import FORUM_ENV # Handle version suffix in nightly mode, just like tutor core if __version_suffix__: @@ -18,213 +12,45 @@ config = { "defaults": { "VERSION": __version__, - "DOCKER_IMAGE": "{{ DOCKER_REGISTRY }}overhangio/openedx-forum:{{ FORUM_VERSION }}", - "MONGODB_DATABASE": "cs_comments_service", - "PORT": "4567", - "API_KEY": "forumapikey", - "REPOSITORY": "https://github.com/openedx/cs_comments_service.git", - "REPOSITORY_VERSION": "{{ OPENEDX_COMMON_VERSION }}", }, } -FORUM_ENV_BASE: dict[str, str] = { - "SEARCH_SERVER": "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}", - "MONGODB_AUTH": "{{ get_mongo_auth(MONGODB_USERNAME, MONGODB_PASSWORD) }}", - "MONGODB_HOST": "{{ MONGODB_HOST|forum_mongodb_host }}", - "MONGODB_PORT": "{{ MONGODB_PORT }}", - "MONGODB_DATABASE": "{{ FORUM_MONGODB_DATABASE }}", - "MONGOID_AUTH_SOURCE": "{{ MONGODB_AUTH_SOURCE }}", - "MONGOID_AUTH_MECH": "{{ MONGODB_AUTH_MECHANISM|auth_mech_as_ruby }}", - "MONGOID_USE_SSL": "{{ 'true' if MONGODB_USE_SSL else 'false' }}", - "MONGOHQ_URL": "{{ get_mongohq_url(MONGODB_HOST, MONGODB_PORT,MONGODB_DATABASE, get_mongo_auth(MONGODB_USERNAME, MONGODB_PASSWORD)) }}", -} - -with open( - str( - importlib_resources.files("tutorforum") - / os.path.join("templates", "forum", "tasks", "forum", "init") - ), - encoding="utf8", -) as f: - tutor_hooks.Filters.CLI_DO_INIT_TASKS.add_item(("forum", f.read())) +# Auto-mount forum repository +tutor_hooks.Filters.MOUNTED_DIRECTORIES.add_item(("openedx", "forum")) -tutor_hooks.Filters.IMAGES_BUILD.add_item( - ( - "forum", - ("plugins", "forum", "build", "forum"), - "{{ FORUM_DOCKER_IMAGE }}", - (), - ) -) -tutor_hooks.Filters.IMAGES_PULL.add_item( - ( - "forum", - "{{ FORUM_DOCKER_IMAGE }}", - ) +tutor_hooks.Filters.ENV_PATCHES.add_items( + [ + # Patch edx-platform + # https://github.com/openedx/edx-platform/pull/35671 + # TODO after this PR has been merged, remove this patch + ( + "openedx-dockerfile-post-git-checkout", + """ +RUN git remote add edly https://github.com/edly-io/edx-platform \ + && git fetch edly edly/forumv2 \ + && git merge edly/edly/forumv2""", + ), + # Enable forum feature + ( + "openedx-common-settings", + """FEATURES["ENABLE_DISCUSSION_SERVICE"] = True""", + ), + ] ) -tutor_hooks.Filters.IMAGES_PUSH.add_item( + +# Enable forum v2 +tutor_hooks.Filters.CLI_DO_INIT_TASKS.add_item( ( - "forum", - "{{ FORUM_DOCKER_IMAGE }}", + "lms", + # TODO at some point, maybe this flag will be renamed to "forum_v2.enable". + """ +(./manage.py lms waffle_flag --list | grep forum_v2.enable_forum_v2) || ./manage.py lms waffle_flag --create --everyone forum_v2.enable_forum_v2 +(./manage.py lms waffle_flag --list | grep forum_v2.enable_mysql_backend) || ./manage.py lms waffle_flag --create --everyone forum_v2.enable_mysql_backend +""", ) ) - -def get_mongohq_url( - mongo_host: str, mongo_port: str, mongo_database: str, mongo_auth: str -) -> str: - """ - Used to return mongo url, to handle the special case when mongodb+srv used - For more info look at the following PR https://github.com/overhangio/tutor-forum/pull/10 - """ - if "mongodb+srv" in mongo_host: - return f"{mongo_host}/{mongo_database}" - return f"mongodb://{mongo_auth}{mongo_host}:{mongo_port}/{mongo_database}" - - -def get_mongo_auth(monogo_username: str, mongo_password: str) -> str: - if monogo_username and mongo_password: - return f"{monogo_username}:{mongo_password}@" - return "" - - -tutor_hooks.Filters.ENV_TEMPLATE_VARIABLES.add_items( - [("get_mongohq_url", get_mongohq_url), ("get_mongo_auth", get_mongo_auth)] -) - - -# Bind-mount repo at runtime -@tutor_hooks.Filters.COMPOSE_MOUNTS.add() -def _mount_cs_comments_service( - volumes: list[tuple[str, str]], name: str -) -> list[tuple[str, str]]: - """ - When mounting cs_comments_service with `--mount=/path/to/cs_comments_service`, - bind-mount the host repo in the forum container. - """ - if name == "cs_comments_service": - repo_path = "/app/cs_comments_service" - volumes += [ - ("forum", repo_path), - ("forum-job", repo_path), - ] - return volumes - - -# Bind-mount repo at build-time -@tutor_hooks.Filters.IMAGES_BUILD_MOUNTS.add() -def _mount_forum_on_build( - mounts: list[tuple[str, str]], host_path: str -) -> list[tuple[str, str]]: - path_basename = os.path.basename(host_path) - if path_basename == "cs_comments_service": - mounts.append(("forum", "forum-src")) - return mounts - - -# Add the "templates" folder as a template root -tutor_hooks.Filters.ENV_TEMPLATE_ROOTS.add_item( - str(importlib_resources.files("tutorforum") / "templates") -) - - -def auth_mech_as_ruby(auth_mech: str) -> str: - """ - Convert the authentication mechanism from the format - specified for the Python version to the Ruby version. - - https://pymongo.readthedocs.io/en/stable/api/pymongo/database.html#pymongo.auth.MECHANISMS - https://github.com/mongodb/mongo-ruby-driver/blob/932b06b7564a5e5ae8d4ad08fe8d6ceee629e4eb/lib/mongo/auth.rb#L69 - """ - return { - "GSSAPI": ":gssapi", - "MONGODB-AWS": ":aws", - "MONGODB-CR": ":mongodb_cr", - "MONGODB-X509": ":mongodb_x509", - "PLAIN": ":plain", - "SCRAM-SHA-1": ":scram", - # SCRAM-256 is only supported from v2.6 of the ruby driver onwards. - # See https://github.com/mongodb/mongo-ruby-driver/releases/tag/v2.6.0 - "SCRAM-SHA-256": ":scram", - }.get(auth_mech) or "" - - -def forum_mongodb_host(host: str) -> str: - """ - Remove the querystring parameters from the mongodb host url. These parameters are - not supported by the outdated mongodb gem. Thus we just trim them out. - """ - # 0 = scheme - # 1 = netloc - # 2 = path - # 3 = params - # 4 = query - # 5 = fragment - parsed = [*urllib.parse.urlparse(host)] - parsed[4] = "" - # We also remove the trailing "/" from the path, which will not play well with the - # full url where we concatenate the database name. - parsed[2] = parsed[2].rstrip("/") - return urllib.parse.urlunparse(parsed) - - -@FORUM_ENV.add(priority=tutor_hooks.priorities.HIGH) -def _add_base_forum_env(forum_env: dict[str, str]) -> dict[str, str]: - """ - Add environment variables needed for standard build of forum service. - """ - forum_env.update(FORUM_ENV_BASE) - return forum_env - - -@tutor_hooks.Filters.ENV_PATCHES.add(priority=tutor_hooks.priorities.HIGH) -def _forum_env_patches(patches: list[tuple[str, str]]) -> list[tuple[str, str]]: - """ - Adds environment variables from FORUM_ENV filter to patches. - """ - # The forum service is configured entirely via environment variables. Docker - # Compose and Kubernetes use different syntax to specify environment - # variables. The following code reads environment variables from the - # `FORUM_ENV` filter and rendered in the appropriate format for both so they - # can be included as patches. - k8s_env_patch = "" - local_env_patch = "" - for key, value in FORUM_ENV.apply({}).items(): - # Kubernetes - k8s_env_patch += f'- name: {key}\n value: "{value}"\n' - local_env_patch += f'{key}: "{value}"\n' - patches += [("forum-k8s-env", k8s_env_patch), ("forum-local-env", local_env_patch)] - return patches - - -tutor_hooks.Filters.ENV_TEMPLATE_FILTERS.add_items( - [ - ("auth_mech_as_ruby", auth_mech_as_ruby), - ("forum_mongodb_host", forum_mongodb_host), - ] -) -# Render the "build" and "apps" folders -tutor_hooks.Filters.ENV_TEMPLATE_TARGETS.add_items( - [ - ("forum/build", "plugins"), - ("forum/apps", "plugins"), - ], -) -# Load patches from files -for path in glob( - os.path.join( - str(importlib_resources.files("tutorforum") / "patches"), - "*", - ) -): - with open(path, encoding="utf-8") as patch_file: - tutor_hooks.Filters.ENV_PATCHES.add_item( - (os.path.basename(path), patch_file.read()) - ) # Add configuration entries tutor_hooks.Filters.CONFIG_DEFAULTS.add_items( [(f"FORUM_{key}", value) for key, value in config.get("defaults", {}).items()] ) -tutor_hooks.Filters.CONFIG_OVERRIDES.add_items( - list(config.get("overrides", {}).items()) -) diff --git a/tutorforum/templates/forum/build/forum/Dockerfile b/tutorforum/templates/forum/build/forum/Dockerfile deleted file mode 100644 index e9fcabc..0000000 --- a/tutorforum/templates/forum/build/forum/Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -# syntax=docker/dockerfile:1 -########## Base image -# https://hub.docker.com/_/ruby/tags -FROM docker.io/ruby:3.3.5-slim-bookworm AS base - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt update && \ - apt upgrade -y && \ - apt install -y wget curl autoconf bison build-essential libffi-dev libgdbm-dev libncurses5-dev libssl-dev libyaml-dev libreadline6-dev zlib1g-dev git - -# Create unprivileged "app" user -# From then on, run as unprivileged app user -RUN useradd --home-dir /app --create-home --shell /bin/bash --uid 1000 app -USER app - -# Install rake and bundler -ENV PATH=/app/.gem/ruby/3.3.5/bin:$PATH -RUN gem install --user-install bundler --version 2.3.26 -RUN gem install --user-install rake --version 13.1.0 - -########## Clone repo -FROM base AS forum-git -ARG FORUM_REPOSITORY={{ FORUM_REPOSITORY }} -ARG FORUM_REPOSITORY_VERSION={{ FORUM_REPOSITORY_VERSION }} -ADD --keep-git-dir=true $FORUM_REPOSITORY#$FORUM_REPOSITORY_VERSION /app/cs_comments_service - -########## Empty layer with just the repo at the root. -# This is useful when overriding the build context with a host repo: -FROM scratch AS forum-src -COPY --from=forum-git /app/cs_comments_service / - -########## Production image -FROM base AS production -COPY --link --chown=1000:1000 --from=forum-src / /app/cs_comments_service - -# Install ruby requirements -WORKDIR /app/cs_comments_service -RUN bundle config set --local deployment 'true' -RUN bundle install - -ENV SINATRA_ENV=staging -ENV NEW_RELIC_ENABLE=false -ENV API_KEY=forumapikey -ENV SEARCH_SERVER_ES7=http://elasticsearch:9200 -ENV MONGODB_AUTH="" -ENV MONGODB_HOST=mongodb -ENV MONGODB_PORT=27017 -ENV MONGODB_DATABASE=cs_comments_service -# Duplicating the function of dockerize -ENV MONGOID_CONNECT_TIMEOUT=600 - -EXPOSE 4567 -CMD ./bin/unicorn -c config/unicorn_tcp.rb -I '.' diff --git a/tutorforum/templates/forum/tasks/forum/init b/tutorforum/templates/forum/tasks/forum/init deleted file mode 100644 index 69d8b5f..0000000 --- a/tutorforum/templates/forum/tasks/forum/init +++ /dev/null @@ -1,8 +0,0 @@ -# Install dependencies in case repository was bind-mounted -if [ ! -e vendor/bundle/ruby ]; then - echo "Installing dependencies in bind-mounted repository..." - bundle install -fi - -bundle exec rake search:initialize -bundle exec rake search:rebuild_indices