Tidy up, fix warnings and use a Makefile (#508)

* Tidy up, fix warnings and use a Makefile

* Update README.md

* Remove make dependency in docker image

* Remove unused USE_VENV variable

* Can't run specific tests with docker-compose

* Fix pagination with no items

Author: andyhd

Dockerfile

@@ -2,63 +2,42 @@ FROM alpine:3.7
 
 MAINTAINER Andy Driver <andy.driver@digital.justice.gov.uk>
 
-# install build dependencies (they'll be uninstalled after pip install)
-RUN apk add --no-cache \
-        --virtual build-deps \
-        gcc \
-        musl-dev
+ENV HELM_VERSION 2.9.1
+ENV HELM_HOME /tmp/helm
+ENV DJANGO_SETTINGS_MODULE "control_panel_api.settings"
 
-# install python3 and 'ca-certificates' so that HTTPS works consistently
+WORKDIR /home/control-panel
+
+# install build dependencies (they'll be uninstalled after pip install)
 RUN apk add --no-cache \
+        build-base \
         openssl \
         ca-certificates \
         libffi-dev \
-        python3-dev
-
-# Temporary bugfix for libressl
-# Postgres needs libressl-dev, but cryptography only works with openssl-dev
-RUN apk add --no-cache --virtual temp-ssl-fix \
-        openssl-dev \
-    && pip3 install cryptography==2.2.2 \
-    && apk del temp-ssl-fix \
-    && apk add --no-cache \
+        python3-dev \
         libressl-dev \
         postgresql-dev
 
-# Install helm
-ENV HELM_VERSION 2.9.1
-RUN wget https://storage.googleapis.com/kubernetes-helm/helm-v$HELM_VERSION-linux-amd64.tar.gz \
-    && tar xzf helm-v$HELM_VERSION-linux-amd64.tar.gz \
-    && mv linux-amd64/helm /usr/local/bin \
-    && rm -rf helm-v$HELM_VERSION-linux-amd64.tar.gz linux-amd64
-
-# Configure helm
-ENV HELM_HOME /tmp/helm
-RUN helm init --client-only
+# Install and configure helm
 COPY helm-repositories.yaml /tmp/helm/repository/repositories.yaml
-RUN helm repo update
-
-WORKDIR /home/control-panel
-
-# install python dependencies
-ADD requirements.txt requirements.txt
-RUN pip3 install -r requirements.txt
-
-# uninstall build dependencies
-RUN apk del build-deps
-
-ENV DJANGO_SETTINGS_MODULE "control_panel_api.settings"
-
-ADD manage.py manage.py
-ADD run_api run_api
-ADD run_tests run_tests
-ADD wait_for_db wait_for_db
-ADD control_panel_api control_panel_api
-ADD moj_analytics moj_analytics
+RUN wget https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz -O helm.tgz \
+ && tar fxz helm.tgz \
+ && mv linux-amd64/helm /usr/local/bin \
+ && rm -rf helm.tgz linux-amd64 \
+ && helm init --client-only \
+ && helm repo update
+
+# install python dependencies (and then remove build dependencies)
+COPY requirements.txt ./
+RUN pip3 install -r requirements.txt \
+ && apk del build-base
+
+COPY control_panel_api control_panel_api
+COPY moj_analytics moj_analytics
+COPY manage.py wait_for_db ./
 
 # collect static files for deployment
 RUN python3 manage.py collectstatic
 
 EXPOSE 8000
-
-CMD ["./run_api"]
+CMD ["gunicorn", "-b", "0.0.0.0:8000", "control_panel_api.wsgi:application"]

Makefile

@@ -0,0 +1,64 @@
+HOST=0.0.0.0
+PORT=8000
+PROJECT=control-panel
+MODULE=control_panel_api
+VENV=venv
+BIN=${VENV}/bin
+
+-include .env
+export
+
+.PHONY: collectstatic dependencies help run test wait_for_db
+
+venv/bin:
+	@if ${USE_VENV} && [ ! -d "${VENV}" ] ; then python3 -m venv ${VENV} ; fi
+
+## dependencies: Install dependencies
+dependencies: ${BIN} requirements.txt
+	@echo
+	@echo "> Fetching dependencies..."
+	@${BIN}/pip3 install -r requirements.txt
+
+## collectstatic: Collect assets into static folder
+collectstatic: dependencies
+	@echo
+	@echo "> Collecting static assets..."
+	@${BIN}/python3 manage.py collectstatic --noinput
+
+## run: Run webapp
+run: collectstatic
+	@echo
+	@echo "> Running webapp..."
+	@${BIN}/gunicorn -b ${HOST}:${PORT} ${MODULE}.wsgi:application
+
+wait_for_db:
+	@echo
+	@echo "> Waiting for database..."
+	@${BIN}/python3 wait_for_db
+
+## test: Run tests
+test: export DJANGO_SETTINGS_MODULE=${MODULE}.settings.test
+test: wait_for_db
+	@echo
+	@echo "> Running tests..."
+	@NAMED_TESTS="$(shell if [ -n "${TEST_NAME}" ]; then echo "-k ${TEST_NAME}" ; fi)" && \
+	${BIN}/pytest --color=yes ${MODULE} $$NAMED_TESTS
+
+## docker-image: Build docker image
+docker-image:
+	@echo
+	@echo "> Building docker image..."
+	@docker build -t ${PROJECT} .
+
+## docker-test: Run tests in Docker container
+docker-test:
+	@echo
+	@echo "> Running tests in Docker..."
+	@docker-compose -f docker-compose.test.yml up --abort-on-container-exit
+
+help: Makefile
+	@echo
+	@echo " Commands in "$(PROJECT)":"
+	@echo
+	@sed -n 's/^##//p' $< | column -t -s ':' | sed -e 's/^/ /'
+	@echo

README.md

@@ -9,7 +9,7 @@ https://github.com/ministryofjustice/analytics-platform-control-panel-frontend
 ## Running with Docker
 
 ```sh
-docker-compose build
+docker-compose build  # OR make docker-image
 docker-compose up
 ```
 and then in a separate terminal window,
@@ -21,12 +21,7 @@ Then browse to http://localhost:8000/
 ### Running tests with docker
 
 ```sh
-docker-compose -f docker-compose.test.yml up --build --abort-on-container-exit
-```
-
-You can run a particular test using the pytest '-k' parameter:
-```sh
-docker-compose -f docker-compose.test.yml build && docker-compose -f docker-compose.test.yml run app ./run_tests -k TEST-NAME
+make docker-test
 ```
 
 ## Running directly on your machine
@@ -35,11 +30,11 @@ docker-compose -f docker-compose.test.yml build && docker-compose -f docker-comp
 
 The Control Panel app requires Python 3.6+
 
-It is best to use a virtual environment to install python dependencies, eg:
+Install dependencies with the following command:
 ```sh
-python -m venv venv
-. venv/bin/activate
-pip install -r requirements.txt
+python3 -m venv venv
+source venv/bin/activate
+pip3 install -r requirements.txt
 ```
 
 ### Kubernetes setup
@@ -90,40 +85,50 @@ export DJANGO_SETTINGS_MODULE=control_panel_api.settings
 
 The Control Panel app connects to a PostgreSQL database, which should have a database with the expected name:
 ```sh
-createdb $DB_NAME
+createuser -d controlpanel
+createdb -U controlpanel controlpanel
 ```
 
 Then you can run migrations:
 ```sh
-python manage.py migrate
+python3 manage.py migrate
 ```
 
 ### Create superuser (on first run only)
 
 ```sh
-python manage.py createsuperuser
+python3 manage.py createsuperuser
 ```
 NB `Username` needs to be your GitHub username
 
 ### Compile Sass and Javascript
 
 Before the first run (or after changes to static assets), you need to run
 ```sh
-python manage.py collectstatic
+python3 manage.py collectstatic
 ```
 
 ### Run the app
 
-You can run the app with
+You can run the app with the Django development server with
+```sh
+python3 manage.py runserver
+```
+Or with Gunicorn WSGI server:
 ```sh
-./run_api
+gunicorn -b 0.0.0.0:8000 control_panel_api.wsgi:application
 ```
 Go to http://localhost:8000/
 
 ### How to run the tests
 
 ```sh
-./run_tests
+make test
+```
+
+You can run a specific test class or function by passing the `TEST_NAME` parameter, eg:
+```sh
+make test TEST_NAME=test_something
 ```
 
 # Deployment

control_panel_api/k8s_patch.py

@@ -7,24 +7,19 @@
 from kubernetes.config.kube_config import _is_expired
 
 
-def load_token(self):
-    if 'auth-provider' not in self._user:
-        return
-
-    provider = self._user['auth-provider']
-
-    if ('name' not in provider
-            or 'config' not in provider
-            or provider['name'] != 'oidc'):
+def load_token(self, provider):
+    if 'config' not in provider:
         return
 
     parts = provider['config']['id-token'].split('.')
 
     if len(parts) != 3:  # Not a valid JWT
         return None
 
+    padding = (4 - len(parts[1]) % 4) * '='
+
     jwt_attributes = json.loads(
-        base64.b64decode(parts[1] + '==').decode('utf-8')
+        base64.b64decode(parts[1] + padding).decode('utf-8')
     )
 
     expire = jwt_attributes.get('exp')

control_panel_api/pagination.py

@@ -17,7 +17,7 @@ def paginate_queryset(self, queryset, request, view=None):
         if not self._page_size_is_all(page_size):
             return super().paginate_queryset(queryset, request, view)
 
-        paginator = self.django_paginator_class(queryset, queryset.count())
+        paginator = self.django_paginator_class(queryset, queryset.count() or 1)
         self.page = paginator.page(1)
 
         return list(self.page)

control_panel_api/serializers.py

@@ -295,7 +295,7 @@ def to_representation(self, bucket_hits):
         return sorted(results, key=itemgetter('count'), reverse=True)
 
     def _get_accessed_by(self, key):
-        match = re.search(f"{settings.ENV}_(app|user)_([\w-]+)/", key)
+        match = re.search(rf"{settings.ENV}_(app|user)_([\w-]+)/", key)
 
         if match:
             return match.group(1), match.group(2)

control_panel_api/settings/base.py

@@ -116,9 +116,9 @@ def is_enabled(value):
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/1.11/howto/static-files/
 
-STATIC_ROOT = os.path.join(BASE_DIR, 'static/')
-
-STATIC_URL = '/static/'
+STATIC_ROOT = os.path.join(os.path.dirname(BASE_DIR), 'static')
+STATIC_HOST = os.environ.get('DJANGO_STATIC_HOST', '')
+STATIC_URL = STATIC_HOST + '/static/'
 
 REST_FRAMEWORK = {
     'DEFAULT_AUTHENTICATION_CLASSES': [

control_panel_api/settings/test.py

@@ -21,3 +21,7 @@
 }
 
 LOGGING['handlers']['console']['level'] = 'CRITICAL'
+ENABLED = {
+    'k8s_rbac': False,
+    'write_to_cluster': True,
+}

control_panel_api/views.py

@@ -123,7 +123,7 @@ class AppViewSet(viewsets.ModelViewSet):
     filter_backends = (DjangoFilterBackend,)
     permission_classes = (AppPermissions,)
 
-    filter_fields = ('name', 'repo_url', 'slug')
+    filterset_fields = ('name', 'repo_url', 'slug')
 
     @handle_external_exceptions
     @transaction.atomic
@@ -238,7 +238,7 @@ class S3BucketViewSet(viewsets.ModelViewSet):
     serializer_class = S3BucketSerializer
     filter_backends = (S3BucketFilter,)
     permission_classes = (S3BucketPermissions,)
-    filter_fields = ('is_data_warehouse',)
+    filterset_fields = ('is_data_warehouse',)
 
     @handle_external_exceptions
     @transaction.atomic

docker-compose.test.yml

@@ -2,7 +2,7 @@ version: '3'
 services:
   app:
     build: .
-    image: "app"
+    image: "control-panel"
     ports:
       - "8000:8000"
     depends_on:
@@ -15,7 +15,7 @@ services:
       DB_USER: "postgres"
       DEBUG: "True"
       DJANGO_SETTINGS_MODULE: "control_panel_api.settings.test"
-    command: ["./run_tests"]
+    command: sh -c "python3 wait_for_db && pytest --color=yes control_panel_api"
   db:
     image: "postgres:9.6.2"
     logging:

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3'
 services:
   app:
     build: .
-    image: app
+    image: control-panel
     ports:
       - "8000:8000"
     depends_on:
@@ -22,8 +22,8 @@ services:
       POSTGRES_USER: "controlpanel"
       POSTGRES_DB: "controlpanel"
   migration:
-    image: app:latest
-    command: ["sh", "-c", "python3 wait_for_db && python3 manage.py migrate"]
+    image: control-panel
+    command: sh -c "python3 wait_for_db && python3 manage.py migrate"
     environment:
       DB_HOST: "db"
       DB_NAME: "controlpanel"

requirements.txt

@@ -30,7 +30,7 @@ kubernetes==8.0.1
 MarkupSafe==1.1.0
 model-mommy==1.6.0
 openapi-codec==1.3.2
-psycopg2==2.7.7
+psycopg2-binary==2.7.7
 py==1.7.0
 pyasn1==0.4.4
 pyasn1-modules==0.2.2