Test case verification using testinfra (#17)

Author: Rickkwa

.gitignore

@@ -0,0 +1,2 @@
+.pytest_cache/
+*.pyc

.travis.yml

@@ -7,8 +7,8 @@ language: python
 python: 3.6
 
 env:
-  - ANSIBLE_VER=2.5.2
-  - ANSIBLE_VER=2.6.1
+  - ANSIBLE_VER=2.5.2 TESTINFRA_VER=1.14.0
+  - ANSIBLE_VER=2.6.1 TESTINFRA_VER=1.14.0
 
 sudo: true
 
@@ -30,18 +30,20 @@ before_install:
 
 install:
   - sudo apt-get install sshpass
+  - pip install pytest
   - pip install ansible==$ANSIBLE_VER
-  - pip install ansible-lint
+  - pip install testinfra==$TESTINFRA_VER
 
 script:
   - |
     for testcase in $(ls tests/cases/*.{yml,yaml}); do
-      echo -e "\033[35m ===================================================== \033[0m"
-      echo -e "\033[35m === $testcase \033[0m"
-      echo -e "\033[35m ===================================================== \033[0m"
-      docker-compose -f tests/docker-compose.yml up -d
-      ansible-playbook tests/test-role.yml --extra-vars="test_case=$(basename ${testcase})" -v || exit 1
-      docker-compose -f tests/docker-compose.yml down
+      echo -e "\033[35m ==================================================================== \033[0m"
+      echo -e "\033[35m ===== $(basename ${testcase%.*}) \033[0m"
+      echo -e "\033[35m ==================================================================== \033[0m"
+      docker-compose -f tests/docker-compose.yml up -d &> /dev/null
+      ansible-playbook tests/cases/$(basename $testcase) -v || true
+      py.test --hosts='ansible://containers' --ansible-inventory='/etc/ansible/hosts' tests/cases/$(basename ${testcase%.*})_spec.py || exit 1
+      docker-compose -f tests/docker-compose.yml down &> /dev/null
     done
 
 jobs:
@@ -52,6 +54,8 @@ jobs:
   include:
     - stage: lint
       before_install: []
+      install:
+        - pip install ansible-lint
       script:
         - ansible-lint -p .
 

tests/README.md

@@ -13,14 +13,22 @@ Then in `docker-compose.yml`, add this environment as a new container using a ne
 
 ## New Test Case
 
-Create a new test case file in `cases/<test-name>.yml`. Test cases should follow the following principles:
+Test cases consists of 2 components.
 
-* Filename should be meaningful, and use dashes (-) to delimit words.
-* First line of the test case file should contain a description of the test (as comments).
+1. Ansible playbook to prepare and execute the role.
+2. Pytest file to verify the state of the container after running the role.
+
+### Ansible playbook test
+
+Create a new test case file in `cases/<test-name>.yml`. The test case should follow the following principles:
+
+* Filename should be meaningful to the test, using dashes (-) to delimit words.
+* The playbook should have some comments at the top to describe the test.
+* Use `pre_tasks` to do any setup for the test, and use `tasks` to actually execute the role.
 * The test should not rely on variables defined in the role. Instead, the test should override the variables.
-* The test case should be split into 3 parts (with comments in the file indicating each part)
-    * **Setup** stage which prepares your variables, and/or environment
-    * **Run** stage which imports and runs the role
-    * **Verification** stage which checks the changes were made correctly
 
-After creating the test case file under `cases/`, the travis ci should automatically pick up the test case and run it against all environments.
+### Verification using Pytest
+
+Create a new test case spec file in `cases/<test-name>_spec.py`. The `<test-name>` should match the one for the corresponding ansible playbook file so that the CI can associate the two. Make use of the `pytest` testing framework with the `testinfra` plugin to write your verification.
+
+After creating the test case files under `tests/cases/`, the travis ci should automatically pick up the test case and run it against all environments.

tests/cases/add-user-group.yml

@@ -3,59 +3,38 @@
 ### Description
 # Test a generic adding of both user and group.
 
-### Setup
+- hosts: containers
+  pre_tasks:
+    - name: Set variables
+      set_fact:
+        um_uid_base: 10000
+        um_gid_base: 10000
+        um_group_inventory:
+          - name: customgroup
+            rel_gid: 1
+            present:
+              - all
+            absent: []
+            sudoer: |
+              %customgroup    ALL=(ALL) ALL
+        um_user_inventory:
+          - name: foo
+            rel_uid: 1
+            present:
+              - all
+            absent: []
+            passwd: $6$CH4pzUgP$PJHNOCKVcWA5ai0kU5qwTkCeK08sx86T6o9exHz7Ki7zN/ip6pPBecr6RYqCaBNDSVobuzwwREHu8KbgNncxf1
+            groups:
+              - customgroup
+          - name: bar
+            rel_uid: 2
+            present:
+              - all
+            absent: []
+            passwd: $6$CH4pzUgP$PJHNOCKVcWA5ai0kU5qwTkCeK08sx86T6o9exHz7Ki7zN/ip6pPBecr6RYqCaBNDSVobuzwwREHu8KbgNncxf1
+
+  tasks:
+    - name: Run role
+      include_role:
+        name: "user_management"
 
-- name: Set variables
-  set_fact:
-    um_uid_base: 10000
-    um_gid_base: 10000
-    um_group_inventory:
-      - name: customgroup
-        rel_gid: 1
-        present:
-          - all
-        absent: []
-        sudoer: |
-          %customgroup    ALL=(ALL) ALL
-    um_user_inventory:
-      - name: foo
-        rel_uid: 1
-        present:
-          - all
-        absent: []
-        passwd: $6$CH4pzUgP$PJHNOCKVcWA5ai0kU5qwTkCeK08sx86T6o9exHz7Ki7zN/ip6pPBecr6RYqCaBNDSVobuzwwREHu8KbgNncxf1
-        groups:
-          - customgroup
-      - name: bar
-        rel_uid: 2
-        present:
-          - all
-        absent: []
-        passwd: $6$CH4pzUgP$PJHNOCKVcWA5ai0kU5qwTkCeK08sx86T6o9exHz7Ki7zN/ip6pPBecr6RYqCaBNDSVobuzwwREHu8KbgNncxf1
-
-
-### Run
-
-- name: Run role
-  include_role:
-    name: "user_management"
-
-
-### Verification
-
-- name: Verify user exists and has correct UID
-  shell: "id -u {{ item.name }} | grep '^{{ item.rel_uid + um_uid_base }}$'"
-  loop: "{{ um_user_inventory }}"
-
-- name: Verify group exists and has correct GID
-  shell: "getent group {{ item.name }} | cut -d: -f3 | grep '^{{ item.rel_gid + um_gid_base }}$'"
-  loop: "{{ um_group_inventory }}"
-
-- name: Verify user 'foo' is in group
-  shell: "id -Gn foo | grep -w customgroup"
-
-- name: Verify user 'bar' is not in group
-  shell: "! id -Gn bar | grep -w customgroup"
-
-- name: Verify sudoer file exists
-  shell: "ls -1 /etc/sudoers.d/ | grep '^rel_gid_1$'"

tests/cases/add-user-group_spec.py

@@ -0,0 +1,33 @@
+import pytest
+
+
+@pytest.mark.parametrize("user,uid", [
+    ("foo", 10001),
+    ("bar", 10002),
+])
+def test_user_exists_and_uid(host, user, uid):
+    user = host.user(user)
+    assert user.exists
+    assert user.uid == uid
+
+
+def test_group_exists_and_gid(host):
+    group = host.group("customgroup")
+    assert group.exists
+    assert group.gid == 10001
+
+
+def test_user1_in_group(host):
+    user = host.user("foo")
+    assert "customgroup" in user.groups
+
+
+def test_user2_not_in_group(host):
+    user = host.user("bar")
+    assert "customgroup" not in user.groups
+
+
+def test_group_sudoers_file(host):
+    file = host.file("/etc/sudoers.d/rel_gid_1")
+    assert file.exists and file.is_file
+    assert file.mode == 0o440

tests/cases/delete-user-group.yml

@@ -3,60 +3,47 @@
 ### Description
 # Test setting absent to 'all' for both user and groups.
 
-### Setup
-
-- name: Set variables
-  set_fact:
-    um_uid_base: 10000
-    um_gid_base: 10000
-    um_group_inventory:
-      - name: customgroup
-        rel_gid: 1
-        present:
-          - all
-        absent:
-          - all
-    um_user_inventory:
-      - name: foo
-        rel_uid: 1
-        present:
-          - all
-        absent:
-          - all
-      - name: bar
-        rel_uid: 2
-        present:
-          - all
-        absent:
-          - all
-
-- name: Set the group to be removed
-  loop: "{{ um_group_inventory }}"
-  group:
-    name: "{{ item.name }}"
-    gid: "{{ um_gid_base + item.rel_gid }}"
-
-- name: Set the users to be removed
-  loop: "{{ um_user_inventory }}"
-  user:
-    name: "{{ item.name }}"
-    uid: "{{ um_uid_base + item.rel_uid }}"
-    group: customgroup
-
-
-### Run
-
-- name: Run role
-  include_role:
-    name: "user_management"
-
-
-### Verification
-
-- name: Verify users don't exist
-  shell: "! id {{ item.name }}"
-  loop: "{{ um_user_inventory }}"
-
-- name: Verify groups don't exist
-  shell: "! getent group {{ item.name }}"
-  loop: "{{ um_group_inventory }}"
+- hosts: containers
+  pre_tasks:
+    - name: Set variables
+      set_fact:
+        um_uid_base: 10000
+        um_gid_base: 10000
+        um_group_inventory:
+          - name: customgroup
+            rel_gid: 1
+            present:
+              - all
+            absent:
+              - all
+        um_user_inventory:
+          - name: foo
+            rel_uid: 1
+            present:
+              - all
+            absent:
+              - all
+          - name: bar
+            rel_uid: 2
+            present:
+              - all
+            absent:
+              - all
+
+    - name: Set the group to be removed
+      loop: "{{ um_group_inventory }}"
+      group:
+        name: "{{ item.name }}"
+        gid: "{{ um_gid_base + item.rel_gid }}"
+
+    - name: Set the users to be removed
+      loop: "{{ um_user_inventory }}"
+      user:
+        name: "{{ item.name }}"
+        uid: "{{ um_uid_base + item.rel_uid }}"
+        group: customgroup
+
+  tasks:
+    - name: Run role
+      include_role:
+        name: "user_management"

tests/cases/delete-user-group_spec.py

@@ -0,0 +1,15 @@
+import pytest
+
+
+@pytest.mark.parametrize("user", [
+    ("foo"),
+    ("bar"),
+])
+def test_user_not_exist(host, user):
+    user = host.user(user)
+    assert not user.exists
+
+
+def test_group_not_exist(host):
+    group = host.group("customgroup")
+    assert not group.exists

tests/cases/minimal-fields.yml

@@ -3,43 +3,24 @@
 ### Description
 # Test adding user and group using the minimal set of fields to pass into user/group inventory.
 
-### Setup
-
-- name: Set variables
-  set_fact:
-    um_uid_base: 10000
-    um_gid_base: 10000
-    um_group_inventory:
-      - name: customgroup
-        rel_gid: 1
-    um_user_inventory:
-      - name: foo
-        rel_uid: 1
-        passwd: $6$CH4pzUgP$PJHNOCKVcWA5ai0kU5qwTkCeK08sx86T6o9exHz7Ki7zN/ip6pPBecr6RYqCaBNDSVobuzwwREHu8KbgNncxf1
-      - name: bar
-        rel_uid: 2
-        passwd: $6$CH4pzUgP$PJHNOCKVcWA5ai0kU5qwTkCeK08sx86T6o9exHz7Ki7zN/ip6pPBecr6RYqCaBNDSVobuzwwREHu8KbgNncxf1
-
-
-### Run
-
-- name: Run role
-  include_role:
-    name: "user_management"
-
-
-### Verification
-
-- name: Verify user exists and has correct UID
-  shell: "id -u {{ item.name }} | grep '^{{ item.rel_uid + um_uid_base }}$'"
-  loop: "{{ um_user_inventory }}"
-
-- name: Verify group exists and has correct GID
-  shell: "getent group {{ item.name }} | cut -d: -f3 | grep '^{{ item.rel_gid + um_gid_base }}$'"
-  loop: "{{ um_group_inventory }}"
-
-- name: Verify user 'foo' is not in group
-  shell: "! id -Gn foo | grep -w customgroup"
-
-- name: Verify sudoer file does not exist
-  shell: "! ls -1 /etc/sudoers.d/ | grep '^rel_gid_1$'"
+- hosts: containers
+  pre_tasks:
+    - name: Set variables
+      set_fact:
+        um_uid_base: 10000
+        um_gid_base: 10000
+        um_group_inventory:
+          - name: customgroup
+            rel_gid: 1
+        um_user_inventory:
+          - name: foo
+            rel_uid: 1
+            passwd: $6$CH4pzUgP$PJHNOCKVcWA5ai0kU5qwTkCeK08sx86T6o9exHz7Ki7zN/ip6pPBecr6RYqCaBNDSVobuzwwREHu8KbgNncxf1
+          - name: bar
+            rel_uid: 2
+            passwd: $6$CH4pzUgP$PJHNOCKVcWA5ai0kU5qwTkCeK08sx86T6o9exHz7Ki7zN/ip6pPBecr6RYqCaBNDSVobuzwwREHu8KbgNncxf1
+
+  tasks:
+    - name: Run role
+      include_role:
+        name: "user_management"