From 7dc3e11a157c948d3ec4f9591a741d4498ea902c Mon Sep 17 00:00:00 2001 From: Caila Finn Date: Tue, 14 Jan 2025 13:45:30 +0000 Subject: [PATCH 1/5] Add ansible playbook to clean nodes in inventory RE #103 --- .../ansible/clean-jenkins-agents.yml | 7 +++++ .../ansible/roles/clean-agent/tasks/main.yml | 27 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 Linux/jenkins-node/ansible/clean-jenkins-agents.yml create mode 100644 Linux/jenkins-node/ansible/roles/clean-agent/tasks/main.yml diff --git a/Linux/jenkins-node/ansible/clean-jenkins-agents.yml b/Linux/jenkins-node/ansible/clean-jenkins-agents.yml new file mode 100644 index 0000000..416a056 --- /dev/null +++ b/Linux/jenkins-node/ansible/clean-jenkins-agents.yml @@ -0,0 +1,7 @@ +- name: Clean jenkins agents by removing workspaces. + hosts: all + + # Tags available: pr, nightly, package, docs + roles: + - role: clean-agent + become: yes diff --git a/Linux/jenkins-node/ansible/roles/clean-agent/tasks/main.yml b/Linux/jenkins-node/ansible/roles/clean-agent/tasks/main.yml new file mode 100644 index 0000000..6525aee --- /dev/null +++ b/Linux/jenkins-node/ansible/roles/clean-agent/tasks/main.yml @@ -0,0 +1,27 @@ +- name: Remove PR directories + community.docker.docker_container_exec: + container: "{{ agent_name }}" + command: bash -l -c "rm -rf pull_requests*" + chdir: "/jenkins_workdir/workspace" + tags: pr + +- name: Remove Nightly directories + community.docker.docker_container_exec: + container: "{{ agent_name }}" + command: bash -l -c "rm -rf *_nightly_deployment*" + chdir: "/jenkins_workdir/workspace" + tags: nightly + +- name: Remove Packages from Branch directories + community.docker.docker_container_exec: + container: "{{ agent_name }}" + command: bash -l -c "rm -rf build_packages_from_branch*" + chdir: "/jenkins_workdir/workspace" + tags: package + +- name: Remove Docs Build directories + community.docker.docker_container_exec: + container: "{{ agent_name }}" + command: bash -l -c "rm -rf build_and_publish_docs*" + chdir: "/jenkins_workdir/workspace" + tags: docs From 88b4e6c395af98e844abc6d58bbd24387b4a0e6e Mon Sep 17 00:00:00 2001 From: Caila Finn Date: Tue, 14 Jan 2025 14:01:51 +0000 Subject: [PATCH 2/5] Add documentation RE #103 --- Linux/jenkins-node/ansible/readme.md | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/Linux/jenkins-node/ansible/readme.md b/Linux/jenkins-node/ansible/readme.md index 91a55a6..e556f52 100644 --- a/Linux/jenkins-node/ansible/readme.md +++ b/Linux/jenkins-node/ansible/readme.md @@ -36,13 +36,27 @@ ansible-playbook -i inventory.txt jenkins-agent-staging.yml -u FedID -K -t agent ## Cleaning nodes -Before cleaning any nodes mark them temporarily offline on Jenkins and ensure no jobs are running on them before cleaning. +- Before cleaning any nodes mark them temporarily offline on Jenkins and ensure no jobs are running on them before cleaning. -The easiest way to clean nodes is using a groovy script on Jenkins. Use the links below for guidance -- [`Remove directories across multiple nodes`](https://developer.mantidproject.org/JenkinsConfiguration.html#remove-directories-across-multiple-nodes) -- [`Remove directories from single node`](https://developer.mantidproject.org/JenkinsConfiguration.html#remove-directories-from-single-node) +- Update the `inventory.txt` file as above, including only the nodes you need to clean. -If this does not work you may need to spin up a new docker container. Use the instructions for Changing docker image below. +- The tasks in the cleaning role are defined by tags: + + - `pr`: Pull Requests. + - `nightly`: Nightly deployments for main and release next. + - `package`: Build Packages from Branch + - `docs`: Docs build and publish. + +- Run the following with the desired tags, uses a comma-separated list. **If no tags are provided all workspaces will be deleted.**: +```sh +ansible-playbook -i inventory.txt clean-jenkins-agents.yml -u FedID -K -t pr,nightly,package,docs +``` + +- Mark the shutdown nodes back online. + +### Troubleshooting + +- If this does not work you may need to spin up a new docker container. Use the instructions for Changing docker image below. ## Changing docker image From d1096bc40078f057efa7c80dfe2d9ed9a564f972 Mon Sep 17 00:00:00 2001 From: Caila Finn Date: Tue, 14 Jan 2025 14:05:46 +0000 Subject: [PATCH 3/5] Improve docs formatting RE #103 --- Linux/jenkins-node/ansible/readme.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Linux/jenkins-node/ansible/readme.md b/Linux/jenkins-node/ansible/readme.md index e556f52..5c8bf0e 100644 --- a/Linux/jenkins-node/ansible/readme.md +++ b/Linux/jenkins-node/ansible/readme.md @@ -7,17 +7,17 @@ New Linux nodes for use on Jenkins are now set up using ansible scripts in this - Ensure you have activated a conda environment with ansible (you may need to use the conda environment set up for use with the ansible-linode repo). - Set up a new node on Jenkins. The easiest way to set up a jenkins node is to copy an existing node: From the jenkins menu, select `New Item`, type in the name for your node, then scroll down to the `copy from` box and enter the node name you wish to copy. - To get the secret for your newly set up node, select your new node from the `Build Executor Status` pane on the left hand side of the jenkins home page. The secret should be displayed as part of the command in the box below `Run from agent command line:`. Note, if you are setting up a node directly on the staging server you will have to use the following command in the jenkins console (`/script`) to obtain the secret: -``` +```java jenkins.model.Jenkins.getInstance().getComputer("").getJnlpMac() ``` - navigate to the [`Ansible folder`](https://github.com/mantidproject/dockerfiles/tree/main/Linux/jenkins-node/ansible) - Update the `inventory.txt` file with the IP address, agent name and agent secret (i.e. Jenkins secret code). Be sure to save this file when update complete! - If creating staging nodes, update the `jenkins-agent-staging.yml` file to specify the correct `jenkins_url`, and `jenkins_identity` variables. To get the `jenkins_identity` use the following command in the jenkins console: -``` +```java hudson.remoting.Base64.encode(org.jenkinsci.main.modules.instance_identity.InstanceIdentity.get().getPublic().getEncoded()) ``` - Run the following command, replacing `staging` with `production` if appropriate and replacing `FedID` with your FedID. -``` +```sh ansible-playbook -i inventory.txt jenkins-agent-staging.yml -u FedID -K ``` - you will be asked for a password - enter your FedID password @@ -30,7 +30,7 @@ When running the `jenkins-agent-staging.yml` or `jenkins-agent-production.yml` p - `agent`: roles tagged with the `agent` tag deploy the docker container that constitutes the jenkins agent. To use a tag, you pass it in to the `ansible-playbook` command with the `-t` flag. For example, if you have already set up the host machine and just want to deploy a jenkins agent: -``` +```sh ansible-playbook -i inventory.txt jenkins-agent-staging.yml -u FedID -K -t agent ``` @@ -40,14 +40,14 @@ ansible-playbook -i inventory.txt jenkins-agent-staging.yml -u FedID -K -t agent - Update the `inventory.txt` file as above, including only the nodes you need to clean. -- The tasks in the cleaning role are defined by tags: +- The tasks in the cleaning role make use of tags to restrict what is cleaned: - `pr`: Pull Requests. - `nightly`: Nightly deployments for main and release next. - `package`: Build Packages from Branch - `docs`: Docs build and publish. -- Run the following with the desired tags, uses a comma-separated list. **If no tags are provided all workspaces will be deleted.**: +- Run the following with the desired tags (which use a comma-separated list). **If no tags are provided all workspaces will be deleted.**: ```sh ansible-playbook -i inventory.txt clean-jenkins-agents.yml -u FedID -K -t pr,nightly,package,docs ``` @@ -64,17 +64,17 @@ If you need to update the docker image or spin up a new docker container on a Li - ssh into the node - Stop and remove the container using the following command, replacing machinename with the appropriate name e.g. `isis-cloud-linux-1` -``` +```sh docker stop machinename && docker rm machinename -``` +```sh - Remove any associated volumes using the following command, again replacing machinename with the appropriate name -``` +```sh docker volume rm machinename ``` - for cloud machines, close the ssh connection and follow the instructions for setting up cloud nodes above. - for physical machines navigate to the folder that contains the `deploy.sh` script. This is normally `dockerfiles/jenkins-node/bin`. - Run the following command (you may be able to find it using reverse search) -``` +```sh ./deploy.sh machinename agent_secret "https://builds.mantidproject.org" latest 50G ``` - on success close the ssh connection and check node connects on Jenkins From e9ffa94d4c295637483bbfde7cffe70c70a8052d Mon Sep 17 00:00:00 2001 From: Caila Finn Date: Tue, 14 Jan 2025 15:39:08 +0000 Subject: [PATCH 4/5] Refactor to allow easier tag usage RE #103 --- .../ansible/clean-jenkins-agents.yml | 37 +++++++++++++++++-- Linux/jenkins-node/ansible/readme.md | 8 ++-- .../ansible/roles/clean-agent/tasks/main.yml | 27 -------------- 3 files changed, 37 insertions(+), 35 deletions(-) delete mode 100644 Linux/jenkins-node/ansible/roles/clean-agent/tasks/main.yml diff --git a/Linux/jenkins-node/ansible/clean-jenkins-agents.yml b/Linux/jenkins-node/ansible/clean-jenkins-agents.yml index 416a056..c377807 100644 --- a/Linux/jenkins-node/ansible/clean-jenkins-agents.yml +++ b/Linux/jenkins-node/ansible/clean-jenkins-agents.yml @@ -1,7 +1,36 @@ -- name: Clean jenkins agents by removing workspaces. +- name: Playbook to clean jenkins agents by removing workspaces. hosts: all # Tags available: pr, nightly, package, docs - roles: - - role: clean-agent - become: yes + tasks: + - name: Remove PR directories + community.docker.docker_container_exec: + container: "{{ agent_name }}" + command: bash -l -c "rm -rf pull_requests*" + chdir: "/jenkins_workdir/workspace" + become: yes + tags: [never, pr] + + - name: Remove Nightly directories + community.docker.docker_container_exec: + container: "{{ agent_name }}" + command: bash -l -c "rm -rf *_nightly_deployment*" + chdir: "/jenkins_workdir/workspace" + become: yes + tags: [never, nightly] + + - name: Remove Packages from Branch directories + community.docker.docker_container_exec: + container: "{{ agent_name }}" + command: bash -l -c "rm -rf build_packages_from_branch*" + chdir: "/jenkins_workdir/workspace" + become: yes + tags: [never, package] + + - name: Remove Docs Build directories + community.docker.docker_container_exec: + container: "{{ agent_name }}" + command: bash -l -c "rm -rf build_and_publish_docs*" + chdir: "/jenkins_workdir/workspace" + become: yes + tags: [never, docs] diff --git a/Linux/jenkins-node/ansible/readme.md b/Linux/jenkins-node/ansible/readme.md index 5c8bf0e..f21336b 100644 --- a/Linux/jenkins-node/ansible/readme.md +++ b/Linux/jenkins-node/ansible/readme.md @@ -38,21 +38,21 @@ ansible-playbook -i inventory.txt jenkins-agent-staging.yml -u FedID -K -t agent - Before cleaning any nodes mark them temporarily offline on Jenkins and ensure no jobs are running on them before cleaning. -- Update the `inventory.txt` file as above, including only the nodes you need to clean. +- Update the `inventory.txt` file as above, including only the nodes you intend to clean. -- The tasks in the cleaning role make use of tags to restrict what is cleaned: +- The tasks in the cleaning playbook make use of tags to restrict what is cleaned: - `pr`: Pull Requests. - `nightly`: Nightly deployments for main and release next. - `package`: Build Packages from Branch - `docs`: Docs build and publish. -- Run the following with the desired tags (which use a comma-separated list). **If no tags are provided all workspaces will be deleted.**: +- Run the following with the desired tags (which use a comma-separated list): ```sh ansible-playbook -i inventory.txt clean-jenkins-agents.yml -u FedID -K -t pr,nightly,package,docs ``` -- Mark the shutdown nodes back online. +- Set the nodes you shut down back online. ### Troubleshooting diff --git a/Linux/jenkins-node/ansible/roles/clean-agent/tasks/main.yml b/Linux/jenkins-node/ansible/roles/clean-agent/tasks/main.yml deleted file mode 100644 index 6525aee..0000000 --- a/Linux/jenkins-node/ansible/roles/clean-agent/tasks/main.yml +++ /dev/null @@ -1,27 +0,0 @@ -- name: Remove PR directories - community.docker.docker_container_exec: - container: "{{ agent_name }}" - command: bash -l -c "rm -rf pull_requests*" - chdir: "/jenkins_workdir/workspace" - tags: pr - -- name: Remove Nightly directories - community.docker.docker_container_exec: - container: "{{ agent_name }}" - command: bash -l -c "rm -rf *_nightly_deployment*" - chdir: "/jenkins_workdir/workspace" - tags: nightly - -- name: Remove Packages from Branch directories - community.docker.docker_container_exec: - container: "{{ agent_name }}" - command: bash -l -c "rm -rf build_packages_from_branch*" - chdir: "/jenkins_workdir/workspace" - tags: package - -- name: Remove Docs Build directories - community.docker.docker_container_exec: - container: "{{ agent_name }}" - command: bash -l -c "rm -rf build_and_publish_docs*" - chdir: "/jenkins_workdir/workspace" - tags: docs From e7f138a547aa23697a9e783c81d0c53c86791afb Mon Sep 17 00:00:00 2001 From: Caila Finn Date: Thu, 13 Feb 2025 11:16:55 +0000 Subject: [PATCH 5/5] Add logging step before removing directories --- .../jenkins-node/ansible/clean-jenkins-agents.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Linux/jenkins-node/ansible/clean-jenkins-agents.yml b/Linux/jenkins-node/ansible/clean-jenkins-agents.yml index c377807..64da0a7 100644 --- a/Linux/jenkins-node/ansible/clean-jenkins-agents.yml +++ b/Linux/jenkins-node/ansible/clean-jenkins-agents.yml @@ -3,6 +3,20 @@ # Tags available: pr, nightly, package, docs tasks: + - name: Collect workspace sizes + community.docker.docker_container_exec: + container: "{{ agent_name }}" + command: bash -l -c "du -sh * | column -t" + chdir: "/jenkins_workdir/workspace" + register: du_result + become: yes + tags: [always] + + - name: Display workspace sizes + debug: + msg: "{{ du_result.stdout_lines }}" + tags: [always] + - name: Remove PR directories community.docker.docker_container_exec: container: "{{ agent_name }}"