From f12b5f67d7ac6f44fe69d843cb41423a4dd42864 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Thu, 13 Jun 2024 15:41:36 +0200 Subject: [PATCH 1/4] Replace conflicting repository-service-tuf dep Previously, `repository-service-tuf` (i.e. the RSTUF cli) was used to bootstrap an RSTUF repo for development. This PR re-implements the relevant parts of the cli locally in Warehouse and removes the `repository-service-tuf` dependency, which conflicts with other dependencies. Change details - Add lightweight RSTUF API client library (can be re-used for #15815) - Add local `warehouse tuf bootstrap` cli subcommand, to wraps lib calls - Invoke local cli via `make inittuf` - Remove dependency supersedes #15958 (cc @facutuesca @woodruffw) Signed-off-by: Lukas Puehringer --- Makefile | 2 +- requirements/dev.txt | 1 - warehouse/cli/tuf.py | 33 +++++++++++++++++ warehouse/tuf/__init__.py | 74 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 warehouse/cli/tuf.py create mode 100644 warehouse/tuf/__init__.py diff --git a/Makefile b/Makefile index e85d88322ba4..9edd5246bc6f 100644 --- a/Makefile +++ b/Makefile @@ -128,7 +128,7 @@ initdb: .state/docker-build-base .state/db-populated inittuf: .state/db-migrated docker compose up -d rstuf-api docker compose up -d rstuf-worker - docker compose run --rm web rstuf admin ceremony -b -u -f dev/rstuf/bootstrap.json --api-server http://rstuf-api + docker compose run --rm web python -m warehouse tuf bootstrap dev/rstuf/bootstrap.json --api-server http://rstuf-api runmigrations: .state/docker-build-base docker compose run --rm web python -m warehouse db upgrade head diff --git a/requirements/dev.txt b/requirements/dev.txt index b9b34ac968f8..d233587fc74f 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -3,4 +3,3 @@ hupper>=1.9 pip-tools>=1.0 pyramid_debugtoolbar>=2.5 pip-api -repository-service-tuf \ No newline at end of file diff --git a/warehouse/cli/tuf.py b/warehouse/cli/tuf.py new file mode 100644 index 000000000000..1dc7bfc31df8 --- /dev/null +++ b/warehouse/cli/tuf.py @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import click + +from warehouse.cli import warehouse +from warehouse.tuf import post_bootstrap, wait_for_success + + +@warehouse.group() +def tuf(): + """Manage TUF.""" + + +@tuf.command() +@click.argument("payload", type=click.File("rb"), required=True) +@click.option("--api-server", required=True) +def bootstrap(payload, api_server): + """Use payload file to bootstrap RSTUF server.""" + task_id = post_bootstrap(api_server, json.load(payload)) + wait_for_success(api_server, task_id) + print(f"Bootstrap completed using `{payload.name}`. 🔐 🎉") diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py new file mode 100644 index 000000000000..d03c032ccaa8 --- /dev/null +++ b/warehouse/tuf/__init__.py @@ -0,0 +1,74 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +RSTUF API client library +""" + +import time + +from typing import Any + +import requests + + +class RSTUFError(Exception): + pass + + +def get_task_state(server: str, task_id: str) -> str: + resp = requests.get(f"{server}/api/v1/task?task_id={task_id}") + resp.raise_for_status() + return resp.json()["data"]["state"] + + +def post_bootstrap(server: str, payload: Any) -> str: + resp = requests.post(f"{server}/api/v1/bootstrap", json=payload) + resp.raise_for_status() + + # TODO: Ask upstream to not return 200 on error + resp_json = resp.json() + resp_data = resp_json.get("data") + if not resp_data: + raise RSTUFError(f"Error in RSTUF job: {resp_json}") + + return resp_data["task_id"] + + +def wait_for_success(server: str, task_id: str): + """Poll RSTUF task state API until success or error.""" + + retries = 20 + delay = 1 + + for _ in range(retries): + state = get_task_state(server, task_id) + + match state: + case "SUCCESS": + break + + case "PENDING" | "RUNNING" | "RECEIVED" | "STARTED": + time.sleep(delay) + continue + + case "FAILURE": + raise RSTUFError("RSTUF job failed, please check payload and retry") + + case "ERRORED" | "REVOKED" | "REJECTED": + raise RSTUFError("RSTUF internal problem, please check RSTUF health") + + case _: + raise RSTUFError(f"RSTUF job returned unexpected state: {state}") + + else: + raise RSTUFError("RSTUF job failed, please check payload and retry") From 1b1fc7de819910db44243239f62560c716c37b77 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Fri, 14 Jun 2024 10:08:22 +0200 Subject: [PATCH 2/4] Make payload arg in tuf cli "lazy" Other than the regular click File, the LazyFile also has the "name" attribute, when passing stdin via "-". We print the name on success. Signed-off-by: Lukas Puehringer --- warehouse/cli/tuf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warehouse/cli/tuf.py b/warehouse/cli/tuf.py index 1dc7bfc31df8..1f9763ba1209 100644 --- a/warehouse/cli/tuf.py +++ b/warehouse/cli/tuf.py @@ -24,7 +24,7 @@ def tuf(): @tuf.command() -@click.argument("payload", type=click.File("rb"), required=True) +@click.argument("payload", type=click.File("rb", lazy=True), required=True) @click.option("--api-server", required=True) def bootstrap(payload, api_server): """Use payload file to bootstrap RSTUF server.""" From 30c50c263c3a713f64c05ce811b8c0b695358264 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Fri, 14 Jun 2024 10:11:04 +0200 Subject: [PATCH 3/4] Add minimal unittest for TUF bootstrap cli Signed-off-by: Lukas Puehringer --- tests/unit/cli/test_tuf.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/unit/cli/test_tuf.py diff --git a/tests/unit/cli/test_tuf.py b/tests/unit/cli/test_tuf.py new file mode 100644 index 000000000000..5486455f7ceb --- /dev/null +++ b/tests/unit/cli/test_tuf.py @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from pretend import call, call_recorder + +from warehouse.cli import tuf + + +class TestTUF: + def test_bootstrap(self, cli, monkeypatch): + task_id = "123456" + server = "rstuf.api" + payload = ["foo"] + + post = call_recorder(lambda *a: task_id) + wait = call_recorder(lambda *a: None) + monkeypatch.setattr(tuf, "post_bootstrap", post) + monkeypatch.setattr(tuf, "wait_for_success", wait) + + result = cli.invoke( + tuf.bootstrap, args=["--api-server", server, "-"], input=json.dumps(payload) + ) + + assert result.exit_code == 0 + + assert post.calls == [call(server, payload)] + assert wait.calls == [call(server, task_id)] From 36b861cc4dbd76c4ad9996095a0a4b959c4923e9 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Fri, 14 Jun 2024 11:30:28 +0200 Subject: [PATCH 4/4] Add unit tests for RSTUF API client lib Signed-off-by: Lukas Puehringer --- tests/unit/tuf/test_tuf.py | 90 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 tests/unit/tuf/test_tuf.py diff --git a/tests/unit/tuf/test_tuf.py b/tests/unit/tuf/test_tuf.py new file mode 100644 index 000000000000..45ddd2c9ade7 --- /dev/null +++ b/tests/unit/tuf/test_tuf.py @@ -0,0 +1,90 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from pretend import call, call_recorder, stub + +from warehouse import tuf + + +class TestTUF: + server = "rstuf.api" + task_id = "123456" + + def test_get_task_state(self, monkeypatch): + state = "SUCCESS" + + resp_json = {"data": {"state": state}} + resp = stub( + raise_for_status=(lambda *a: None), json=(lambda *a, **kw: resp_json) + ) + get = call_recorder(lambda *a: resp) + monkeypatch.setattr(tuf.requests, "get", get) + + result = tuf.get_task_state(self.server, self.task_id) + + assert result == state + assert get.calls == [call(f"{self.server}/api/v1/task?task_id={self.task_id}")] + + def test_post_bootstrap(self, monkeypatch): + payload = ["foo"] + + resp_json = {"data": {"task_id": self.task_id}} + resp = stub( + raise_for_status=(lambda *a: None), json=(lambda *a, **kw: resp_json) + ) + post = call_recorder(lambda *a, **kw: resp) + monkeypatch.setattr(tuf.requests, "post", post) + + # Test success + result = tuf.post_bootstrap(self.server, payload) + + assert result == self.task_id + assert post.calls == [call(f"{self.server}/api/v1/bootstrap", json=payload)] + + # Test fail with incomplete response json + del resp_json["data"] + with pytest.raises(tuf.RSTUFError): + tuf.post_bootstrap(self.server, payload) + + def test_wait_for_success(self, monkeypatch): + get_task_state = call_recorder(lambda *a: "SUCCESS") + monkeypatch.setattr(tuf, "get_task_state", get_task_state) + tuf.wait_for_success(self.server, self.task_id) + + assert get_task_state.calls == [call(self.server, self.task_id)] + + @pytest.mark.parametrize( + "state, iterations", + [ + ("PENDING", 20), + ("RUNNING", 20), + ("RECEIVED", 20), + ("STARTED", 20), + ("FAILURE", 1), + ("ERRORED", 1), + ("REVOKED", 1), + ("REJECTED", 1), + ("bogus", 1), + ], + ) + def test_wait_for_success_error(self, state, iterations, monkeypatch): + monkeypatch.setattr(tuf.time, "sleep", lambda *a: None) + + get_task_state = call_recorder(lambda *a: state) + monkeypatch.setattr(tuf, "get_task_state", get_task_state) + + with pytest.raises(tuf.RSTUFError): + tuf.wait_for_success(self.server, self.task_id) + + assert get_task_state.calls == [call(self.server, self.task_id)] * iterations