From 77d61eb3599405cf850f388fc8a0f3c6031ade51 Mon Sep 17 00:00:00 2001 From: Ivan Bulyk Date: Mon, 19 Feb 2024 17:27:48 +0100 Subject: [PATCH] added initial ci/cd configuration --- .dockerignore | 39 +++++ .github/workflows/audit.yml | 16 ++ .github/workflows/general.yml | 141 ++++++++++++++++++ .idea/rustymailer.iml | 1 + ...afd6439db2d49d0cb506618ae9e780c7e0558.json | 17 +++ Cargo.lock | 12 ++ Cargo.toml | 1 + Dockerfile | 28 ++++ LICENSE-MIT | 21 +++ configuration.yaml | 7 - configuration/base.yaml | 10 ++ configuration/local.yaml | 4 + configuration/production.yaml | 4 + spec.yaml | 57 +++++++ src/configuration.rs | 94 ++++++++++-- src/main.rs | 12 +- tests/health_check.rs | 7 +- 17 files changed, 440 insertions(+), 31 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/audit.yml create mode 100644 .github/workflows/general.yml create mode 100644 .sqlx/query-793f0df728d217c204123f12e4eafd6439db2d49d0cb506618ae9e780c7e0558.json create mode 100644 Dockerfile create mode 100644 LICENSE-MIT delete mode 100644 configuration.yaml create mode 100644 configuration/base.yaml create mode 100644 configuration/local.yaml create mode 100644 configuration/production.yaml create mode 100644 spec.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ba7fb71 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,39 @@ +# Include any files or directories that you don't want to be copied to your +# container here (e.g., local build artifacts, temporary files, etc.). +# +# For more help, visit the .dockerignore file reference guide at +# https://docs.docker.com/go/build-context-dockerignore/ + +**/.DS_Store +**/.classpath +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/charts +**/docker-compose* +**/compose* +**/Dockerfile* +**/node_modules +**/npm-debug.log +**/secrets.dev.yaml +**/values.dev.yaml +/bin +/target +spec.yaml +deploy/ +tests/ +Dockerfile +scripts/ +migrations/ +LICENSE +README.md +README.Docker.md diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 0000000..7c7cb7d --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,16 @@ +name: Security audit +on: + schedule: + - cron: '0 0 * * *' + push: + paths: + - '**/Cargo.toml' + - '**/Cargo.lock' +jobs: + security_audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: taiki-e/install-action@cargo-deny + - name: Scan for vulnerabilities + run: cargo deny check advisories \ No newline at end of file diff --git a/.github/workflows/general.yml b/.github/workflows/general.yml new file mode 100644 index 0000000..28df59e --- /dev/null +++ b/.github/workflows/general.yml @@ -0,0 +1,141 @@ +name: Rust + +on: + # These settings allow us to run this specific CI pipeline for PRs against + # this specific branch + push: + branches: + - main + pull_request: + types: [ opened, synchronize, reopened ] + branches: + - main + +env: + CARGO_TERM_COLOR: always + SQLX_VERSION: 0.7.3 + SQLX_FEATURES: "rustls,postgres" + +jobs: + test: + name: Test + runs-on: ubuntu-latest + services: + postgres: + image: postgres:14 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: postgres + ports: + - 5432:5432 + redis: + image: redis:7 + ports: + - 6379:6379 + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + key: sqlx-${{ env.SQLX_VERSION }} + - name: Install sqlx-cli + run: + cargo install sqlx-cli + --version=${{ env.SQLX_VERSION }} + --features ${{ env.SQLX_FEATURES }} + --no-default-features + --locked + - name: Migrate database + run: | + sudo apt-get install libpq-dev -y + SKIP_DOCKER=true ./scripts/init_db.sh + - name: Check sqlx-data.json is up-to-date + run: | + cargo sqlx prepare --workspace --check + - name: Run tests + run: cargo test + + fmt: + name: Rustfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + - name: Enforce formatting + run: cargo fmt --check + + clippy: + name: Clippy + runs-on: ubuntu-latest + services: + postgres: + image: postgres:14 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: postgres + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + with: + key: sqlx-${{ env.SQLX_VERSION }} + - name: Install sqlx-cli + run: + cargo install sqlx-cli + --version=${{ env.SQLX_VERSION }} + --features ${{ env.SQLX_FEATURES }} + --no-default-features + --locked + - name: Migrate database + run: | + sudo apt-get install libpq-dev -y + SKIP_DOCKER=true ./scripts/init_db.sh + - name: Linting + run: cargo clippy -- -D warnings + + coverage: + name: Code coverage + runs-on: ubuntu-latest + services: + postgres: + image: postgres:14 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: postgres + ports: + - 5432:5432 + redis: + image: redis:7 + ports: + - 6379:6379 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@stable + - name: Install libpq + run: sudo apt-get update && sudo apt-get install postgresql-client -y + - uses: Swatinem/rust-cache@v2 + with: + key: sqlx-${{ env.SQLX_VERSION }} + - name: Install tarpaulin + run: cargo install cargo-tarpaulin + - name: Install sqlx-cli + run: + cargo install sqlx-cli + --version=${{ env.SQLX_VERSION }} + --features ${{ env.SQLX_FEATURES }} + --no-default-features + --locked + - name: Migrate database + run: SKIP_DOCKER=true ./scripts/init_db.sh + - name: Generate code coverage + run: cargo tarpaulin --verbose --workspace \ No newline at end of file diff --git a/.idea/rustymailer.iml b/.idea/rustymailer.iml index bbe0a70..215f902 100644 --- a/.idea/rustymailer.iml +++ b/.idea/rustymailer.iml @@ -4,6 +4,7 @@ + diff --git a/.sqlx/query-793f0df728d217c204123f12e4eafd6439db2d49d0cb506618ae9e780c7e0558.json b/.sqlx/query-793f0df728d217c204123f12e4eafd6439db2d49d0cb506618ae9e780c7e0558.json new file mode 100644 index 0000000..349e0dd --- /dev/null +++ b/.sqlx/query-793f0df728d217c204123f12e4eafd6439db2d49d0cb506618ae9e780c7e0558.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO subscriptions (id, email, name, subscribed_at)\n VALUES ($1, $2, $3, $4)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "793f0df728d217c204123f12e4eafd6439db2d49d0cb506618ae9e780c7e0558" +} diff --git a/Cargo.lock b/Cargo.lock index b002e5c..6bf18ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1858,6 +1858,7 @@ dependencies = [ "once_cell", "reqwest", "serde", + "serde-aux", "sqlx", "tokio", "tracing", @@ -1937,6 +1938,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-aux" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a86348501c129f3ad50c2f4635a01971f76974cd8a3f335988a0f1581c082765" +dependencies = [ + "chrono", + "serde", + "serde_json", +] + [[package]] name = "serde_derive" version = "1.0.196" diff --git a/Cargo.toml b/Cargo.toml index b5d4514..e138062 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ actix-web = "4.5.1" config = "0.14.0" reqwest = "0.11.24" serde = { version = "1.0.196", features = ["derive"] } +serde-aux = "4.4.0" sqlx = { version = "0.7.3", default-features = false, features = ["runtime-tokio-rustls", "macros", "postgres", "uuid", "chrono", "migrate"] } tracing = "0.1.40" tracing-actix-web = "0.7.9" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c6a5fea --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM rust:1.76.0 AS builder + +WORKDIR /app + +COPY . . + +ENV SQLX_OFFLINE true + +RUN cargo build --release + +FROM debian:bullseye-slim AS runtime + +WORKDIR /app + +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends openssl \ +# Clean up + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /app/target/release/rustymailer rustymailer + +COPY configuration configuration + +ENV APP_ENVIRONMENT production + +ENTRYPOINT ["./rustymailer"] \ No newline at end of file diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 0000000..0c1bfde --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Ivan Bulyk + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/configuration.yaml b/configuration.yaml deleted file mode 100644 index 04dc5ef..0000000 --- a/configuration.yaml +++ /dev/null @@ -1,7 +0,0 @@ -application_port: 8000 -database: - host: "localhost" - port: 5432 - username: "postgres" - password: "password" - database_name: "newsletter" \ No newline at end of file diff --git a/configuration/base.yaml b/configuration/base.yaml new file mode 100644 index 0000000..ceb833a --- /dev/null +++ b/configuration/base.yaml @@ -0,0 +1,10 @@ +application: + port: 8000 + host: 0.0.0.0 +database: + host: "127.0.0.1" + port: 5432 + username: "postgres" + password: "password" + database_name: "newsletter" + require_ssl: false \ No newline at end of file diff --git a/configuration/local.yaml b/configuration/local.yaml new file mode 100644 index 0000000..3b77405 --- /dev/null +++ b/configuration/local.yaml @@ -0,0 +1,4 @@ +application: + host: 127.0.0.1 +database: + require_ssl: false \ No newline at end of file diff --git a/configuration/production.yaml b/configuration/production.yaml new file mode 100644 index 0000000..f3ac210 --- /dev/null +++ b/configuration/production.yaml @@ -0,0 +1,4 @@ +application: + host: 0.0.0.0 +database: + require_ssl: true \ No newline at end of file diff --git a/spec.yaml b/spec.yaml new file mode 100644 index 0000000..46b4853 --- /dev/null +++ b/spec.yaml @@ -0,0 +1,57 @@ +name: rustymailer +# See https://www.digitalocean.com/docs/app-platform/#regional-availability for the available options +# You can get region slugs from https://www.digitalocean.com/docs/platform/availability-matrix/ +# `fra` stands for Frankfurt (Germany - EU) +region: ams3 +services: + - name: rustymailer + # Relative to the repository root + dockerfile_path: Dockerfile + source_dir: . + github: + branch: main + deploy_on_push: true + repo: ivanbulyk/rustymailer + # Active probe used by DigitalOcean's to ensure our application is healthy + health_check: + # The path to our health check endpoint! It turned out to be useful in the end! + http_path: /health_check + # The port the application will be listening on for incoming requests + # It should match what we specify in our configuration.yaml file! + http_port: 8000 + # For production workloads we'd go for at least two! + instance_count: 1 + # Let's keep the bill lean for now... + instance_size_slug: basic-xxs + # All incoming requests should be routed to our app + routes: + - path: / + envs: + - key: APP_APPLICATION__BASE_URL + scope: RUN_TIME + value: ${APP_URL} + - key: APP_DATABASE__USERNAME + scope: RUN_TIME + value: ${newsletter.USERNAME} + - key: APP_DATABASE__PASSWORD + scope: RUN_TIME + value: ${newsletter.PASSWORD} + - key: APP_DATABASE__HOST + scope: RUN_TIME + value: ${newsletter.HOSTNAME} + - key: APP_DATABASE__PORT + scope: RUN_TIME + value: ${newsletter.PORT} + - key: APP_DATABASE__DATABASE_NAME + scope: RUN_TIME + value: ${newsletter.DATABASE} +databases: + # PG = Postgres + - engine: PG + # Database name + name: newsletter + # Again, let's keep the bill lean + num_nodes: 1 + size: db-s-dev-database + # Postgres version - using the latest here + version: "14" \ No newline at end of file diff --git a/src/configuration.rs b/src/configuration.rs index 950d671..93ee132 100644 --- a/src/configuration.rs +++ b/src/configuration.rs @@ -1,41 +1,107 @@ +use serde_aux::field_attributes::deserialize_number_from_string; +use sqlx::postgres::{PgConnectOptions, PgSslMode}; + #[derive(serde::Deserialize)] pub struct Settings { pub database: DatabaseSettings, - pub application_port: u16, + pub application: ApplicationSettings, +} + +#[derive(serde::Deserialize)] +pub struct ApplicationSettings { + #[serde(deserialize_with = "deserialize_number_from_string")] + pub port: u16, + pub host: String, } #[derive(serde::Deserialize)] pub struct DatabaseSettings { pub username: String, pub password: String, + #[serde(deserialize_with = "deserialize_number_from_string")] pub port: u16, pub host: String, pub database_name: String, + pub require_ssl: bool, } impl DatabaseSettings { - pub fn connection_string(&self) -> String { - format!( - "postgres://{}:{}@{}:{}/{}", - self.username, self.password, self.host, self.port, self.database_name - ) + pub fn without_db(&self) -> PgConnectOptions { + let ssl_mode = if self.require_ssl { + PgSslMode::Require + } else { + PgSslMode::Prefer + }; + + PgConnectOptions::new() + .host(&self.host) + .username(&self.username) + .password(&self.password) + .port(self.port) + .ssl_mode(ssl_mode) } - pub fn connection_string_without_db(&self) -> String { - format!( - "postgres://{}:{}@{}:{}", - self.username, self.password, self.host, self.port - ) + pub fn with_db(&self) -> PgConnectOptions { + self.without_db().database(&self.database_name) } } pub fn get_configuration() -> Result { + let base_path = std::env::current_dir().expect("failed to determine current directory"); + let configuration_directory = base_path.join("configuration"); + + // Detect the running environment. + // Default to `local` if unspecified. + let environment: Environment = std::env::var("APP_ENVIRONMENT") + .unwrap_or_else(|_| "local".into()) + .try_into() + .expect("failed to parse APP_ENVIRONMENT"); + let environment_filename = format!("{}.yaml", environment.as_str()); let settings = config::Config::builder() - .add_source(config::File::new( - "configuration.yaml", - config::FileFormat::Yaml, + .add_source(config::File::from( + configuration_directory.join("base.yaml"), + )) + .add_source(config::File::from( + configuration_directory.join(environment_filename), )) + // Add in settings from environment variables (with a prefix of APP and '__' as separator) + // E.g. `APP_APPLICATION__PORT=5001 would set `Settings.application.port` + .add_source( + config::Environment::with_prefix("APP") + .prefix_separator("_") + .separator("__"), + ) .build()?; settings.try_deserialize::() } + +/// The possible runtime environment for our application. +pub enum Environment { + Local, + Production, +} + +impl Environment { + pub fn as_str(&self) -> &'static str { + match self { + Environment::Local => "local", + Environment::Production => "production", + } + } +} + +impl TryFrom for Environment { + type Error = String; + + fn try_from(s: String) -> Result { + match s.to_lowercase().as_str() { + "local" => Ok(Self::Local), + "production" => Ok(Self::Production), + other => Err(format!( + "{} is not a supported environment. Use either `local` or `production`.", + other + )), + } + } +} diff --git a/src/main.rs b/src/main.rs index c1ee7f8..78fdbec 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ use rustymailer::configuration::get_configuration; use rustymailer::startup::run; use rustymailer::telemetry::{get_subscriber, init_subscriber}; -use sqlx::PgPool; +use sqlx::postgres::PgPoolOptions; use std::net::TcpListener; #[actix_web::main] @@ -9,10 +9,12 @@ async fn main() -> std::io::Result<()> { let subscriber = get_subscriber("rustymailer".into(), "info".into(), std::io::stdout); init_subscriber(subscriber); let configuration = get_configuration().expect("failed to read configuration"); - let connection_pull = PgPool::connect(&configuration.database.connection_string()) - .await - .expect("failed to connect Postgres"); - let address = format!("127.0.0.1:{}", configuration.application_port); + let connection_pull = PgPoolOptions::new().connect_lazy_with(configuration.database.with_db()); + + let address = format!( + "{}:{}", + configuration.application.host, configuration.application.port + ); let listener = TcpListener::bind(address)?; run(listener, connection_pull)?.await?; Ok(()) diff --git a/tests/health_check.rs b/tests/health_check.rs index 9c8f826..62b440e 100644 --- a/tests/health_check.rs +++ b/tests/health_check.rs @@ -6,7 +6,6 @@ use sqlx::{Connection, Executor, PgConnection, PgPool}; use std::net::TcpListener; use uuid::Uuid; -// Ensure that the `tracing` stack is only initialised once using `once_cell` static TRACING: Lazy<()> = Lazy::new(|| { let default_filter_level = "info".to_string(); let subscriber_name = "test".to_string(); @@ -25,8 +24,6 @@ pub struct TestApp { } async fn spawn_app() -> TestApp { - // The first time `initialize` is invoked the code in `TRACING` is executed. - // All other invocations will instead skip execution. Lazy::force(&TRACING); let listener = TcpListener::bind("127.0.0.1:0").expect("Failed to bind random port"); @@ -47,7 +44,7 @@ async fn spawn_app() -> TestApp { pub async fn configure_database(config: &DatabaseSettings) -> PgPool { // Create database - let mut connection = PgConnection::connect(&config.connection_string_without_db()) + let mut connection = PgConnection::connect_with(&config.without_db()) .await .expect("failed to connect to Postgres"); connection @@ -56,7 +53,7 @@ pub async fn configure_database(config: &DatabaseSettings) -> PgPool { .expect("failed to create database"); // Migrate database - let connection_pool = PgPool::connect(&config.connection_string()) + let connection_pool = PgPool::connect_with(config.with_db()) .await .expect("failed to connect to Postgres"); sqlx::migrate!("./migrations")