Need to b64 decode cockroach config secret (#1336)

Author: michaeljguarino

Makefile

@@ -64,8 +64,8 @@ test-certs:
 	cockroach cert create-client root --certs-dir test-certs --ca-key test-certs/ca.key && \
 	cockroach cert list --certs-dir test-certs
 
-testup: test-certs ## sets up dependent services for test
-	docker compose up -d
+testup: ## sets up dependent services for test
+	docker compose up -d --remove-orphans
 
 testdown: ## tear down test dependencies
 	docker compose down

apps/core/lib/core/clients/console.ex

@@ -41,6 +41,18 @@ defmodule Core.Clients.Console do
   }
   """
 
+  @stack_q """
+  query Stack($id: ID!) {
+    infrastructureStack(id: $id) {
+      id
+      output {
+        name
+        value
+      }
+    }
+  }
+  """
+
   def new(url, token) do
     Req.new(base_url: with_gql(url), auth: "Token #{token}")
     |> AbsintheClient.attach()
@@ -49,7 +61,8 @@ defmodule Core.Clients.Console do
   def clusters(client) do
     Req.post(client, graphql: @clusters_q)
     |> case do
-      {:ok, %Req.Response{body: %{"clusters" => %{"edges" => edges}}}} -> {:ok, Enum.map(edges, & &1["node"])}
+      {:ok, %Req.Response{body: %{"data" => %{"clusters" => %{"edges" => edges}}}}} ->
+        {:ok, Enum.map(edges, & &1["node"])}
       res ->
         Logger.warn "Failed to fetch clusters: #{inspect(res)}"
         {:error, "could not fetch clusters"}
@@ -59,7 +72,7 @@ defmodule Core.Clients.Console do
   def repo(client, url) do
     Req.post(client, graphql: {@repo_q, %{url: url}})
     |> case do
-      {:ok, %Req.Response{body: %{"gitRepository" => %{"id" => id}}}} -> {:ok, id}
+      {:ok, %Req.Response{body: %{"data" => %{"gitRepository" => %{"id" => id}}}}} -> {:ok, id}
       res ->
         Logger.warn "Failed to fetch clusters: #{inspect(res)}"
         {:error, "could not fetch repo"}
@@ -81,7 +94,18 @@ defmodule Core.Clients.Console do
     |> service_resp("deleteServiceDeployment")
   end
 
-  defp service_resp({:ok, %Req.Response{status: 200, body: body}}, field) do
+  def stack(client, id) do
+    Req.post(client, graphql: {@stack_q, %{id: id}})
+    |> case do
+      {:ok, %Req.Response{body: %{"data" => %{"infrastructureStack" => stack}}}} ->
+        {:ok, stack}
+      res ->
+        Logger.warn "Failed to fetch stack: #{inspect(res)}"
+        {:error, "could not fetch stack"}
+    end
+  end
+
+  defp service_resp({:ok, %Req.Response{status: 200, body: %{"data" => body}}}, field) do
     case body[field] do
       %{"id" => id} -> {:ok, id}
       err ->

apps/core/lib/core/schema/console_instance.ex

@@ -1,7 +1,7 @@
 defmodule Core.Schema.ConsoleInstance do
   use Piazza.Ecto.Schema
   alias Piazza.Ecto.EncryptedString
-  alias Core.Schema.{CockroachCluster, CloudCluster, User}
+  alias Core.Schema.{PostgresCluster, CloudCluster, User}
 
   defenum Size, small: 0, medium: 1, large: 2
   defenum Status,
@@ -41,6 +41,7 @@ defmodule Core.Schema.ConsoleInstance do
       field :dbpassword,     EncryptedString
       field :subdomain,      :string
       field :jwt_secret,     EncryptedString
+      field :erlang_secret,  EncryptedString
       field :owner_name,     :string
       field :owner_email,    :string
       field :admin_password, EncryptedString
@@ -54,7 +55,7 @@ defmodule Core.Schema.ConsoleInstance do
       field :kas_redis,      EncryptedString
     end
 
-    belongs_to :cockroach, CockroachCluster
+    belongs_to :postgres,  PostgresCluster
     belongs_to :cluster,   CloudCluster
     belongs_to :owner,     User
 
@@ -91,7 +92,7 @@ defmodule Core.Schema.ConsoleInstance do
 
   def regions(), do: @region_map
 
-  @valid ~w(name cloud size region status subdomain url external_id cockroach_id cluster_id owner_id)a
+  @valid ~w(name cloud size region status subdomain url external_id postgres_id cluster_id owner_id)a
 
   def changeset(model, attrs \\ %{}) do
     model
@@ -117,7 +118,7 @@ defmodule Core.Schema.ConsoleInstance do
   end
 
   @conf_valid ~w(
-    database dbuser dbpassword
+    database dbuser dbpassword erlang_secret
     subdomain jwt_secret owner_name owner_email admin_password aes_key
     encryption_key client_id client_secret plural_token
     kas_api kas_private kas_redis

apps/core/lib/core/schema/cockroach_cluster.ex renamed to apps/core/lib/core/schema/postgres_cluster.ex

@@ -1,15 +1,16 @@
-defmodule Core.Schema.CockroachCluster do
+defmodule Core.Schema.PostgresCluster do
   use Piazza.Ecto.Schema
   alias Piazza.Ecto.EncryptedString
   alias Core.Schema.CloudCluster
 
   @saturation 1000
 
-  schema "cockroach_clusters" do
+  schema "postgres_clusters" do
     field :name,        :string
     field :cloud,       CloudCluster.Cloud
     field :region,      :string
     field :url,         EncryptedString
+    field :host,        :string
     field :certificate, :string
     field :endpoints,   :map
     field :count,       :integer, default: 0
@@ -27,8 +28,8 @@ defmodule Core.Schema.CockroachCluster do
 
   def changeset(model, attrs \\ %{}) do
     model
-    |> cast(attrs, ~w(name cloud region url certificate endpoints)a)
+    |> cast(attrs, ~w(name cloud region url host certificate endpoints)a)
     |> unique_constraint(:name)
-    |> validate_required(~w(name cloud region url certificate endpoints)a)
+    |> validate_required(~w(name cloud url host)a)
   end
 end

apps/core/lib/core/services/cloud.ex

@@ -4,12 +4,12 @@ defmodule Core.Services.Cloud do
   alias Core.Repo
   alias Core.PubSub
   alias Core.Services.{Accounts, Users, Repositories, Shell}
-  alias Core.Schema.{CloudCluster, CockroachCluster, ConsoleInstance, User, OIDCProvider}
+  alias Core.Schema.{CloudCluster, PostgresCluster, ConsoleInstance, User, OIDCProvider}
 
   @type error :: {:error, term}
   @type console_resp :: {:ok, ConsoleInstance.t} | error
   @type cluster_resp :: {:ok, CloudCluster.t} | error
-  @type cockroach_resp :: {:ok, CockroachCluster.t} | error
+  @type postgres_resp :: {:ok, PostgresCluster.t} | error
 
   def get_instance!(id), do: Repo.get!(ConsoleInstance, id)
 
@@ -23,13 +23,13 @@ defmodule Core.Services.Cloud do
     |> Repo.insert_or_update()
   end
 
-  @spec upsert_cockroach(map, binary) :: cockroach_resp
-  def upsert_cockroach(attrs, name) do
-    case Repo.get_by(CockroachCluster, name: name) do
-      %CockroachCluster{} = cluster -> cluster
-      nil -> %CockroachCluster{name: name}
+  @spec upsert_postgres(map, binary) :: postgres_resp
+  def upsert_postgres(attrs, name) do
+    case Repo.get_by(PostgresCluster, name: name) do
+      %PostgresCluster{} = cluster -> cluster
+      nil -> %PostgresCluster{name: name}
     end
-    |> CockroachCluster.changeset(attrs)
+    |> PostgresCluster.changeset(attrs)
     |> Repo.insert_or_update()
   end
 
@@ -41,7 +41,7 @@ defmodule Core.Services.Cloud do
     start_transaction()
     |> add_operation(:auth, fn _ -> allow(%ConsoleInstance{}, user, :create) end)
     |> add_operation(:cluster, fn _ -> select_cluster(attrs[:cloud], attrs[:region]) end)
-    |> add_operation(:cockroach, fn _ -> select_roach(attrs[:cloud]) end)
+    |> add_operation(:postgres, fn _ -> select_roach(attrs[:cloud]) end)
     |> add_operation(:sa, fn _ ->
       Accounts.create_service_account(%{name: "#{name}-cloud-sa", email: "#{name}-cloud-sa@srv.plural.sh"}, user)
     end)
@@ -58,11 +58,11 @@ defmodule Core.Services.Cloud do
       Repositories.upsert_oidc_provider(%{
         auth_method: :post,
         bindings: Shell.oidc_bindings(inst.oidc_provider, user),
-        redirect_uris: Shell.merge_uris(["https://console.#{name}.cloud.plural.sh/oauth/callback"], inst.oidc_provider)
+        redirect_uris: Shell.merge_uris(["https://console-#{name}.cloud.plural.sh/oauth/callback"], inst.oidc_provider)
       }, inst.id, sa)
     end)
-    |> add_operation(:instance, fn %{oidc: oidc, token: token, cluster: cluster, cockroach: roach, sa: sa} ->
-      %ConsoleInstance{status: :pending, cluster_id: cluster.id, cockroach_id: roach.id, owner_id: sa.id}
+    |> add_operation(:instance, fn %{oidc: oidc, token: token, cluster: cluster, postgres: roach, sa: sa} ->
+      %ConsoleInstance{status: :pending, cluster_id: cluster.id, postgres_id: roach.id, owner_id: sa.id}
       |> ConsoleInstance.changeset(add_configuration(attrs, name, token.token, oidc, user))
       |> Repo.insert()
     end)
@@ -135,24 +135,25 @@ defmodule Core.Services.Cloud do
   end
 
   defp add_configuration(attrs, name, token, %OIDCProvider{} = oidc, %User{} = user) do
-    Map.merge(attrs, %{subdomain: "#{name}.cloud.plural.sh", url: "console.#{name}.cloud.plural.sh"})
+    Map.merge(attrs, %{subdomain: "#{name}.cloud.plural.sh", url: "console-#{name}.cloud.plural.sh"})
     |> Map.put(:configuration, %{
       aes_key:        aes_key(),
       encryption_key: encryption_key(),
       database:       "#{name}_cloud",
       dbuser:         "#{name}_user",
-      dbpassword:     Core.random_alphanum(30),
+      dbpassword:     Core.random_alphanum(32),
       subdomain:      "#{name}.cloud.plural.sh",
-      jwt_secret:     Core.random_alphanum(30),
+      jwt_secret:     Core.random_alphanum(32) |> Base.encode64(),
       owner_name:     user.name,
       owner_email:    user.email,
-      admin_password: Core.random_alphanum(30),
+      admin_password: Core.random_alphanum(32) |> Base.encode64(),
+      erlang_secret:  Core.random_alphanum(32) |> Base.encode64(),
       client_id:      oidc.client_id,
       client_secret:  oidc.client_secret,
       plural_token:   token,
-      kas_api:        Core.random_alphanum(30),
-      kas_private:    Core.random_alphanum(30),
-      kas_redis:      Core.random_alphanum(30)
+      kas_api:        Core.random_alphanum(64) |> Base.encode64(),
+      kas_private:    Core.random_alphanum(64) |> Base.encode64(),
+      kas_redis:      Core.random_alphanum(64) |> Base.encode64(),
     })
   end
 
@@ -165,8 +166,8 @@ defmodule Core.Services.Cloud do
   end
 
   defp select_roach(cloud) do
-    CockroachCluster.for_cloud(cloud)
-    |> CockroachCluster.unsaturated()
+    PostgresCluster.for_cloud(cloud)
+    |> PostgresCluster.unsaturated()
     |> Repo.all()
     |> random_choice("Could not place in #{cloud}")
   end

apps/core/lib/core/services/cloud/configuration.ex

@@ -1,5 +1,5 @@
 defmodule Core.Services.Cloud.Configuration do
-  alias Core.Schema.{ConsoleInstance, CockroachCluster}
+  alias Core.Schema.{ConsoleInstance, PostgresCluster}
 
   def build(%ConsoleInstance{configuration: conf, size: size} = inst) do
     Map.take(conf, ~w(
@@ -16,23 +16,24 @@ defmodule Core.Services.Cloud.Configuration do
       kas_api
       kas_private
       kas_redis
+      erlang_secret
     )a)
     |> Map.merge(%{
       postgres_url: build_pg_url(inst),
+      cloud: "#{inst.cloud}",
+      cluster_name: inst.name,
       size: "#{size}",
-      postgres_certificate: certificate(inst)
     })
     |> Map.put(:size, "#{size}")
-    |> Enum.map(fn {k, v} -> %{name: Macro.camelize("#{k}"), value: v} end)
+    |> Enum.map(fn {k, v} -> %{name: k, value: v} end)
   end
 
-  defp certificate(%ConsoleInstance{cockroach: %CockroachCluster{certificate: cert}}), do: cert
+  # defp certificate(%ConsoleInstance{postgres: %PostgresCluster{certificate: cert}}), do: cert
 
   defp build_pg_url(%ConsoleInstance{
     configuration: %{dbuser: u, dbpassword: p, database: database},
-    region: region,
-    cockroach: %CockroachCluster{endpoints: endpoints}
+    postgres: %PostgresCluster{host: host}
   }) do
-    "postgresql://#{u}:#{p}@#{endpoints[region]}/#{database}"
+    "postgresql://#{u}:#{p}@#{host}/#{database}"
   end
 end

apps/core/lib/core/services/cloud/poller.ex

@@ -2,10 +2,9 @@ defmodule Core.Services.Cloud.Poller do
   use GenServer
   alias Core.Clients.Console
   alias Core.Services.Cloud
-  alias Kazan.Apis.Core.V1, as: CoreV1
   require Logger
 
-  @poll :timer.minutes(5)
+  @poll :timer.minutes(2)
 
   defmodule State, do: defstruct [:client, :repo]
 
@@ -15,16 +14,16 @@ defmodule Core.Services.Cloud.Poller do
 
   def init(_) do
     :timer.send_interval(@poll, :clusters)
-    :timer.send_interval(@poll, :roaches)
+    :timer.send_interval(@poll, :pgs)
     send self(), :repo
     {:ok, %State{client: Console.new(Core.conf(:console_url), Core.conf(:console_token))}}
   end
 
   def repository(), do: GenServer.call(__MODULE__, :repo)
 
-  def handle_call(:repo, %{repo: id} = state) when is_binary(id),
+  def handle_call(:repo, _, %{repo: id} = state) when is_binary(id),
     do: {:reply, {:ok, id}, state}
-  def handle_call(:repo, state), do: {:reply, {:error, "repo not pulled"}, state}
+  def handle_call(:repo, _, state), do: {:reply, {:error, "repo not pulled"}, state}
 
   def handle_info(:repo, %{client: client} = state) do
     case Console.repo(client, Core.conf(:mgmt_repo)) do
@@ -43,12 +42,13 @@ defmodule Core.Services.Cloud.Poller do
     {:noreply, state}
   end
 
-  def handle_info(:roaches, state) do
-    case read_secret() do
-      {:ok, roaches} ->
-        Enum.each(roaches, &upsert_roach/1)
-      err ->
-        Logger.warn "failed to fetch available cockroach clusters: #{inspect(err)}"
+  def handle_info(:pgs, %{client: client} = state) do
+    with {:ok, stack} <- Console.stack(client, Core.conf(:stack_id)),
+         %{"value" => v} <- Enum.find(stack["output"], & &1["name"] == "clusters"),
+         {:ok, pgs} <- Jason.decode(v) do
+      Enum.each(pgs, fn {k, v} -> upsert_pg(k, v) end)
+    else
+      err -> Logger.warn "failed to fetch cluster info: #{inspect(err)}"
     end
     {:noreply, state}
   end
@@ -61,29 +61,24 @@ defmodule Core.Services.Cloud.Poller do
       cloud: to_cloud(distro),
       region: meta["region"]
     }, name)
+    |> log_err("failed to insert cloud cluster")
   end
 
-  defp upsert_roach(%{"name" => name} = roach) do
-    Cloud.upsert_cockroach(%{
-      cloud: roach["cloud"],
-      url: roach["url"],
-      certificate: roach["certificate"],
-      endpoints: roach["endpoints"]
+  defp upsert_pg(name, pg) do
+    Cloud.upsert_postgres(%{
+      cloud: pg["cloud"],
+      url: pg["url"],
+      # certificate: pg["certificate"],
+      host: pg["host"]
     }, name)
-  end
-
-  defp read_secret() do
-    CoreV1.read_namespaced_secret!("plural", "plrl-cloud-config")
-    |> Kazan.run()
-    |> case do
-      {:ok, %CoreV1.Secret{data: %{"cockroaches" => roaches}}} ->
-        Jason.decode(roaches)
-      _ -> {:error, "could not find secret"}
-    end
+    |> log_err("failed to insert postgres cluster")
   end
 
   defp to_cloud("EKS"), do: :aws
   defp to_cloud("GKE"), do: :gcp
   defp to_cloud("AKS"), do: :azure
   defp to_cloud(_), do: :aws
+
+  defp log_err({:error, _} = err, msg), do: "#{msg}: #{inspect(err)}"
+  defp log_err(pass, _), do: pass
 end