Add external_id to contacts

Author: wmnnd

assets/static/downloads/keila_import_template.csv

@@ -1,3 +1,3 @@
-Email,First name,Last name,Data
-jane@example.com,Jane,Doe,"{""tags"":[""rocket-scientist""]}"
-john@example.com,John,Smith,"{""tags"":[""book-lover""]}"
+Email,First name,Last name,Data,External ID
+jane@example.com,Jane,Doe,"{""marketing_consent"": true, ""age"": 34, ""birthplace"": ""Paris"", ""tags"":[""rocket-scientist""]}",
+john@example.com,John,Smith,"{""birthplace"": ""Berlin"", ""tags"":[""book-enthusiast""]}",

assets/static/downloads/keila_import_template.ods

@@ -64,10 +64,7 @@ defmodule Keila.Contacts do
   """
   @spec get_project_contact(Project.id(), Contact.id()) :: Contact.t() | nil
   def get_project_contact(project_id, contact_id) do
-    case get_contact(contact_id) do
-      contact = %{project_id: ^project_id} -> contact
-      _other -> nil
-    end
+    Repo.get_by(Contact, project_id: project_id, id: contact_id)
   end
 
   @doc """
@@ -79,6 +76,16 @@ defmodule Keila.Contacts do
     Repo.get_by(Contact, project_id: project_id, email: email)
   end
 
+  @doc """
+  Gets specified Contact within Project context. Returns `nil` if Contact couldn‘t be found
+  or belongs to a different Project.
+  """
+  @spec get_project_contact_by_external_id(Project.id(), external_id :: String.t()) ::
+          Contact.t() | nil
+  def get_project_contact_by_external_id(project_id, external_id) do
+    Repo.get_by(Contact, project_id: project_id, external_id: external_id)
+  end
+
   @doc """
   Returns Contacts for specified Project.
 

lib/keila/contacts/contacts.ex

@@ -59,15 +59,13 @@ defmodule Keila.Contacts.Import do
     lines = read_file_line_count!(filename)
     send(notify_pid, {:contacts_import_progress, 0, lines})
 
-    insert_opts = insert_opts(on_conflict)
-
     File.stream!(filename)
     |> parser.parse_stream()
     |> Stream.map(row_function)
     |> Stream.reject(&is_nil/1)
     |> Stream.with_index()
     |> Stream.map(fn {changeset, n} ->
-      case Repo.insert(changeset, insert_opts) do
+      case insert(changeset, n, project_id, on_conflict) do
         {:ok, %{id: id}} ->
           Keila.Tracking.log_event("import", id, %{})
           n
@@ -107,6 +105,7 @@ defmodule Keila.Contacts.Import do
     columns =
       [
         email: find_header_column(headers, ~r{email.?(address)?}i),
+        external_id: find_header_column(headers, ~r{external.?id}i),
         first_name: find_header_column(headers, ~r{first.?name}i),
         last_name: find_header_column(headers, ~r{last.?name}i),
         data: find_header_column(headers, ~r{data}i),
@@ -162,20 +161,65 @@ defmodule Keila.Contacts.Import do
     |> then(fn lines -> max(lines - 1, 0) end)
   end
 
-  defp insert_opts(on_conflict) do
-    conflict_opts =
-      case on_conflict do
-        :replace -> [on_conflict: {:replace_all_except, [:id, :email, :project_id]}]
-        :ignore -> [on_conflict: :nothing]
-      end
+  defp insert(changeset, _n, _project_id, :ignore) do
+    Repo.insert(changeset, on_conflict: :nothing)
+  end
+
+  defp insert(changeset, n, project_id, :replace) do
+    external_id = get_change(changeset, :external_id)
 
-    [returning: false, conflict_target: [:email, :project_id]] ++ conflict_opts
+    if not is_nil(external_id) do
+      maybe_pre_set_external_id(changeset, project_id, external_id)
+    end
+
+    insert_opts = replace_insert_opts(changeset, external_id)
+    Repo.insert(changeset, insert_opts)
+  rescue
+    e in Postgrex.Error ->
+      raise_import_error!(changeset, e, n + 1)
   end
 
-  defp raise_import_error!(changeset, line) do
+  @replace_fields [:email, :external_id, :first_name, :last_name, :data, :updated_at, :status]
+  defp replace_insert_opts(changeset, external_id) do
+    external_id? = not is_nil(external_id)
+
+    replace_fields =
+      @replace_fields
+      |> Enum.filter(&(not is_nil(get_change(changeset, &1))))
+
+    conflict_target =
+      if external_id?, do: [:external_id, :project_id], else: [:email, :project_id]
+
+    [
+      conflict_target: conflict_target,
+      on_conflict: {:replace, replace_fields},
+      returning: false
+    ]
+  end
+
+  # This is necessary because Postgres doesn't allow using both email and
+  # external_id as conflict targets. Because of this and to allow updating
+  # existing Contacts that don't have an external ID yet, this function
+  # sets the external ID for such contacts before they are updated.
+  defp maybe_pre_set_external_id(changeset, project_id, external_id) do
+    email = get_change(changeset, :email)
+
+    if not is_nil(email) do
+      from(c in Contact,
+        where: c.project_id == ^project_id and c.email == ^email and is_nil(c.external_id),
+        update: [set: [external_id: ^external_id]]
+      )
+      |> Repo.update_all([])
+    end
+  end
+
+  defp raise_import_error!(changeset, exception \\ nil, line) do
     message =
-      case changeset.errors do
-        [{field, {message, _}} | _] ->
+      case {changeset, exception} do
+        {_, %Postgrex.Error{postgres: %{code: :unique_violation}}} ->
+          gettext("duplicate entry")
+
+        {%{errors: [{field, {message, _}} | _]}, _} ->
           gettext("Field %{field}: %{message}", field: field, message: message)
 
         _other ->

lib/keila/contacts/import.ex

@@ -11,6 +11,7 @@ defmodule Keila.Contacts.Contact do
 
   schema "contacts" do
     field(:email, :string)
+    field(:external_id, :string)
     field(:first_name, :string)
     field(:last_name, :string)
     field(:status, Ecto.Enum, values: @statuses, default: :active)
@@ -24,10 +25,11 @@ defmodule Keila.Contacts.Contact do
           Ecto.Changeset.t(t())
   def creation_changeset(struct \\ %__MODULE__{}, params, project_id) do
     struct
-    |> cast(params, [:email, :first_name, :last_name, :project_id, :data])
+    |> cast(params, [:email, :external_id, :first_name, :last_name, :project_id, :data])
     |> put_change(:project_id, project_id)
     |> validate_email()
     |> validate_max_name_length()
+    |> validate_external_id()
     |> check_data_size_constraint()
   end
 
@@ -41,9 +43,10 @@ defmodule Keila.Contacts.Contact do
   @spec update_changeset(t(), Ecto.Changeset.data()) :: Ecto.Changeset.t(t())
   def update_changeset(struct \\ %__MODULE__{}, params) do
     struct
-    |> cast(params, [:email, :first_name, :last_name, :data])
+    |> cast(params, [:email, :external_id, :first_name, :last_name, :data])
     |> validate_email()
     |> validate_max_name_length()
+    |> validate_external_id()
     |> check_data_size_constraint()
     |> maybe_remove_double_opt_in_at()
   end
@@ -139,4 +142,10 @@ defmodule Keila.Contacts.Contact do
     |> validate_length(:first_name, max: 50)
     |> validate_length(:last_name, max: 50)
   end
+
+  defp validate_external_id(changeset) do
+    changeset
+    |> validate_length(:external_id, max: 40)
+    |> unique_constraint([:external_id, :project_id])
+  end
 end

lib/keila/contacts/schemas/contact.ex

@@ -17,7 +17,7 @@ defmodule KeilaWeb.ContactsCsvExport do
       |> send_chunked(200)
 
     header =
-      [["Email", "First name", "Last name", "Data", "Status"]]
+      [["Email", "First name", "Last name", "Data", "Status", "External ID"]]
       |> NimbleCSV.RFC4180.dump_to_iodata()
       |> IO.iodata_to_binary()
 
@@ -28,7 +28,16 @@ defmodule KeilaWeb.ContactsCsvExport do
       |> Stream.map(fn contact ->
         data = if is_nil(contact.data), do: nil, else: Jason.encode!(contact.data)
 
-        [[contact.email, contact.first_name, contact.last_name, data, contact.status]]
+        [
+          [
+            contact.email,
+            contact.first_name,
+            contact.last_name,
+            data,
+            contact.status,
+            contact.external_id
+          ]
+        ]
         |> NimbleCSV.RFC4180.dump_to_iodata()
         |> IO.iodata_to_binary()
       end)

lib/keila_web/helpers/contacts_csv_export.ex

@@ -98,6 +98,21 @@
           <% end %>
         <% end %>
       </div>
+      <div class="form-row">
+        <%= label(f, :email, gettext("External ID")) %>
+        <span class="block text-sm mb-2">
+          <%= gettext(
+            "The external ID allows updating the contact profile from an external source even if its email address changes."
+          ) %>
+        </span>
+        <%= with_validation(f, :external_id) do %>
+          <%= text_input(f, :external_id,
+            placeholder: gettext("920207"),
+            class: "text-black",
+            autofocus: true
+          ) %>
+        <% end %>
+      </div>
     </.form>
   </div>
 

lib/keila_web/templates/contact/edit.html.heex

@@ -0,0 +1,11 @@
+defmodule Keila.Repo.Migrations.AddContactExternalId do
+  use Ecto.Migration
+
+  def change do
+    alter table("contacts") do
+      add :external_id, :string, size: 40
+    end
+
+    create unique_index("contacts", [:external_id, :project_id])
+  end
+end

priv/repo/migrations/20250223090326_add_contact_external_id.exs

@@ -112,12 +112,19 @@ defmodule Keila.ContactsTest do
   end
 
   @tag :contacts
-  test "Get project contact by ID and email", %{project: project} do
+  test "Get project contact by ID, email, and external ID", %{project: project} do
     contact1 = insert!(:contact, %{project_id: project.id})
     contact2 = insert!(:contact, %{project_id: project.id})
+    contact3 = insert!(:contact, %{project_id: project.id, external_id: "ext"})
 
     assert contact1 == Contacts.get_project_contact(contact1.project_id, contact1.id)
     assert contact2 == Contacts.get_project_contact_by_email(contact2.project_id, contact2.email)
+
+    assert contact3 ==
+             Contacts.get_project_contact_by_external_id(
+               contact3.project_id,
+               contact3.external_id
+             )
   end
 
   @tag :contacts
@@ -274,6 +281,75 @@ defmodule Keila.ContactsTest do
     refute Repo.get_by(Contacts.Contact, email: "empty@example.com")
   end
 
+  @tag :contacts
+  test "Import RFC 4180 CSV with external IDs and on_conflict: :ignore", %{project: project} do
+    assert :ok ==
+             Contacts.import_csv(
+               project.id,
+               "test/keila/contacts/import_external_ids.csv",
+               on_conflict: :ignore
+             )
+
+    contacts = Contacts.get_project_contacts(project.id)
+
+    expected = [
+      %{email: "foo@example.com", external_id: nil},
+      %{email: "foo2@example.com", external_id: "1"},
+      %{email: "foo3@example.com", external_id: "3"}
+    ]
+
+    for %{email: email, external_id: external_id} <- expected do
+      assert Enum.find(contacts, fn
+               %{email: ^email, external_id: ^external_id} -> true
+               _ -> false
+             end)
+    end
+
+    assert length(contacts) == length(expected)
+  end
+
+  @tag :contacts
+  test "Import CSV with external IDs and on_conflict: :replace", %{project: project} do
+    assert :ok ==
+             Contacts.import_csv(
+               project.id,
+               "test/keila/contacts/import_external_ids.csv",
+               on_conflict: :replace
+             )
+
+    contacts = Contacts.get_project_contacts(project.id)
+
+    expected = [
+      %{email: "foo2@example.com", external_id: "1"},
+      %{email: "foo3@example.com", external_id: "3"}
+    ]
+
+    for %{email: email, external_id: external_id} <- expected do
+      assert Enum.find(contacts, fn
+               %{email: ^email, external_id: ^external_id} -> true
+               _ -> false
+             end)
+    end
+
+    assert length(contacts) == length(expected)
+  end
+
+  @tag :contacts
+  test "Gracefully handle duplicates with external IDs and on_conflict: :replace", %{
+    project: project
+  } do
+    assert {:error, message} =
+             Contacts.import_csv(
+               project.id,
+               "test/keila/contacts/import_external_ids_duplicate.csv",
+               on_conflict: :replace
+             )
+
+    assert message =~ "duplicate entry"
+
+    assert Enum.empty?(Contacts.get_project_contacts(project.id))
+  end
+
   @tag :contacts
   test "Import Excel TSV/CSV", %{project: project} do
     assert :ok == Contacts.import_csv(project.id, "test/keila/contacts/import_excel.csv")

test/keila/contacts/contacts_test.exs

@@ -0,0 +1,5 @@
+Email,External ID
+foo@example.com,
+foo@example.com,1
+foo2@example.com,1
+foo3@example.com,3

test/keila/contacts/import_external_ids.csv

@@ -0,0 +1,5 @@
+Email,External ID
+foo@example.com,
+foo@example.com,1
+foo2@example.com
+foo2@example.com,1

test/keila/contacts/import_external_ids_duplicate.csv

@@ -103,7 +103,8 @@ defmodule Keila.Factory do
     %Keila.Contacts.Contact{
       email: "contact-#{get_counter_value()}@example.org",
       first_name: "First-#{get_counter_value()}",
-      last_name: "Last-#{get_counter_value()}"
+      last_name: "Last-#{get_counter_value()}",
+      external_id: Ecto.UUID.generate()
     }
   end