Initial elasticsearch changes

- Removed duplicate entry
- Moved some code and a couple small refactors of the helper module
- Updated to not pass the document index when creating
- cleaning up some unneeded changes
- Updated search function
- Changed elixir version back to 1.3.4
- Updates per comments, added add_documents() function to helper
- added some convenience functions
- Removing unnecessary code, eliminated 1.4 warning messages so it's done when we upgrade
- Added tests for fuzzy search, updates some map formats
- Added match all function
- Added search endpoint for elasticsearch, updated to return maps
- Added comment about using -- with lists

Author: paulsullivanjr

config/dev.exs

@@ -65,6 +65,9 @@ config :code_corps,
   postmark_project_request_template: "123",
   postmark_receipt_template: "123"
 
+# Configure elasticsearch
+config :code_corps, :elasticsearch_url, "http://0.0.0.0:9200"
+
 # If the dev environment has no CLOUDEX_API_KEY set, we want the app
 # to still run, with cloudex in test API mode
 if System.get_env("CLOUDEX_API_KEY") == nil do

config/prod.exs

@@ -52,6 +52,9 @@ config :code_corps, :analytics, CodeCorps.Analytics.SegmentAPI
 config :code_corps, :stripe, Stripe
 config :code_corps, :stripe_env, :prod
 
+# Configure elasticsearch
+config :code_corps, :elasticsearch_url, "http://0.0.0.0:9200"
+
 config :sentry,
   environment_name: Mix.env || :prod
 

config/test.exs

@@ -38,6 +38,9 @@ config :code_corps, :stripe_env, :test
 
 config :code_corps, :icon_color_generator, CodeCorps.RandomIconColor.TestGenerator
 
+# Configure elasticsearch
+config :code_corps, :elasticsearch_url, "http://0.0.0.0:9200"
+
 # Set Corsica logging to output no console warning when rejecting a request
 config :code_corps, :corsica_log_level, [rejected: :debug]
 
@@ -54,6 +57,10 @@ config :code_corps,
   github_app_client_secret: System.get_env("GITHUB_TEST_APP_CLIENT_SECRET"),
   github_app_pem: pem
 
+# Configure elasticsearch
+config :code_corps, :elasticsearch_url, "http://0.0.0.0:9200"
+config :code_corps, :elasticsearch_index,  "skills"
+
 config :sentry,
   environment_name: Mix.env || :test
 

lib/code_corps_web/controllers/skill_controller.ex

@@ -33,6 +33,14 @@ defmodule CodeCorpsWeb.SkillController do
     end
   end
 
+  @elasticsearch_index "skills"
+  @elasticsearch_type "title"
+  @elasticsearch_url  Application.get_env(:code_corps, :elasticsearch_url)
+
+  def search(_conn, params) do
+    CodeCorps.ElasticSearchHelper.search(@elasticsearch_url, @elasticsearch_index, @elasticsearch_type, query)
+  end
+
   @spec load_skills(map) :: list(Skill.t)
   defp load_skills(%{} = params) do
     Skill

lib/code_corps_web/router.ex

@@ -82,6 +82,7 @@ defmodule CodeCorpsWeb.Router do
     resources "/role-skills", RoleSkillController, only: [:create, :delete]
     resources "/roles", RoleController, only: [:create]
     resources "/skills", SkillController, only: [:create]
+    resources "/skills/search", SkillController, only: [:show]
     resources "/stripe-connect-accounts", StripeConnectAccountController, only: [:show, :create, :update]
     resources "/stripe-connect-plans", StripeConnectPlanController, only: [:show, :create]
     resources "/stripe-connect-subscriptions", StripeConnectSubscriptionController, only: [:show, :create]

mix.exs

@@ -81,7 +81,8 @@ defmodule CodeCorps.Mixfile do
       {:sweet_xml, "~> 0.5"},
       {:timber, "~> 2.0"}, # Logging
       {:timex, "~> 3.0"},
-      {:timex_ecto, "~> 3.0"}
+      {:timex_ecto, "~> 3.0"},
+      {:elastix, git: "https://github.com/paulsullivanjr/elastix.git"} # for elastic search
     ]
   end
 

mix.lock

@@ -20,13 +20,14 @@
   "earmark": {:hex, :earmark, "1.2.3", "206eb2e2ac1a794aa5256f3982de7a76bf4579ff91cb28d0e17ea2c9491e46a4", [:mix], []},
   "ecto": {:hex, :ecto, "2.2.6", "3fd1067661d6d64851a0d4db9acd9e884c00d2d1aa41cc09da687226cf894661", [:mix], [{:db_connection, "~> 1.1", [hex: :db_connection, optional: true]}, {:decimal, "~> 1.2", [hex: :decimal, optional: false]}, {:mariaex, "~> 0.8.0", [hex: :mariaex, optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, optional: true]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: false]}, {:postgrex, "~> 0.13.0", [hex: :postgrex, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]},
   "ecto_ordered": {:hex, :ecto_ordered, "0.2.0-beta1", "cb066bc608f1c8913cea85af8293261720e6a88e3c99061e6877d7025352f045", [:mix], [{:ecto, "~> 2.0", [hex: :ecto, optional: false]}]},
-  "elixir_make": {:hex, :elixir_make, "0.4.0", "992f38fabe705bb45821a728f20914c554b276838433349d4f2341f7a687cddf", [:mix], []},
-  "ex_aws": {:hex, :ex_aws, "1.1.4", "4bdc4fff91f8d35c7fe2355b9da54cc51f980c92f1137715d8b2d70d8e8511cc", [:mix], [{:configparser_ex, "~> 0.2.1", [hex: :configparser_ex, optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, optional: true]}]},
-  "ex_doc": {:hex, :ex_doc, "0.18.1", "37c69d2ef62f24928c1f4fdc7c724ea04aecfdf500c4329185f8e3649c915baf", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, optional: false]}]},
-  "ex_machina": {:hex, :ex_machina, "2.1.0", "4874dc9c78e7cf2d429f24dc3c4005674d4e4da6a08be961ffccc08fb528e28b", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, optional: true]}]},
+  "elastix": {:git, "https://github.com/paulsullivanjr/elastix.git", "72441f08d59491ec1101b8bb9afe56463a5cbd75", []},
+  "elixir_make": {:hex, :elixir_make, "0.4.0", "992f38fabe705bb45821a728f20914c554b276838433349d4f2341f7a687cddf", [:mix], [], "hexpm"},
+  "ex_aws": {:hex, :ex_aws, "1.1.5", "789173f385934f7e27f9ef36692a6c5f7dde06fd6e6f64d4cd92cda613d34bf9", [:mix], [{:configparser_ex, "~> 0.2.1", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"},
+  "ex_doc": {:hex, :ex_doc, "0.18.1", "37c69d2ef62f24928c1f4fdc7c724ea04aecfdf500c4329185f8e3649c915baf", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, repo: "hexpm", optional: false]}], "hexpm"},
+  "ex_machina": {:hex, :ex_machina, "2.1.0", "4874dc9c78e7cf2d429f24dc3c4005674d4e4da6a08be961ffccc08fb528e28b", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
   "excoveralls": {:hex, :excoveralls, "0.7.5", "339e433e5d3bce09400dc8de7b9040741a409c93917849916c136a0f51fdc183", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: false]}, {:hackney, ">= 0.12.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
   "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm"},
-  "file_system": {:hex, :file_system, "0.2.2", "7f1e9de4746f4eb8a4ca8f2fbab582d84a4e40fa394cce7bfcb068b988625b06", [], [], "hexpm"},
+  "file_system": {:hex, :file_system, "0.2.2", "7f1e9de4746f4eb8a4ca8f2fbab582d84a4e40fa394cce7bfcb068b988625b06", [:mix], [], "hexpm"},
   "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], []},
   "gettext": {:hex, :gettext, "0.13.1", "5e0daf4e7636d771c4c71ad5f3f53ba09a9ae5c250e1ab9c42ba9edccc476263", [:mix], []},
   "guardian": {:hex, :guardian, "1.0.0", "21bae2a8c0b4ed5943d9da0c6aeb16e52874c1f675de5d7920ae35471c6263f9", [:mix], [{:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.0 or ~> 1.2 or ~> 1.3", [hex: :phoenix, repo: "hexpm", optional: true]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}, {:uuid, ">= 1.1.1", [hex: :uuid, repo: "hexpm", optional: false]}], "hexpm"},

test/integration/skill_controller_search_test.exs

@@ -0,0 +1,59 @@
+defmodule SkillControllerSearchIntegrationTest do
+  use ExUnit.Case, async: true
+  alias CodeCorps.ElasticSearchHelper
+
+  @test_url Application.get_env(:code_corps, :elasticsearch_url)
+  @test_index  "skills"
+  @type_value "title"
+
+  @elixir %{"id" => 1, "description" => "Elixir is an awesome functional language", "title" =>  "Elixir", "original_row" => 1}
+  @ruby %{"id" => 2, "description" => "Ruby is an awesome OO language", "title" => "Ruby", "original_row" => 2}
+  @rails %{"id" => 3, "description" => "Rails is a modern framework", "title" => "Rails", "original_row" => 3}
+  @css %{"id" => 4, "description" => "CSS is pretty cool too", "title" => "CSS", "original_row" => 4}
+  @phoenix %{"id" => 5, "description" => "Phoenix is a super framework", "title" => "Phoenix", "original_row" => 5}
+
+  setup do
+    ElasticSearchHelper.delete(@test_url, @test_index)
+    ElasticSearchHelper.create_index(@test_url, @test_index, @type_value)
+    init()
+    :ok
+  end
+
+  test "search partial word" do
+   results = ElasticSearchHelper.search(@test_url, @test_index, "title", "ru")
+   assert results == [@ruby]
+  end
+
+  test "fuzzy search partial word" do
+    results = ElasticSearchHelper.search(@test_url, @test_index, "title", "rj")
+    # Two lists can be concatenated or subtracted using the ++/2 and --/2
+    # see: http://elixir-lang.org/getting-started/basic-types.html#linked-lists
+    # This allows us to confirm the values we want regardless of the order the values are returned in.
+    assert results -- ["Ruby", "Rails"] == []
+  end
+
+  test "search whole word" do
+    results = ElasticSearchHelper.search(@test_url, @test_index, "title", "css")
+    assert results == [@css]
+  end
+
+  test "fuzzy search whole word" do
+    results = ElasticSearchHelper.search(@test_url, @test_index, "title", "csw")
+    assert results == [@css]
+  end
+
+  test "search no matches" do
+    results = ElasticSearchHelper.search(@test_url, @test_index, "title", "foo")
+    assert results == []
+  end
+
+  test "match all entries" do
+    results = ElasticSearchHelper.match_all(@test_url, @test_index, "title")
+    assert results -- [@elixir, @ruby, @rails, @css] == []
+  end
+
+  def init do
+    ElasticSearchHelper.add_documents(@test_url, @test_index, @type_value,
+      [@elixir, @css, @ruby], [refresh: true])
+  end
+end

web/helpers/elastic_search_helper.ex

@@ -0,0 +1,94 @@
+defmodule CodeCorps.ElasticSearchHelper do
+  alias Elastix.Search
+  alias Elastix.Index
+  alias Elastix.Document
+
+  def delete(url, index) do
+    Index.delete(url, index)
+  end
+
+  def create_index(url, index, type) do
+    Index.settings(url, index, settings_map())
+    Index.settings(url, "#{index}/_mapping/#{type}", field_filter(type))
+  end
+
+  def add_documents(url, index, type, documents) when is_list(documents) do
+    add_documents(url, index, type, documents, [])
+  end
+
+  def add_documents(url, index, type, documents, query) when is_list(documents) do
+    Enum.each(documents, fn(x) -> add_document(url, index, type, x, query) end)
+  end
+
+  def add_document(url, index, type, data) do
+    add_document(url, index, type, data, [])
+  end
+
+  def add_document(url, index, type, data, query) do
+    Document.index_new(url, index, type, data, query)
+  end
+
+  def search(url, index, type, search_query) do
+    data = %{
+      query: %{
+        match: %{"#{type}": search_query}
+      }
+    }
+    Search.search(url, index, [], data) |> process_response(type)
+  end
+
+  def match_all(url, index, type) do
+    data = %{
+      query: %{
+        match_all: %{}
+      }
+    }
+    Search.search(url, index, [], data) |> process_response(type)
+  end
+
+  def process_response(%HTTPoison.Response{status_code: 200} = response, type) do
+    response.body["hits"]["hits"] |> Enum.map(fn(x) -> x["_source"] end)
+  end
+
+  def process_response(_), do: []
+
+  defp settings_map do
+    %{
+        settings: %{
+          number_of_shards: 5,
+          analysis: %{
+            filter: %{
+              autocomplete_filter: %{
+                type:     "edge_ngram",
+                min_gram: 2,
+                max_gram: 20
+              }
+            },
+            analyzer: %{
+              autocomplete: %{
+                type:      "custom",
+                tokenizer: "standard",
+                filter: [
+                  "lowercase",
+                  "autocomplete_filter"
+                ]
+              }
+            }
+          }
+        }
+      }
+  end
+
+  def field_filter(type) do
+    %{
+      "#{type}" => %{
+        "properties" => %{
+          "#{type}" => %{
+            "type" =>     "string",
+            "analyzer" =>  "autocomplete"
+          }
+        }
+      }
+    }
+  end
+end