diff --git a/README.md b/README.md index 662b25d..30107f7 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,10 @@ # KVX -This is a simple/basic [Elixir](http://elixir-lang.org/) in-memory Key/Value Store -using [Shards](https://github.com/cabol/shards) – which is the default adapter. +This is a simple/basic in-memory Key/Value Store written in [**Elixir**](http://elixir-lang.org/) +and using [**Shards**](https://github.com/cabol/shards) as default adapter. + +Again, **KVX** is a simple library, most of the work is done by **Shards**, and +its typical use case might be as a **Cache**. ## Usage @@ -9,7 +12,7 @@ Add `kvx` to your Mix dependencies: ```elixir defp deps do - [{:kvx, "~> 0.1.0"}] + [{:kvx, "~> 0.1.1"}] end ``` @@ -21,6 +24,16 @@ defmodule MyTestMod do end ``` +## Getting Started! + +Let's try it out, compile your project and start an interactive console: + +``` +$ mix deps.get +$ mix compile +$ iex -S mix +``` + Now let's play with `kvx`: ```elixir @@ -76,6 +89,23 @@ config :kvx, shards_mod: :shards ``` +Besides, you can define bucket options in the config: + +```elixir +config :kvx, + adapter: KVX.Bucket.Shards, + ttl: 43200, + shards_mod: :shards, + buckets: [ + mybucket1: [ + n_shards: 4 + ], + mybucket2: [ + n_shards: 8 + ] + ] +``` + In case of Shards adapter, run-time options when calling `new/2` function, are the same as `shards:new/2`. E.g.: @@ -84,3 +114,163 @@ MyModule.new(:mybucket, [n_shards: 4]) ``` > **NOTE:** For more information check [KVX.Bucket.Shards](./lib/kvx/adapters/shards/bucket_shards.ex). + +## Running Tests + +``` +$ mix test +``` + +### Coverage + +``` +$ mix coveralls +``` + + > **NOTE:** For more coverage options check [**excoveralls**](https://github.com/parroty/excoveralls). + +## Example + +As we mentioned before, one of the most typical use case might be +use **KVX** as a **Cache**. Now, let's suppose you're working with +[**Ecto**](https://github.com/elixir-ecto/ecto), and you want to be +able to cache data when you call `Ecto.Repo.get/3`, and on other hand, +be able to handle eviction, remove/update cached data when they +change or mutate – typically when you call `Ecto.Repo.insert/2`, +`Ecto.Repo.update/2`, etc. + +To do so, let's implement our own `CacheableRepo` to encapsulate +data access and caching logic. First let's create our bucket and +the `Ecto.Repo` in two separated modules: + +```elixir +defmodule MyApp.Cache do + use KVX.Bucket +end + +defmodule MyApp.Repo do + use Ecto.Repo, otp_app: :myapp +end +``` + +Now, let's code our `CacheableRepo`, re-implementing some `Ecto.Repo` +functions but adding caching. It is as simple as this: + +```elixir +defmodule MyApp.CacheableRepo do + alias MyApp.Repo + alias MyApp.Cache + + require Logger + + def get(queryable, id, opts \\ []) do + get(&Repo.get/3, queryable, id, opts) + end + + def get!(queryable, id, opts \\ []) do + get(&Repo.get!/3, queryable, id, opts) + end + + def get_by(queryable, clauses, opts \\ []) do + get(&Repo.get_by/3, queryable, clauses, opts) + end + + def get_by!(queryable, clauses, opts \\ []) do + get(&Repo.get_by!/3, queryable, clauses, opts) + end + + defp get(fun, queryable, key, opts) do + b = bucket(queryable) + case Cache.get(b, key) do + nil -> + value = fun.(queryable, key, opts) + if value != nil do + Logger.debug "CACHING : #{inspect key} => #{inspect value}" + Cache.set(b, key, value) + end + value + value -> + Logger.debug "CACHED : #{inspect key} => #{inspect value}" + value + end + end + + def insert(struct, opts \\ []) do + case Repo.insert(struct, opts) do + {:ok, schema} = rs -> + schema + |> bucket + |> Cache.del(schema.id) + rs + error -> + error + end + end + + def insert!(struct, opts \\ []) do + rs = Repo.insert!(struct, opts) + rs + |> bucket + |> Cache.del(rs.id) + rs + end + + def update(struct, opts \\ []) do + case Repo.update(struct, opts) do + {:ok, schema} = rs -> + schema + |> bucket + |> Cache.set(schema.id, schema) + rs + error -> + error + end + end + + def update!(struct, opts \\ []) do + rs = Repo.update!(struct, opts) + rs + |> bucket + |> Cache.set(rs.id, rs) + rs + end + + def delete(struct, opts \\ []) do + case Repo.delete(struct, opts) do + {:ok, schema} = rs -> + schema + |> bucket + |> Cache.del(schema.id) + rs + error -> + error + end + end + + def delete!(struct, opts \\ []) do + rs = Repo.delete!(struct, opts) + rs + |> bucket + |> Cache.del(rs.id) + rs + end + + # function to resolve what bucket depending on the given schema + def bucket(MyApp.ModelA), do: :b1 + def bucket(%MyApp.ModelA{}), do: :b1 + def bucket(MyApp.ModelB), do: :b2 + def bucket(%MyApp.ModelB{}), do: :b2 + def bucket(_), do: :default +end +``` + +Now that we have our `CacheableRepo`, it can be used instead of `Ecto.Repo` +(since it is a wrapper on top of it, but it adds caching) for data you +consider can be cached, for example, you can use it from your +**Phoenix Controllers** – in case you're using [Phoenix](http://www.phoenixframework.org/). + +## Copyright and License + +Copyright (c) 2016 Carlos Andres Bolaños R.A. + +**KVX** source code is licensed under the [**MIT License**](LICENSE.md). diff --git a/config/test.exs b/config/test.exs index 972eb3b..5030506 100644 --- a/config/test.exs +++ b/config/test.exs @@ -3,4 +3,9 @@ use Mix.Config # KVX config config :kvx, adapter: KVX.Bucket.Shards, - ttl: 1 + ttl: 1, + buckets: [ + mybucket: [ + n_shards: 2 + ] + ] diff --git a/lib/kvx.ex b/lib/kvx.ex index 497d542..8280e23 100644 --- a/lib/kvx.ex +++ b/lib/kvx.ex @@ -1,2 +1,44 @@ defmodule KVX do + @moduledoc """ + This is a simple/basic in-memory Key/Value Store written in + [**Elixir**](http://elixir-lang.org/) and using + [**Shards**](https://github.com/cabol/shards) + as default adapter. + + Again, **KVX** is a simple library, most of the work + is done by **Shards**, and its typical use case might + be as a **Cache**. + + ## Adapters + + **KVX** was designed to be flexible and support multiple + backends. We currently ship with one backend: + + * `KVX.Bucket.Shards` - uses [Shards](https://github.com/cabol/shards), + to implement the `KVX.Bucket` interface. + + **KVX** adapters config might looks like: + + config :kvx, + adapter: KVX.Bucket.Shards, + ttl: 43200, + shards_mod: :shards, + buckets: [ + mybucket1: [ + n_shards: 4 + ], + mybucket2: [ + n_shards: 8 + ] + ] + + In case of Shards adapter, run-time options when calling `new/2` + function, are the same as `shards:new/2`. E.g.: + + MyModule.new(:mybucket, [n_shards: 4]) + + ## Example + + Check the example [**HERE**](https://github.com/cabol/kvx#example). + """ end diff --git a/lib/kvx/adapters/shards/bucket_shards.ex b/lib/kvx/adapters/shards/bucket_shards.ex index f8720b6..543634d 100644 --- a/lib/kvx/adapters/shards/bucket_shards.ex +++ b/lib/kvx/adapters/shards/bucket_shards.ex @@ -8,12 +8,29 @@ defmodule KVX.Bucket.Shards do * `:shards_mod` - internal Shards module to use. By default, `:shards` module is used, which is a wrapper on top of `:shards_local` and `:shards_dist`. + * `:buckets` - this can be used to set bucket options in config, + so it can be loaded when the bucket is created. See example below. Run-time options when calling `new/2` function, are the same as `shards:new/2`. For example: MyModule.new(:mybucket, [n_shards: 4]) + ## Example: + + config :kvx, + adapter: KVX.Bucket.Shards, + ttl: 43200, + shards_mod: :shards, + buckets: [ + mybucket1: [ + n_shards: 4 + ], + mybucket2: [ + n_shards: 8 + ] + ] + For more information about `shards`: * [GitHub](https://github.com/cabol/shards) @@ -29,7 +46,7 @@ defmodule KVX.Bucket.Shards do ## Setup Commands - def new(bucket, opts \\ []) do + def new(bucket, opts \\ []) when is_atom(bucket) do case Process.whereis(bucket) do nil -> new_bucket(bucket, opts) _ -> bucket @@ -37,10 +54,17 @@ defmodule KVX.Bucket.Shards do end defp new_bucket(bucket, opts) do - {^bucket, _} = @shards.new(bucket, opts) - bucket + opts = maybe_get_bucket_opts(bucket, opts) + @shards.new(bucket, opts) end + defp maybe_get_bucket_opts(bucket, []) do + :kvx + |> Application.get_env(:buckets, []) + |> Keyword.get(bucket, []) + end + defp maybe_get_bucket_opts(_, opts), do: opts + ## Storage Commands def add(bucket, key, value, ttl \\ @default_ttl) do @@ -116,11 +140,20 @@ defmodule KVX.Bucket.Shards do bucket end - def flush!(bucket) do + def delete(bucket) do + true = @shards.delete(bucket) + bucket + end + + def flush(bucket) do true = @shards.delete_all_objects(bucket) bucket end + ## Extended functions + + def __shards_mod__, do: @shards + ## Private functions defp seconds_since_epoch(diff) do diff --git a/lib/kvx/bucket.ex b/lib/kvx/bucket.ex index 474678e..b92a896 100644 --- a/lib/kvx/bucket.ex +++ b/lib/kvx/bucket.ex @@ -85,8 +85,12 @@ defmodule KVX.Bucket do @adapter.delete(bucket, key) end - def flush!(bucket) do - @adapter.flush!(bucket) + def delete(bucket) do + @adapter.delete(bucket) + end + + def flush(bucket) do + @adapter.flush(bucket) end end end @@ -107,7 +111,7 @@ defmodule KVX.Bucket do @doc """ Store this data, only if it does not already exist. If an item already - exists and an add fails with an exception. + exists and an add fails with a `KVX.ConflictError` exception. If `bucket` doesn't exist, it will raise an argument error. @@ -192,6 +196,17 @@ defmodule KVX.Bucket do """ defcallback delete(bucket, key) :: bucket + @doc """ + Deletes an entire bucket, if it exists. + + If `bucket` doesn't exist, it will raise an argument error. + + ## Example + + MyBucket.delete(:mybucket) + """ + defcallback delete(bucket) :: bucket + @doc """ Invalidate all existing cache items. @@ -199,7 +214,7 @@ defmodule KVX.Bucket do ## Example - MyBucket.flush!(:mybucket) + MyBucket.flush(:mybucket) """ - defcallback flush!(bucket) :: bucket + defcallback flush(bucket) :: bucket end diff --git a/mix.exs b/mix.exs index e85abe3..efcbb2c 100644 --- a/mix.exs +++ b/mix.exs @@ -3,7 +3,7 @@ defmodule KVX.Mixfile do def project do [app: :kvx, - version: "0.1.0", + version: "0.1.1", elixir: "~> 1.3", build_embedded: Mix.env == :prod, start_permanent: Mix.env == :prod, @@ -21,7 +21,7 @@ defmodule KVX.Mixfile do end defp deps do - [{:shards, "~> 0.3.0"}, + [{:shards, "~> 0.3.1"}, {:ex2ms, "~> 1.4.0"}, {:ex_doc, ">= 0.0.0", only: :dev}, {:excoveralls, "~> 0.5.6", only: :test}] diff --git a/mix.lock b/mix.lock index 3f4ad0b..fa4a9f4 100644 --- a/mix.lock +++ b/mix.lock @@ -9,5 +9,5 @@ "jsx": {:hex, :jsx, "2.6.2", "213721e058da0587a4bce3cc8a00ff6684ced229c8f9223245c6ff2c88fbaa5a", [:mix, :rebar], []}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []}, "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], []}, - "shards": {:hex, :shards, "0.3.0", "688131a0ab6f85dda0a517c7aef029b71ca9f5abe855523cf7267c4f91c4a39f", [:rebar3, :make], []}, + "shards": {:hex, :shards, "0.3.1", "e8a116641d517bcf57a1aeba900dff5dfd39bcad4fbe2f213f76c45792dde9a5", [:rebar3, :make], []}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.0", "edee20847c42e379bf91261db474ffbe373f8acb56e9079acb6038d4e0bf414f", [:rebar, :make], []}} diff --git a/test/bucket_shards_test.exs b/test/bucket_shards_test.exs index 0276745..8a94822 100644 --- a/test/bucket_shards_test.exs +++ b/test/bucket_shards_test.exs @@ -4,20 +4,30 @@ defmodule KVX.Bucket.ShardsTest do doctest KVX + @bucket __MODULE__ + require Ex2ms + setup do + @bucket = @bucket + |> new([n_shards: 4]) + |> flush + + on_exit fn -> + assert_raise ArgumentError, fn -> + @bucket + |> delete + |> find_all + end + end + :ok + end + test "default config" do assert KVX.Bucket.Shards === __adapter__ assert 1 === __ttl__ end - test "new bucket" do - rs = :set - |> new - |> new - assert :set === rs - end - test "invalid bucket error" do assert_raise ArgumentError, fn -> set(:invalid, :k1, 1) @@ -26,18 +36,22 @@ defmodule KVX.Bucket.ShardsTest do assert_raise ArgumentError, fn -> get(:invalid, :k1) end + + assert_raise ArgumentError, fn -> + delete(:invalid, :k1) + end end test "flush bucket" do - :temp + @bucket |> new - |> flush! + |> flush |> mset([k1: 1, k2: 2, k3: 3, k4: 4, k1: 1]) - assert [k1: 1, k2: 2, k3: 3, k4: 4] === find_all(:temp) |> Enum.sort + assert [k1: 1, k2: 2, k3: 3, k4: 4] === find_all(@bucket) |> Enum.sort - rs = :temp - |> flush! + rs = @bucket + |> flush |> find_all |> Enum.sort @@ -45,47 +59,62 @@ defmodule KVX.Bucket.ShardsTest do end test "storage and retrieval commands test" do - :set - |> new([n_shards: 4]) - |> flush! + @bucket |> mset([k1: 1, k2: 2, k3: 3, k4: 4, k1: 1]) - assert 1 === get(:set, :k1) - assert [1, 2] === mget(:set, [:k1, :k2]) - assert nil === get(:set, :k11) + assert 1 === get(@bucket, :k1) + assert [1, 2] === mget(@bucket, [:k1, :k2]) + assert nil === get(@bucket, :k11) assert_raise KVX.ConflictError, fn -> - add(:set, :k1, 11) + add(@bucket, :k1, 11) end - add(:set, :kx, 123) - assert 123 === get(:set, :kx) + add(@bucket, :kx, 123) + assert 123 === get(@bucket, :kx) - assert [k1: 1, k2: 2, k3: 3, k4: 4, kx: 123] === find_all(:set) |> Enum.sort + assert [k1: 1, k2: 2, k3: 3, k4: 4, kx: 123] === find_all(@bucket) |> Enum.sort ms1 = Ex2ms.fun do {_, v, _} = obj when rem(v, 2) == 0 -> obj end - assert [k2: 2, k4: 4] === find_all(:set, ms1) |> Enum.sort + assert [k2: 2, k4: 4] === find_all(@bucket, ms1) |> Enum.sort - nil = :set + nil = @bucket |> delete(:k1) |> get(:k1) end test "ttl test" do - :ttl_test - |> new - |> flush! + @bucket |> mset([k1: 1, k2: 2, k3: 3], 2) |> set(:k4, 4, 3) |> set(:k5, 5, :infinity) - assert [k1: 1, k2: 2, k3: 3, k4: 4, k5: 5] === find_all(:ttl_test) |> Enum.sort + assert [k1: 1, k2: 2, k3: 3, k4: 4, k5: 5] === find_all(@bucket) |> Enum.sort - :timer.sleep(2100) - assert [k4: 4, k5: 5] === find_all(:ttl_test) |> Enum.sort + :timer.sleep(2000) + assert [k4: 4, k5: 5] === find_all(@bucket) |> Enum.sort :timer.sleep(1000) - assert nil === get(:ttl_test, :k4) - assert [k5: 5] === find_all(:ttl_test) |> Enum.sort - assert 5 === get(:ttl_test, :k5) + assert nil === get(@bucket, :k4) + assert [k5: 5] === find_all(@bucket) |> Enum.sort + assert 5 === get(@bucket, :k5) + end + + test "cleanup commands test" do + @bucket + |> mset([k1: 1, k2: 2, k3: 3]) + + assert [k1: 1, k2: 2, k3: 3] === find_all(@bucket) |> Enum.sort + + nil = @bucket + |> delete(:k1) + |> get(:k1) + + assert [k2: 2, k3: 3] === find_all(@bucket) |> Enum.sort + end + + test "load bucket opts from config test" do + assert :mybucket === new(:mybucket) + assert 2 === :shards_state.n_shards(:mybucket) + assert :shards === KVX.Bucket.Shards.__shards_mod__ end end