v0.4.0 unsorted mode

Author: djthread

CHANGELOG.md

@@ -6,6 +6,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic
 Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.0] - 2023-02-27
+
+### Changed
+- `fields` list was optional, but the behavior when none declared has changed.
+  - Before, the `id_key` (default `:id`) was automatically used in the
+    background. This meant that presorted indexes (ascending and descending)
+    were auto-maintained and used as the default when calling `get_records` etc. 
+  - Now, no such presorted indexes will be maintained. When querying,
+    `:ets.tab_to_list/2` will be used, with no particular order being promised.
+
+## [0.3.3] - 2023-02-07
+
+### Changed
+- Minor dialyzer fix.
+
 ## [0.3.2] - 2023-02-03
 
 ### Added

lib/indexed.ex

@@ -8,7 +8,7 @@ defmodule Indexed do
   @moduledoc """
   Tools for creating an index.
   """
-  alias Indexed.View
+  alias Indexed.{Entity, View}
   alias __MODULE__
 
   # Baseline opts. Others such as :named_table may be added.
@@ -21,7 +21,7 @@ defmodule Indexed do
   * `:index_ref` - ETS table reference for all indexed data.
   """
   @type t :: %Indexed{
-          entities: %{atom => Indexed.Entity.t()},
+          entities: %{atom => Entity.t()},
           index_ref: table_ref
         }
 
@@ -83,7 +83,7 @@ defmodule Indexed do
   @doc "Get a record by id from the index."
   @spec get(t, atom, id, any) :: any
   def get(index, entity_name, id, default \\ nil) do
-    case :ets.lookup(Map.fetch!(index.entities, entity_name).ref, id) do
+    case :ets.lookup(ref(index, entity_name), id) do
       [{_, val}] -> val
       [] -> default
     end
@@ -170,17 +170,17 @@ defmodule Indexed do
   sub-section of the data should be queried. Default is `nil` - no prefilter.
   """
   @spec get_records(t, atom, prefilter, order_hint | nil) :: [record]
-  def get_records(index, entity_name, prefilter, order_hint \\ nil) do
-    default_order_hint = fn ->
-      path = [Access.key(:entities), entity_name, Access.key(:fields)]
-      index |> get_in(path) |> hd() |> elem(0)
+  def get_records(index, entity_name, prefilter \\ nil, order_hint \\ nil) do
+    if ord = order_hint || default_order_hint(index, entity_name) do
+      index
+      |> get_index(entity_name, prefilter, ord)
+      |> Enum.map(&get(index, entity_name, &1))
+    else
+      index
+      |> ref(entity_name)
+      |> :ets.tab2list()
+      |> Enum.map(&elem(&1, 1))
     end
-
-    order_hint = order_hint || default_order_hint.()
-
-    index
-    |> get_index(entity_name, prefilter, order_hint)
-    |> Enum.map(&get(index, entity_name, &1))
   end
 
   @doc "Cache key for a given entity, field and direction."
@@ -243,12 +243,17 @@ defmodule Indexed do
 
   @doc """
   Get the name of the first indexed field for an entity.
+  If none, return `nil`, meaning there are no ordering indexes.
   Good order_hint default.
   """
   @spec default_order_hint(t, atom) :: atom
   def default_order_hint(index, entity_name) do
-    k = &Access.key(&1)
-    index |> get_in([k.(:entities), entity_name, k.(:fields)]) |> hd() |> elem(0)
+    path = [Access.key(:entities), entity_name, Access.key(:fields)]
+
+    case get_in(index, path) do
+      [{name, _} | _] -> name
+      _ -> nil
+    end
   end
 
   @doc "Delete all ETS tables associated with the given index."

lib/indexed/actions/create_view.ex

@@ -4,7 +4,6 @@ defmodule Indexed.Actions.CreateView do
   """
   alias Indexed.Actions.Warm
   alias Indexed.View
-  require Logger
 
   @typep id :: any
 
@@ -29,8 +28,6 @@ defmodule Indexed.Actions.CreateView do
   """
   @spec run(Indexed.t(), atom, View.fingerprint(), keyword) :: {:ok, View.t()} | :error
   def run(index, entity_name, fingerprint, opts \\ []) do
-    Logger.debug("Creating #{entity_name} view: #{fingerprint}")
-
     entity = Map.fetch!(index.entities, entity_name)
     prefilter = opts[:prefilter]
 
@@ -113,8 +110,6 @@ defmodule Indexed.Actions.CreateView do
       map_key = Indexed.uniques_map_key(entity_name, fingerprint, field_name)
       list_key = Indexed.uniques_list_key(entity_name, fingerprint, field_name)
 
-      Logger.debug("  * Saving #{field_name} uniques with #{map_size(counts_map)} values.")
-
       :ets.insert(index.index_ref, {map_key, counts_map})
       :ets.insert(index.index_ref, {list_key, list})
     end
@@ -141,8 +136,6 @@ defmodule Indexed.Actions.CreateView do
       asc_key = Indexed.index_key(entity_name, fingerprint, {:asc, field_name})
       desc_key = Indexed.index_key(entity_name, fingerprint, {:desc, field_name})
 
-      Logger.debug("  * Saving #{field_name} index with #{length(sorted_ids)} ids.")
-
       :ets.insert(index.index_ref, {asc_key, sorted_ids})
       :ets.insert(index.index_ref, {desc_key, Enum.reverse(sorted_ids)})
     end

lib/indexed/actions/put.ex

@@ -3,19 +3,18 @@ defmodule Indexed.Actions.Put do
   import Indexed.Helpers, only: [add_to_lookup: 4, id: 1]
   alias Indexed.{Entity, UniquesBundle, View}
   alias __MODULE__
-  require Logger
 
   defstruct [:current_view, :entity_name, :index, :previous, :pubsub, :record]
 
   @typedoc """
-  * `:current_view` - View struct currently being updated.
-  * `:entity_name` - Entity name being operated on.
-  * `:index` - See `t:Indexed.t/0`.
-  * `:previous` - The previous version of the record. `nil` if none.
-  * `:pubsub` - If configured, a Phoenix.PubSub module to send view updates.
-  * `:record` - The new record being added in the put operation.
+  - `current_view` : View struct currently being updated.
+  - `entity_name` : Entity name being operated on.
+  - `index` : See `t:Indexed.t/0`.
+  - `previous` : The previous version of the record. `nil` if none.
+  - `pubsub` : If configured, a Phoenix.PubSub module to send view updates.
+  - `record` : The new record being added in the put operation.
   """
-  @type t :: %__MODULE__{
+  @type t :: %Put{
           current_view: View.t() | nil,
           entity_name: atom,
           index: Indexed.t(),
@@ -48,8 +47,6 @@ defmodule Indexed.Actions.Put do
   end
 
   defp do_run(%{entity_name: name, index: index} = put, id) do
-    Logger.debug("Putting into #{put.entity_name}: id #{id}")
-
     %{fields: fields, lookups: lookups, prefilters: prefilters, ref: ref} =
       Map.fetch!(index.entities, name)
 
@@ -63,8 +60,6 @@ defmodule Indexed.Actions.Put do
         update_all_uniques(put, pf_opts[:maintain_unique] || [], nil, false)
 
       {pf_key, pf_opts} ->
-        Logger.debug("--> Getting UB for #{name} prefilter nil, field: #{pf_key}")
-
         handle_prefilter_value = fn pnb, value, new_value? ->
           prefilter = {pf_key, value}
           update_index_for_fields(put, prefilter, fields, new_value?)
@@ -199,8 +194,6 @@ defmodule Indexed.Actions.Put do
     %{previous: previous, record: record} = put
 
     Enum.each(fields, fn {field_name, _} = field ->
-      Logger.debug("--> Updating index for PF #{inspect(prefilter)}, field: #{field_name}")
-
       this_under_pf = under_prefilter?(put, record, prefilter)
       prev_under_pf = previous && under_prefilter?(put, previous, prefilter)
       record_value = Map.get(record, field_name)

lib/indexed/actions/warm.ex

@@ -3,7 +3,6 @@ defmodule Indexed.Actions.Warm do
   import Indexed.Helpers, only: [add_to_lookup: 4, id: 2]
   alias Indexed.{Entity, UniquesBundle}
   alias __MODULE__
-  require Logger
 
   defstruct [:data_tuple, :entity_name, :id_key, :index_ref]
 
@@ -55,9 +54,10 @@ defmodule Indexed.Actions.Warm do
   as keys and keyword lists of options are values. Allowed options are as
   follows:
 
-  * `:fields` - List of field name atoms to index by. At least one required.
+  * `:fields` - List of field name atoms to index by.
     * If field is a DateTime, use sort: `{:my_field, sort: :datetime}`.
     * Ascending and descending will be indexed for each field.
+    * If none, sorted results will not be available.
   * `:id_key` - Primary key to use in indexes and for accessing the records of
     this entity.  See `t:Indexed.Entity.t/0`. Default: `:id`.
   * `:lookups` - See `t:Indexed.Entity.t/0`.
@@ -145,13 +145,11 @@ defmodule Indexed.Actions.Warm do
         index_ref: index.index_ref
       }
 
-      Logger.debug("Warming #{entity_name}...")
-
       # Create and insert the caches for this entity: for each prefilter
       # configured, build & store indexes for each indexed field.
-      # Internally, a `t:prefilter/0` refers to a `{:my_field, "possible
-      # value"}` tuple or `nil` which we implicitly include, where no
-      # prefilter is applied.
+      # Internally, a `t:prefilter/0` refers to a `{:my_field, "some value"}`
+      # tuple or `nil` which we implicitly include,
+      # where no prefilter is applied.
       for prefilter_config <- entity.prefilters,
           do: warm_prefilter(warm, prefilter_config, entity.fields)
 
@@ -190,11 +188,6 @@ defmodule Indexed.Actions.Warm do
       warm_sorted(warm, prefilter, field, data_tuple)
     end
 
-    Logger.debug("""
-      * Putting index (PF #{pf_key || "nil"}) for \
-    #{inspect(Enum.map(fields, &elem(&1, 0)))}\
-    """)
-
     if nil == pf_key do
       Enum.each(fields, &warm_sorted.(nil, &1, full_data))
 
@@ -216,8 +209,6 @@ defmodule Indexed.Actions.Warm do
       bundle = UniquesBundle.new(counts_map, Enum.sort(Enum.uniq(list)))
       UniquesBundle.put(bundle, index_ref, entity_name, nil, pf_key, new?: true)
 
-      Logger.debug("--> Putting UB (for #{pf_key}) with #{map_size(counts_map)} elements.")
-
       # For each value found for the prefilter, create a set of indexes.
       Enum.each(grouped, fn {pf_val, data} ->
         prefilter = {pf_key, pf_val}
@@ -276,21 +267,14 @@ defmodule Indexed.Actions.Warm do
       list = counts_map |> Map.keys() |> Enum.sort()
       bundle = UniquesBundle.new(counts_map, list)
 
-      Logger.debug("""
-      --> Putting UB (PF #{inspect(prefilter)}, #{field_name}) \
-      with #{map_size(counts_map)} elements."\
-      """)
-
       UniquesBundle.put(bundle, index_ref, entity_name, prefilter, field_name, new?: true)
     end)
   end
 
   @doc "Normalize fields."
-  @spec resolve_fields_opt([atom | Entity.field()], atom) :: [Entity.field()]
+  @spec resolve_fields_opt([atom | Entity.field()] | nil, atom) :: [Entity.field()]
   def resolve_fields_opt(fields, _entity_name) do
-    # match?([_ | _], fields) || raise "At least one field to index is required on #{entity_name}."
-
-    Enum.map(fields, fn
+    Enum.map(List.wrap(fields), fn
       {_name, _opts} = f -> f
       name when is_atom(name) -> {name, []}
     end)

lib/indexed/entity.ex

@@ -4,7 +4,9 @@ defmodule Indexed.Entity do
 
   @typedoc """
   - `fields` : List of `t:field/0`s for which pairs of lists should be
-    maintained with the ID sorted ascending and descending.
+    maintained with the ID sorted ascending and descending. If none, then no
+    such indexes will be maintained and queries will default to
+    `order_hint: nil`, meaning that result ordering is not needed.
   - `id_key` : Specifies how to find the id for a record.  It can be an atom
     field name to access, a function, or a tuple in the form `{module,
     function_name}`. In the latter two cases, the record will be passed in.
@@ -20,16 +22,16 @@ defmodule Indexed.Entity do
       unique values under the prefilter will be managed. These lists can be
       fetched via `Indexed.get_uniques_list/4` and
       `Indexed.get_uniques_map/4`.
-  - `ref` : ETS table reference where records of this entity type are
-    stored, keyed by id. This will be nil in the version compiled into a managed
-    module.
+  - `ref` : ETS table reference where records of
+    this entity type are stored, keyed by id.
+    (`nil` in the instances compiled into a managed module.)
   """
   @type t :: %__MODULE__{
-          fields: [field],
+          fields: [field] | nil,
           id_key: any,
           lookups: [field_name],
           prefilters: [prefilter_config],
-          ref: :ets.tid() | atom | nil
+          ref: Indexed.table_ref() | nil
         }
 
   @typedoc """

lib/indexed/managed.ex

@@ -146,7 +146,7 @@ defmodule Indexed.Managed do
   """
   @type t :: %Managed{
           children: children,
-          fields: [atom | Entity.field()],
+          fields: [atom | Entity.field()] | nil,
           id_key: id_key,
           lookups: [Entity.field_name()],
           query: (Ecto.Queryable.t() -> Ecto.Queryable.t()) | nil,

lib/indexed/managed/prepare.ex

@@ -22,7 +22,6 @@ defmodule Indexed.Managed.Prepare do
     manageds
     |> map_put.(:children, &do_rewrite_children/2)
     |> map_put.(:prefilters, &do_rewrite_prefilters/2)
-    |> map_put.(:fields, &do_rewrite_fields/2)
     |> map_put.(:tracked, &do_rewrite_tracked/2)
   end
 
@@ -84,16 +83,6 @@ defmodule Indexed.Managed.Prepare do
       else: finish.(required)
   end
 
-  # If :fields is empty, use the id key or the first field given by Ecto.
-  @spec do_rewrite_fields(Managed.t(), [Managed.t()]) :: [atom | Entity.field()]
-  defp do_rewrite_fields(%{fields: [], id_key: id_key}, _) when is_atom(id_key),
-    do: [id_key]
-
-  defp do_rewrite_fields(%{fields: [], module: mod}, _),
-    do: [hd(mod.__schema__(:fields))]
-
-  defp do_rewrite_fields(%{fields: fields}, _), do: fields
-
   # Return true for tracked if another entity has a :one association to us.
   @spec do_rewrite_tracked(Managed.t(), [Managed.t()]) :: boolean
   defp do_rewrite_tracked(%{name: name}, manageds) do

lib/indexed/uniques_bundle.ex

@@ -9,7 +9,6 @@ defmodule Indexed.UniquesBundle do
   of these values.
   """
   alias __MODULE__
-  require Logger
 
   @typedoc """
   A 3-element tuple defines unique values under a prefilter:
@@ -66,11 +65,6 @@ defmodule Indexed.UniquesBundle do
     map = Indexed.get_uniques_map(index, entity_name, prefilter, field_name)
     list = Indexed.get_uniques_list(index, entity_name, prefilter, field_name)
 
-    Logger.debug(fn ->
-      key = Indexed.uniques_map_key(entity_name, prefilter, field_name)
-      "UB: Getting #{key}: #{inspect(map)}"
-    end)
-
     {map, list, [], false}
   end
 
@@ -121,17 +115,13 @@ defmodule Indexed.UniquesBundle do
     field_pf? = is_tuple(prefilter)
 
     if not new? and field_pf? and Enum.empty?(list) do
-      Logger.debug("UB: Dropping #{counts_key}")
-
       # This prefilter has ran out of records -- delete the ETS table.
       # Note that for views (binary prefilter) and the global prefilter (nil)
       # we allow the uniques to remain in the empty state. They go when the
       # view or entire index, respectively, is destroyed.
       :ets.delete(index_ref, list_key)
       :ets.delete(index_ref, counts_key)
     else
-      Logger.debug("UB: Putting #{counts_key}: #{inspect(counts_map)}")
-
       if new? or list_updated?,
         do: :ets.insert(index_ref, {list_key, list}),
         else: list

mix.exs

@@ -2,7 +2,7 @@ defmodule Indexed.MixProject do
   use Mix.Project
 
   @source_url "https://github.com/djthread/indexed"
-  @version "0.3.3"
+  @version "0.3.4"
 
   def project do
     [