normalization can be selected for KNN

Author: kortirso

CHANGELOG.md

@@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 - preprocessing normalization for data set
 
+### Modified
+- normalization can be selected for KNN
+
 ## [0.1.4] - 2018-12-17
 ### Modified
 - errors conditions and tests for KNN

lib/learn_kit/knn.ex

@@ -85,6 +85,7 @@ defmodule LearnKit.Knn do
     - k: number of nearest neighbours, default is 3, optional
     - algorithm: brute, optional
     - weight: uniform/distance, default is uniform, optional
+    - normalization: none/minimax/z_normalization, default is none, optional
 
   ## Examples
 
@@ -106,7 +107,7 @@ defmodule LearnKit.Knn do
         {:error, "K option must be positive integer"}
 
       true ->
-        options = Keyword.merge([k: 3, algorithm: "brute", weight: "uniform"], options)
+        options = Keyword.merge([k: 3, algorithm: "brute", weight: "uniform", normalization: "none"], options)
         {label, _} = prediction(data_set, options)
         {:ok, label}
     end

lib/learn_kit/knn/classify.ex

@@ -3,18 +3,39 @@ defmodule LearnKit.Knn.Classify do
   Module for knn classify functions
   """
 
-  alias LearnKit.Math
+  alias LearnKit.{Preprocessing, Math}
 
   defmacro __using__(_opts) do
     quote do
       defp prediction(data_set, options) do
         data_set
+        |> filter_features_by_size(Keyword.get(options, :feature))
+        |> check_normalization(options)
         |> calc_distances_for_features(options)
         |> sort_distances()
         |> select_closest_features(options)
         |> check_zero_distance(options)
       end
 
+      # knn uses only features with the same size as current feature
+      defp filter_features_by_size(data_set, current_feature) do
+        Enum.map(data_set, fn {key, features} ->
+          {
+            key,
+            Enum.filter(features, fn feature -> length(feature) == length(current_feature) end)
+          }
+        end)
+      end
+
+      # normalize features
+      defp check_normalization(data_set, options) do
+        type = Keyword.get(options, :normalization)
+        case type do
+          t when t in ["minimax", "z_normalization"] -> normalize(data_set, options, type)
+          _ -> data_set
+        end
+      end
+
       # select algorithm for prediction
       defp calc_distances_for_features(data_set, options) do
         case Keyword.get(options, :algorithm) do
@@ -23,14 +44,17 @@ defmodule LearnKit.Knn.Classify do
         end
       end
 
+      # sort distances
       defp sort_distances(features) do
         Enum.sort(features, &(elem(&1, 0) <= elem(&2, 0)))
       end
 
+      # take closest features
       defp select_closest_features(features, options) do
         Enum.take(features, Keyword.get(options, :k))
       end
 
+      # check existeness of current feature in data set
       defp check_zero_distance(closest_features, options) do
         {distance, label} = Enum.at(closest_features, 0)
         cond do
@@ -39,13 +63,33 @@ defmodule LearnKit.Knn.Classify do
         end
       end
 
+      # select best result based on weights
       defp select_best_label(features, options) do
         features
         |> calc_feature_weights(options)
         |> accumulate_weight_of_labels([])
         |> sort_result()
       end
 
+      # normalize each feature
+      defp normalize(data_set, options, type) do
+        coefficients = find_coefficients_for_normalization(data_set, type)
+        Enum.map(data_set, fn {key, features} ->
+          {
+            key,
+            Enum.map(features, fn feature -> Preprocessing.normalize_feature(feature, coefficients, type) end)
+          }
+        end)
+      end
+
+      # find coefficients for normalization
+      defp find_coefficients_for_normalization(data_set, type) do
+        Enum.reduce(data_set, [], fn {_, features}, acc ->
+          Enum.reduce(features, acc, fn feature, acc -> [feature | acc] end)
+        end)
+        |> Preprocessing.coefficients(type)
+      end
+
       defp calc_feature_weights(features, options) do
         Enum.map(features, fn feature ->
           Tuple.append(feature, calc_feature_weight(Keyword.get(options, :weight), elem(feature, 0)))
@@ -70,17 +114,10 @@ defmodule LearnKit.Knn.Classify do
         Enum.map(keys, fn key ->
           data_set
           |> Keyword.get(key)
-          |> filter_features_by_size(current_feature)
           |> calc_distances_in_label(current_feature, key)
         end)
       end
 
-      defp filter_features_by_size(features, current_feature) do
-        Enum.filter(features, fn feature ->
-          length(feature) == length(current_feature)
-        end)
-      end
-
       defp calc_distances_in_label(features, current_feature, key) do
         Enum.reduce(features, [], fn feature, acc ->
           distance = calc_distance_between_features(feature, current_feature)

lib/learn_kit/preprocessing.ex

@@ -3,7 +3,7 @@ defmodule LearnKit.Preprocessing do
   Module for data preprocessing
   """
 
-  alias LearnKit.Preprocessing
+  alias LearnKit.{Preprocessing, Math}
 
   use Preprocessing.Normalize
 
@@ -64,4 +64,57 @@ defmodule LearnKit.Preprocessing do
       _ -> normalization(features, "minimax")
     end
   end
+
+  @doc """
+  Prepare coefficients for normalization
+
+  ## Parameters
+
+    - features: features grouped by index
+    - type: minimax/z_normalization
+
+  ## Examples
+
+      iex> LearnKit.Preprocessing.coefficients([[1, 2], [3, 4], [5, 6]], "minimax")
+      [{1, 5}, {2, 6}]
+
+      iex> LearnKit.Preprocessing.coefficients([[1, 2], [3, 4], [5, 6]], "z_normalization")
+      [{3.0, 1.632993161855452}, {4.0, 1.632993161855452}]
+
+  """
+  @spec coefficients(matrix, String.t()) :: matrix
+
+  def coefficients(features, type) when is_list(features) and is_binary(type) do
+    features
+    |> Math.transpose()
+    |> Enum.map(fn list -> return_params(list, type) end)
+  end
+
+  @doc """
+  Normalize 1 feature with predefined coefficients
+
+  ## Parameters
+
+    - feature: feature for normalization
+    - coefficients: predefined coefficients
+    - type: minimax/z_normalization
+
+  ## Examples
+
+      iex> LearnKit.Preprocessing.normalize_feature([1, 2], [{1, 5}, {2, 6}], "minimax")
+      [0.0, 0.0]
+
+  """
+  @spec normalize_feature(list, list(tuple), String.t()) :: list
+
+  def normalize_feature(feature, coefficients, type) when is_list(feature) and is_list(coefficients) and is_binary(type) do
+    Enum.zip(feature, coefficients)
+    |> Enum.map(fn {point, params_for_point} ->
+      divider = define_divider(params_for_point, type)
+      case divider do
+        0 -> point
+        _ -> (point - elem(params_for_point, 0)) / divider
+      end
+    end)
+  end
 end

lib/learn_kit/preprocessing/normalize.ex

@@ -8,9 +8,9 @@ defmodule LearnKit.Preprocessing.Normalize do
   defmacro __using__(_opts) do
     quote do
       defp normalization(features, type) do
-        features_by_index = Math.transpose(features)
-        list_of_params = Enum.map(features_by_index, fn list -> return_params(list, type) end)
-        features_by_index
+        list_of_params = coefficients(features, type)
+        features
+        |> Math.transpose()
         |> Enum.with_index()
         |> Enum.map(fn {feature, index} -> transform_feature(feature, Enum.at(list_of_params, index), type) end)
         |> Math.transpose()

test/learn_kit/knn_test.exs

@@ -63,5 +63,13 @@ defmodule LearnKit.KnnTest do
     test "classify new feature, for existed point", state do
       assert {:ok, :b1} = Knn.classify(state[:classifier], [feature: [-2, -2], k: 3, weight: "uniform"])
     end
+
+    test "classify new feature, minimax normalization", state do
+      assert {:ok, :a1} = Knn.classify(state[:classifier], [feature: [-1, -2], k: 3, weight: "distance", normalization: "minimax"])
+    end
+
+    test "classify new feature, z normalization", state do
+      assert {:ok, :a1} = Knn.classify(state[:classifier], [feature: [-1, -2], k: 3, weight: "distance", normalization: "z_normalization"])
+    end
   end
 end

test/learn_kit/preprocessing_test.exs

@@ -18,6 +18,24 @@ defmodule LearnKit.PreprocessingTest do
   end
 
   describe "for valid data" do
+    test "prepare coefficients for normalization, minimax" do
+      result = Preprocessing.coefficients([[1, 2], [3, 4], [5, 6]], "minimax")
+
+      assert result == [{1, 5}, {2, 6}]
+    end
+
+    test "prepare coefficients for normalization, z_normalization" do
+      result = Preprocessing.coefficients([[1, 2], [3, 4], [5, 6]], "z_normalization")
+
+      assert result == [{3.0, 1.632993161855452}, {4.0, 1.632993161855452}]
+    end
+
+    test "normalize 1 feature with predefined coefficients" do
+      result = Preprocessing.normalize_feature([1, 2], [{1, 5}, {2, 6}], "minimax")
+
+      assert result == [0.0, 0.0]
+    end
+
     test "normalize data set with minimax normalization" do
       result = Preprocessing.normalize([[1, 2], [3, 4], [5, 6]])