Merge pull request #2 from kortirso/feature/linear_regression_gradient_descent

Feature/linear regression gradient descent

Author: kortirso

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+### Modified
+- Linear Regression, fit with gradient descent
+
 ## [0.1.2] - 2018-11-22
 ### Added
 - CHANGELOG.md file

README.md

@@ -33,12 +33,18 @@ Initialize predictor with data:
   predictor = Linear.new([1, 2, 3, 4], [3, 6, 10, 15])
 ```
 
-Fit data set:
+Fit data set with least squares method:
 
 ```elixir
   predictor = predictor |> Linear.fit
 ```
 
+Fit data set with gradient descent method:
+
+```elixir
+  predictor = predictor |> Linear.fit([method: "gradient descent"])
+```
+
 Predict using the linear model:
 
 ```elixir

lib/learn_kit/knn.ex

@@ -97,9 +97,7 @@ defmodule LearnKit.Knn do
 
   def classify(%Knn{data_set: data_set}, options \\ []) do
     try do
-      unless Keyword.has_key?(options, :feature) do
-        raise "Feature option is required"
-      end
+      unless Keyword.has_key?(options, :feature), do: raise "Feature option is required"
       # modification of options
       options = Keyword.merge([k: 3, algorithm: "brute", weight: "uniform"], options)
       # prediction

lib/learn_kit/knn/classify.ex

@@ -2,6 +2,9 @@ defmodule LearnKit.Knn.Classify do
   @moduledoc """
   Module for knn classify functions
   """
+
+  alias LearnKit.Math
+
   defmacro __using__(_opts) do
     quote do
       defp prediction(data_set, options) do
@@ -34,7 +37,8 @@ defmodule LearnKit.Knn.Classify do
       defp calc_feature_weights(features, options) do
         features
         |> Enum.map(fn feature ->
-          Tuple.append(feature, calc_feature_weight(Keyword.get(options, :weight), elem(feature, 0)))
+          feature
+          |> Tuple.append(calc_feature_weight(Keyword.get(options, :weight), elem(feature, 0)))
         end)
       end
 
@@ -78,9 +82,7 @@ defmodule LearnKit.Knn.Classify do
         features
         |> Enum.reduce([], fn feature, acc ->
           distance = feature |> calc_distance_between_features(current_feature)
-          if distance == 0 do
-            raise "Feature exists in train data set with label #{key}"
-          end
+          if distance == 0, do: raise "Feature exists in train data set with label #{key}"
           acc = [{distance, key} | acc]
         end)
       end
@@ -92,19 +94,15 @@ defmodule LearnKit.Knn.Classify do
       defp calc_distance_between_points(acc, feature_from_data_set, feature, current_index, size) when current_index <= size do
         Enum.at(feature_from_data_set, current_index) - Enum.at(feature, current_index)
         |> :math.pow(2)
-        |> summ(acc)
+        |> Math.summ(acc)
         |> calc_distance_between_points(feature_from_data_set, feature, current_index + 1, size)
       end
 
-      defp calc_distance_between_points(acc, _feature_from_data_set, _feature, _current_index, _size) do
+      defp calc_distance_between_points(acc, _, _, _, _) do
         acc
         |> :math.sqrt
       end
 
-      defp summ(a, b) do
-        a + b
-      end
-
       defp calc_feature_weight(weight, distance) do
         case weight do
           "uniform" -> 1
@@ -117,7 +115,7 @@ defmodule LearnKit.Knn.Classify do
         acc
       end
 
-      defp accumulate_weight_of_labels([{_distance, key, weight} | tail], acc) do
+      defp accumulate_weight_of_labels([{_, key, weight} | tail], acc) do
         previous = if Keyword.has_key?(acc, key), do: Keyword.get(acc, key), else: 0
         acc = Keyword.put(acc, key, previous + weight)
         accumulate_weight_of_labels(tail, acc)

lib/learn_kit/math.ex

@@ -6,6 +6,21 @@ defmodule LearnKit.Math do
   @type row :: [number]
   @type matrix :: [row]
 
+  @doc """
+  Sum of 2 numbers
+
+  ## Examples
+
+      iex> LearnKit.Math.summ(1, 2)
+      3
+
+  """
+  @spec summ(number, number) :: number
+
+  def summ(a, b) do
+    a + b
+  end
+
   @doc """
   Calculate the mean from a list of numbers
 
@@ -125,11 +140,43 @@ defmodule LearnKit.Math do
   defp swap_rows_cols([head | _]) when head == [], do: []
 
   defp swap_rows_cols(rows) do
-    firsts = Enum.map(rows, fn(x) -> hd(x) end)
-    others = Enum.map(rows, fn(x) -> tl(x) end)
+    firsts = Enum.map(rows, fn x -> hd(x) end)
+    others = Enum.map(rows, fn x -> tl(x) end)
     [firsts | swap_rows_cols(others)]
   end
 
+  @doc """
+  Scalar multiplication
+
+  ## Examples
+
+      iex> LearnKit.Math.scalar_multiply(10, [5, 6])
+      [50, 60]
+
+  """
+  @spec scalar_multiply(integer, list) :: list
+
+  def scalar_multiply(multiplicator, list) when is_list(list) do
+    list
+    |> Enum.map(fn x -> x * multiplicator end)
+  end
+
+  @doc """
+  Vector subtraction
+
+  ## Examples
+
+      iex> LearnKit.Math.vector_subtraction([40, 50, 60], [35, 5, 40])
+      [5, 45, 20]
+
+  """
+  @spec vector_subtraction(list, list) :: list
+
+  def vector_subtraction(x, y) when is_list(x) and is_list(y) and length(x) == length(y) do
+    Enum.zip(x, y)
+    |> Enum.map(fn {xi, yi} -> xi - yi end)
+  end
+
   @doc """
   Division for 2 elements
 
@@ -162,8 +209,7 @@ defmodule LearnKit.Math do
     size = length(x)
 
     Enum.zip(x, y)
-    |> Enum.map(fn {xi, yi} -> (xi - mean_x) * (yi - mean_y) end)
-    |> Enum.sum
+    |> Enum.reduce(0, fn {xi, yi}, acc -> acc + (xi - mean_x) * (yi - mean_y) end)
     |> division(size - 1)
   end
 
@@ -182,15 +228,9 @@ defmodule LearnKit.Math do
     mean_x = mean(x)
     mean_y = mean(y)
 
-    divider = Enum.zip(x, y)
-            |> Enum.map(fn {xi, yi} -> (xi - mean_x) * (yi - mean_y) end)
-            |> Enum.sum
-    denom_x = x
-              |> Enum.map(fn xi -> :math.pow(xi - mean_x, 2) end)
-              |> Enum.sum
-    denom_y = y
-              |> Enum.map(fn yi -> :math.pow(yi - mean_y, 2) end)
-              |> Enum.sum
+    divider = Enum.zip(x, y) |> Enum.reduce(0, fn {xi, yi}, acc -> acc + (xi - mean_x) * (yi - mean_y) end)
+    denom_x = x |> Enum.reduce(0, fn xi, acc -> acc + :math.pow(xi - mean_x, 2) end)
+    denom_y = y |> Enum.reduce(0, fn yi, acc -> acc + :math.pow(yi - mean_y, 2) end)
 
     divider / :math.sqrt(denom_x * denom_y)
   end

lib/learn_kit/naive_bayes/gaussian/classify.ex

@@ -7,7 +7,7 @@ defmodule LearnKit.NaiveBayes.Gaussian.Classify do
       # classify data
       # returns data like [label1: 0.03592747361085857, label2: 0.00399309643713954]
       defp classify_data(fit_data, feature) do
-        labels_count = length(Keyword.keys(fit_data))
+        labels_count = fit_data |> Keyword.keys |> length
         fit_data
         |> Enum.map(fn {label, fit_results} ->
           {label, class_probability(labels_count, feature, fit_results)}
@@ -23,7 +23,7 @@ defmodule LearnKit.NaiveBayes.Gaussian.Classify do
       end
 
       # multiply together the feature probabilities for all of the features in a label for given values
-      defp feature_mult([], _fit_results, acc, _index), do: acc
+      defp feature_mult([], _, acc, _), do: acc
 
       defp feature_mult([head | tail], fit_results, acc, index) do
         acc = acc * feature_probability(index, head, fit_results)
@@ -38,7 +38,7 @@ defmodule LearnKit.NaiveBayes.Gaussian.Classify do
           if fit_result.mean == value, do: 1.0, else: 0.0
         else
         # calculate the gaussian probability
-          exp = - :math.pow((value - fit_result.mean), 2) / (2 * fit_result.variance)
+          exp = - :math.pow(value - fit_result.mean, 2) / (2 * fit_result.variance)
           :math.exp(exp) / :math.sqrt(2 * :math.pi * fit_result.variance)
         end
       end

lib/learn_kit/regression/linear.ex

@@ -7,8 +7,7 @@ defmodule LearnKit.Regression.Linear do
 
   alias LearnKit.Regression.Linear
 
-  use Linear.Fit
-  use Linear.Predict
+  use Linear.Calculations
 
   @type factors :: [number]
   @type results :: [number]
@@ -55,6 +54,11 @@ defmodule LearnKit.Regression.Linear do
   ## Parameters
 
     - predictor: %LearnKit.Regression.Linear{}
+    - options: keyword list with options
+
+  ## Options
+
+    - method: method for fit, "least squares"/"gradient descent", default is "least squares", optional
 
   ## Examples
 
@@ -65,11 +69,28 @@ defmodule LearnKit.Regression.Linear do
         results: [3, 6, 10, 15]
       }
 
+      iex> predictor = predictor |> LearnKit.Regression.Linear.fit([method: "gradient descent"])
+      %LearnKit.Regression.Linear{
+        coefficients: [-1.4975720508482548, 3.9992148848913356],
+        factors: [1, 2, 3, 4],
+        results: [3, 6, 10, 15]
+      }
+
   """
   @spec fit(%Linear{factors: factors, results: results}) :: %Linear{factors: factors, results: results, coefficients: coefficients}
 
-  def fit(%Linear{factors: factors, results: results}) do
-    %Linear{factors: factors, results: results, coefficients: fit_data(factors, results)}
+  def fit(%Linear{factors: factors, results: results}, options \\ []) do
+    coefficients = Keyword.merge([method: ""], options)
+                    |> define_method_for_fit
+                    |> fit_data(factors, results)
+    %Linear{factors: factors, results: results, coefficients: coefficients}
+  end
+
+  defp define_method_for_fit(options) do
+    case Keyword.get(options, :method) do
+      "gradient descent" -> "gradient descent"
+      _ -> ""
+    end
   end
 
   @doc """

lib/learn_kit/regression/linear/calculations.ex

@@ -0,0 +1,89 @@
+defmodule LearnKit.Regression.Linear.Calculations do
+  @moduledoc """
+  Module for fit functions
+  """
+
+  alias LearnKit.Math
+
+  defmacro __using__(_opts) do
+    quote do
+      defp fit_data(method, factors, results) when method == "gradient descent" do
+        gradient_descent_iteration([:rand.uniform, :rand.uniform], 0.0001, nil, 1000000, Enum.zip(factors, results), 0)
+      end
+
+      defp fit_data(_, factors, results) do
+        beta = Math.correlation(factors, results) * Math.standard_deviation(results) / Math.standard_deviation(factors)
+        alpha = Math.mean(results) - beta * Math.mean(factors)
+        [alpha, beta]
+      end
+
+      defp predict_sample(sample, [alpha, beta]) do
+        sample * beta + alpha
+      end
+
+      defp calculate_score([], _, _), do: raise("There was no fit for model")
+
+      defp calculate_score(coefficients, factors, results) do
+        1.0 - sum_of_squared_errors(coefficients, factors, results) / total_sum_of_squares(results)
+      end
+
+      defp total_sum_of_squares(list) do
+        mean_list = Math.mean(list)
+        list
+        |> Enum.reduce(0, fn x, acc -> acc + :math.pow(x - mean_list, 2) end)
+      end
+
+      defp sum_of_squared_errors(coefficients, factors, results) do
+        Enum.zip(factors, results)
+        |> Enum.reduce(0, fn {xi, yi}, acc -> acc + squared_prediction_error(coefficients, xi, yi) end)
+      end
+
+      defp squared_prediction_error(coefficients, x, y) do
+        coefficients
+        |> prediction_error(x, y)
+        |> :math.pow(2)
+      end
+
+      defp squared_error_gradient(coefficients, x, y) do
+        error_variable = coefficients |> prediction_error(x, y)
+        [
+          -2 * error_variable,
+          -2 * error_variable * x
+        ]
+      end
+
+      defp prediction_error(coefficients, x, y) do
+        y - predict_sample(x, coefficients)
+      end
+
+      defp gradient_descent_iteration(_, _, min_theta, _, _, iterations_with_no_improvement) when iterations_with_no_improvement >= 100, do: min_theta
+
+      defp gradient_descent_iteration(theta, alpha, min_theta, min_value, data, iterations_with_no_improvement) do
+        [
+          min_theta,
+          min_value,
+          iterations_with_no_improvement,
+          alpha
+        ] = data |> check_value(min_value, theta, min_theta, iterations_with_no_improvement, alpha)
+
+        theta = data
+                |> Enum.shuffle
+                |> Enum.reduce(theta, fn {xi, yi}, acc ->
+                  gradient_i = squared_error_gradient(acc, xi, yi)
+                  acc |> Math.vector_subtraction(alpha |> Math.scalar_multiply(gradient_i))
+                end)
+        gradient_descent_iteration(theta, alpha, min_theta, min_value, data, iterations_with_no_improvement)
+      end
+
+      defp check_value(data, min_value, theta, min_theta, iterations_with_no_improvement, alpha) do
+        value = data |> Enum.reduce(0, fn {xi, yi}, acc -> acc + squared_prediction_error(theta, xi, yi) end)
+        cond do
+          value < min_value ->
+            [theta, value, 0, 0.0001]
+          true ->
+            [min_theta, min_value, iterations_with_no_improvement + 1, alpha * 0.9]
+        end
+      end
+    end
+  end
+end