add gradient descent calculation

Author: kortirso

lib/learn_kit/math.ex

@@ -130,6 +130,38 @@ defmodule LearnKit.Math do
     [firsts | swap_rows_cols(others)]
   end
 
+  @doc """
+  Scalar multiplication
+
+  ## Examples
+
+      iex> LearnKit.Math.scalar_multiply(10, [5, 6])
+      [50, 60]
+
+  """
+  @spec scalar_multiply(integer, list) :: list
+
+  def scalar_multiply(multiplicator, list) when is_list(list) do
+    list
+    |> Enum.map(fn x -> x * multiplicator end)
+  end
+
+  @doc """
+  Vector subtraction
+
+  ## Examples
+
+      iex> LearnKit.Math.vector_subtraction([40, 50, 60], [35, 5, 40])
+      [5, 45, 20]
+
+  """
+  @spec vector_subtraction(list, list) :: list
+
+  def vector_subtraction(x, y) when is_list(x) and is_list(y) and length(x) == length(y) do
+    Enum.zip(x, y)
+    |> Enum.map(fn {xi, yi} -> xi - yi end)
+  end
+
   @doc """
   Division for 2 elements
 

lib/learn_kit/regression/linear.ex

@@ -7,8 +7,7 @@ defmodule LearnKit.Regression.Linear do
 
   alias LearnKit.Regression.Linear
 
-  use Linear.Fit
-  use Linear.Predict
+  use Linear.Calculations
 
   @type factors :: [number]
   @type results :: [number]
@@ -72,7 +71,7 @@ defmodule LearnKit.Regression.Linear do
 
       iex> predictor = predictor |> LearnKit.Regression.Linear.fit([method: "gradient descent"])
       %LearnKit.Regression.Linear{
-        coefficients: [-1.5, 4.0],
+        coefficients: [-1.4975720508482548, 3.9992148848913356],
         factors: [1, 2, 3, 4],
         results: [3, 6, 10, 15]
       }

lib/learn_kit/regression/linear/calculations.ex

@@ -0,0 +1,93 @@
+defmodule LearnKit.Regression.Linear.Calculations do
+  @moduledoc """
+  Module for fit functions
+  """
+
+  alias LearnKit.Math
+
+  defmacro __using__(_opts) do
+    quote do
+      defp fit_data(method, factors, results) when method == "gradient descent" do
+        gradient_descent_iteration([:rand.uniform, :rand.uniform], 0.0001, nil, 1000000, Enum.zip(factors, results), 0)
+      end
+
+      defp fit_data(_, factors, results) do
+        beta = Math.correlation(factors, results) * Math.standard_deviation(results) / Math.standard_deviation(factors)
+        alpha = Math.mean(results) - beta * Math.mean(factors)
+        [alpha, beta]
+      end
+
+      defp predict_sample(sample, [alpha, beta]) do
+        sample * beta + alpha
+      end
+
+      defp calculate_score([], _, _), do: raise("There was no fit for model")
+
+      defp calculate_score(coefficients, factors, results) do
+        1.0 - sum_of_squared_errors(coefficients, factors, results) / total_sum_of_squares(results)
+      end
+
+      defp total_sum_of_squares(list) do
+        mean_list = Math.mean(list)
+        list
+        |> Enum.map(fn x -> :math.pow(x - mean_list, 2) end)
+        |> Enum.sum
+      end
+
+      defp sum_of_squared_errors(coefficients, factors, results) do
+        Enum.zip(factors, results)
+        |> Enum.map(fn {xi, yi} -> squared_prediction_error(coefficients, xi, yi) end)
+        |> Enum.sum
+      end
+
+      defp squared_prediction_error(coefficients, x, y) do
+        coefficients
+        |> prediction_error(x, y)
+        |> :math.pow(2)
+      end
+
+      defp squared_error_gradient(coefficients, x, y) do
+        error_variable = coefficients |> prediction_error(x, y)
+        [
+          -2 * error_variable,
+          -2 * error_variable * x
+        ]
+      end
+
+      defp prediction_error(coefficients, x, y) do
+        y - predict_sample(x, coefficients)
+      end
+
+      defp gradient_descent_iteration(_, _, min_theta, _, _, iterations_with_no_improvement) when iterations_with_no_improvement >= 100, do: min_theta
+
+      defp gradient_descent_iteration(theta, alpha, min_theta, min_value, data, iterations_with_no_improvement) do
+        [
+          min_theta,
+          min_value,
+          iterations_with_no_improvement,
+          alpha
+        ] = data |> check_value(min_value, theta, min_theta, iterations_with_no_improvement, alpha)
+
+        theta = data
+                |> Enum.shuffle
+                |> Enum.reduce(theta, fn {xi, yi}, acc ->
+                  gradient_i = squared_error_gradient(acc, xi, yi)
+                  acc |> Math.vector_subtraction(alpha |> Math.scalar_multiply(gradient_i))
+                end)
+        gradient_descent_iteration(theta, alpha, min_theta, min_value, data, iterations_with_no_improvement)
+      end
+
+      defp check_value(data, min_value, theta, min_theta, iterations_with_no_improvement, alpha) do
+        value = data
+                |> Enum.map(fn {xi, yi} -> squared_prediction_error(theta, xi, yi) end)
+                |> Enum.sum
+        cond do
+          value < min_value ->
+            [theta, value, 0, 0.0001]
+          true ->
+            [min_theta, min_value, iterations_with_no_improvement + 1, alpha * 0.9]
+        end
+      end
+    end
+  end
+end