Merge pull request #3 from davidrichey/polynomial_regression

Polynomial Regression Predictor

Author: kortirso

CHANGELOG.md

@@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## Unreleased
+### Added
+- Polynomial Regression predictor
+
 ### Modified
 - code refactoring
 
@@ -41,4 +44,4 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [0.1.0] - 2018-11-15
 ### Added
-- K-Nearest Neighbours algorithm
+- K-Nearest Neighbours algorithm

lib/learn_kit/regression/linear.ex

@@ -8,6 +8,7 @@ defmodule LearnKit.Regression.Linear do
   alias LearnKit.Regression.Linear
 
   use Linear.Calculations
+  use LearnKit.Regression.Score
 
   @type factors :: [number]
   @type results :: [number]
@@ -42,7 +43,8 @@ defmodule LearnKit.Regression.Linear do
   """
   @spec new(factors, results) :: %Linear{factors: factors, results: results, coefficients: []}
 
-  def new(factors, results) when is_list(factors) and is_list(results), do: %Linear{factors: factors, results: results}
+  def new(factors, results) when is_list(factors) and is_list(results),
+    do: %Linear{factors: factors, results: results}
 
   @doc """
   Fit train data
@@ -73,13 +75,19 @@ defmodule LearnKit.Regression.Linear do
       }
 
   """
-  @spec fit(%Linear{factors: factors, results: results}) :: %Linear{factors: factors, results: results, coefficients: coefficients}
-
-  def fit(%Linear{factors: factors, results: results}, options \\ []) when is_list(options) do
+  @spec fit(%Linear{factors: factors, results: results}) :: %Linear{
+          factors: factors,
+          results: results,
+          coefficients: coefficients
+        }
+
+  def fit(linear = %Linear{factors: factors, results: results}, options \\ [])
+      when is_list(options) do
     coefficients =
       Keyword.merge([method: ""], options)
       |> define_method_for_fit()
-      |> do_fit(factors, results)
+      |> do_fit(linear)
+
     %Linear{factors: factors, results: results, coefficients: coefficients}
   end
 
@@ -106,32 +114,30 @@ defmodule LearnKit.Regression.Linear do
   """
   @spec predict(%Linear{coefficients: coefficients}, list) :: {:ok, list}
 
-  def predict(%Linear{coefficients: coefficients}, samples) when is_list(samples) do
+  def predict(linear = %Linear{coefficients: _}, samples) when is_list(samples) do
     {
       :ok,
-      Enum.map(samples, fn sample -> predict_sample(sample, coefficients) end)
+      do_predict(linear, samples)
     }
   end
 
   @doc """
-  Returns the coefficient of determination R^2 of the prediction
+  Predict using the linear model
 
   ## Parameters
 
     - predictor: %LearnKit.Regression.Linear{}
+    - sample: Sample variable
 
   ## Examples
 
-      iex> predictor |> LearnKit.Regression.Linear.score
-      {:ok, 0.9876543209876543}
+      iex> predictor |> LearnKit.Regression.Linear.predict(4)
+      {:ok, 14.5}
 
   """
-  @spec score(%Linear{factors: factors, results: results, coefficients: coefficients}) :: {:ok, number}
+  @spec predict(%Linear{coefficients: coefficients}, list) :: {:ok, list}
 
-  def score(%Linear{factors: factors, results: results, coefficients: coefficients}) do
-    {
-      :ok,
-      calculate_score(coefficients, factors, results)
-    }
+  def predict(%Linear{coefficients: [alpha, beta]}, sample) do
+    {:ok, sample * beta + alpha}
   end
 end

lib/learn_kit/regression/linear/calculations.ex

@@ -4,60 +4,59 @@ defmodule LearnKit.Regression.Linear.Calculations do
   """
 
   alias LearnKit.Math
+  alias LearnKit.Regression.Linear
 
   defmacro __using__(_opts) do
     quote do
-      defp do_fit(method, factors, results) when method == "gradient descent" do
-        gradient_descent_iteration([:rand.uniform, :rand.uniform], 0.0001, nil, 1000000, Enum.zip(factors, results), 0)
+      defp do_fit(method, %Linear{factors: factors, results: results})
+           when method == "gradient descent" do
+        gradient_descent_iteration(
+          [:rand.uniform(), :rand.uniform()],
+          0.0001,
+          nil,
+          1_000_000,
+          Enum.zip(factors, results),
+          0
+        )
       end
 
-      defp do_fit(_, factors, results) do
-        beta = Math.correlation(factors, results) * Math.standard_deviation(results) / Math.standard_deviation(factors)
+      defp do_fit(_, %Linear{factors: factors, results: results}) do
+        beta =
+          Math.correlation(factors, results) * Math.standard_deviation(results) /
+            Math.standard_deviation(factors)
+
         alpha = Math.mean(results) - beta * Math.mean(factors)
         [alpha, beta]
       end
 
-      defp predict_sample(sample, [alpha, beta]) do
-        sample * beta + alpha
+      defp do_predict(linear, samples) do
+        Enum.map(samples, fn sample ->
+          {:ok, prediction} = predict(linear, sample)
+          prediction
+        end)
       end
 
-      defp calculate_score([], _, _), do: raise("There was no fit for model")
-
-      defp calculate_score(coefficients, factors, results) do
-        1.0 - sum_of_squared_errors(coefficients, factors, results) / total_sum_of_squares(results)
-      end
-
-      defp total_sum_of_squares(list) do
-        mean_list = Math.mean(list)
-        Enum.reduce(list, 0, fn x, acc -> acc + :math.pow(x - mean_list, 2) end)
-      end
-
-      defp sum_of_squared_errors(coefficients, factors, results) do
-        Enum.zip(factors, results)
-        |> Enum.reduce(0, fn {xi, yi}, acc -> acc + squared_prediction_error(coefficients, xi, yi) end)
-      end
-
-      defp squared_prediction_error(coefficients, x, y) do
-        coefficients
-        |> prediction_error(x, y)
-        |> :math.pow(2)
-      end
+      defp squared_error_gradient(linear, x, y) do
+        error_variable = prediction_error(linear, x, y)
 
-      defp squared_error_gradient(coefficients, x, y) do
-        error_variable = prediction_error(coefficients, x, y)
         [
           -2 * error_variable,
           -2 * error_variable * x
         ]
       end
 
-      defp prediction_error(coefficients, x, y) do
-        y - predict_sample(x, coefficients)
-      end
-
-      defp gradient_descent_iteration(_, _, min_theta, _, _, iterations_with_no_improvement) when iterations_with_no_improvement >= 100, do: min_theta
-
-      defp gradient_descent_iteration(theta, alpha, min_theta, min_value, data, iterations_with_no_improvement) do
+      defp gradient_descent_iteration(_, _, min_theta, _, _, iterations_with_no_improvement)
+           when iterations_with_no_improvement >= 100,
+           do: min_theta
+
+      defp gradient_descent_iteration(
+             theta,
+             alpha,
+             min_theta,
+             min_value,
+             data,
+             iterations_with_no_improvement
+           ) do
         [
           min_theta,
           min_value,
@@ -69,14 +68,26 @@ defmodule LearnKit.Regression.Linear.Calculations do
           data
           |> Enum.shuffle()
           |> Enum.reduce(theta, fn {xi, yi}, acc ->
-            gradient_i = squared_error_gradient(acc, xi, yi)
+            gradient_i = squared_error_gradient(%Linear{coefficients: theta}, xi, yi)
             acc |> Math.vector_subtraction(alpha |> Math.scalar_multiply(gradient_i))
           end)
-        gradient_descent_iteration(theta, alpha, min_theta, min_value, data, iterations_with_no_improvement)
+
+        gradient_descent_iteration(
+          theta,
+          alpha,
+          min_theta,
+          min_value,
+          data,
+          iterations_with_no_improvement
+        )
       end
 
       defp check_value(data, min_value, theta, min_theta, iterations_with_no_improvement, alpha) do
-        value = Enum.reduce(data, 0, fn {xi, yi}, acc -> acc + squared_prediction_error(theta, xi, yi) end)
+        value =
+          Enum.reduce(data, 0, fn {xi, yi}, acc ->
+            acc + squared_prediction_error(%Linear{coefficients: theta}, xi, yi)
+          end)
+
         cond do
           value < min_value ->
             [theta, value, 0, 0.0001]

lib/learn_kit/regression/polynomial.ex

@@ -0,0 +1,130 @@
+defmodule LearnKit.Regression.Polynomial do
+  @moduledoc """
+  Module for Polynomial Regression algorithm
+  """
+
+  defstruct factors: [], results: [], coefficients: [], degree: 2
+
+  alias LearnKit.Regression.Polynomial
+
+  use Polynomial.Calculations
+  use LearnKit.Regression.Score
+
+  @type factors :: [number]
+  @type results :: [number]
+  @type coefficients :: [number]
+  @type degree :: integer
+
+  @doc """
+  Creates polynomial predictor with data_set
+
+  ## Parameters
+
+    - factors: Array of predictor variables
+    - results: Array of criterion variables
+
+  ## Examples
+
+      iex> predictor = LearnKit.Regression.Polynomial.new([1, 2, 3, 4], [3, 6, 10, 15])
+      %LearnKit.Regression.Polynomial{factors: [1, 2, 3, 4], results: [3, 6, 10, 15], coefficients: [], degree: 2}
+
+  """
+  @spec new(factors, results) :: %Polynomial{
+          factors: factors,
+          results: results,
+          coefficients: [],
+          degree: 2
+        }
+  def new(factors, results) when is_list(factors) and is_list(results) do
+    %Polynomial{factors: factors, results: results}
+  end
+
+  def new(_, _), do: Polynomial.new([], [])
+  def new, do: Polynomial.new([], [])
+
+  @doc """
+  Fit train data
+
+  ## Parameters
+
+    - predictor: %LearnKit.Regression.Polynomial{}
+    - options: keyword list with options
+
+  ## Options
+
+    - degree: nth degree of polynomial model, default set to 2
+
+  ## Examples
+
+      iex> predictor = predictor |> LearnKit.Regression.Polynomial.fit
+      %LearnKit.Regression.Polynomial{
+        coefficients: [0.9999999999998295, 1.5000000000000853, 0.4999999999999787],
+        degree: 2,
+        factors: [1, 2, 3, 4],
+        results: [3, 6, 10, 15]
+      }
+
+      iex> predictor = predictor |> LearnKit.Regression.Polynomial.fit([degree: 3])
+      %LearnKit.Regression.Polynomial{
+        coefficients: [1.0000000000081855, 1.5000000000013642, 0.5,
+         8.526512829121202e-14],
+        degree: 3,
+        factors: [1, 2, 3, 4],
+        results: [3, 6, 10, 15]
+      }
+
+  """
+  @spec fit(%Polynomial{factors: factors, results: results}) :: %Polynomial{
+          factors: factors,
+          results: results,
+          coefficients: coefficients,
+          degree: degree
+        }
+  def fit(%Polynomial{factors: factors, results: results}, options \\ []) do
+    degree = options[:degree] || 2
+    matrix = matrix(factors, degree)
+    xys = x_y_matrix(factors, results, degree + 1, [])
+    coefficients = matrix |> Matrix.inv() |> Matrix.mult(xys) |> List.flatten()
+    %Polynomial{factors: factors, results: results, coefficients: coefficients, degree: degree}
+  end
+
+  @doc """
+  Predict using the polynomial model
+
+  ## Parameters
+
+    - predictor: %LearnKit.Regression.Polynomial{}
+    - samples: Array of variables
+
+  ## Examples
+
+      iex> predictor |> LearnKit.Regression.Polynomial.predict([5,6])
+      {:ok, [20.999999999999723, 27.999999999999574]}
+
+  """
+  @spec predict(%Polynomial{coefficients: coefficients, degree: degree}, list) :: {:ok, list}
+  def predict(polynomial = %Polynomial{coefficients: _, degree: _}, samples)
+      when is_list(samples) do
+    {:ok, do_predict(polynomial, samples)}
+  end
+
+  @doc """
+  Predict using the polynomial model
+
+  ## Parameters
+
+    - predictor: %LearnKit.Regression.Polynomial{}
+    - sample: Sample variable
+
+  ## Examples
+
+      iex> predictor |> LearnKit.Regression.Polynomial.predict(5)
+      {:ok, 20.999999999999723}
+
+  """
+  @spec predict(%Polynomial{coefficients: coefficients, degree: degree}, number) :: {:ok, number}
+  def predict(%Polynomial{coefficients: coefficients, degree: degree}, sample) do
+    ordered_coefficients = coefficients |> Enum.reverse()
+    {:ok, substitute_coefficients(ordered_coefficients, sample, degree, 0.0)}
+  end
+end