Merge pull request #1 from kortirso/feature/linear_regression

Feature/linear regression

Author: kortirso

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [Unreleased]
 ### Added
 - CHANGELOG.md file
+- Add simple Linear Regression predictor
 
 ## [0.1.1] - 2018-11-19
 ### Added

README.md

@@ -2,7 +2,11 @@
 
 Elixir package for machine learning
 
-Available algorithms:
+Available algorithms for prediction:
+
+- Linear Regression
+
+Available algorithms for classification:
 
 - K-Nearest Neighbours
 - Gaussian Naive Bayes
@@ -20,21 +24,50 @@ def deps do
 end
 ```
 
+### Linear Regression
+
+Initialize predictor with data:
+
+```elixir
+  alias LearnKit.Regression.Linear
+  predictor = Linear.new([1, 2, 3, 4], [3, 6, 10, 15])
+```
+
+Fit data set:
+
+```elixir
+  predictor = predictor |> Linear.fit
+```
+
+Predict using the linear model:
+
+```elixir
+  predictor |> Linear.predict([4, 8, 13])
+```
+    samples - array of variables, required
+
+Returns the coefficient of determination R^2 of the prediction:
+
+```elixir
+  predictor |> Linear.score
+```
+
 ### K-Nearest Neighbours classification
 
 Initialize classificator with data set consists from labels and features:
 
 ```elixir
-  classificator = LearnKit.Knn.new
-                  |> LearnKit.Knn.add_train_data({:a1, [-1, -1]})
-                  |> LearnKit.Knn.add_train_data({:a1, [-2, -1]})
-                  |> LearnKit.Knn.add_train_data({:a2, [1, 1]})
+  alias LearnKit.Knn
+  classificator = Knn.new
+                  |> Knn.add_train_data({:a1, [-1, -1]})
+                  |> Knn.add_train_data({:a1, [-2, -1]})
+                  |> Knn.add_train_data({:a2, [1, 1]})
 ```
 
 Predict label for new feature:
 
 ```elixir
-  LearnKit.Knn.classify(classificator, [feature: [-1, -2], k: 3, weight: "distance"])
+  Knn.classify(classificator, [feature: [-1, -2], k: 3, weight: "distance"])
 ```
     feature - new feature for prediction, required
     k - number of nearest neighbors, optional, default - 3
@@ -46,36 +79,37 @@ Predict label for new feature:
 Initialize classificator with data set consists from labels and features:
 
 ```elixir
-  classificator = LearnKit.NaiveBayes.Gaussian.new
-                  |> LearnKit.NaiveBayes.Gaussian.add_train_data({:a1, [-1, -1]})
-                  |> LearnKit.NaiveBayes.Gaussian.add_train_data({:a1, [-2, -1]})
-                  |> LearnKit.NaiveBayes.Gaussian.add_train_data({:a2, [1, 1]})
+  alias LearnKit.NaiveBayes.Gaussian
+  classificator = Gaussian.new
+                  |> Gaussian.add_train_data({:a1, [-1, -1]})
+                  |> Gaussian.add_train_data({:a1, [-2, -1]})
+                  |> Gaussian.add_train_data({:a2, [1, 1]})
 ```
 
-Fit data set
+Fit data set:
 
 ```elixir
-  classificator = classificator |> LearnKit.NaiveBayes.Gaussian.fit
+  classificator = classificator |> Gaussian.fit
 ```
 
-Return probability estimates for the feature
+Return probability estimates for the feature:
 
 ```elixir
-  classificator = classificator |> LearnKit.NaiveBayes.Gaussian.predict_proba([1, 2])
+  classificator |> Gaussian.predict_proba([1, 2])
 ```
     feature - new feature for prediction, required
 
-Return exact prediction for the feature
+Return exact prediction for the feature:
 
 ```elixir
-  classificator = classificator |> LearnKit.NaiveBayes.Gaussian.predict([1, 2])
+  classificator |> Gaussian.predict([1, 2])
 ```
     feature - new feature for prediction, required
 
-Returns the mean accuracy on the given test data and labels
+Returns the mean accuracy on the given test data and labels:
 
 ```elixir
-  classificator = classificator |> LearnKit.NaiveBayes.Gaussian.score
+  classificator |> Gaussian.score
 ```
 
 ## Contributing

lib/learn_kit/math.ex

@@ -65,10 +65,7 @@ defmodule LearnKit.Math do
 
   def variance(list, list_mean) when is_list(list) do
     list
-    |> Enum.map(fn x ->
-      list_mean - x
-      |> :math.pow(2)
-    end)
+    |> Enum.map(fn x -> :math.pow(list_mean - x, 2) end)
     |> mean
   end
 
@@ -132,4 +129,69 @@ defmodule LearnKit.Math do
     others = Enum.map(rows, fn(x) -> tl(x) end)
     [firsts | swap_rows_cols(others)]
   end
+
+  @doc """
+  Division for 2 elements
+
+  ## Examples
+
+      iex> LearnKit.Math.division(10, 2)
+      5.0
+
+  """
+  @spec division(number, number) :: number
+
+  def division(x, y) when y != 0 do
+    x / y
+  end
+
+  @doc """
+  Calculate the covariance of two lists
+
+  ## Examples
+
+      iex> LearnKit.Math.covariance([1, 2, 3], [14, 17, 25])
+      5.5
+
+  """
+  @spec covariance(list, list) :: number
+
+  def covariance(x, y) when length(x) == length(y) do
+    mean_x = mean(x)
+    mean_y = mean(y)
+    size = length(x)
+
+    Enum.zip(x, y)
+    |> Enum.map(fn {xi, yi} -> (xi - mean_x) * (yi - mean_y) end)
+    |> Enum.sum
+    |> division(size - 1)
+  end
+
+  @doc """
+  Correlation of two lists
+
+  ## Examples
+
+      iex> LearnKit.Math.correlation([1, 2, 3], [14, 17, 25])
+      0.9672471299049061
+
+  """
+  @spec correlation(list, list) :: number
+
+  def correlation(x, y) when length(x) == length(y) do
+    mean_x = mean(x)
+    mean_y = mean(y)
+
+    divider = Enum.zip(x, y)
+            |> Enum.map(fn {xi, yi} -> (xi - mean_x) * (yi - mean_y) end)
+            |> Enum.sum
+    denom_x = x
+              |> Enum.map(fn xi -> :math.pow(xi - mean_x, 2) end)
+              |> Enum.sum
+    denom_y = y
+              |> Enum.map(fn yi -> :math.pow(yi - mean_y, 2) end)
+              |> Enum.sum
+
+    divider / :math.sqrt(denom_x * denom_y)
+  end
 end

lib/learn_kit/regression/linear.ex

@@ -0,0 +1,114 @@
+defmodule LearnKit.Regression.Linear do
+  @moduledoc """
+  Module for Linear Regression algorithm
+  """
+
+  defstruct factors: [], results: [], coefficients: []
+
+  alias LearnKit.Regression.Linear
+
+  use Linear.Fit
+  use Linear.Predict
+
+  @type factors :: [number]
+  @type results :: [number]
+  @type coefficients :: [number]
+
+  @doc """
+  Creates predictor with empty data_set
+
+  ## Examples
+
+      iex> predictor = LearnKit.Regression.Linear.new
+      %LearnKit.Regression.Linear{factors: [], results: [], coefficients: []}
+
+  """
+  @spec new() :: %LearnKit.Regression.Linear{factors: [], results: [], coefficients: []}
+
+  def new do
+    Linear.new([], [])
+  end
+
+  @doc """
+  Creates predictor with data_set
+
+  ## Parameters
+
+    - factors: Array of predictor variables
+    - results: Array of criterion variables
+
+  ## Examples
+
+      iex> predictor = LearnKit.Regression.Linear.new([1, 2, 3, 4], [3, 6, 10, 15])
+      %LearnKit.Regression.Linear{factors: [1, 2, 3, 4], results: [3, 6, 10, 15], coefficients: []}
+
+  """
+  @spec new(factors, results) :: %LearnKit.Regression.Linear{factors: factors, results: results, coefficients: []}
+
+  def new(factors, results) do
+    %Linear{factors: factors, results: results}
+  end
+
+  @doc """
+  Fit train data
+
+  ## Parameters
+
+    - predictor: %LearnKit.Regression.Linear{}
+
+  ## Examples
+
+      iex> predictor = predictor |> LearnKit.Regression.Linear.fit
+      %LearnKit.Regression.Linear{
+        coefficients: [-1.5, 4.0],
+        factors: [1, 2, 3, 4],
+        results: [3, 6, 10, 15]
+      }
+
+  """
+  @spec fit(%LearnKit.Regression.Linear{factors: factors, results: results}) :: %LearnKit.Regression.Linear{factors: factors, results: results, coefficients: coefficients}
+
+  def fit(%Linear{factors: factors, results: results}) do
+    %Linear{factors: factors, results: results, coefficients: fit_data(factors, results)}
+  end
+
+  @doc """
+  Predict using the linear model
+
+  ## Parameters
+
+    - predictor: %LearnKit.Regression.Linear{}
+    - samples: Array of variables
+
+  ## Examples
+
+      iex> predictor |> LearnKit.Regression.Linear.predict([4, 8, 13])
+      {:ok, [14.5, 30.5, 50.5]}
+
+  """
+  @spec predict(%LearnKit.Regression.Linear{coefficients: coefficients}, list) :: {:ok, list}
+
+  def predict(%Linear{coefficients: coefficients}, samples) do
+    result = samples |> Enum.map(fn sample -> predict_sample(sample, coefficients) end)
+    {:ok, result}
+  end
+
+  @doc """
+  Returns the coefficient of determination R^2 of the prediction
+
+  ## Parameters
+
+    - predictor: %LearnKit.Regression.Linear{}
+
+  ## Examples
+
+      iex> predictor |> LearnKit.Regression.Linear.score
+      {:ok, 0.9876543209876543}
+
+  """
+  @spec score(%LearnKit.Regression.Linear{factors: factors, results: results, coefficients: coefficients}) :: {:ok, number}
+
+  def score(%Linear{factors: factors, results: results, coefficients: coefficients}) do
+    {:ok, calculate_score(coefficients, factors, results)}
+  end
+end

lib/learn_kit/regression/linear/fit.ex

@@ -0,0 +1,17 @@
+defmodule LearnKit.Regression.Linear.Fit do
+  @moduledoc """
+  Module for fit functions
+  """
+
+  alias LearnKit.Math
+
+  defmacro __using__(_opts) do
+    quote do
+      defp fit_data(factors, results) do
+        beta = Math.correlation(factors, results) * Math.standard_deviation(results) / Math.standard_deviation(factors)
+        alpha = Math.mean(results) - beta * Math.mean(factors)
+        [alpha, beta]
+      end
+    end
+  end
+end

lib/learn_kit/regression/linear/predict.ex

@@ -0,0 +1,38 @@
+defmodule LearnKit.Regression.Linear.Predict do
+  @moduledoc """
+  Module for fit functions
+  """
+
+  alias LearnKit.Math
+
+  defmacro __using__(_opts) do
+    quote do
+      defp predict_sample(sample, [alpha, beta]) do
+        sample * beta + alpha
+      end
+
+      defp calculate_score([], _, _), do: raise("There was no fit for model")
+
+      defp calculate_score(coefficients, factors, results) do
+        1.0 - sum_of_squared_errors(coefficients, factors, results) / total_sum_of_squares(results)
+      end
+
+      defp total_sum_of_squares(list) do
+        mean_list = Math.mean(list)
+        list
+        |> Enum.map(fn x -> :math.pow(x - mean_list, 2) end)
+        |> Enum.sum
+      end
+
+      defp sum_of_squared_errors(coefficients, factors, results) do
+        Enum.zip(factors, results)
+        |> Enum.map(fn {xi, yi} -> :math.pow(prediction_error(coefficients, xi, yi), 2) end)
+        |> Enum.sum
+      end
+
+      defp prediction_error(coefficients, x, y) do
+        y - predict_sample(x, coefficients)
+      end
+    end
+  end
+end