evaluate.R

# MDP: A motif detector and predictor.
#
#    Copyright (c) 2018 Grzegorz Stepien
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
# 
#    For more details, see './LICENSE.md'
#    (where '.' represents this program's root directory).

# This script creates a number of plots based on statistical data generated by an MDP run.
# Does so by providing the configuration data structures to the 'evaluate_all()' method from
# './mdp/mdp/src/main/r/data_evaluator.R'.

r_parameters <- list(
  #' Set to `TRUE` if `stderr` and `stdout` shall be forwarded to a log file (in the same folder as this script)
  print_to_sink = FALSE
)

packages <- c("rprojroot")
for(pkg in packages){
  if(!require(pkg, character.only = TRUE)){
    install.packages(pkg, dependencies = TRUE, repos = "http://cran.us.r-project.org")
    library(pkg, character.only = TRUE)
  }
}
thisfile1 <- normalizePath(thisfile())
thisfolder <- dirname(thisfile1)
r_parameters[["thisfolder"]] <- thisfolder
dir <- file.path(thisfolder, "..", "..","..")
cat(paste0("Setting working directory to \"", dir, "\"\n"))
setwd(dir)

options(error = function() traceback(2))
source(file = file.path(".","mdp","mdp","src","main","r","data_evaluator.R"))


#--------- OPTIONS:----------

options <- list(
  common_options = list(
    main_title_size = 1.3,
    window_title = "Evaluation plot",
    # Root dir of experiment
    root_dir = file.path(r_parameters[["thisfolder"]], ".."),
    dir_pattern = "^01$",
    
    #' Each folder below is assumed to be relative to "./<directory_matching_dir_pattern>/" 
    #' uniquely identifying each subexperiment. Therefore, "*" is implicitly used as the key identifying each subexperiment's dataset
    x_axis = list(type = "csv", 
                  file = "stats/lm_error.csv",
                  rows = "Runtime / ms",
                  title = "runtime / s",
                  selector = function(x){return(x / 1000)},
                  short_title = "sec",
                  title_size = 1.3,
                  num_ticks = 100,
                  labels_every_tick = 10,
                  label_size = 1.3,
                  plot_width = 1200,
                  limits = NULL),
    x_axis_experiment = list(),#json_param_name = list("synth_ds_its","noise_decorator_params","noise_mean")),
    
    y_axis = list(type = "csv",
                  selector = function(x){return(tail(x, n = 1))},
                  title_size = 1.3,
                  num_ticks = 10,
                  labels_every_tick = 2,
                  label_size = 1.3,
                  plot_height = 700,
                  limits = c(0,1)),
    
    group_by = list(),#primary_group = list(type = "json",
                    #                 file = "config/synth_ds_global_options.json",
                    #                 json_param_name = list("trainer_multivar_count_lm", "transition_to_existing_state_weight", 1),
                    #                 legend_name = "tes",
                    #                 order = 1)),
    
    legend_size = 1.3,
    
    #' Sort w.r.t. `sort(x, decreasing = FALSE, index.return = TRUE)` method (which must be applicable)
    sort_x_values = TRUE,
    
    is_continuous = TRUE,
    y_is_numeric = TRUE,
    x_is_numeric = TRUE,
    
    #' If multiple y values are assigned to same x value, keep only one according to strategy below
    x_remove_duplicates = TRUE, 
    #' Only required if x_remove_duplicates is TRUE. Should return a value to be used as y.
    x_remove_duplicates_strategy = function(x,ys) {
      stopifnot(length(ys) > 1)
      #' Simply take the first element of ys
      return(ys[[1]])
    },
    # gtools for 'mixedsort'
    packages = c("rjson", "gtools"),
    
    rnd_color_seed = 1323414,
    
    #' Point type
    pt_type = 21,
    #' Point size
    pt_size = 0.8,
    #' Line width
    l_width = 1.4#,
    
    # Main title postfix ("<main_title_prefix> w.r.t. <main_title_postfix>")
    #main_title_postfix = "noise mean",
    #file_name_postfix = "noise",
    
    
    #exp_stationary_error_name = "expected stationary prediction error"
  )
)

specific_options <- list(
  lm_err_development_options = list(
    main_title = "trained lm error over time",
    plot_out_path = file.path(r_parameters[["thisfolder"]], "graphs", "trained_lm_error_over_time.svg"),
    #dir_pattern = ".*",
    pt_size = 0.2,
    # x_axis = list(type = "csv", 
    #               file = "stats/lm_error.csv",
    #               rows = "Runtime / ms",
    #               title = "runtime / s",
    #               selector = function(x){return(x / 1000)}),
    y_axis = list(file = "stats/lm_error.csv",
                  #' Multiple rows implies additional grouping with row name having highest grouping priority
                  rows = c("Error", "CharacterError"),
                  #' Two params below only required if length of rows parameter > 1
                  row_group_legend_values = c("full", "char"),
                  row_group_legend_name = "error mode",
                  title = "prediction error",
                  selector = function(x){return(x)}),
    group_by = list(#primary_group = list(order = 3),
      #second_group = list(type = "json",
      #                    file = "config/synth_ds_global_options.json",
      #                    json_param_name = list("trainer_multivar_count_lm", "transition_to_new_state_weight", 1),
      #                    legend_name = "tns",
      #                    order = 2),
      # third_group = list(type = options$common_options$x_axis$type, 
      #                    file = options$common_options$x_axis$file,
      #                    json_param_name = options$common_options$x_axis_experiment$json_param_name,
      #                    legend_name = options$common_options$x_axis$short_title,
      #                    order = 1,
      #                    round_digits = 2)
      )
  ),
  # Throughput (num data / runtime processed by motif detector transceiver - estimated via lm_error.csv)
  lm_data_no_to_timestamp_ratio_options = list(
    main_title = "num processed data points per min over time",
    plot_out_path = file.path(r_parameters[["thisfolder"]], "graphs", "num_processed_pts_per_min_over_time.svg"),
    #x_axis = options$common_options$x_axis_experiment,
    y_axis = list(file = "stats/lm_error.csv",
                  rows = NULL,
                  selector = function(x){
                    return(list(x[["DataNumber"]] / x[["Runtime / ms"]] * 60000))},
                  title = "# processed points per min",
                  limits = NULL),
    group_by = list()
  ),
  f1_options = list(
    #x_axis = options$common_options$x_axis_experiment,
    main_title = "motif detection f1 over time",
    plot_out_path = file.path(r_parameters[["thisfolder"]], "graphs", "motif_f1_over_time.svg"),
    #' Append x,y values of datasets having identical values assigned to the parameters provided below
    #' New indices: Values of the parameters provided below in that order
    group_by = list(),
    x_axis = list(file = "stats/f1.csv"),
    y_axis = list(file = "stats/f1.csv",
                  rows = "OverallF1",
                  title = "f1",
                  selector = function(x){return(x)})
  )
  #,
  #' lm_err_options = list(
  #'   x_axis = options$common_options$x_axis_experiment,
  #'   main_title_prefix = "trained lm error over last 100 predictions",
  #'   plot_out_path = file.path(r_parameters[["thisfolder"]], "graphs", paste0("trained_lm_err_vs_", options$common_options$file_name_postfix, ".svg")),
  #'   #' Append x,y values of datasets having identical values assigned to the parameters provided below
  #'   #' New indices: Values of the parameters provided below in that order
  #'   group_by = list(),
  #' 
  #'   y_axis = list(file = "stats/lm_error.csv",
  #'                 #' Multiple rows implies additional grouping with row name having highest grouping priority
  #'                 rows = c("Error"),#, "CharacterError"),
  #'                 #' Two params below only required if length of rows parameter > 1
  #'                 row_group_legend_values = c("full", "char"),
  #'                 row_group_legend_name = "error mode",
  #'                 title = "prediction error")
  #' ),
  #' lm_char_err_options = list(
  #'   x_axis = options$common_options$x_axis_experiment,
  #'   main_title_prefix = "trained lm error over last 100 character label predictions",
  #'   plot_out_path = file.path(r_parameters[["thisfolder"]], "graphs", paste0("trained_lm_char_err_vs_", options$common_options$file_name_postfix, ".svg")),
  #'   #' Append x,y values of datasets having identical values assigned to the parameters provided below
  #'   #' New indices: Values of the parameters provided below in that order
  #'   group_by = list(),
  #' 
  #'   y_axis = list(file = "stats/lm_error.csv",
  #'                 rows = "CharacterError",
  #'                 title = "prediction error")
  #' ),
  #' exp_err_options = list(
  #'   x_axis = options$common_options$x_axis_experiment,
  #'   main_title_prefix = "expected stationary prediction error of trained lm",
  #'   plot_out_path = file.path(r_parameters[["thisfolder"]], "graphs", paste0("trained_lm_exp_err_vs_", options$common_options$file_name_postfix, ".svg")),
  #'   #' Append x,y values of datasets having identical values assigned to the parameters provided below
  #'   #' New indices: Values of the parameters provided below in that order
  #'   group_by = list(),
  #'   y_axis = list(type = "json",
  #'                 file = "models/trained_lm.lm",
  #'                 json_param_name = list("min_error"),
  #'                 title = "expected stationary prediction error",
  #'                 limits = NULL)
  #' ),
  #' perplex_options = list(
  #'   x_axis = options$common_options$x_axis_experiment,
  #'   main_title_prefix = "perplexity rate of trained lm",
  #'   plot_out_path = file.path(r_parameters[["thisfolder"]], "graphs", paste0("trained_lm_perplex_vs_", options$common_options$file_name_postfix, ".svg")),
  #'   #' Append x,y values of datasets having identical values assigned to the parameters provided below
  #'   #' New indices: Values of the parameters provided below in that order
  #'   group_by = list(),
  #' 
  #'   y_axis = list(type = "json",
  #'                 file = "models/trained_lm.lm",
  #'                 json_param_name = list("perplexity_rate"),
  #'                 title = "perplexity rate",
  #'                 limits = NULL)
  #' ),
  #' perplex_vs_exp_err = list(
  #'   main_title = "perplexity rate (trained lm) w.r.t. expected stationary prediction error (trained lm)",
  #'   plot_out_path = file.path(r_parameters[["thisfolder"]], "graphs", "trained_lm_perplex_vs_trained_lm_exp_err.svg"),
  #'   
  #'   #' Each folder is assumed to be relative to "./<directory_matching_dir_pattern>/" 
  #'   #' uniquely identifying each subexperiment. Therefore, "*" is implicitly used as the key identifying each subexperiment's dataset
  #'   x_axis = list(type = "json", 
  #'                 file = "models/trained_lm.lm", 
  #'                 json_param_name = list("min_error"),
  #'                 title = options$common_options$exp_stationary_error_name,
  #'                 limits = NULL),
  #'   group_by = list(),
  #'   y_axis = list(type = "json", 
  #'                 file = "models/trained_lm.lm", 
  #'                 json_param_name = list("perplexity_rate"),
  #'                 title = "perplexity rate",
  #'                 limits = NULL),
  #'   
  #'   #' Sort w.r.t. `sort(x, decreasing = FALSE, index.return = TRUE)` method (which must be applicable)
  #'   sort_x_values = TRUE,
  #'   
  #'   is_continuous = FALSE,
  #'   #' If multiple y values are assigned to same x value, keep only one according to strategy below
  #'   x_remove_duplicates = FALSE
  #'),
  # exp_err_vs_tns = list(
  #  main_title_prefix = "expected stationary model error",
  #  main_title_postfix = "'transition to new state weight'",
  #  plot_out_path = file.path(r_parameters[["thisfolder"]], "graphs", paste0("exp_model_err_vs_", "tns", ".svg")),
  #  
  #  x_axis = list(type = "json", 
  #                file = "config/synth_ds_global_options.json", 
  #                json_param_name = list("trainer_multivar_count_lm","transition_to_new_state_weight", 1),
  #                title = "transition to new state weight",
  #                limits = NULL),
  #  
  #  y_axis = list(type = "json", 
  #                file = "models/lm.lm", 
  #                json_param_name = list("min_error"),
  #                title = options$common_options$exp_stationary_error_name,
  #                limits = NULL)
  #  
  #)
)

for(experiment_name in names(specific_options)){
  has_main <- "main_title" %in% names(specific_options[[experiment_name]])
  has_pre <- "main_title_prefix" %in% names(specific_options[[experiment_name]])
  has_post <- "main_title_postfix" %in% names(specific_options[[experiment_name]])
  
  stopifnot(has_main && !has_pre && !has_post || !has_main && has_pre)
  
  if(has_pre){
    prefix <- specific_options[[experiment_name]][["main_title_prefix"]]
    if(has_post){
      postfix <- specific_options[[experiment_name]][["main_title_postfix"]]
    }
    else{
      postfix <- options$common_options$main_title_postfix
    }
    specific_options[[experiment_name]][["main_title"]] <- paste0(prefix, " w.r.t. ", postfix)
  }
}

evaluate_all(append(options, specific_options))
#---------END OF OPTIONS---------