loo/touchstone/script.R at 4e9c78ef61072798fc69e503cf432c3b1c8b1f47 · stan-dev/loo · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
# see `help(run_script, package = 'touchstone')` on how to run this
# interactively

# installs branches to benchmark
touchstone::branch_install()

touchstone::pin_assets("touchstone/wine.rds")

# These synthetic workloads are large enough to expose real slowdowns in the
# core `loo()` paths, but still short enough to keep PR feedback reasonably fast.
touchstone::benchmark_run(
  expr_before_benchmark = {
    suppressPackageStartupMessages(library(loo))
    # benchmark_run() evaluates in a callr subprocess, so load pinned assets here.
    wine_log_lik_matrix <- readRDS(touchstone::path_pinned_asset(
      "touchstone/wine.rds"
    ))
    matrix_r_eff <- rep(1, ncol(wine_log_lik_matrix))
  },
  loo_matrix = {
    suppressWarnings(
      loo(
        wine_log_lik_matrix,
        r_eff = matrix_r_eff,
        cores = 1
      )
    )
  },
  n = 10
)

touchstone::benchmark_run(
  expr_before_benchmark = {
    suppressPackageStartupMessages(library(loo))
    wine_log_lik_matrix <- readRDS(touchstone::path_pinned_asset(
      "touchstone/wine.rds"
    ))
    function_r_eff <- rep(1, ncol(wine_log_lik_matrix))
    wine_data <- data.frame(obs = seq_len(ncol(wine_log_lik_matrix)))
    wine_llfun <- function(data_i, draws) draws[, data_i$obs, drop = FALSE]
  },
  loo_function = {
    suppressWarnings(
      loo(
        wine_llfun,
        data = wine_data,
        draws = wine_log_lik_matrix,
        r_eff = function_r_eff,
        cores = 1
      )
    )
  },
  n = 10
)

# create artifacts used downstream in the GitHub Action
touchstone::benchmark_analyze()