diff --git a/NEWS.md b/NEWS.md index 00023b0fbc..8444d0df14 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,14 @@ # tern 0.9.10.9017 ### Enhancements +* Added `factor_level_method` argument to `df_explicit_na()` to control factor level ordering + when converting character or logical columns. Supported methods: `"sort_auto"` (default, + locale-aware, preserves original behavior), `"sort_radix"` (byte-order / ASCII sort), and + `"data"` (first-appearance order). (#1322) +* Added `factor_as_factor` argument to `df_explicit_na()` to allow re-encoding of existing + factor columns using `factor_level_method`. Defaults to `FALSE` to preserve original behavior. +* Added `factor_level_last_pattern` argument to `df_explicit_na()` to move factor levels + matching a regular expression to the end (before `na_level`). * Added `alternative` argument to `s_coxph_pairwise()` to allow one-sided hypothesis testing. * Added `lr_stat_df` to the parameters return list of `s_coxph_pairwise()`. * Added `uncond_exact_diff` method to `estimate_proportion_diff()` for the unconditional exact confidence interval for the difference in proportions by inverting one-sided tail tests over a nuisance parameter. diff --git a/R/df_explicit_na.R b/R/df_explicit_na.R index f5023db166..de5b8b8ad4 100644 --- a/R/df_explicit_na.R +++ b/R/df_explicit_na.R @@ -21,6 +21,28 @@ #' in `data` to factors. #' @param na_level (`string`)\cr string used to replace all `NA` or empty #' values inside non-`omit_columns` columns. +#' @param factor_as_factor (`flag`)\cr whether to re-encode existing factor variables +#' using `factor_level_method`. When `FALSE` (default), existing factor levels are +#' preserved as-is (original behavior). +#' @param factor_level_method (`string`)\cr method used to order factor levels when +#' converting character or logical variables (or existing factors when +#' `factor_as_factor = TRUE`). One of: +#' \describe{ +#' \item{`"sort_auto"`}{`sort(unique(x))` — default R sort, locale-aware (default). +#' Preserves the original behavior of this function.} +#' \item{`"sort_radix"`}{`sort(unique(x), method = "radix")` — byte-order (ASCII) sort. +#' Unlike `"sort_auto"`, this is not locale-sensitive: uppercase letters always sort +#' before lowercase. On data where all values share the same case (e.g. all-caps +#' ADaM variables) the two methods produce identical results.} +#' \item{`"data"`}{`unique(x)` — levels in order of first appearance in the data.} +#' } +#' @param factor_level_last_pattern (`string` or `NULL`)\cr regular expression. Any +#' factor levels matching this pattern are moved to the end (before `na_level`). +#' `NULL` (default) disables this behaviour. Note: this parameter only takes effect +#' when factor levels are being re-encoded (i.e. for character/logical columns with +#' `char_as_factor`/`logical_as_factor`, or for existing factor columns with +#' `factor_as_factor = TRUE`). Existing factor columns where `factor_as_factor = FALSE` +#' are not affected. #' #' @return A `data.frame` with the chosen modifications applied. #' @@ -66,17 +88,34 @@ #' adsl$AGE[adsl$AGE < 30] <- NA #' adsl <- df_explicit_na(adsl) #' +#' # Example 4: Control factor level ordering +#' # Use radix sort to match SAS PROC SORT behavior. +#' df_explicit_na(my_data, factor_level_method = "sort_radix") +#' # Use data order (first appearance). +#' df_explicit_na(my_data, factor_level_method = "data") +#' +#' # Example 5: Move matching levels to the end +#' # Levels matching "^Other" are placed last (before na_level). +#' df_explicit_na(my_data, factor_level_last_pattern = "^Other") +#' #' @export df_explicit_na <- function(data, omit_columns = NULL, char_as_factor = TRUE, logical_as_factor = FALSE, - na_level = "") { + na_level = "", + factor_as_factor = FALSE, + factor_level_method = c("sort_auto", "sort_radix", "data"), + factor_level_last_pattern = NULL) { checkmate::assert_character(omit_columns, null.ok = TRUE, min.len = 1, any.missing = FALSE) checkmate::assert_data_frame(data) checkmate::assert_flag(char_as_factor) checkmate::assert_flag(logical_as_factor) + checkmate::assert_flag(factor_as_factor) checkmate::assert_string(na_level) + checkmate::assert_string(factor_level_last_pattern, null.ok = TRUE) + factor_level_method <- factor_level_method[[1]] + checkmate::assert_choice(factor_level_method, c("sort_auto", "sort_radix", "data")) target_vars <- if (is.null(omit_columns)) { names(data) @@ -99,6 +138,7 @@ df_explicit_na <- function(data, # Determine whether to convert character or logical input. do_char_conversion <- is.character(xi) && char_as_factor do_logical_conversion <- is.logical(xi) && logical_as_factor + do_factor_conversion <- is.factor(xi) && factor_as_factor # Pre-convert logical to character to deal correctly with replacing NA # values below. @@ -112,8 +152,23 @@ df_explicit_na <- function(data, # Convert to factors if requested for the original type, # set na_level as the last value. - if (do_char_conversion || do_logical_conversion) { - levels_xi <- setdiff(sort(unique(xi)), na_level) + if (do_char_conversion || do_logical_conversion || do_factor_conversion) { + if (do_factor_conversion) { + xi <- as.character(xi) + } + + sort_xi <- switch(factor_level_method, + "data" = unique(xi), + "sort_radix" = sort(unique(xi), method = "radix"), + sort(unique(xi)) + ) + + if (!is.null(factor_level_last_pattern)) { + last_levels <- grep(factor_level_last_pattern, sort_xi, value = TRUE) + sort_xi <- c(setdiff(sort_xi, last_levels), last_levels) + } + + levels_xi <- setdiff(sort_xi, na_level) if (na_level %in% unique(xi)) { levels_xi <- c(levels_xi, na_level) } diff --git a/inst/WORDLIST b/inst/WORDLIST index b581d3c8c0..9210cb89aa 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -1,3 +1,4 @@ +ADaM ADAE ADLB ADPP @@ -37,6 +38,8 @@ Satterthwaite Schouten TLG TLGs +Tord +behaviour binom biomarker biomarkers diff --git a/man/df_explicit_na.Rd b/man/df_explicit_na.Rd index 654f136ab2..d3a11f9091 100644 --- a/man/df_explicit_na.Rd +++ b/man/df_explicit_na.Rd @@ -9,7 +9,10 @@ df_explicit_na( omit_columns = NULL, char_as_factor = TRUE, logical_as_factor = FALSE, - na_level = "" + na_level = "", + factor_as_factor = FALSE, + factor_level_method = c("sort_auto", "sort_radix", "data"), + factor_level_last_pattern = NULL ) } \arguments{ @@ -26,6 +29,31 @@ in \code{data} to factors.} \item{na_level}{(\code{string})\cr string used to replace all \code{NA} or empty values inside non-\code{omit_columns} columns.} + +\item{factor_as_factor}{(\code{flag})\cr whether to re-encode existing factor variables +using \code{factor_level_method}. When \code{FALSE} (default), existing factor levels are +preserved as-is (original behavior).} + +\item{factor_level_method}{(\code{string})\cr method used to order factor levels when +converting character or logical variables (or existing factors when +\code{factor_as_factor = TRUE}). One of: +\describe{ +\item{\code{"sort_auto"}}{\code{sort(unique(x))} — default R sort, locale-aware (default). +Preserves the original behavior of this function.} +\item{\code{"sort_radix"}}{\code{sort(unique(x), method = "radix")} — byte-order (ASCII) sort. +Unlike \code{"sort_auto"}, this is not locale-sensitive: uppercase letters always sort +before lowercase. On data where all values share the same case (e.g. all-caps +ADaM variables) the two methods produce identical results.} +\item{\code{"data"}}{\code{unique(x)} — levels in order of first appearance in the data.} +}} + +\item{factor_level_last_pattern}{(\code{string} or \code{NULL})\cr regular expression. Any +factor levels matching this pattern are moved to the end (before \code{na_level}). +\code{NULL} (default) disables this behaviour. Note: this parameter only takes effect +when factor levels are being re-encoded (i.e. for character/logical columns with +\code{char_as_factor}/\code{logical_as_factor}, or for existing factor columns with +\code{factor_as_factor = TRUE}). Existing factor columns where \code{factor_as_factor = FALSE} +are not affected.} } \value{ A \code{data.frame} with the chosen modifications applied. @@ -84,6 +112,16 @@ adsl <- tern_ex_adsl adsl$AGE[adsl$AGE < 30] <- NA adsl <- df_explicit_na(adsl) +# Example 4: Control factor level ordering +# Use radix sort to match SAS PROC SORT behavior. +df_explicit_na(my_data, factor_level_method = "sort_radix") +# Use data order (first appearance). +df_explicit_na(my_data, factor_level_method = "data") + +# Example 5: Move matching levels to the end +# Levels matching "^Other" are placed last (before na_level). +df_explicit_na(my_data, factor_level_last_pattern = "^Other") + } \seealso{ \code{\link[=sas_na]{sas_na()}} and \code{\link[=explicit_na]{explicit_na()}} for other missing data helper functions. diff --git a/man/tern-package.Rd b/man/tern-package.Rd index 1bbdae1894..b75b0050ec 100644 --- a/man/tern-package.Rd +++ b/man/tern-package.Rd @@ -38,6 +38,7 @@ Authors: Other contributors: \itemize{ + \item David Munoz Tord \email{david.munoztord@mailbox.org} [contributor] \item F. Hoffmann-La Roche AG [copyright holder, funder] } diff --git a/tests/testthat/test-df_explicit_na.R b/tests/testthat/test-df_explicit_na.R index 3ec9d1d792..043309e45b 100644 --- a/tests/testthat/test-df_explicit_na.R +++ b/tests/testthat/test-df_explicit_na.R @@ -90,3 +90,75 @@ testthat::test_that("df_explicit_na just returns unmodified data if all columns )) testthat::expect_identical(result, my_data) }) + +testthat::test_that("factor_level_method = 'sort_auto' gives same result as default", { + my_data <- example_data + result_default <- df_explicit_na(my_data, char_as_factor = TRUE) + result_auto <- df_explicit_na(my_data, char_as_factor = TRUE, factor_level_method = "sort_auto") + testthat::expect_identical(result_default, result_auto) +}) + +testthat::test_that("factor_level_method = 'sort_radix' uses byte-order sort (uppercase before lowercase)", { + # Mixed-case values: sort_auto is locale-aware (case-insensitive on en_US.UTF-8), + # sort_radix is byte-order (uppercase < lowercase, consistent with SAS PROC SORT). + my_data <- data.frame(x = c("UNKNOWN", "Unknown", "unknown", NA), stringsAsFactors = FALSE) + result_radix <- df_explicit_na(my_data, factor_level_method = "sort_radix") + result_auto <- df_explicit_na(my_data, factor_level_method = "sort_auto") + # radix: UNKNOWN < Unknown < unknown (ASCII byte order) + testthat::expect_equal(levels(result_radix$x), c("UNKNOWN", "Unknown", "unknown", "")) +}) + +testthat::test_that("factor_level_method = 'data' preserves first-appearance order", { + my_data <- data.frame(x = c("C", "A", "B", NA), stringsAsFactors = FALSE) + result <- df_explicit_na(my_data, factor_level_method = "data") + testthat::expect_equal(levels(result$x), c("C", "A", "B", "")) +}) + +testthat::test_that("factor_as_factor re-encodes existing factor levels using factor_level_method", { + my_data <- data.frame( + x = factor(c("C", "A", "B"), levels = c("C", "A", "B")), + stringsAsFactors = FALSE + ) + result <- df_explicit_na(my_data, factor_as_factor = TRUE, factor_level_method = "sort_auto") + testthat::expect_equal(levels(result$x), c("A", "B", "C")) +}) + +testthat::test_that("factor_as_factor = FALSE preserves existing factor levels (original behavior)", { + my_data <- data.frame( + x = factor(c("C", "A", "B"), levels = c("C", "A", "B")), + stringsAsFactors = FALSE + ) + result <- df_explicit_na(my_data, factor_as_factor = FALSE) + testthat::expect_equal(levels(result$x), c("C", "A", "B")) +}) + +testthat::test_that("factor_level_last_pattern moves matching levels to end", { + my_data <- data.frame(x = c("Other", "A", "B", "Other2"), stringsAsFactors = FALSE) + result <- df_explicit_na(my_data, factor_level_last_pattern = "^Other") + lvls <- levels(result$x) + # "A", "B" before "Other*" + testthat::expect_true(which(lvls == "A") < which(lvls == "Other")) + testthat::expect_true(which(lvls == "B") < which(lvls == "Other2")) +}) + +testthat::test_that("factor_level_last_pattern = NULL does not change level order", { + my_data <- data.frame(x = c("C", "A", "B"), stringsAsFactors = FALSE) + result_with <- df_explicit_na(my_data, factor_level_last_pattern = NULL) + result_without <- df_explicit_na(my_data) + testthat::expect_identical(levels(result_with$x), levels(result_without$x)) +}) + +testthat::test_that("factor_level_last_pattern combined with na_level: na_level stays last", { + my_data <- data.frame(x = c("Other", "A", NA), stringsAsFactors = FALSE) + result <- df_explicit_na(my_data, factor_level_last_pattern = "^Other") + lvls <- levels(result$x) + testthat::expect_equal(tail(lvls, 1), "") + testthat::expect_true(which(lvls == "Other") < which(lvls == "")) +}) + +testthat::test_that("Check new parameter errors", { + my_data <- example_data + testthat::expect_error(df_explicit_na(my_data, factor_as_factor = "TRUE"), "logical") + testthat::expect_error(df_explicit_na(my_data, factor_level_last_pattern = 123), "string") + testthat::expect_error(df_explicit_na(my_data, factor_level_method = "invalid"), "Must be element") +})