Force NLS_LANG to AL32UTF8 for Oracle (OCI) connections (tam#35892)

Multibyte data came back as literal question marks in the app. Measured against a
real database with a column of Japanese text:

  NLS_LANG unset, or *.US7ASCII  -> every non-ASCII character arrives as 0x3f
  *.JA16SJIS                     -> Shift-JIS bytes, despite unicode_as_utf8=TRUE
  *.AL32UTF8                     -> correct UTF-8

The server converts before the data reaches the client, so the characters are
gone rather than merely mislabeled, and the design's assumption that the existing
NLS_LANG setting would simply carry over was wrong in both directions: an unset
value loses data, and a value a user set for the ODBC based Oracle data source
(where odbc handles encoding separately) actively breaks OCI.

The character set is therefore forced, not defaulted, keeping the user's language
and territory. NLS_LANG is set only around the connect and restored immediately
after: OCI fixes the client character set when it creates its environment on that
first connect, so later queries and connections in the session stay UTF-8 while
the ODBC path keeps the user's own setting. Verified with a pre-existing
JA16SJIS value present.

Fetched character columns are now also marked as UTF-8. ROracle leaves them
unmarked, which R reads in the session's native encoding - wrong on a machine
running an en_US locale, as this one does.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: hidekoji

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: exploratory
 Type: Package
 Title: R package for Exploratory
-Version: 16.0.19
+Version: 16.0.20
 Date: 2026-07-28
 Authors@R: c(person("Hideaki", "Hayashi", email = "hideaki@exploratory.io", role = c("aut", "cre")), person("Hide", "Kojima", email = "hide@exploratory.io", role = c("aut")), person("Kan", "Nishida", email = "kan@exploratory.io", role = c("aut")), person("Kei", "Saito", email = "kei@exploratory.io", role = c("aut")), person("Yosuke", "Yasuda", email = "double.y.919.quick@gmail.com", role = c("aut")))
 URL: https://github.com/exploratory-io/exploratory_func

R/system.R

@@ -1034,6 +1034,31 @@ applyOracleClientEnv <- function(explicitDir = Sys.getenv("EXPLORATORY_ORACLE_CL
   invisible(TRUE)
 }
 
+# Returns an NLS_LANG value whose character set is AL32UTF8, preserving the language and
+# territory the user configured (if any).
+#
+# The character set half of NLS_LANG decides how the Oracle server converts character data
+# before it reaches the client, and getting it wrong loses data rather than merely mislabeling
+# it. Measured against a real database, with a column holding Japanese text:
+#   NLS_LANG unset / *.US7ASCII  -> every non-ASCII character arrives as "?" (0x3f). Gone.
+#   *.JA16SJIS                   -> Shift-JIS bytes, even though the driver is asked for UTF-8
+#                                   via unicode_as_utf8, so the strings are then mislabeled.
+#   *.AL32UTF8                   -> correct UTF-8.
+# So the character set has to be forced, not merely defaulted: a user who set NLS_LANG for the
+# ODBC based Oracle data source would otherwise silently corrupt Oracle (OCI) results.
+oracleOCINlsLang <- function(currentNlsLang = Sys.getenv("NLS_LANG")) {
+  if (is.null(currentNlsLang) || length(currentNlsLang) == 0 || is.na(currentNlsLang[[1]]) ||
+      currentNlsLang[[1]] == "") {
+    return("American_America.AL32UTF8")
+  }
+  value <- as.character(currentNlsLang[[1]])
+  if (!grepl(".", value, fixed = TRUE)) {
+    # Language and territory only, no character set part.
+    return(paste0(value, ".AL32UTF8"))
+  }
+  paste0(sub("\\.[^.]*$", "", value), ".AL32UTF8")
+}
+
 # Single source of truth for the Oracle (OCI) connection pool key so that getDBConnection
 # and clearDBConnection can never drift apart. Every component is normalized here because
 # paste() silently drops NULL, which would produce a key that no clear call can match.
@@ -1990,6 +2015,16 @@ getDBConnection <- function(type, host = NULL, port = "", databaseName = "", use
       if (is.na(bulkReadRows) || bulkReadRows <= 0) {
         bulkReadRows <- 10000L
       }
+      # Force the NLS_LANG character set to AL32UTF8 for the duration of the connect, then put
+      # the user's own value back. OCI fixes the client character set when it creates its
+      # environment, which happens on this first connect, so restoring afterwards leaves this
+      # connection (and every later one in the session) reading UTF-8 while the ODBC based
+      # Oracle data source keeps whatever NLS_LANG the user configured for it.
+      previousNlsLang <- Sys.getenv("NLS_LANG")
+      Sys.setenv(NLS_LANG = oracleOCINlsLang(previousNlsLang))
+      on.exit({
+        if (previousNlsLang == "") Sys.unsetenv("NLS_LANG") else Sys.setenv(NLS_LANG = previousNlsLang)
+      }, add = TRUE)
       conn <- ROracle::dbConnect(
         ROracle::Oracle(unicode_as_utf8 = TRUE),
         username = username,
@@ -2311,6 +2346,17 @@ queryOracleOCI <- function(host = "", port = 1521, databaseName = "", username =
     })
   }
   colnames(df) <- iconv(colnames(df), from = "utf8", to = "utf8") # Work around to read multibyte column names without getting garbled.
+  # The connection forces the NLS_LANG character set to AL32UTF8, so character data is UTF-8
+  # here. ROracle leaves it unmarked ("unknown"), which R then reads in the session's native
+  # encoding - wrong whenever that is not UTF-8, as on a machine running an en_US locale.
+  # Marking it keeps multibyte values intact through the rest of the pipeline.
+  characterColumns <- vapply(df, is.character, logical(1))
+  if (any(characterColumns)) {
+    df[characterColumns] <- lapply(df[characterColumns], function(column) {
+      Encoding(column) <- "UTF-8"
+      column
+    })
+  }
   df
 }
 

tests/testthat/test_system.R

@@ -1300,3 +1300,24 @@ test_that("applyOracleClientEnv reports FALSE when no Oracle client can be found
   dir.create(emptyDir)
   expect_false(exploratory:::applyOracleClientEnv(emptyDir))
 })
+
+test_that("oracleOCINlsLang always forces the AL32UTF8 character set", {
+  # Measured against a real database: an unset or US7ASCII NLS_LANG makes the server replace
+  # every non-ASCII character with "?" before it reaches the client, and JA16SJIS returns
+  # Shift-JIS bytes even though the driver is asked for UTF-8. Only AL32UTF8 is safe, so the
+  # character set is forced rather than merely defaulted.
+  expect_equal(exploratory:::oracleOCINlsLang(""), "American_America.AL32UTF8")
+  expect_equal(exploratory:::oracleOCINlsLang(NULL), "American_America.AL32UTF8")
+  expect_equal(exploratory:::oracleOCINlsLang(NA), "American_America.AL32UTF8")
+
+  # A user's language and territory are preserved; only the character set is replaced.
+  expect_equal(exploratory:::oracleOCINlsLang("Japanese_Japan.JA16SJIS"), "Japanese_Japan.AL32UTF8")
+  expect_equal(exploratory:::oracleOCINlsLang("American_America.US7ASCII"), "American_America.AL32UTF8")
+  expect_equal(exploratory:::oracleOCINlsLang("Japanese_Japan.AL32UTF8"), "Japanese_Japan.AL32UTF8")
+
+  # Language and territory with no character set part at all.
+  expect_equal(exploratory:::oracleOCINlsLang("Japanese_Japan"), "Japanese_Japan.AL32UTF8")
+
+  # A territory containing a dot must not confuse the character set split.
+  expect_equal(exploratory:::oracleOCINlsLang("A_B.C.JA16SJIS"), "A_B.C.AL32UTF8")
+})