make methods on ENV safe if they load an encoding

Most of the time this should work fine, but if a native extension calls
`setlocale(SOME_LOCALE)`, and that encoding is not loaded, it will load
next time `rb_locale_encoding()` is called. This is called for methods on ENV
like ENV#[]. Since these ENV methods acquire the VM lock, we need to
make sure the encoding is loaded prior to locking it.

Calling `setlocale()` like this is supported. The documentation for
`rb_locale_encoding()` states:

```
 * This  is dynamic.   If  you  change the  process'  locale  by e.g.   calling
 * `setlocale(3)`, that should also change the return value of this function.
 *
 * There is no official way for Ruby scripts to manipulate locales, though.
 rb_encoding *rb_locale_encoding(void);
```

Author: luke-gruber

encoding.c

@@ -856,6 +856,7 @@ rb_enc_autoload_p(rb_encoding *enc)
 int
 rb_enc_find_index(const char *name)
 {
+    ASSERT_vm_unlocking(); // it needs to be unlocked so it can call `load_encoding` if necessary
     size_t input_len = strlen(name);
     switch(input_len) {
         case 5:
@@ -873,7 +874,6 @@ rb_enc_find_index(const char *name)
         default:
             break;
     }
-    ASSERT_vm_unlocking(); // it needs to be unlocked so it can call `load_encoding` if necessary
     int i;
     GLOBAL_ENC_TABLE_LOCKING(enc_table) {
         i = enc_registered(enc_table, name);
@@ -1556,6 +1556,9 @@ int rb_locale_charmap_index(void);
 int
 rb_locale_encindex(void)
 {
+    // `rb_locale_charmap_index` can call `enc_find_index`, which can
+    // load an encoding. This needs to be done without VM lock held.
+    ASSERT_vm_unlocking();
     int idx = rb_locale_charmap_index();
 
     if (idx < 0) idx = ENCINDEX_UTF_8;

hash.c

@@ -5192,25 +5192,26 @@ env_enc_str_new(const char *ptr, long len, rb_encoding *enc)
 }
 
 static VALUE
-env_str_new(const char *ptr, long len)
+env_str_new(const char *ptr, long len, rb_encoding *enc)
 {
-    return env_enc_str_new(ptr, len, env_encoding());
+    return env_enc_str_new(ptr, len, enc);
 }
 
 static VALUE
-env_str_new2(const char *ptr)
+env_str_new2(const char *ptr, rb_encoding *enc)
 {
     if (!ptr) return Qnil;
-    return env_str_new(ptr, strlen(ptr));
+    return env_str_new(ptr, strlen(ptr), enc);
 }
 
 static VALUE
 getenv_with_lock(const char *name)
 {
     VALUE ret;
+    rb_encoding *enc = env_encoding();
     ENV_LOCKING() {
         const char *val = getenv(name);
-        ret = env_str_new2(val);
+        ret = env_str_new2(val, enc);
     }
     return ret;
 }
@@ -5773,13 +5774,14 @@ env_values(void)
 {
     VALUE ary = rb_ary_new();
 
+    rb_encoding *enc = env_encoding();
     ENV_LOCKING() {
         char **env = GET_ENVIRON(environ);
 
         while (*env) {
             char *s = strchr(*env, '=');
             if (s) {
-                rb_ary_push(ary, env_str_new2(s+1));
+                rb_ary_push(ary, env_str_new2(s+1, enc));
             }
             env++;
         }
@@ -5865,14 +5867,15 @@ env_each_pair(VALUE ehash)
 
     VALUE ary = rb_ary_new();
 
+    rb_encoding *enc = env_encoding();
     ENV_LOCKING() {
         char **env = GET_ENVIRON(environ);
 
         while (*env) {
             char *s = strchr(*env, '=');
             if (s) {
-                rb_ary_push(ary, env_str_new(*env, s-*env));
-                rb_ary_push(ary, env_str_new2(s+1));
+                rb_ary_push(ary, env_str_new(*env, s-*env, enc));
+                rb_ary_push(ary, env_str_new2(s+1, enc));
             }
             env++;
         }
@@ -6255,13 +6258,14 @@ env_to_a(VALUE _)
 {
     VALUE ary = rb_ary_new();
 
+    rb_encoding *enc = env_encoding();
     ENV_LOCKING() {
         char **env = GET_ENVIRON(environ);
         while (*env) {
             char *s = strchr(*env, '=');
             if (s) {
-                rb_ary_push(ary, rb_assoc_new(env_str_new(*env, s-*env),
-                                              env_str_new2(s+1)));
+                rb_ary_push(ary, rb_assoc_new(env_str_new(*env, s-*env, enc),
+                                              env_str_new2(s+1, enc)));
             }
             env++;
         }
@@ -6509,14 +6513,15 @@ env_key(VALUE dmy, VALUE value)
     StringValue(value);
     VALUE str = Qnil;
 
+    rb_encoding *enc = env_encoding();
     ENV_LOCKING() {
         char **env = GET_ENVIRON(environ);
         while (*env) {
             char *s = strchr(*env, '=');
             if (s++) {
                 long len = strlen(s);
                 if (RSTRING_LEN(value) == len && strncmp(s, RSTRING_PTR(value), len) == 0) {
-                    str = env_str_new(*env, s-*env-1);
+                    str = env_str_new(*env, s-*env-1, enc);
                     break;
                 }
             }
@@ -6533,13 +6538,14 @@ env_to_hash(void)
 {
     VALUE hash = rb_hash_new();
 
+    rb_encoding *enc = env_encoding();
     ENV_LOCKING() {
         char **env = GET_ENVIRON(environ);
         while (*env) {
             char *s = strchr(*env, '=');
             if (s) {
-                rb_hash_aset(hash, env_str_new(*env, s-*env),
-                             env_str_new2(s+1));
+                rb_hash_aset(hash, env_str_new(*env, s-*env, enc),
+                             env_str_new2(s+1, enc));
             }
             env++;
         }
@@ -6684,14 +6690,15 @@ env_shift(VALUE _)
     VALUE result = Qnil;
     VALUE key = Qnil;
 
+    rb_encoding *enc = env_encoding();
     ENV_LOCKING() {
         char **env = GET_ENVIRON(environ);
         if (*env) {
             const char *p = *env;
             char *s = strchr(p, '=');
             if (s) {
-                key = env_str_new(p, s-p);
-                VALUE val = env_str_new2(getenv(RSTRING_PTR(key)));
+                key = env_str_new(p, s-p, enc);
+                VALUE val = env_str_new2(getenv(RSTRING_PTR(key)), enc);
                 result = rb_assoc_new(key, val);
             }
         }