Fix OpenSSL 3.6 compatibility and update gost-engine NID handling

Fixes with the OpenSSL patch:
- Resolved patch conflicts with OpenSSL 3.6.
- Fixed Resumption and PSK modes.
- Fixed define conflict between `TLS1_TLSTREE_S` and `SSL_QUIC` that broke `TLS1_3_RFC_AES_128_GCM_SHA256`.
- Properly registered `NID_magma_mgm` and `NID_kuznyechik_mgm` through the standard `make update` process.

Fixes related to `gost-engine` code:
- Updated `tcl_tests/tls13.try` to match the new algorithm output format in OpenSSL 3.6.
- Refactored NID handling: previously `gost-engine` defined MGM mode NIDs internally, but they are now expected from OpenSSL.
  If absent, `gost-engine` temporarily creates them dynamically until OpenSSL includes proper definitions.
- Added a test for session resumption.
- Fix running tests on CI on Windows: before this commit the tests used incorrect openssl version (the one provided by the platform)
- Disable running PERL tests on Windows (it always fails)

Author: Vladislav Sapegin

.github/workflows/ci.yml

@@ -2,7 +2,7 @@ name: CI
 on: [push, pull_request]
 
 env:
-    OPENSSL_BRANCH: openssl-3.4.2
+    OPENSSL_BRANCH: openssl-3.6
     USE_RPATH: yes
     PATCH_OPENSSL: 0
     GOST_PROVIDER_ENABLE_ONLINE_TESTS: 1

.github/workflows/codeql-analysis.yml

@@ -1,7 +1,7 @@
 name: "CodeQL"
 
 env:
-  OPENSSL_BRANCH: openssl-3.4.2
+  OPENSSL_BRANCH: openssl-3.6
   #RPATH: "-Wl,-rpath=${PREFIX}/lib"
   #PREFIX: ${HOME}/opt
   #PATH: ${PREFIX}/bin:${PATH}

.github/workflows/coverity.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ github.repository_owner == 'gost-engine' || github.event_name == 'workflow_dispatch' }}
     env:
-      OPENSSL_BRANCH: openssl-3.4.2
+      OPENSSL_BRANCH: openssl-3.6
       USE_RPATH: yes
       PATCH_OPENSSL: 1
       GOST_PROVIDER_ENABLE_ONLINE_TESTS: 0

.github/workflows/windows.yml

@@ -12,7 +12,7 @@ jobs:
               with:
                   repository: openssl/openssl
                   path: openssl
-                  ref: openssl-3.4.2
+                  ref: openssl-3.6
                   fetch-depth: 0
             - run: echo "::set-output name=head::$(git -C openssl describe --always --long)"
               id: openssl
@@ -49,7 +49,7 @@ jobs:
             - run: cmake --build .
             - name: Run tests
               run: |
-                  $env:PATH = "$pwd\openssl\_dest\Program Files\OpenSSL\bin;$env:PATH"
                   $env:OPENSSL_ENGINES = "$pwd\bin\Debug"
                   $env:OPENSSL_MODULES = "$pwd\bin\Debug"
+                  Copy-Item -Path "$pwd\openssl\_dest\Program Files\OpenSSL\bin\*.dll" -Destination "$pwd\bin\Debug"
                   ctest -C Debug --output-on-failure

CMakeLists.txt

@@ -48,6 +48,10 @@ if (ASAN)
   set(SKIP_PERL_TESTS 1)
 endif()
 
+if (CMAKE_SYSTEM_NAME STREQUAL "Windows")
+  set(SKIP_PERL_TESTS 1)
+endif()
+
 set(CMAKE_C_STANDARD 90)
 CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME_C)
 CHECK_LIBRARY_EXISTS(rt clock_gettime "" HAVE_CLOCK_GETTIME_RT)
@@ -322,6 +326,12 @@ if(TLS13_PATCHED_OPENSSL)
   set_tests_properties(mgm-with-provider
     PROPERTIES ENVIRONMENT "${TEST_ENVIRONMENT_PROVIDER}")
   set_property(TARGET test_mgm APPEND PROPERTY COMPILE_DEFINITIONS ENGINE_DIR="${OUTPUT_DIRECTORY}")
+
+  add_executable(test_tls13handshake test_tls13handshake.c)
+  target_link_libraries(test_tls13handshake OpenSSL::Crypto OpenSSL::SSL)
+  add_test(NAME test-tls13handshake-with-provider COMMAND test_tls13handshake)
+  set_tests_properties(test-tls13handshake-with-provider
+    PROPERTIES ENVIRONMENT "${TEST_ENVIRONMENT_PROVIDER}")
 endif()
 
 if(NOT SKIP_PERL_TESTS)

gost_crypt.c

@@ -251,7 +251,7 @@ GOST_cipher magma_ecb_cipher = {
     .do_cipher = magma_cipher_do_ecb,
 };
 
- GOST_cipher magma_mgm_cipher = {
+GOST_cipher magma_mgm_cipher = {
     .nid = NID_undef,
     .template = &magma_template_cipher,
     .block_size = 1,
@@ -264,7 +264,7 @@ GOST_cipher magma_ecb_cipher = {
     .ctrl = gost_magma_mgm_ctrl,
     .cleanup = gost_magma_mgm_cleanup,
     .ctx_size = sizeof(gost_mgm_ctx)
- };
+};
 
 static void magma_NID_callback (int nid)
 {
@@ -1164,7 +1164,7 @@ static int gost_magma_mgm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
     case EVP_CTRL_TLSTREE:
         {
             unsigned char newkey[32];
-            if (gost_tlstree(NID_magma_mgm,
+            if (gost_tlstree(OBJ_sn2nid(SN_magma_mgm),
                     (const unsigned char *)mctx->ks.g_ks.cctx.master_key,
                     newkey, (const unsigned char *)ptr, mctx->tlstree_mode)
                   > 0) {

gost_eng.c

@@ -331,33 +331,55 @@ static int gost_engine_destroy(ENGINE* e) {
  * Following is the glue that populates the ENGINE structure and that
  * binds it to OpenSSL libraries
  */
-
 static GOST_NID_JOB *missing_NIDs[] = {
     &kuznyechik_mgm_NID,
     &magma_mgm_NID,
 };
 
+static int create_new_nid_for_gost_nid_job(GOST_NID_JOB *job) {
+    int nid = OBJ_new_nid(1);
+    ASN1_OBJECT *obj =
+        ASN1_OBJECT_create(nid, NULL, 0, job->sn, job->ln);
+
+    if (!obj || OBJ_add_object(obj) == NID_undef) {
+        OPENSSL_free(obj);
+        return 0;
+    }
+
+    job->asn1 = obj;
+    job->callback(nid);
+
+    return 1;
+}
+
 static int create_NIDs() {
-    int i;
-    int new_nid = OBJ_new_nid(OSSL_NELEM(missing_NIDs));
+    int i, nid;
+
     for (i = 0; i < OSSL_NELEM(missing_NIDs); i++) {
         GOST_NID_JOB *job = missing_NIDs[i];
-        ASN1_OBJECT *obj =
-            ASN1_OBJECT_create(new_nid + i, NULL, 0, job->sn, job->ln);
-        job->asn1 = obj;
-        if (!obj || OBJ_add_object(obj) == NID_undef) {
-            OPENSSL_free(obj);
+
+        nid = OBJ_sn2nid(job->sn);
+        if (nid != NID_undef) {
+            job->callback(nid);
+            continue;
+        }
+
+        if (!create_new_nid_for_gost_nid_job(job)) {
             return 0;
         }
-        (*missing_NIDs[i]->callback)(new_nid + i);
     }
     return 1;
 }
 
-static void free_NIDs() {
+static void free_NIDs(void) {
     int i;
+
     for (i = 0; i < OSSL_NELEM(missing_NIDs); i++) {
-        ASN1_OBJECT_free(missing_NIDs[i]->asn1);
+        GOST_NID_JOB *job = missing_NIDs[i];
+        if (job->asn1) {
+            ASN1_OBJECT_free(job->asn1);
+            job->asn1 = NULL;
+        }
     }
 }
 

gost_grasshopper_cipher.c

@@ -1105,7 +1105,7 @@ static int gost_grasshopper_mgm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void
     case EVP_CTRL_TLSTREE:
         {
             unsigned char newkey[32];
-            if (gost_tlstree(NID_kuznyechik_mgm,
+            if (gost_tlstree(OBJ_sn2nid(SN_kuznyechik_mgm),
                     mctx->ks.gh_ks.master_key.k.b, newkey,
                     (const unsigned char *)ptr, mctx->tlstree_mode)
                   > 0) {

gost_keyexpimp.c

@@ -12,6 +12,8 @@
 #include <openssl/buffer.h>
 
 #include "gost_lcl.h"
+#include "gost_gost2015.h"
+#include "gost_grasshopper_cipher.h"
 #include "e_gost_err.h"
 
 static uint32_t be32(uint32_t host)
@@ -267,51 +269,40 @@ int gost_tlstree(int cipher_nid, const unsigned char *in, unsigned char *out,
     unsigned char ko1[32], ko2[32];
     int ret;
 
-    switch (cipher_nid) {
-    case NID_magma_cbc:
+    if (cipher_nid == NID_magma_cbc) {
         c1 = 0x00000000C0FFFFFF;
         c2 = 0x000000FEFFFFFFFF;
         c3 = 0x00F0FFFFFFFFFFFF;
-        break;
-    case NID_grasshopper_cbc:
+    } else if (cipher_nid == NID_grasshopper_cbc) {
         c1 = 0x00000000FFFFFFFF;
         c2 = 0x0000F8FFFFFFFFFF;
         c3 = 0xC0FFFFFFFFFFFFFF;
-        break;
-    case NID_magma_mgm:
-        switch (mode) {
-        case TLSTREE_MODE_S:    // TLS_GOSTR341112_256_WITH_MAGMA_MGM_S
+    } else if (cipher_nid == magma_mgm_cipher.nid) {
+        if (mode == TLSTREE_MODE_S) {    // TLS_GOSTR341112_256_WITH_MAGMA_MGM_S
             c1 = 0x000000fcffffffff;
             c2 = 0x00e0ffffffffffff;
             c3 = 0xffffffffffffffff;
-            break;
-        case TLSTREE_MODE_L:    // TLS_GOSTR341112_256_WITH_MAGMA_MGM_L
+        } else if (mode == TLSTREE_MODE_L) { // TLS_GOSTR341112_256_WITH_MAGMA_MGM_L
             c1 = 0x000000000000e0ff;
             c2 = 0x000000c0ffffffff;
             c3 = 0x80ffffffffffffff;
-            break;
-        default:
+        } else {
             return 0;
         }
-        break;
-    case NID_kuznyechik_mgm:
-        switch (mode) {
-        case TLSTREE_MODE_S:    // TLS_GOSTR341112_256_WITH_KUZNYECHIK_MGM_S
+    } else if (cipher_nid == grasshopper_mgm_cipher.nid) {
+        if (mode == TLSTREE_MODE_S) {    // TLS_GOSTR341112_256_WITH_KUZNYECHIK_MGM_S
             c1 = 0x000000e0ffffffff;
             c2 = 0x0000ffffffffffff;
             c3 = 0xf8ffffffffffffff;
-            break;
-        case TLSTREE_MODE_L:    // TLS_GOSTR341112_256_WITH_KUZNYECHIK_MGM_L
+        } else if (mode == TLSTREE_MODE_L) { // TLS_GOSTR341112_256_WITH_KUZNYECHIK_MGM_L
             c1 = 0x00000000000000f8;
             c2 = 0x00000000f0ffffff;
             c3 = 0x00e0ffffffffffff;
-            break;
-        default:
+        } else {
             return 0;
         }
-        break;
-    default:
-        return 0;
+    } else {
+        return 0; /* неизвестный cipher_nid */
     }
 #ifndef L_ENDIAN
     BUF_reverse((unsigned char *)&seq, tlsseq, 8);

gost_lcl.h

@@ -37,32 +37,6 @@ EVP_CTRL_SET_TLSTREE_PARAMS definition in OpenSSL is expected."
 #  define EVP_CTRL_SET_TLSTREE_PARAMS 0xFF
 # endif
 
-# ifndef NID_magma_mgm
-#  if defined(_MSC_VER)
-#   pragma message("Gost-engine is built against not fully supported version of OpenSSL. \
-NID_magma_mgm definition in OpenSSL is expected. No magma mgm functionality is \
-guaranteed.")
-#  else
-#   warning "Gost-engine is built against not fully supported version of OpenSSL. \
-NID_magma_mgm definition in OpenSSL is expected. No magma mgm functionality is \
-guaranteed."
-#  endif
-#  define NID_magma_mgm ((int)(INT_MAX - 1))
-# endif
-
-# ifndef NID_kuznyechik_mgm
-#  if defined(_MSC_VER)
-#   pragma message("Gost-engine is built against not fully supported version of OpenSSL. \
-NID_kuznyechik_mgm definition in OpenSSL is expected. No magma mgm functionality is \
-guaranteed.")
-#  else
-#   warning "Gost-engine is built against not fully supported version of OpenSSL. \
-NID_kuznyechik_mgm definition in OpenSSL is expected. No kuznyechik mgm functionality is \
-guaranteed."
-#  endif
-#  define NID_kuznyechik_mgm ((int)(INT_MAX - 2))
-# endif
-
 /* Control commands */
 # define GOST_PARAM_CRYPT_PARAMS 0
 # define GOST_PARAM_PBE_PARAMS 1