MBEDTLS / ESP32 hardware hash engine fault on concurrent use with WolfSSH/WolfSSL

[dexterbg]

Crash signature:

• Mongoose TLS connect → MBEDTLS SSL handshake (SHA256 calc) → esp_sha_read_digest_state() (ESP32 HW hash engine)
  → abort() "SHA peripheral returned all zero state, probably due to fault injection"

Reproduction:

• open ssh to module, enable log monitoring (!)
• on any console or web UI (or the ssh):
  repeat starting & stopping server v2 with ~3-4 seconds in between
  → crash occurs eventually

Observations:

• only happens with a running ssh session with log monitoring enabled / running transmissions
• does not happen with server V2 & V3 TLS connections starting concurrently, so seems to be wolfssh specific

Crash backtrace from test at commit ec1448e:

Using elf file: /home/balzer/esp/Open-Vehicle-Monitoring-System-3/vehicle/OVMS.V3/build/ovms3.elf
0x4008ddae is in invoke_abort (/home/balzer/esp/esp-idf/components/esp32/panic.c:156).
151	#endif
152	    while (1) {
153	        if (esp_cpu_in_ocd_debug_mode()) {
154	            __asm__ ("break 0,0");
155	        }
156	        *((int *) 0) = 0;
157	    }
158	}
159	
160	void abort()
0x4008e049 is in abort (/home/balzer/esp/esp-idf/components/esp32/panic.c:171).
166	     * don't overwrite that.
167	     */
168	    if (esp_reset_reason_get_hint() == ESP_RST_UNKNOWN) {
169	        esp_reset_reason_set_hint(ESP_RST_PANIC);
170	    }
171	    invoke_abort();
172	}
173	
174	
175	static const char *edesc[] = {
0x403541b9 is in esp_sha_read_digest_state (/home/balzer/esp/esp-idf/components/esp32/hwcrypto/sha.c:272).
267	    for (int i = 0; i < word_len; i++) {
268	        if (digest_state_words[i] != 0) {
269	            return;
270	        }
271	    }
272	    abort(); // SHA peripheral returned all zero state, probably due to fault injection
273	}
274	
275	void esp_sha_block(esp_sha_type sha_type, const void *data_block, bool is_first_block)
276	{
0x40339762 is in mbedtls_sha256_clone (/home/balzer/esp/esp-idf/components/mbedtls/port/esp_sha256.c:106).
101	
102	    if (src->mode == ESP_MBEDTLS_SHA256_HARDWARE) {
103	        /* Copy hardware digest state out to cloned state,
104	           which will become a software digest.
105	        */
106	        esp_sha_read_digest_state(SHA2_256, dst->state);
107	        dst->mode = ESP_MBEDTLS_SHA256_SOFTWARE;
108	    }
109	}
110	
0x403281e5 is in ssl_calc_finished_tls_sha256 (/home/balzer/esp/esp-idf/components/mbedtls/mbedtls/library/ssl_tls.c:6276).
6271	
6272	    mbedtls_sha256_init( &sha256 );
6273	
6274	    MBEDTLS_SSL_DEBUG_MSG( 2, ( "=> calc  finished tls sha256" ) );
6275	
6276	    mbedtls_sha256_clone( &sha256, &ssl->handshake->fin_sha256 );
6277	
6278	    /*
6279	     * TLSv1.2:
6280	     *   hash = PRF( master, finished_label,
0x40329d73 is in mbedtls_ssl_parse_finished (/home/balzer/esp/esp-idf/components/mbedtls/mbedtls/library/ssl_tls.c:6577).
6572	    unsigned int hash_len;
6573	    unsigned char buf[SSL_MAX_HASH_LEN];
6574	
6575	    MBEDTLS_SSL_DEBUG_MSG( 2, ( "=> parse finished" ) );
6576	
6577	    ssl->handshake->calc_finished( ssl, buf, ssl->conf->endpoint ^ 1 );
6578	
6579	    if( ( ret = mbedtls_ssl_read_record( ssl, 1 ) ) != 0 )
6580	    {
6581	        MBEDTLS_SSL_DEBUG_RET( 1, "mbedtls_ssl_read_record", ret );
0x4032d9fa is in mbedtls_ssl_handshake_client_step (/home/balzer/esp/esp-idf/components/mbedtls/mbedtls/library/ssl_cli.c:3894).
3889	       case MBEDTLS_SSL_SERVER_CHANGE_CIPHER_SPEC:
3890	           ret = mbedtls_ssl_parse_change_cipher_spec( ssl );
3891	           break;
3892	
3893	       case MBEDTLS_SSL_SERVER_FINISHED:
3894	           ret = mbedtls_ssl_parse_finished( ssl );
3895	           break;
3896	
3897	       case MBEDTLS_SSL_FLUSH_BUFFERS:
3898	           MBEDTLS_SSL_DEBUG_MSG( 2, ( "handshake: done" ) );
0x4032a066 is in mbedtls_ssl_handshake_step (/home/balzer/esp/esp-idf/components/mbedtls/mbedtls/library/ssl_tls.c:8117).
8112	    if( ssl == NULL || ssl->conf == NULL )
8113	        return( MBEDTLS_ERR_SSL_BAD_INPUT_DATA );
8114	
8115	#if defined(MBEDTLS_SSL_CLI_C)
8116	    if( ssl->conf->endpoint == MBEDTLS_SSL_IS_CLIENT )
8117	        ret = mbedtls_ssl_handshake_client_step( ssl );
8118	#endif
8119	#if defined(MBEDTLS_SSL_SRV_C)
8120	    if( ssl->conf->endpoint == MBEDTLS_SSL_IS_SERVER )
8121	        ret = mbedtls_ssl_handshake_server_step( ssl );
0x4032a091 is in mbedtls_ssl_handshake (/home/balzer/esp/esp-idf/components/mbedtls/mbedtls/library/ssl_tls.c:8141).
8136	
8137	    MBEDTLS_SSL_DEBUG_MSG( 2, ( "=> handshake" ) );
8138	
8139	    while( ssl->state != MBEDTLS_SSL_HANDSHAKE_OVER )
8140	    {
8141	        ret = mbedtls_ssl_handshake_step( ssl );
8142	
8143	        if( ret != 0 )
8144	            break;
8145	    }
0x4015bdfc is in mg_ssl_if_handshake (/home/balzer/esp/Open-Vehicle-Monitoring-System-3/vehicle/OVMS.V3/components/mongoose/mongoose/mongoose.c:5018).
5013	  int err;
5014	  /* If bio is not yet set, do it now. */
5015	  if (ctx->ssl->p_bio == NULL) {
5016	    mbedtls_ssl_set_bio(ctx->ssl, nc, ssl_socket_send, ssl_socket_recv, NULL);
5017	  }
5018	  err = mbedtls_ssl_handshake(ctx->ssl);
5019	  if (err != 0) return mg_ssl_if_mbed_err(nc, err);
5020	#ifdef MG_SSL_IF_MBEDTLS_FREE_CERTS
5021	  /*
5022	   * Free the peer certificate, we don't need it after handshake.
0x4015d5d1 is in mg_ssl_begin (/home/balzer/esp/Open-Vehicle-Monitoring-System-3/vehicle/OVMS.V3/components/mongoose/mongoose/mongoose.c:3765).
3760	}
3761	
3762	#if MG_ENABLE_SSL
3763	static void mg_ssl_begin(struct mg_connection *nc) {
3764	  int server_side = (nc->listener != NULL);
3765	  enum mg_ssl_if_result res = mg_ssl_if_handshake(nc);
3766	  DBG(("%p %d res %d", nc, server_side, res));
3767	
3768	  if (res == MG_SSL_OK) {
3769	    nc->flags |= MG_F_SSL_HANDSHAKE_DONE;
0x4015eeae is in mg_mgr_handle_conn (/home/balzer/esp/Open-Vehicle-Monitoring-System-3/vehicle/OVMS.V3/components/mongoose/mongoose/mongoose.c:3820).
3815	       */
3816	      err = nc->err;
3817	#endif
3818	#if MG_ENABLE_SSL
3819	      if ((nc->flags & MG_F_SSL) && err == 0) {
3820	        mg_ssl_begin(nc);
3821	      } else {
3822	        mg_if_connect_cb(nc, err);
3823	      }
3824	#else
0x4015f3ea is in mg_socket_if_poll (/home/balzer/esp/Open-Vehicle-Monitoring-System-3/vehicle/OVMS.V3/components/mongoose/mongoose/mongoose.c:4042).
4037	        fd_flags |= _MG_F_FD_CAN_WRITE;
4038	      }
4039	#endif
4040	    }
4041	    tmp = nc->next;
4042	    mg_mgr_handle_conn(nc, fd_flags, now);
4043	  }
4044	
4045	  for (nc = mgr->active_connections; nc != NULL; nc = tmp) {
4046	    tmp = nc->next;
0x4015b5aa is in mg_mgr_poll (/home/balzer/esp/Open-Vehicle-Monitoring-System-3/vehicle/OVMS.V3/components/mongoose/mongoose/mongoose.c:2538).
2533	    LOG(LL_ERROR, ("cannot poll: no interfaces"));
2534	    return 0;
2535	  }
2536	
2537	  for (i = 0; i < m->num_ifaces; i++) {
2538	    now = m->ifaces[i]->vtable->poll(m->ifaces[i], timeout_ms);
2539	  }
2540	  return now;
2541	}
2542	
0x4013b72f is in OvmsNetManager::MongooseTask() (/home/balzer/esp/Open-Vehicle-Monitoring-System-3/vehicle/OVMS.V3/main/ovms_netmanager.cpp:985).
980	
981	  // Main event loop
982	  while (!m_mongoose_stopping)
983	    {
984	    // poll interfaces:
985	    if (mg_mgr_poll(&m_mongoose_mgr, 250) == 0)
986	      {
987	      ESP_LOGD(TAG, "MongooseTask: no interfaces available => exit");
988	      break;
989	      }
0x4013b815 is in MongooseRawTask(void*) (/home/balzer/esp/Open-Vehicle-Monitoring-System-3/vehicle/OVMS.V3/main/ovms_netmanager.cpp:966).
961	#ifdef CONFIG_OVMS_SC_GPL_MONGOOSE
962	
963	static void MongooseRawTask(void *pvParameters)
964	  {
965	  OvmsNetManager* me = (OvmsNetManager*)pvParameters;
966	  me->MongooseTask();
967	  }
968	
969	void OvmsNetManager::MongooseTask()
970	  {

[leres]

Here's another way to tickle the abort() at the end of esp_sha_read_digest_state():

• ssh into the module
• start a ping of the module in another window
• reboot the module ("module reset" in the ssh session)
• note when the ping returns stop
• when the ping returns resume, hit CR twice in the ssh session

The first CR will connection reset the ssh session.
The second CR will initiate a new ssh session.
Within a few seconds the module panics.

This crash is possible during the first 10 seconds or so after ping returns resume.

Here is a backtrace: backtrace.txt