Prevent sanitizer warnings for lj_tab_new*() and table.new().

Reported by Sergey Bronnikov.

(cherry picked from commit 8f421c8)

The Undefined Behaviour Sanitizer [1] produces a warning because
the function `lua_createtable()` takes signed integer arguments,
but the `lj_tab_new_ah()` was not properly validating or converting
these signed values before using them in unsigned arithmetic.

The fix changes the signature of `lj_tab_new_ah()` to accept
uint32_t directly, and adjusts `lua_createtable()` to cast the
incoming signed int values to uint32_t before passing them.

[1]: https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html

Sergey Bronnikov:
* added the description and the test for the problem

Part of tarantool/tarantool#12480

Reviewed-by: Evgeniy Temirgaleev <e.temirgaleev@tarantool.org>
Reviewed-by: Sergey Kaplun <skaplun@tarantool.org>
Signed-off-by: Sergey Kaplun <skaplun@tarantool.org>

Author: Mike Pall

src/lj_api.c

@@ -746,7 +746,7 @@ LUA_API void lua_pushlightuserdata(lua_State *L, void *p)
 LUA_API void lua_createtable(lua_State *L, int narray, int nrec)
 {
   lj_gc_check(L);
-  settabV(L, L->top, lj_tab_new_ah(L, narray, nrec));
+  settabV(L, L->top, lj_tab_new_ah(L, (uint32_t)narray, (uint32_t)nrec));
   incr_top(L);
 }
 

src/lj_tab.c

@@ -165,9 +165,9 @@ GCtab *lj_tab_new(lua_State *L, uint32_t asize, uint32_t hbits)
 }
 
 /* The API of this function conforms to lua_createtable(). */
-GCtab *lj_tab_new_ah(lua_State *L, int32_t a, int32_t h)
+GCtab *lj_tab_new_ah(lua_State *L, uint32_t a, uint32_t h)
 {
-  return lj_tab_new(L, (uint32_t)(a > 0 ? a+1 : 0), hsize2hbits(h));
+  return lj_tab_new(L, a ? a+1 : 0, hsize2hbits(h));
 }
 
 #if LJ_HASJIT

src/lj_tab.h

@@ -34,7 +34,7 @@ static LJ_AINLINE uint32_t hashrot(uint32_t lo, uint32_t hi)
 #define hsize2hbits(s)	((s) ? ((s)==1 ? 1 : 1+lj_fls((uint32_t)((s)-1))) : 0)
 
 LJ_FUNCA GCtab *lj_tab_new(lua_State *L, uint32_t asize, uint32_t hbits);
-LJ_FUNC GCtab *lj_tab_new_ah(lua_State *L, int32_t a, int32_t h);
+LJ_FUNC GCtab *lj_tab_new_ah(lua_State *L, uint32_t a, uint32_t h);
 #if LJ_HASJIT
 LJ_FUNC GCtab * LJ_FASTCALL lj_tab_new1(lua_State *L, uint32_t ahsize);
 #endif

test/tarantool-tests/lj-1458-ub-table-new.test.lua

@@ -0,0 +1,30 @@
+local tap = require('tap')
+
+-- The test file to demonstrate UBSan warning for `table.new()`
+-- with a minimal and maximum array and hash parts values.
+-- See also: https://github.com/LuaJIT/LuaJIT/issues/1458.
+local test = tap.test('lj-1458-ub-table-new')
+
+local INT_MAX = 2 ^ 31 - 1
+local INT_MIN = -2 ^ 31
+
+local table_sizes = {
+  { 0, INT_MIN },
+  { 0, INT_MAX },
+  { INT_MIN, 0 },
+  { INT_MAX, 0 },
+}
+
+test:plan(#table_sizes * 2)
+
+local table_new = require('table.new')
+
+for _, case in ipairs(table_sizes) do
+  local apart, hpart = unpack(case)
+  local ok, err = pcall(table_new, apart, hpart)
+  local message = ('table.new(%d, %d)'):format(apart, hpart)
+  test:is(ok, false, message .. ' is ok')
+  test:ok(err:match('table overflow'), message .. ' correct error message')
+end
+
+test:done(true)