Fix integer overflows

Author: JukkaL

mypyc/lib-rt/vecs/vec_nested.c

@@ -329,6 +329,11 @@ VecNested VecNested_Extend(VecNested vec, PyObject *iterable) {
 VecNested VecNested_ExtendVec(VecNested dst, VecNested src) {
     if (src.len == 0)
         return dst;
+    if (src.len > PY_SSIZE_T_MAX - dst.len) {
+        PyErr_NoMemory();
+        VEC_DECREF(dst);
+        return vec_error();
+    }
     Py_ssize_t new_len = dst.len + src.len;
     // VecNested buf is never NULL (even for empty vecs), so no NULL guard needed
     Py_ssize_t cap = VEC_CAP(dst);
@@ -346,8 +351,13 @@ VecNested VecNested_ExtendVec(VecNested dst, VecNested src) {
     }
     // Need to reallocate (or dst and src share a buffer)
     Py_ssize_t new_cap = cap;
-    while (new_cap < new_len)
+    while (new_cap < new_len) {
+        if (new_cap > (PY_SSIZE_T_MAX - 1) / 2) {
+            new_cap = new_len;
+            break;
+        }
         new_cap = 2 * new_cap + 1;
+    }
     int aliased = dst.buf == src.buf;
     VecNested new = vec_alloc(new_cap, dst.buf->item_type, dst.buf->depth);
     if (VEC_IS_ERROR(new)) {

mypyc/lib-rt/vecs/vec_t.c

@@ -342,6 +342,11 @@ VecT VecT_Extend(VecT vec, PyObject *iterable, size_t item_type) {
 VecT VecT_ExtendVec(VecT dst, VecT src, size_t item_type) {
     if (src.len == 0)
         return dst;
+    if (src.len > PY_SSIZE_T_MAX - dst.len) {
+        PyErr_NoMemory();
+        VEC_DECREF(dst);
+        return vec_error();
+    }
     Py_ssize_t new_len = dst.len + src.len;
     if (dst.buf == NULL) {
         // dst is empty, allocate new buf
@@ -369,8 +374,13 @@ VecT VecT_ExtendVec(VecT dst, VecT src, size_t item_type) {
     }
     // Need to reallocate (or dst and src share a buffer)
     Py_ssize_t new_cap = cap;
-    while (new_cap < new_len)
+    while (new_cap < new_len) {
+        if (new_cap > (PY_SSIZE_T_MAX - 1) / 2) {
+            new_cap = new_len;
+            break;
+        }
         new_cap = 2 * new_cap + 1;
+    }
     int aliased = dst.buf == src.buf;
     VecT new = vec_alloc(new_cap, dst.buf->item_type);
     if (VEC_IS_ERROR(new)) {

mypyc/lib-rt/vecs/vec_template.c

@@ -372,6 +372,11 @@ VEC FUNC(Extend)(VEC vec, PyObject *iterable) {
 VEC FUNC(ExtendVec)(VEC dst, VEC src) {
     if (src.len == 0)
         return dst;
+    if (unlikely(src.len > PY_SSIZE_T_MAX - dst.len)) {
+        PyErr_NoMemory();
+        VEC_DECREF(dst);
+        return vec_error();
+    }
     Py_ssize_t new_len = dst.len + src.len;
     Py_ssize_t cap = dst.buf ? VEC_CAP(dst) : 0;
     if (new_len <= cap && dst.buf != src.buf) {
@@ -382,8 +387,13 @@ VEC FUNC(ExtendVec)(VEC dst, VEC src) {
     }
     // Need to reallocate (or dst and src share a buffer)
     Py_ssize_t new_cap = cap;
-    while (new_cap < new_len)
+    while (new_cap < new_len) {
+        if (unlikely(new_cap > (PY_SSIZE_T_MAX - 1) / 2)) {
+            new_cap = new_len;
+            break;
+        }
         new_cap = 2 * new_cap + 1;
+    }
     VEC new = vec_alloc(new_cap);
     if (VEC_IS_ERROR(new)) {
         VEC_DECREF(dst);