Fix heap overflow on computing parametrizations in non-shape-position case

wegank · wegank · commit 0128fa0c409b · 2025-12-15T04:36:06.000+01:00
diff --git a/src/fglm/fglm_core.c b/src/fglm/fglm_core.c
@@ -1282,7 +1282,7 @@ int compute_parametrizations_non_shape_position_case(param_t *param,
         }
         else{
           nmod_poly_fit_length(param->coords[nvars-2-nc],
-                               param->elim->length-1 );
+                               FLINT_MAX(2, param->elim->length-1) );
           param->coords[nvars-2-nc]->length = data_bms->BMS->R1->length ;
           param->coords[nvars-2-nc]->coeffs[0] = 0;
           param->coords[nvars-2-nc]->coeffs[1] = 0;

Original file line number	Diff line number	Diff line change
`@@ -1282,7 +1282,7 @@ int compute_parametrizations_non_shape_position_case(param_t *param,`
`1282`	`1282`	`}`
`1283`	`1283`	`else{`
`1284`	`1284`	`nmod_poly_fit_length(param->coords[nvars-2-nc],`
`1285`		`- param->elim->length-1 );`
	`1285`	`+ FLINT_MAX(2, param->elim->length-1) );`
`1286`	`1286`	`param->coords[nvars-2-nc]->length = data_bms->BMS->R1->length ;`
`1287`	`1287`	`param->coords[nvars-2-nc]->coeffs[0] = 0;`
`1288`	`1288`	`param->coords[nvars-2-nc]->coeffs[1] = 0;`