Skip to content

Commit d213143

Browse files
jeggermontjeggermont
committed
Change index computation to explicitely use 64-bit so there are no overflow problems with large datasets.
Change index computation to explicitely use 64-bit so there are no overflow problems with large datasets.
1 parent b5a68be commit d213143

1 file changed

Lines changed: 61 additions & 25 deletions

File tree

‎src/Common/DataContainerInterface.cpp‎

Lines changed: 61 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
#include <cassert>
2222
#include <numeric>
2323
#include <algorithm>
24+
#include <cassert>
25+
#include <utility>
2426

2527
#if defined(_OPENMP)
2628
#include <omp.h>
@@ -30,6 +32,38 @@ using namespace mv;
3032

3133
namespace local
3234
{
35+
template<typename outType, typename inType>
36+
constexpr outType safe_numeric_cast(const inType& in) {
37+
38+
static_assert(std::numeric_limits<inType>::is_specialized &&
39+
std::numeric_limits<outType>::is_specialized);
40+
41+
if constexpr (!std::is_same_v<inType, outType>) {
42+
43+
// 1. Range Check for Integral Types
44+
if constexpr (std::is_integral_v<inType> && std::is_integral_v<outType>) {
45+
assert(std::cmp_greater_equal(in, std::numeric_limits<outType>::lowest()));
46+
assert(std::cmp_less_equal(in, std::numeric_limits<outType>::max()));
47+
}
48+
49+
// 2. Range Check for Floating Point (Inputs or Outputs)
50+
else {
51+
// We use lowest() here because min() on a float is the smallest POSITIVE number
52+
assert(in >= static_cast<inType>(std::numeric_limits<outType>::lowest()));
53+
assert(in <= static_cast<inType>(std::numeric_limits<outType>::max()));
54+
}
55+
56+
// 3. Fraction Check (Floating Point -> Integer)
57+
if constexpr (std::is_integral_v<outType> && std::is_floating_point_v<inType>) {
58+
// Using epsilon to allow for tiny rounding errors (e.g., 4.99999999999999)
59+
assert(std::abs(in - std::round(in)) < std::numeric_limits<inType>::epsilon());
60+
}
61+
}
62+
63+
return static_cast<outType>(in);
64+
}
65+
66+
3367
class Progress
3468
{
3569
mv::DataHierarchyItem& dataHierarcyItem;
@@ -66,14 +100,14 @@ namespace local
66100
static_assert(!std::is_same<T3, std::int64_t>::value, "");
67101
static_assert(!std::is_same<T3, std::uint64_t>::value, "");
68102

69-
long long lrows = ((long long)m_data->getNumPoints());
103+
std::int64_t lrows = local::safe_numeric_cast<std::int64_t>(m_data->getNumPoints());
70104
auto columns = m_data->getNumDimensions();
71105

72106
local::Progress progress(m_data->getDataHierarchyItem(), "Loading Data", lrows);
73107
m_data->visitFromBeginToEnd([&column_index, &row_offset, &data, transformType, lrows, columns, &progress](const auto beginOfData, const auto endOfData)
74108
{
75109
#pragma omp parallel for schedule(dynamic,1)
76-
for (long long row = 0; row < lrows; ++row)
110+
for (std::int64_t row = 0; row < lrows; ++row)
77111
{
78112
uint64_t start = row_offset[row];
79113
uint64_t end = row_offset[row + 1];
@@ -203,16 +237,17 @@ void DataContainerInterface::set_sparse_row_data(H5Utils::VectorHolder& column_i
203237

204238
void DataContainerInterface::increase_sparse_row_data(std::vector<uint64_t> &column_index, std::vector<uint32_t> &row_offset, std::vector<float> &data, TRANSFORM::Type transformType)
205239
{
206-
long long lrows = ((long long)m_data->getNumPoints());
240+
241+
std::int64_t lrows = local::safe_numeric_cast<std::int64_t>(m_data->getNumPoints());
207242
auto columns = m_data->getNumDimensions();
208243
local::Progress progress(m_data->getDataHierarchyItem(), "Loading Data", lrows);
209244
m_data->visitFromBeginToEnd([&column_index, &row_offset, &data, transformType, lrows, columns, &progress](const auto beginOfData, const auto endOfData)
210245
{
211246
#pragma omp parallel for
212-
for (long long row = 0; row < lrows; ++row)
247+
for (std::int64_t row = 0; row < lrows; ++row)
213248
{
214-
uint32_t start = row_offset[row];
215-
uint32_t end = row_offset[row + 1];
249+
uint64_t start = row_offset[row];
250+
uint64_t end = row_offset[row + 1];
216251
uint64_t points_offset = row * columns;
217252

218253
for (uint64_t i = start; i < end; ++i)
@@ -238,18 +273,18 @@ void DataContainerInterface::increase_sparse_row_data(std::vector<uint64_t> &col
238273

239274
void DataContainerInterface::set_sparse_column_data(std::vector<uint64_t> &row_index, std::vector<uint32_t> &column_offset, std::vector<float> &data, TRANSFORM::Type transformType /*= TRANSFORM::NONE*/)
240275
{
241-
auto columns = m_data->getNumDimensions();
276+
const std::int64_t columns = local::safe_numeric_cast<std::int64_t>(m_data->getNumDimensions());
242277
local::Progress progress(m_data->getDataHierarchyItem(), "Loading Data", columns);
243278
m_data->visitFromBeginToEnd([&column_offset, &row_index, &data, transformType, columns, &progress](const auto beginOfData, const auto endOfData)
244279
{
245280
#pragma omp parallel for
246-
for (long long column = 0; column < columns; ++column)
281+
for (std::int64_t column = 0; column < columns; ++column)
247282
{
248-
uint32_t start = column_offset[column];
249-
uint32_t end = column_offset[column + 1];
250-
for (uint32_t i = start; i < end; ++i)
283+
const auto start = column_offset[column];
284+
const auto end = column_offset[column + 1];
285+
for (auto i = start; i < end; ++i)
251286
{
252-
uint32_t r = row_index[i];
287+
uint64_t r = row_index[i];
253288
float value = data[i];
254289
if (value != 0)
255290
{
@@ -271,18 +306,18 @@ void DataContainerInterface::set_sparse_column_data(std::vector<uint64_t> &row_i
271306
void DataContainerInterface::increase_sparse_column_data(std::vector<uint64_t> &row_index, std::vector<uint32_t> &column_offset, std::vector<float> &data, TRANSFORM::Type transformType /*= TRANSFORM::NONE*/)
272307

273308
{
274-
auto columns = m_data->getNumDimensions();
309+
const std::int64_t columns = local::safe_numeric_cast<std::int64_t>(m_data->getNumDimensions());
275310
local::Progress progress(m_data->getDataHierarchyItem(), "Loading Data", columns);
276311
m_data->visitFromBeginToEnd([&row_index, &column_offset, &data, transformType, columns, &progress](const auto beginOfData, const auto endOfData)
277312
{
278313
#pragma omp parallel for
279-
for (long long column = 0; column < columns; ++column)
314+
for (std::int64_t column = 0; column < columns; ++column)
280315
{
281-
uint32_t start = column_offset[column];
282-
uint32_t end = column_offset[column + 1];
283-
for (uint32_t i = start; i < end; ++i)
316+
const auto start = column_offset[column];
317+
const auto end = column_offset[column + 1];
318+
for (auto i = start; i < end; ++i)
284319
{
285-
uint32_t r = row_index[i];
320+
uint64_t r = row_index[i];
286321
float value = data[i];
287322
if (value != 0)
288323
{
@@ -303,8 +338,8 @@ void DataContainerInterface::increase_sparse_column_data(std::vector<uint64_t> &
303338

304339
void DataContainerInterface::applyTransform(TRANSFORM::Type transformType, bool normalized_and_cpm)
305340
{
306-
const auto rows = m_data->getNumPoints();
307-
const auto columns = m_data->getNumDimensions();
341+
const std::int64_t rows = local::safe_numeric_cast<std::int64_t>(m_data->getNumPoints());
342+
const std::int64_t columns = local::safe_numeric_cast<std::int64_t>(m_data->getNumDimensions());
308343
if ((transformType.first == TRANSFORM::NONE) && !normalized_and_cpm)
309344
return;
310345
local::Progress progress(m_data->getDataHierarchyItem(), "Processing Data", rows);
@@ -345,7 +380,8 @@ mv::Dataset<Points> DataContainerInterface::points()
345380

346381
void DataContainerInterface::set(RowID row, ColumnID column, const ValueType & value)
347382
{
348-
m_data->setValueAt((row*m_data->getNumDimensions()) + column, value);
383+
std::uint64_t index = static_cast<std::uint64_t>(row) * m_data->getNumDimensions() + column;
384+
m_data->setValueAt(index, value);
349385
}
350386

351387
void DataContainerInterface::addRow(RowID row, const std::vector<uint32_t> &columns, const std::vector<float> &data, TRANSFORM::Type transformType)
@@ -354,7 +390,7 @@ void DataContainerInterface::addRow(RowID row, const std::vector<uint32_t> &colu
354390
if (dataSize != columns.size())
355391
throw std::out_of_range("DataContainerInterface::addRow vectors have different sizes!");
356392

357-
auto points_offset = row * m_data->getNumDimensions();
393+
const std::uint64_t points_offset = row* m_data->getNumDimensions();
358394
m_data->visitFromBeginToEnd([&columns, &data, transformType, points_offset](const auto beginOfData, const auto endOfData)
359395
{
360396
auto d = data.cbegin();
@@ -385,13 +421,13 @@ void DataContainerInterface::add(std::vector<uint32_t> *rows, std::vector<uint32
385421
assert(false);
386422

387423
}
388-
long long lrows = ((long long)m_rows);
424+
std::int64_t lrows = local::safe_numeric_cast<std::int64_t>(m_rows);
389425
#pragma omp parallel for
390-
for (long long row = 0; row < lrows; ++row)
426+
for (std::int64_t row = 0; row < lrows; ++row)
391427
{
392428
uint32_t start = (*rows)[row];
393429
uint32_t end = (*rows)[row + 1];
394-
std::uint64_t points_offset = row * m_data->getNumDimensions();
430+
const std::uint64_t points_offset = row * m_data->getNumDimensions();
395431

396432
m_data->visitFromBeginToEnd([columns, data, transformType, start, end, points_offset](const auto beginOfData, const auto endOfData)
397433
{

0 commit comments

Comments
 (0)