Skip to content

Commit f1a9f40

Browse files
Qi ChenUbuntu
Qi Chen
authored and
Ubuntu
committed
fix headvector version
1 parent 4693c1e commit f1a9f40

1 file changed

Lines changed: 21 additions & 31 deletions

File tree

‎AnnService/inc/Core/SPANN/ExtraDynamicSearcher.h‎

Lines changed: 21 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -540,9 +540,9 @@ namespace SPTAG::SPANN {
540540

541541
SPANN::Index<ValueType>* GetHeadIndex() const { return m_headIndex; }
542542

543-
bool CheckIsNeedReassign(std::vector<std::shared_ptr<std::string>>& newHeadsVec, ValueType* data, std::shared_ptr<std::string> splitHeadVec, float_t headToSplitHeadDist, float_t currentHeadDist, bool isInSplitHead)
543+
bool CheckIsNeedReassign(std::vector<std::shared_ptr<std::string>>& newHeadsVec, ValueType* data, ValueType* splitHeadVec, float_t headToSplitHeadDist, float_t currentHeadDist, bool isInSplitHead)
544544
{
545-
float_t splitHeadDist = m_headIndex->ComputeDistance(data, splitHeadVec->data());
545+
float_t splitHeadDist = m_headIndex->ComputeDistance(data, splitHeadVec);
546546

547547
if (isInSplitHead) {
548548
if (splitHeadDist >= currentHeadDist) return false;
@@ -752,7 +752,6 @@ namespace SPTAG::SPANN {
752752
std::vector<SizeType> localIndices;
753753
localIndices.reserve(postVectorNum);
754754
uint8_t* vectorId = postingP;
755-
SizeType headj = -1;
756755
bool hasHead = false;
757756
for (SizeType j = 0; j < postVectorNum; j++, vectorId += m_vectorInfoSize)
758757
{
@@ -774,19 +773,18 @@ namespace SPTAG::SPANN {
774773
}
775774
}
776775

777-
if (VID == headID) {
778-
headj = j;
779-
headVec = std::make_shared<std::string>((char*)vectorId + m_metaDataSize, m_vectorDataSize);
780-
}
776+
if (VID == headID) headVec = std::make_shared<std::string>((char*)vectorId, m_vectorInfoSize);
781777
//if (VID >= m_versionMap->Count()) SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "DEBUG: vector ID:%d total size:%d\n", VID, m_versionMap->Count());
782778
if (m_versionMap->Deleted(VID) || m_versionMap->GetVersion(VID) != version) continue;
783779

784780
if (VID == headID) hasHead = true;
785781
localIndices.push_back(j);
786782
}
787-
if (headj < 0) {
783+
if (headVec == nullptr) {
788784
SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Split fail: cannot find head in posting! headID:%lld\n", (std::int64_t)headID);
789785
return ErrorCode::Fail;
786+
} else {
787+
*((uint8_t*)(headVec->data() + sizeof(SizeType))) = m_versionMap->GetVersion(headID);
790788
}
791789
// double gcEndTime = sw.getElapsedMs();
792790
// m_splitGcCost += gcEndTime;
@@ -800,8 +798,8 @@ namespace SPTAG::SPANN {
800798
memcpy(ptr, postingList.data() + localIndices[j] * m_vectorInfoSize, m_vectorInfoSize);
801799
}
802800
if (!hasHead) {
803-
Serialize(ptr, headID, m_versionMap->GetVersion(headID), headVec->data());
804-
localIndices.push_back(headj);
801+
memcpy(ptr, headVec->data(), m_vectorInfoSize);
802+
localIndices.push_back(0); // just to make sure head is included in posting, the index won't be used
805803
}
806804
postingList.resize(localIndices.size() * m_vectorInfoSize);
807805
if ((ret=PutPostingToDB(headID, postingList, MaxTimeout, &(p_exWorkSpace->m_diskRequests))) != ErrorCode::Success) {
@@ -842,7 +840,7 @@ namespace SPTAG::SPANN {
842840
memcpy(ptr, postingList.c_str() + localIndices[j] * m_vectorInfoSize, m_vectorInfoSize);
843841
if (*((SizeType*)(ptr)) == headID) hasHead = true;
844842
}
845-
if (!hasHead) memcpy(newpostingList.data(), postingList.c_str() + headj * m_vectorInfoSize, m_vectorInfoSize);
843+
if (!hasHead) memcpy(newpostingList.data(), headVec->data(), m_vectorInfoSize);
846844
SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Cluserting Failed (The same vector), Only Keep %d vectors.\n", cut);
847845

848846
if ((ret=PutPostingToDB(headID, newpostingList, MaxTimeout, &(p_exWorkSpace->m_diskRequests))) != ErrorCode::Success) {
@@ -857,7 +855,7 @@ namespace SPTAG::SPANN {
857855
}
858856

859857
std::vector<int> ks(2, 0);
860-
if (m_headIndex->ComputeDistance(args.centers, headVec->c_str()) < m_headIndex->ComputeDistance(args.centers + args._D, headVec->c_str())) {
858+
if (m_headIndex->ComputeDistance(args.centers, headVec->c_str() + m_metaDataSize) < m_headIndex->ComputeDistance(args.centers + args._D, headVec->c_str() + m_metaDataSize)) {
861859
ks[0] = 1;
862860
} else {
863861
ks[1] = 1;
@@ -875,14 +873,14 @@ namespace SPTAG::SPANN {
875873
for (int j = 0; j < args.counts[k]; j++, ptr += m_vectorInfoSize)
876874
{
877875
memcpy(ptr, postingList.c_str() + localIndices[first + j] * m_vectorInfoSize, m_vectorInfoSize);
878-
//Serialize(ptr, localIndicesInsert[localIndices[first + j]], localIndicesInsertVersion[localIndices[first + j]], smallSample[localIndices[first + j]]);
879876
}
880-
if (!theSameHead && headVec && m_headIndex->ComputeDistance(args.centers + k * args._D, headVec->c_str()) < Epsilon) {
877+
if (!theSameHead && m_headIndex->ComputeDistance(args.centers + k * args._D, headVec->c_str() + m_metaDataSize) < Epsilon) {
881878
newHeadsID[k] = headID;
882-
newHeadsVec[k] = headVec;
879+
newHeadsVec[k] = headVec->substr(m_metaDataSize, m_vectorDataSize);
883880
newHeadVID = headID;
884881
theSameHead = true;
885-
if (!hasHead && headj != -1) newPostingLists[k] += postingList.substr(headj * m_vectorInfoSize, m_vectorInfoSize);
882+
if (!hasHead) newPostingLists[k] += *headVec;
883+
886884
auto splitPutBegin = std::chrono::high_resolution_clock::now();
887885
if ((ret=PutPostingToDB(newHeadVID, newPostingLists[k], MaxTimeout, &(p_exWorkSpace->m_diskRequests))) != ErrorCode::Success) {
888886
SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Fail to override posting %lld\n", (std::int64_t)(newHeadVID));
@@ -1181,6 +1179,8 @@ namespace SPTAG::SPANN {
11811179
if (headVec == nullptr) {
11821180
SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "MergePostings fail: cannot find head vector in posting! headID:%lld\n", (std::int64_t)headID);
11831181
return ErrorCode::Fail;
1182+
} else {
1183+
*((uint8_t*)(headVec->data() + sizeof(SizeType))) = m_versionMap->GetVersion(headID);
11841184
}
11851185

11861186
if (currentLength > m_mergeThreshold)
@@ -1489,7 +1489,7 @@ namespace SPTAG::SPANN {
14891489
std::vector<std::string> &postingLists, std::vector<SizeType> &newHeadsID, std::vector<std::shared_ptr<std::string>> &newHeadsVec,
14901490
bool theSameHead)
14911491
{
1492-
auto headVector = reinterpret_cast<const ValueType*>(headVec->data());
1492+
auto headVector = reinterpret_cast<const ValueType*>(headVec->data() + m_metaDataSize);
14931493

14941494
// Collect vectors that need reassign, then do RNGSelection inline
14951495
// and batch Append by target head to reduce TiKV RPCs.
@@ -1528,20 +1528,10 @@ namespace SPTAG::SPANN {
15281528
}
15291529
};
15301530

1531-
if (m_opt->m_excludehead && !theSameHead)
1532-
{
1533-
if (!m_versionMap->Deleted(headID))
1534-
{
1535-
std::shared_ptr<std::string> vectorinfo = std::make_shared<std::string>(m_vectorInfoSize, ' ');
1536-
Serialize(vectorinfo->data(), headID, m_versionMap->GetVersion(headID), headVector);
1537-
// excludehead reassign: use the lambda with headPrev=-1
1538-
tryBatchReassign(reinterpret_cast<uint8_t*>(vectorinfo->data()), -1);
1539-
}
1540-
}
15411531
std::vector<float> newHeadsDist(2, 0.0f);
15421532
std::set<SizeType> reAssignVectorsTopK;
1543-
if (newHeadsVec[0]) newHeadsDist[0] = m_headIndex->ComputeDistance(headVec->data(), newHeadsVec[0]->data());
1544-
if (newHeadsVec[1]) newHeadsDist[1] = m_headIndex->ComputeDistance(headVec->data(), newHeadsVec[1]->data());
1533+
if (newHeadsVec[0]) newHeadsDist[0] = m_headIndex->ComputeDistance(headVector, newHeadsVec[0]->data());
1534+
if (newHeadsVec[1]) newHeadsDist[1] = m_headIndex->ComputeDistance(headVector, newHeadsVec[1]->data());
15451535
for (int i = 0; i < postingLists.size(); i++) {
15461536
if (!newHeadsVec[i]) continue;
15471537
auto& postingList = postingLists[i];
@@ -1562,7 +1552,7 @@ namespace SPTAG::SPANN {
15621552
if (reAssignVectorsTopK.find(vid) == reAssignVectorsTopK.end() && !m_versionMap->Deleted(vid) && m_versionMap->GetVersion(vid) == version) {
15631553
m_stat.m_reAssignScanNum++;
15641554
float dist = m_headIndex->ComputeDistance(newHeadsVec[i]->data(), vector);
1565-
if (CheckIsNeedReassign(newHeadsVec, vector, headVec, newHeadsDist[i], dist, true)) {
1555+
if (CheckIsNeedReassign(newHeadsVec, vector, headVector, newHeadsDist[i], dist, true)) {
15661556
tryBatchReassign(vectorId, newHeadsID[i]);
15671557
reAssignVectorsTopK.insert(vid);
15681558
}
@@ -1658,7 +1648,7 @@ namespace SPTAG::SPANN {
16581648
if (reAssignVectorsTopK.find(vid) == reAssignVectorsTopK.end() && !m_versionMap->Deleted(vid) && m_versionMap->GetVersion(vid) == version) {
16591649
m_stat.m_reAssignScanNum++;
16601650
float dist = m_headIndex->ComputeDistance(HeadPrevTopKVec[i]->data(), vector);
1661-
if (CheckIsNeedReassign(newHeadsVec, vector, headVec, newHeadsDist[i], dist, false)) {
1651+
if (CheckIsNeedReassign(newHeadsVec, vector, headVector, newHeadsDist[i], dist, false)) {
16621652
tryBatchReassign(vectorId, HeadPrevTopK[i]);
16631653
reAssignVectorsTopK.insert(vid);
16641654
}

0 commit comments

Comments
 (0)