Skip to content

Commit d51dc04

Browse files
ghallidayghalliday
committed
Merge remote-tracking branch 'origin/candidate-10.0.x' into candidate-10.2.x
Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com> # Conflicts: # helm/hpcc/Chart.yaml # helm/hpcc/templates/_helpers.tpl # version.cmake
2 parents b419dac + 69eea87 commit d51dc04

10 files changed

Lines changed: 396 additions & 23 deletions

File tree

‎common/remote/hooks/azure/azureapiutils.cpp‎

Lines changed: 38 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,14 +34,45 @@ using namespace std::chrono;
3434

3535
bool areManagedIdentitiesEnabled()
3636
{
37-
// Check for Azure AD Workload Identity or legacy managed identity
38-
static bool hasWorkloadIdentity = std::getenv("AZURE_CLIENT_ID") &&
39-
std::getenv("AZURE_TENANT_ID") &&
40-
std::getenv("AZURE_FEDERATED_TOKEN_FILE");
41-
42-
static bool hasManagedIdentity = std::getenv("MSI_ENDPOINT") || std::getenv("IDENTITY_ENDPOINT");
37+
static bool hasIMDS = []() {
38+
// Check for Azure AD Workload Identity or legacy managed identity
39+
if (std::getenv("AZURE_CLIENT_ID") && std::getenv("AZURE_TENANT_ID") && std::getenv("AZURE_FEDERATED_TOKEN_FILE"))
40+
return true;
41+
42+
// Check for App Service / Container Instances managed identity
43+
if (std::getenv("MSI_ENDPOINT") || std::getenv("IDENTITY_ENDPOINT"))
44+
return true;
45+
46+
// Check for Azure VM managed identity via IMDS (Instance Metadata Service) probe (once).
47+
// 169.254.169.254 is a non-routable link-local address that Azure uses to expose the
48+
// IMDS to VMs running on its platform. It is only reachable from within an Azure VM.
49+
// See: https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/how-to-use-vm-token#get-a-token-using-http
50+
try
51+
{
52+
Azure::Core::Http::Request request(Azure::Core::Http::HttpMethod::Get, Azure::Core::Url("http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://management.azure.com/"));
53+
request.SetHeader("Metadata", "true");
54+
55+
auto transport = std::make_shared<Azure::Core::Http::CurlTransport>();
56+
Azure::Core::Context context;
57+
// Use a short timeout to avoid blocking startup if not on Azure
58+
context = context.WithDeadline(std::chrono::system_clock::now() + std::chrono::seconds(2));
59+
60+
auto response = transport->Send(request, context);
61+
auto statusCode = response->GetStatusCode();
62+
// 200 = identity assigned and working
63+
// 400 = IMDS reachable but identity may not be properly configured
64+
if (statusCode == Azure::Core::Http::HttpStatusCode::BadRequest)
65+
OWARNLOG("Azure IMDS probe returned 400 (Bad Request) — managed identity endpoint is reachable but may not be properly configured");
66+
return statusCode == Azure::Core::Http::HttpStatusCode::Ok;
67+
}
68+
catch (...)
69+
{
70+
// Connection refused, timeout, not on Azure — IMDS not available
71+
return false;
72+
}
73+
}();
4374

44-
return hasWorkloadIdentity || hasManagedIdentity;
75+
return hasIMDS;
4576
}
4677

4778
std::shared_ptr<Azure::Storage::StorageSharedKeyCredential> getAzureSharedKeyCredential(const char * accountName, const char * secretName)

‎dali/base/dadfs.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5259,8 +5259,9 @@ protected: friend class CDistributedFilePart;
52595259
StringBuffer newfn;
52605260
newrfn.getRemotePath(newfn);
52615261
Owned<IFile> f = createIFile(oldrfn);
5262+
Owned<IFile> destFile = createIFile(newrfn);
52625263
if (!isrep||f->exists()) { // ignore non-existant replicates
5263-
f->move(newfn.str());
5264+
f->move(destFile->queryFilename());
52645265
PROGLOG("Succeeded rename %s to %s",oldfn.str(),newfn.str());
52655266
}
52665267
done = true;

‎dali/base/dadiags.cpp‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,9 +195,18 @@ class CDaliDiagnosticsServer: public IDaliServer, public Thread
195195
if (success)
196196
mb.append(connectionInfo);
197197
}
198-
else if (0 == stricmp(id, "save")) {
198+
else if (0 == stricmp(id, "save"))
199+
{
199200
PROGLOG("Dalidiag requests SDS save");
200-
querySDSServer().saveRequest();
201+
try
202+
{
203+
querySDSServer().saveRequest();
204+
}
205+
catch (IException *e)
206+
{
207+
serializeException(e, mb);
208+
e->Release();
209+
}
201210
}
202211
else if (0 == stricmp(id, "settracetransactions")) {
203212
PROGLOG("Dalidiag requests Trace Transactions");

‎dali/base/dasds.cpp‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2108,7 +2108,7 @@ class CCovenSDSManager : public CSDSManagerBase, implements ISDSManagerServer, i
21082108
CheckedCriticalSection treeRegCrit;
21092109
Owned<Thread> unhandledThread;
21102110
unsigned writeTransactions;
2111-
bool ignoreExternals;
2111+
std::atomic<bool> ignoreExternals;
21122112
StringAttr dataPath;
21132113
StringAttr daliName;
21142114
Owned<IPropertyTree> properties;
@@ -6832,11 +6832,6 @@ void CCovenSDSManager::loadStore(const bool *abort)
68326832

68336833
void CCovenSDSManager::saveStore(const char *storeName, SaveStoreFlags flags)
68346834
{
6835-
struct CIgnore
6836-
{
6837-
CIgnore() { SDSManager->ignoreExternals=true; }
6838-
~CIgnore() { SDSManager->ignoreExternals=false; }
6839-
} ignore;
68406835
if (hasMask(flags, ssf_stop))
68416836
{
68426837
// Dali is stopping, we don't care about any inflight transactions any more, we're about to save and quit
@@ -6848,6 +6843,14 @@ void CCovenSDSManager::saveStore(const char *storeName, SaveStoreFlags flags)
68486843
// lock blockedSaveCrit after flush, since deltaWriter itself blocks on blockedSaveCrit
68496844
deltaWriter.flush(); // transactions will be blocked at this stage, no new deltas will be added to writer
68506845
}
6846+
6847+
if (ignoreExternals.exchange(true)) // NB: This is set during save to prevent serialization of externals during the save
6848+
{
6849+
Owned<IException> exception = makeStringException(0, "Save already in progress");
6850+
EXCLOG(exception);
6851+
throw exception.getClear();
6852+
}
6853+
COnScopeExit resetIgnoreExternals([&]() { ignoreExternals = false; });
68516854
CHECKEDCRITICALBLOCK(blockedSaveCrit, fakeCritTimeout);
68526855
iStoreHelper->saveStore(root, NULL);
68536856
unsigned initNodeTableSize = allNodes.maxElements()+OVERFLOWSIZE;

‎dali/base/dasds.ipp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -428,7 +428,10 @@ public:
428428
int errorCode() const { return errCode; }
429429
StringBuffer &errorMessage(StringBuffer &out) const
430430
{
431-
return translateCode(out).append("\n").append(errMsg.str());
431+
translateCode(out);
432+
if (errMsg.length())
433+
out.append(" - ").append(errMsg.str());
434+
return out;
432435
}
433436
MessageAudience errorAudience() const { return MSGAUD_user; }
434437

‎dali/dalidiag/dalidiag.cpp‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -640,7 +640,16 @@ int main(int _argc, char* argv[])
640640
MemoryBuffer mb;
641641
mb.append("save");
642642
getDaliDiagnosticValue(mb);
643-
PROGLOG("SDS store saved");
643+
Owned<IException> exception;
644+
if (mb.length())
645+
exception.setown(deserializeException(mb));
646+
if (exception)
647+
{
648+
StringBuffer errMsg;
649+
PROGLOG("Dali SDS save failed: [%d, %s]", exception->errorCode(), exception->errorMessage(errMsg).str());
650+
}
651+
else
652+
PROGLOG("SDS store saved");
644653
break;
645654
}
646655
if (0 == stricmp(arg, "locks")) {

‎testing/regress/ecl/indexfilepos.ecl‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
/*##############################################################################
2+
3+
HPCC SYSTEMS software Copyright (C) 2025 HPCC Systems®.
4+
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
############################################################################## */
17+
18+
//Test the different variants of fileposition fields
19+
// - fixed and variable with records
20+
// - 3 different index types
21+
// - 2 different ways of defining the index (from dataset or from record)
22+
23+
//version compressionType='legacy',variableWidth=false,zeroFilePos=false,hasFilePos=false,fromDataset=false
24+
//version compressionType='legacy',variableWidth=false,zeroFilePos=false,hasFilePos=true,fromDataset=false
25+
//version compressionType='legacy',variableWidth=false,zeroFilePos=true,hasFilePos=true,fromDataset=false
26+
//version compressionType='legacy',variableWidth=true,zeroFilePos=false,hasFilePos=false,fromDataset=false
27+
//version compressionType='legacy',variableWidth=true,zeroFilePos=false,hasFilePos=true,fromDataset=false
28+
//version compressionType='legacy',variableWidth=true,zeroFilePos=true,hasFilePos=true,fromDataset=false
29+
30+
//version compressionType='legacy',variableWidth=false,zeroFilePos=false,hasFilePos=false,fromDataset=true
31+
//version compressionType='legacy',variableWidth=false,zeroFilePos=false,hasFilePos=true,fromDataset=true
32+
//version compressionType='legacy',variableWidth=false,zeroFilePos=true,hasFilePos=true,fromDataset=true
33+
//version compressionType='legacy',variableWidth=true,zeroFilePos=false,hasFilePos=false,fromDataset=true
34+
//version compressionType='legacy',variableWidth=true,zeroFilePos=false,hasFilePos=true,fromDataset=true
35+
//version compressionType='legacy',variableWidth=true,zeroFilePos=true,hasFilePos=true,fromDataset=true
36+
37+
//version compressionType='inplace',variableWidth=false,zeroFilePos=false,hasFilePos=false,fromDataset=false
38+
//version compressionType='inplace',variableWidth=false,zeroFilePos=false,hasFilePos=true,fromDataset=false
39+
//version compressionType='inplace',variableWidth=false,zeroFilePos=true,hasFilePos=true,fromDataset=false
40+
//version compressionType='inplace',variableWidth=true,zeroFilePos=false,hasFilePos=false,fromDataset=false
41+
//version compressionType='inplace',variableWidth=true,zeroFilePos=false,hasFilePos=true,fromDataset=false
42+
//version compressionType='inplace',variableWidth=true,zeroFilePos=true,hasFilePos=true,fromDataset=false
43+
44+
//version compressionType='inplace',variableWidth=false,zeroFilePos=false,hasFilePos=false,fromDataset=true
45+
//version compressionType='inplace',variableWidth=false,zeroFilePos=false,hasFilePos=true,fromDataset=true
46+
//version compressionType='inplace',variableWidth=false,zeroFilePos=true,hasFilePos=true,fromDataset=true
47+
//version compressionType='inplace',variableWidth=true,zeroFilePos=false,hasFilePos=false,fromDataset=true
48+
//version compressionType='inplace',variableWidth=true,zeroFilePos=false,hasFilePos=true,fromDataset=true
49+
//version compressionType='inplace',variableWidth=true,zeroFilePos=true,hasFilePos=true,fromDataset=true
50+
51+
//version compressionType='hybrid',variableWidth=false,zeroFilePos=false,hasFilePos=false,fromDataset=false
52+
//version compressionType='hybrid',variableWidth=false,zeroFilePos=false,hasFilePos=true,fromDataset=false
53+
//version compressionType='hybrid',variableWidth=false,zeroFilePos=true,hasFilePos=true,fromDataset=false
54+
//version compressionType='hybrid',variableWidth=true,zeroFilePos=false,hasFilePos=false,fromDataset=false
55+
//version compressionType='hybrid',variableWidth=true,zeroFilePos=false,hasFilePos=true,fromDataset=false
56+
//version compressionType='hybrid',variableWidth=true,zeroFilePos=true,hasFilePos=true,fromDataset=false
57+
58+
//version compressionType='hybrid',variableWidth=false,zeroFilePos=false,hasFilePos=false,fromDataset=true
59+
//version compressionType='hybrid',variableWidth=false,zeroFilePos=false,hasFilePos=true,fromDataset=true
60+
//version compressionType='hybrid',variableWidth=false,zeroFilePos=true,hasFilePos=true,fromDataset=true
61+
//version compressionType='hybrid',variableWidth=true,zeroFilePos=false,hasFilePos=false,fromDataset=true
62+
//version compressionType='hybrid',variableWidth=true,zeroFilePos=false,hasFilePos=true,fromDataset=true
63+
//version compressionType='hybrid',variableWidth=true,zeroFilePos=true,hasFilePos=true,fromDataset=true
64+
65+
import ^ as root;
66+
import $.setup;
67+
import Std.File AS FileServices;
68+
69+
compressionType := #IFDEFINED(root.compressionType, 'hybrid');
70+
variableWidth := #IFDEFINED(root.variableWidth, true);
71+
zeroFilePos := #IFDEFINED(root.zeroFilePos, true);
72+
hasFilePos := #IFDEFINED(root.hasFilePos, true);
73+
fromDataset := #IFDEFINED(root.fromDataset, true);
74+
75+
idxRecord := RECORD
76+
unsigned uid
77+
=>
78+
unsigned uid2;
79+
#if (variableWidth)
80+
string pad := '<pad>';
81+
#end
82+
#if (not zeroFilePos)
83+
unsigned8 uid3;
84+
#end
85+
END;
86+
87+
numParents := 50;
88+
89+
ds := DATASET(numParents,
90+
TRANSFORM(idxRecord,
91+
SELF.uid := COUNTER;
92+
SELF.uid2 := HASH32(COUNTER);
93+
#if (variableWidth)
94+
SELF.pad := 'Arfle Barfle Gloop!!'[1..COUNTER%20];
95+
#end
96+
#if (not zeroFilePos)
97+
SELF.uid3 := COUNTER;
98+
#end
99+
), DISTRIBUTED);
100+
101+
102+
prefix := setup.Files(false, false).indexPrefix + WORKUNIT;
103+
indexName := prefix+'_fpos_index';
104+
105+
#if (fromDataset)
106+
i := INDEX(ds, { uid }, {ds}, indexName, compressed(compressionType),FILEPOSITION(hasFilePos));
107+
#else
108+
i := INDEX(idxRecord, indexName, compressed(compressionType),FILEPOSITION(hasFilePos));
109+
#end
110+
111+
// NOTE: use recordof(i) in the following to ensure that any implicit file position is included
112+
lhs := PROJECT(SORT(ds, uid2), TRANSFORM(recordof(i), SELF := LEFT));
113+
114+
matchRecord := RECORD
115+
recordof(i) l;
116+
recordof(i) r;
117+
END;
118+
119+
j := JOIN(lhs, i, LEFT.uid=RIGHT.uid, TRANSFORM(matchRecord, SELF.l := LEFT; SELF.r := RIGHT), KEEP(1));
120+
121+
SEQUENTIAL(
122+
BUILD(i, ds, OVERWRITE);
123+
OUTPUT(j(l != r));
124+
125+
// Clean-up
126+
FileServices.DeleteLogicalFile(indexName)
127+
);

‎testing/regress/ecl/key/indexfilepos.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
<Dataset name='Result 1'>
2+
</Dataset>
3+
<Dataset name='Result 2'>
4+
</Dataset>

0 commit comments

Comments
 (0)