Skip to content

Commit df304ab

Browse files
tavisitesindril
tavisit
authored and
esindril
committed
MGM: Honour eos.excludefsid for reading access and consolidate the logic so
that it applies irrespective of the type of scheduler being used. Fixes EOS-5755
1 parent edac8a4 commit df304ab

6 files changed

Lines changed: 125 additions & 64 deletions

File tree

‎mgm/ofs/XrdMgmOfsFile.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2233,6 +2233,7 @@ XrdMgmOfsFile::open(eos::common::VirtualIdentity* invid,
22332233

22342234
std::vector<unsigned int> selectedfs;
22352235
std::vector<unsigned int> excludefs = GetExcludedFsids();
2236+
std::vector<uint32_t> excludefs_access(excludefs.begin(), excludefs.end());
22362237
std::vector<std::string> proxys;
22372238
std::vector<std::string> firewalleps;
22382239
// file systems which are unavailable during a read operation
@@ -2337,6 +2338,7 @@ XrdMgmOfsFile::open(eos::common::VirtualIdentity* invid,
23372338
acsargs.tried_cgi = &tried_cgi;
23382339
acsargs.unavailfs = &unavailfs;
23392340
acsargs.vid = &vid;
2341+
acsargs.exclude_filesystems = &excludefs_access;
23402342

23412343
if (!acsargs.isValid()) {
23422344
// there is something wrong in the arguments of file access

‎mgm/placement/FlatScheduler.cc‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -184,9 +184,6 @@ FlatScheduler::access(const ClusterData& cluster_data,
184184

185185
auto s_index = accessStategyIndex(args.strategy);
186186
mPlacementStrategy[s_index]->access(cluster_data, args);
187-
188-
189187
return 0;
190-
191188
}
192189
} // namespace eos::mgm::placement

‎mgm/placement/PlacementStrategy.hh‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -250,15 +250,19 @@ struct PlacementArguments {
250250
std::string_view geolocation;
251251
std::vector<uint32_t>* unavailfs {nullptr};
252252
const std::vector<uint32_t>& selectedfs;
253-
254-
AccessArguments(size_t& _selectedIndex, ino64_t _inode,
255-
PlacementStrategyT _strategy,
256-
std::string_view _geolocation,
257-
std::vector<uint32_t>* _unavailfs,
258-
const std::vector<uint32_t>& _selectedfs) :
259-
selectedIndex(_selectedIndex), inode(_inode),
260-
strategy(_strategy), geolocation(_geolocation),
261-
unavailfs(_unavailfs), selectedfs(_selectedfs)
253+
const std::vector<uint32_t>* excludefs{nullptr};
254+
255+
AccessArguments(size_t& _selectedIndex, ino64_t _inode, PlacementStrategyT _strategy,
256+
std::string_view _geolocation, std::vector<uint32_t>* _unavailfs,
257+
const std::vector<uint32_t>& _selectedfs,
258+
const std::vector<uint32_t>* _excludefs = nullptr)
259+
: selectedIndex(_selectedIndex)
260+
, inode(_inode)
261+
, strategy(_strategy)
262+
, geolocation(_geolocation)
263+
, unavailfs(_unavailfs)
264+
, selectedfs(_selectedfs)
265+
, excludefs(_excludefs)
262266
{
263267
}
264268

‎mgm/scheduler/Scheduler.cc‎

Lines changed: 61 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -314,14 +314,9 @@ int Scheduler::FlatSchedulerFileAccess(AccessArguments *args) {
314314
return EINVAL;
315315
}
316316

317-
placement::AccessArguments access_args {
318-
*args->fsindex,
319-
args->inode,
320-
strategy,
321-
args->vid->geolocation,
322-
args->unavailfs,
323-
*args->locationsfs
324-
};
317+
placement::AccessArguments access_args{*args->fsindex, args->inode,
318+
strategy, args->vid->geolocation,
319+
args->unavailfs, *args->locationsfs};
325320

326321
return gOFS->mFsScheduler->access(spaceName, access_args);
327322
}
@@ -352,51 +347,81 @@ int Scheduler::FileAccess(AccessArguments* args)
352347
return ENODATA;
353348
}
354349

350+
int rc = 0;
351+
355352
if (!FlatSchedulerFileAccess(args)) {
356353
eos_static_debug("msg=\"successfully accessed file via FlatScheduler\" index=%zu",
357354
*args->fsindex);
358-
return 0;
359355
} else {
360356
eos_static_info("%s", "msg=\"Failed access via FlatScheduler, falling back to geotree\"");
361-
}
357+
eos_static_debug("requesting file access from geolocation %s",
358+
args->vid->geolocation.c_str());
359+
GeoTreeEngine::SchedType st = toGeoTreeSchedtype(args->schedtype, args->isRW);
362360

363-
eos_static_debug("requesting file access from geolocation %s",
364-
args->vid->geolocation.c_str());
365-
GeoTreeEngine::SchedType st = toGeoTreeSchedtype(args->schedtype, args->isRW);
361+
// make sure we have the matching geo location before the not matching one
362+
if (!args->tried_cgi->empty()) {
363+
std::vector<std::string> hosts;
366364

365+
if (!gOFS->mGeoTreeEngine->getInfosFromFsIds(*args->locationsfs, 0, &hosts, 0)) {
366+
eos_static_debug("could not retrieve host for all the avoided fsids");
367+
}
367368

368-
// make sure we have the matching geo location before the not matching one
369-
if (!args->tried_cgi->empty()) {
370-
std::vector<std::string> hosts;
369+
size_t idx = 0;
371370

372-
if (!gOFS->mGeoTreeEngine->getInfosFromFsIds(*args->locationsfs, 0,
373-
&hosts, 0)) {
374-
eos_static_debug("could not retrieve host for all the avoided fsids");
371+
// we store unavailable filesystems in the unavail vector
372+
for (auto it = hosts.begin(); it != hosts.end(); it++) {
373+
if ((!it->empty()) && args->tried_cgi->find((*it) + ",") != std::string::npos) {
374+
// - this matters for RAID layouts because we have to remove there URLs
375+
// to let the RAID driver use only online stripes
376+
args->unavailfs->push_back((*args->locationsfs)[idx]);
377+
}
378+
379+
idx++;
380+
}
375381
}
376382

377-
size_t idx = 0;
383+
rc = gOFS->mGeoTreeEngine->accessHeadReplicaMultipleGroup(
384+
nReqStripes, *args->fsindex, *args->locationsfs, args->inode, args->dataproxys,
385+
args->firewallentpts, st, args->vid->geolocation, args->forcedfsid,
386+
args->unavailfs);
387+
}
378388

379-
// we store unavailable filesystems in the unavail vector
380-
for (auto it = hosts.begin(); it != hosts.end(); it++) {
381-
if ((!it->empty()) && args->tried_cgi->find((*it) + ",") != std::string::npos) {
382-
// - this matters for RAID layouts because we have to remove there URLs
383-
// to let the RAID driver use only online stripes
384-
args->unavailfs->push_back((*args->locationsfs)[idx]);
389+
// Honour the exclusion list: if the chosen file system is excluded, try to
390+
// find another suitable location that is neither excluded nor unavailable.
391+
// If no such location exists then the access request can not be satisfied.
392+
if (rc == 0 && args->exclude_filesystems && !args->exclude_filesystems->empty() &&
393+
*args->fsindex < args->locationsfs->size()) {
394+
auto chosen = (*args->locationsfs)[*args->fsindex];
395+
bool is_excluded =
396+
std::find(args->exclude_filesystems->begin(), args->exclude_filesystems->end(),
397+
chosen) != args->exclude_filesystems->end();
398+
if (is_excluded) {
399+
bool found = false;
400+
401+
for (size_t i = 0; i < args->locationsfs->size(); ++i) {
402+
auto fsid = (*args->locationsfs)[i];
403+
bool excluded = std::find(args->exclude_filesystems->begin(),
404+
args->exclude_filesystems->end(),
405+
fsid) != args->exclude_filesystems->end();
406+
bool unavail =
407+
args->unavailfs && std::find(args->unavailfs->begin(), args->unavailfs->end(),
408+
fsid) != args->unavailfs->end();
409+
if (!excluded && !unavail) {
410+
*args->fsindex = i;
411+
found = true;
412+
break;
413+
}
385414
}
386415

387-
idx++;
416+
if (!found) {
417+
eos_static_err("%s", "msg=\"no accessible file system location left after "
418+
"applying the eos.excludefsid exclusion list\"");
419+
rc = ENODATA;
420+
}
388421
}
389422
}
390423

391-
return gOFS->mGeoTreeEngine->accessHeadReplicaMultipleGroup(nReqStripes,
392-
*args->fsindex,
393-
*args->locationsfs,
394-
args->inode,
395-
args->dataproxys,
396-
args->firewallentpts,
397-
st,
398-
args->vid->geolocation,
399-
args->forcedfsid, args->unavailfs);
424+
return rc;
400425
}
401426

402427
void Scheduler::ReshuffleFs(std::vector<unsigned int> &selectedfs)

‎mgm/scheduler/Scheduler.hh‎

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -235,22 +235,25 @@ public:
235235
unsigned long* fsindex;
236236
//! return filesystems currently unavailable
237237
std::vector<unsigned int>* unavailfs;
238-
239-
AccessArguments() :
240-
forcedfsid(0),
241-
forcedspace(0),
242-
tried_cgi(),
243-
lid(0),
244-
inode(0),
245-
isRW(false),
246-
bookingsize(0),
247-
schedtype(regular),
248-
vid(nullptr),
249-
locationsfs(nullptr),
250-
dataproxys(nullptr),
251-
firewallentpts(nullptr),
252-
fsindex(nullptr),
253-
unavailfs(nullptr)
238+
//! filesystem ids to exclude from access scheduling (eos.excludefsid)
239+
const std::vector<uint32_t>* exclude_filesystems;
240+
241+
AccessArguments()
242+
: forcedfsid(0)
243+
, forcedspace(0)
244+
, tried_cgi()
245+
, lid(0)
246+
, inode(0)
247+
, isRW(false)
248+
, bookingsize(0)
249+
, schedtype(regular)
250+
, vid(nullptr)
251+
, locationsfs(nullptr)
252+
, dataproxys(nullptr)
253+
, firewallentpts(nullptr)
254+
, fsindex(nullptr)
255+
, unavailfs(nullptr)
256+
, exclude_filesystems(nullptr)
254257
{}
255258

256259
bool isValid() const

‎test/eos-instance-test‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2112,6 +2112,36 @@ runtest "### Check " unix SUCCESS "" eosj fileinfo "${ADJUST_TEST_FILE} | jq
21122112
runtest "### Rm " unix SUCCESS "" eos rm -F ${ADJUST_TEST_FILE}
21132113
runtest "### Rmdir " unix SUCCESS "" eos rmdir ${ADJUST_TEST_DIR}
21142114

2115+
# ------------------------------------------------------------------------------
2116+
categorie="excludefsid"
2117+
# ------------------------------------------------------------------------------
2118+
# Check that eos.excludefsid is honoured when scheduling read access
2119+
EXCL_TEST_DIR="/eos/$EOS_TEST_INSTANCE/test/instancetest/excludefsid"
2120+
EXCL_TEST_FILE="${EXCL_TEST_DIR}/file.dat"
2121+
runtest "### Mkdir " unix SUCCESS "" eos mkdir ${EXCL_TEST_DIR}
2122+
runtest "### Attr " unix SUCCESS "" eos attr set default=replica ${EXCL_TEST_DIR}
2123+
runtest "### Upload " unix SUCCESS "" eos cp ${TESTSYSFILE1K} ${EXCL_TEST_FILE}
2124+
# The file has two replicas - exclude the first one and check the other is used
2125+
excl_fsid0=$(command eos -j -b root://$EOS_TEST_REDIRECTOR fileinfo ${EXCL_TEST_FILE} | jq -r .locations[0].fsid)
2126+
excl_fsid1=$(command eos -j -b root://$EOS_TEST_REDIRECTOR fileinfo ${EXCL_TEST_FILE} | jq -r .locations[1].fsid)
2127+
# Download a few times so we don't end up on the surviving replica just by luck
2128+
for i in 1 2 3; do
2129+
download "$EOS_TEST_TESTSYS/dumpit" "${EXCL_TEST_FILE}?eos.excludefsid=${excl_fsid0}"
2130+
done
2131+
excl_used=0
2132+
while read -r line; do
2133+
# the selected file system is target[replicaindex]=(host,fsid)
2134+
idx=$(echo "$line" | grep -oP 'replicaindex=\K[0-9]+')
2135+
chosen_fsid=$(echo "$line" | grep -oP "target\[${idx}\]=\([^,]+,\K[0-9]+")
2136+
[ "${chosen_fsid}" = "${excl_fsid0}" ] && excl_used=1
2137+
done < <(grep "path=${EXCL_TEST_FILE} " /var/log/eos/mgm/xrdlog.mgm | grep replicaindex | tail -3)
2138+
runtest "### ExclFsid " unix SUCCESS "" shell test "${excl_used}" = "0"
2139+
# Excluding both replicas leaves no location available - access must fail
2140+
runtest "### ExclAll " unix ERROR "" download "$EOS_TEST_TESTSYS/dumpit" "${EXCL_TEST_FILE}?eos.excludefsid=${excl_fsid0},${excl_fsid1}"
2141+
runtest "### Rm " unix SUCCESS "" eos rm -F ${EXCL_TEST_FILE}
2142+
runtest "### Rmdir " unix SUCCESS "" eos rmdir ${EXCL_TEST_DIR}
2143+
rm -rf "$EOS_TEST_TESTSYS/dumpit"
2144+
21152145
# --------------------------
21162146
# Parallel sockets (does not work currently with 3.1.0 clients)
21172147
# --------------------------

0 commit comments

Comments
 (0)