Skip to content

Commit 03aecf4

Browse files
kofemannkofemann
committed
drv: add possibility to use direct-IO on flush
Motivation: When file is flushed to tape, we don't expect it to be re-read again. However, during flushing the filesystem cache in invalidated, thus has impact on client that perform random IO. Modification: Add option `-use-dio` to enable direct-i/o on flush
1 parent d445ade commit 03aecf4

6 files changed

Lines changed: 46 additions & 20 deletions

File tree

‎README.md‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -96,20 +96,20 @@ queue define class -expire=0 -pending=0 -total=0 -open <hsmType> *
9696

9797
### The available configuration options:
9898

99-
| Name | Description | required | default |
100-
|:---------------------|:---------------------------------------------------------------|---------:|------------|
101-
| cta-instance-name | The dCache instance name configured in CTA | yes | - |
102-
| cta-frontend-addr | A comma separated list of CTA `cta-dcache` endpoints | yes | - |
103-
| cta-user | The dCache instance associated user in CTA | yes | - |
104-
| cta-group | The dCache instance associated group in CTA | yes | - |
105-
| cta-ca-chain | The path to CA root chain for use with TLS | no | - |
106-
| cta-use-tls | A switch (true/false) to enable TLS for CTA control connection | no | `false` |
107-
| cta-frontend-timeout | How log dCache waits in seconds for CTA frontend to reply | no | 30 |
108-
| io-endpoint | The hostname or IP offered by dCache for IO by CTA | no | `hostname` |
109-
| io-port | The TCP port offered by dCache for IO by CTA | no | - |
99+
| Name | Description | required | default |
100+
|:-------------------------|:---------------------------------------------------------------|---------:|------------|
101+
| cta-instance-name | The dCache instance name configured in CTA | yes | - |
102+
| cta-frontend-addr | A comma separated list of CTA `cta-dcache` endpoints | yes | - |
103+
| cta-user | The dCache instance associated user in CTA | yes | - |
104+
| cta-group | The dCache instance associated group in CTA | yes | - |
105+
| cta-ca-chain | The path to CA root chain for use with TLS | no | - |
106+
| cta-use-tls | A switch (true/false) to enable TLS for CTA control connection | no | `false` |
107+
| cta-frontend-timeout | How log dCache waits in seconds for CTA frontend to reply | no | 30 |
108+
| io-endpoint | The hostname or IP offered by dCache for IO by CTA | no | `hostname` |
109+
| io-port | The TCP port offered by dCache for IO by CTA | no | - |
110+
| use-dio | Use Direct-I/O | no | `false` |
110111
| restore-success-on-close | **obsolete** | - | - |
111112

112-
113113
### Load balancing and failover
114114

115115
If multiple `cta-frontend-addr` are provided, the driver will try to connect to all endpoints and use `round-robin`

‎src/main/java/org/dcache/nearline/cta/CtaNearlineStorage.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ public class CtaNearlineStorage implements NearlineStorage {
5959
public static final String IO_ENDPOINT = "io-endpoint";
6060
public static final String IO_PORT = "io-port";
6161
public static final String CTA_REQUEST_TIMEOUT = "cta-frontend-timeout";
62+
public static final String DIO = "use-dio";
6263

6364
protected final String type;
6465
protected final String name;
@@ -129,6 +130,11 @@ public class CtaNearlineStorage implements NearlineStorage {
129130
*/
130131
private int ctaRequestTimeoutInSec = 30;
131132

133+
/**
134+
* Use direct IO for data mover.
135+
*/
136+
private boolean dio;
137+
132138
public CtaNearlineStorage(String type, String name) {
133139

134140
Objects.requireNonNull(type, "HSM type is not provided");
@@ -446,12 +452,14 @@ public void configure(Map<String, String> properties) throws IllegalArgumentExce
446452
if (timeoutString != null) {
447453
ctaRequestTimeoutInSec = Integer.parseInt(timeoutString);
448454
}
455+
456+
dio = Boolean.parseBoolean(properties.getOrDefault(DIO, "false"));
449457
}
450458

451459
@Override
452460
public void start() {
453461

454-
dataMover = new DataMover(type, name, ioSocketAddress, pendingRequests);
462+
dataMover = new DataMover(type, name, ioSocketAddress, pendingRequests, dio);
455463
dataMover.startAsync().awaitRunning();
456464

457465
ChannelCredentials credentials;

‎src/main/java/org/dcache/nearline/cta/xrootd/DataMover.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,11 +69,17 @@ public class DataMover extends AbstractIdleService implements CtaTransportProvid
6969

7070
private volatile String url;
7171

72+
/**
73+
* Use direct IO for data mover.
74+
*/
75+
private boolean dio;
76+
7277
public DataMover(String type, String name, InetSocketAddress sa,
73-
ConcurrentMap<String, PendingRequest> pendingRequests) {
78+
ConcurrentMap<String, PendingRequest> pendingRequests, boolean useDio) {
7479

7580
hsmType = type;
7681
hsmName = name;
82+
dio = useDio;
7783

7884
try {
7985
this.pendingRequests = pendingRequests;
@@ -181,7 +187,7 @@ protected void initChannel(Channel ch) throws Exception {
181187
ServerProtocolFlags flags = new ServerProtocolFlags(0);
182188
TLSSessionInfo tlsSessionInfo = new TLSSessionInfo(flags);
183189

184-
DataServerHandler dataServer = new DataServerHandler(hsmType, hsmName, pendingRequests);
190+
DataServerHandler dataServer = new DataServerHandler(hsmType, hsmName, pendingRequests, dio);
185191
dataServer.setSigningPolicy(signingPolicy);
186192
dataServer.setTlsSessionInfo(tlsSessionInfo);
187193

‎src/main/java/org/dcache/nearline/cta/xrootd/DataServerHandler.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import static org.dcache.xrootd.protocol.XrootdProtocol.kXR_xset;
3030

3131
import com.google.common.net.InetAddresses;
32+
import com.sun.nio.file.ExtendedOpenOption;
3233
import diskCacheV111.util.Adler32;
3334
import diskCacheV111.util.CacheException;
3435
import io.netty.channel.ChannelHandlerContext;
@@ -41,12 +42,14 @@
4142
import java.nio.channels.ClosedChannelException;
4243
import java.nio.channels.FileChannel;
4344
import java.nio.charset.StandardCharsets;
45+
import java.nio.file.OpenOption;
4446
import java.nio.file.StandardOpenOption;
4547
import java.time.Duration;
4648
import java.time.Instant;
4749
import java.util.ArrayList;
4850
import java.util.Base64;
4951
import java.util.EnumSet;
52+
import java.util.HashSet;
5053
import java.util.List;
5154
import java.util.Set;
5255
import java.util.concurrent.ConcurrentMap;
@@ -168,12 +171,18 @@ public String toString() {
168171
*/
169172
private final String hsmType;
170173

174+
/**
175+
* Use direct IO for data mover.
176+
*/
177+
private final boolean dio;
178+
171179
public DataServerHandler(String type, String name,
172-
ConcurrentMap<String, PendingRequest> pendingRequests) {
180+
ConcurrentMap<String, PendingRequest> pendingRequests, boolean useDio) {
173181

174182
hsmType = type;
175183
hsmName = name;
176184
this.pendingRequests = pendingRequests;
185+
dio = useDio;
177186
}
178187

179188
@Override
@@ -236,7 +245,7 @@ protected OpenResponse doOnOpen(ChannelHandlerContext ctx,
236245
TimeUtils.describe(Duration.between(Instant.now(), pr.getSubmissionTime()).abs())
237246
.orElse("-"));
238247

239-
EnumSet<StandardOpenOption> openOptions = EnumSet.noneOf(StandardOpenOption.class);
248+
Set<OpenOption> openOptions = new HashSet<>();
240249
if (msg.isReadWrite() || msg.isNew() || msg.isDelete()) {
241250
if (!(r instanceof StageRequest)) {
242251
throw new XrootdException(kXR_ArgInvalid,
@@ -254,6 +263,9 @@ protected OpenResponse doOnOpen(ChannelHandlerContext ctx,
254263
"An attempt to open-for-write for flush requests");
255264
}
256265
LOGGER.info("Opening {} for reading from {}.", file, new RemoteAddressHolder(ctx));
266+
if (dio) {
267+
openOptions.add(ExtendedOpenOption.DIRECT);
268+
}
257269
openOptions.add(StandardOpenOption.READ);
258270
}
259271

‎src/test/java/org/dcache/nearline/cta/xrootd/DataMoverTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ public class DataMoverTest {
2424
public void setUp() throws UnknownHostException {
2525
requests = new ConcurrentHashMap<>();
2626
dataMover = new DataMover("cta", "testing",
27-
new InetSocketAddress(InetAddress.getLocalHost(), 0), requests);
27+
new InetSocketAddress(InetAddress.getLocalHost(), 0), requests, false);
2828
dataMover.startAsync().awaitRunning();
2929
}
3030

@@ -37,7 +37,7 @@ public void tearDown() {
3737
public void testBindError() throws UnknownHostException {
3838
URI uri = URI.create(dataMover.getTransport("1", 1).getDstUrl());
3939
new DataMover("foo", "bar",
40-
new InetSocketAddress(InetAddress.getLocalHost(), uri.getPort()), requests)
40+
new InetSocketAddress(InetAddress.getLocalHost(), uri.getPort()), requests, false)
4141
.startAsync()
4242
.awaitRunning();
4343
}

‎src/test/java/org/dcache/nearline/cta/xrootd/DataServerHandlerTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public void setUp() throws Exception {
5959
when(ctx.channel()).thenReturn(cannel);
6060

6161
requests = new ConcurrentHashMap<>();
62-
handler = new DataServerHandler("cta", "test", requests);
62+
handler = new DataServerHandler("cta", "test", requests, false);
6363
}
6464

6565
@Test(expected = XrootdException.class)

0 commit comments

Comments
 (0)