Skip to content

Commit 8651aa4

Browse files
ChenSammiChenSammi
authored
HDDS-13343. Consider delegation token lifetime for secret key expiry (#8742)
1 parent 59c2ec3 commit 8651aa4

11 files changed

Lines changed: 115 additions & 22 deletions

File tree

‎hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,7 @@ public final class HddsConfigKeys {
255255

256256
public static final String HDDS_SECRET_KEY_EXPIRY_DURATION =
257257
"hdds.secret.key.expiry.duration";
258-
public static final String HDDS_SECRET_KEY_EXPIRY_DURATION_DEFAULT = "7d";
258+
public static final String HDDS_SECRET_KEY_EXPIRY_DURATION_DEFAULT = "9d";
259259

260260
public static final String HDDS_SECRET_KEY_ROTATE_DURATION =
261261
"hdds.secret.key.rotate.duration";

‎hadoop-hdds/common/src/main/resources/ozone-default.xml‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2773,8 +2773,14 @@
27732773
<name>ozone.manager.delegation.token.max-lifetime</name>
27742774
<value>7d</value>
27752775
<description>
2776-
Default max time interval after which ozone delegation token will
2777-
not be renewed.
2776+
Default max time interval after which ozone delegation token will not be renewed.
2777+
Delegation Token is signed and verified using secret key which has a max hdds.secret.key.expiry.duration lifetime.
2778+
To guarantee that the delegation token can be properly loaded, verified, and renewed during its lifetime,
2779+
(ozone.manager.delegation.token.max-lifetime + hdds.secret.key.rotate.duration + ozone.manager.delegation.remover.scan.interval)
2780+
must not be greater than hdds.secret.key.expiry.duration.
2781+
If any of ozone.manager.delegation.token.max-lifetime, hdds.secret.key.expiry.duration, hdds.secret.key.rotate.duration
2782+
or ozone.manager.delegation.remover.scan.interval value is changed, The above constrain must be checked and
2783+
values be adjusted accordingly if necessary.
27782784
</description>
27792785
</property>
27802786

@@ -4558,12 +4564,18 @@
45584564
</property>
45594565
<property>
45604566
<name>hdds.secret.key.expiry.duration</name>
4561-
<value>7d</value>
4567+
<value>9d</value>
45624568
<tag>SCM, SECURITY</tag>
45634569
<description>
45644570
The duration for which symmetric secret keys issued by SCM are valid.
4565-
This default value, in combination with hdds.secret.key.rotate.duration=1d, results in 7 secret keys (for the
4566-
last 7 days) are kept valid at any point of time.
4571+
Secret key is used to sign delegation tokens signed by OM, so the secret key must be valid for at least
4572+
(ozone.manager.delegation.token.max-lifetime + hdds.secret.key.rotate.duration + ozone.manager.delegation.remover.scan.interval)
4573+
time to guarantee that delegation tokens can be verified by OM. Considering the default value of three properties
4574+
mentioned and rounding up to days, this property's default value, in combination with hdds.secret.key.rotate.duration=1d,
4575+
results in 9 secret keys (for the last 9 days) are kept valid at any point of time.
4576+
If any of ozone.manager.delegation.token.max-lifetime, hdds.secret.key.rotate.duration or
4577+
ozone.manager.delegation.remover.scan.interval value is changed, this property should be checked, and updated
4578+
accordingly if necessary.
45674579
</description>
45684580
</property>
45694581
<property>

‎hadoop-ozone/dist/src/main/compose/common/security.conf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@ OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.Authent
5656
OZONE-SITE.XML_hdds.secret.key.rotate.duration=5m
5757
OZONE-SITE.XML_hdds.secret.key.rotate.check.duration=1m
5858
OZONE-SITE.XML_hdds.secret.key.expiry.duration=1h
59+
OZONE-SITE.XML_ozone.manager.delegation.token.max-lifetime=30m
60+
OZONE-SITE.XML_ozone.manager.delegation.token.renew-interval=5m
61+
OZONE-SITE.XML_ozone.manager.delegation.remover.scan.interval=1m
5962

6063
OZONE-SITE.XML_ozone.om.http.auth.type=kerberos
6164
OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos

‎hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,3 +171,6 @@ OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true
171171
OZONE-SITE.XML_hdds.secret.key.rotate.duration=5m
172172
OZONE-SITE.XML_hdds.secret.key.rotate.check.duration=1m
173173
OZONE-SITE.XML_hdds.secret.key.expiry.duration=1h
174+
OZONE-SITE.XML_ozone.manager.delegation.token.max-lifetime=30m
175+
OZONE-SITE.XML_ozone.manager.delegation.token.renew-interval=5m
176+
OZONE-SITE.XML_ozone.manager.delegation.remover.scan.interval=1m

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeySnapshot.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY;
3232
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
3333
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
34+
import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_KEY;
35+
import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY;
3436
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE;
3537
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY;
3638
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
@@ -113,6 +115,8 @@ public void init() throws Exception {
113115
ROTATE_CHECK_DURATION_MS + "ms");
114116
conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, ROTATE_DURATION_MS + "ms");
115117
conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, EXPIRY_DURATION_MS + "ms");
118+
conf.set(DELEGATION_TOKEN_MAX_LIFETIME_KEY, ROTATE_DURATION_MS + "ms");
119+
conf.set(DELEGATION_REMOVER_SCAN_INTERVAL_KEY, ROTATE_CHECK_DURATION_MS + "ms");
116120

117121
MiniOzoneHAClusterImpl.Builder builder = MiniOzoneCluster.newHABuilder(conf);
118122
builder

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeysApi.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333
import static org.apache.hadoop.hdds.utils.HddsServerUtil.getSecretKeyClientForDatanode;
3434
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
3535
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
36+
import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_KEY;
37+
import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY;
3638
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE;
3739
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY;
3840
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
@@ -185,6 +187,8 @@ public void testSecretKeyApiSuccess() throws Exception {
185187
conf.set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION, "100ms");
186188
conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, "1s");
187189
conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, "3000ms");
190+
conf.set(DELEGATION_TOKEN_MAX_LIFETIME_KEY, "1500ms");
191+
conf.set(DELEGATION_REMOVER_SCAN_INTERVAL_KEY, "100ms");
188192

189193
startCluster(3);
190194
SecretKeyProtocol secretKeyProtocol = getSecretKeyProtocol();
@@ -258,6 +262,7 @@ public void testSecretKeyAfterSCMFailover() throws Exception {
258262
conf.set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION, "10m");
259263
conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, "1d");
260264
conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, "7d");
265+
conf.set(DELEGATION_TOKEN_MAX_LIFETIME_KEY, "5d");
261266

262267
startCluster(3);
263268
SecretKeyProtocol securityProtocol = getSecretKeyProtocol();

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestBlockTokens.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY;
3636
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
3737
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
38+
import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_KEY;
39+
import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY;
3840
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE;
3941
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY;
4042
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
@@ -309,6 +311,8 @@ private static void setSecretKeysConfig() {
309311
ROTATION_CHECK_DURATION_IN_MS + "ms");
310312
conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, ROTATE_DURATION_IN_MS + "ms");
311313
conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, EXPIRY_DURATION_IN_MS + "ms");
314+
conf.set(DELEGATION_TOKEN_MAX_LIFETIME_KEY, ROTATE_DURATION_IN_MS + "ms");
315+
conf.set(DELEGATION_REMOVER_SCAN_INTERVAL_KEY, ROTATION_CHECK_DURATION_IN_MS + "ms");
312316

313317
// enable tokens
314318
conf.setBoolean(HDDS_BLOCK_TOKEN_ENABLED, true);

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
2121
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_TLS_ENABLED;
22+
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_EXPIRY_DURATION;
2223
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_ACK_TIMEOUT;
2324
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_CHECK_INTERNAL;
2425
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION;
@@ -88,6 +89,7 @@
8889
import java.util.Properties;
8990
import java.util.UUID;
9091
import java.util.concurrent.Callable;
92+
import java.util.concurrent.TimeUnit;
9193
import java.util.regex.Matcher;
9294
import java.util.regex.Pattern;
9395
import org.apache.commons.io.IOUtils;
@@ -535,6 +537,31 @@ void testSecureOMInitializationFailure() throws Exception {
535537
}
536538
}
537539

540+
/**
541+
* Tests the secure om Initialization Failure due to delegation token and secret key configuration don't meet
542+
* requirement.
543+
*/
544+
@Test
545+
void testSecureOMDelegationTokenSecretManagerInitializationFailure() throws Exception {
546+
initSCM();
547+
// Create a secure SCM instance as om client will connect to it
548+
scm = HddsTestUtils.getScmSimple(conf);
549+
try {
550+
scm.start();
551+
conf.setTimeDuration(HDDS_SECRET_KEY_EXPIRY_DURATION, 7, TimeUnit.DAYS);
552+
conf.setTimeDuration(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, 7, TimeUnit.DAYS);
553+
IllegalArgumentException exception = assertThrows(
554+
IllegalArgumentException.class, () -> setupOm(conf));
555+
assertTrue(exception.getMessage().contains("Secret key expiry duration hdds.secret.key.expiry.duration " +
556+
"should be greater than value of (ozone.manager.delegation.token.max-lifetime + " +
557+
"ozone.manager.delegation.remover.scan.interval + hdds.secret.key.rotate.duration"));
558+
} finally {
559+
if (scm != null) {
560+
scm.stop();
561+
}
562+
}
563+
}
564+
538565
/**
539566
* Tests the secure om Initialization success.
540567
*/
@@ -866,7 +893,6 @@ void testSecureOmReInit() throws Exception {
866893
scm.stop();
867894
}
868895
}
869-
870896
}
871897

872898
/**
@@ -913,7 +939,6 @@ void testSecureOmInitSuccess() throws Exception {
913939
}
914940
IOUtils.closeQuietly(om);
915941
}
916-
917942
}
918943

919944
/**

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/checksum/TestContainerCommandReconciliation.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@
4949
import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.assertTreesSortedAndMatch;
5050
import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.buildTestTree;
5151
import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.readChecksumFile;
52+
import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_KEY;
53+
import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY;
5254
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE;
5355
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY;
5456
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
@@ -600,9 +602,11 @@ public static void writeChecksumFileToDatanodes(long containerID, ContainerMerkl
600602

601603
private static void setSecretKeysConfig() {
602604
// Secret key lifecycle configs.
603-
conf.set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION, "500s");
604-
conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, "500s");
605+
conf.set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION, "1s");
606+
conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, "100s");
605607
conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, "500s");
608+
conf.set(DELEGATION_TOKEN_MAX_LIFETIME_KEY, "300s");
609+
conf.set(DELEGATION_REMOVER_SCAN_INTERVAL_KEY, "1s");
606610

607611
// enable tokens
608612
conf.setBoolean(HDDS_BLOCK_TOKEN_ENABLED, true);

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java‎

Lines changed: 41 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RATIS_SNAPSHOT_DIR;
5050
import static org.apache.hadoop.ozone.OzoneConsts.PREPARE_MARKER_KEY;
5151
import static org.apache.hadoop.ozone.OzoneConsts.RPC_PORT;
52+
import static org.apache.hadoop.ozone.OzoneConsts.SCM_CA_CERT_STORAGE_DIR;
5253
import static org.apache.hadoop.ozone.OzoneConsts.TRANSACTION_INFO_KEY;
5354
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT;
5455
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT_DEFAULT;
@@ -191,6 +192,7 @@
191192
import org.apache.hadoop.hdds.security.exception.OzoneSecurityException;
192193
import org.apache.hadoop.hdds.security.symmetric.DefaultSecretKeyClient;
193194
import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
195+
import org.apache.hadoop.hdds.security.symmetric.SecretKeyConfig;
194196
import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSecretManager;
195197
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
196198
import org.apache.hadoop.hdds.server.OzoneAdmins;
@@ -955,7 +957,19 @@ private void instantiateServices(boolean withNewSnapshot) throws IOException {
955957
metadataManager.getLock()
956958
);
957959
if (secConfig.isSecurityEnabled() || testSecureOmFlag) {
958-
delegationTokenMgr = createDelegationTokenSecretManager(configuration);
960+
try {
961+
delegationTokenMgr = createDelegationTokenSecretManager(configuration);
962+
} catch (IllegalArgumentException e) {
963+
if (metadataManager != null) {
964+
// to avoid the unit test leak report failure
965+
try {
966+
metadataManager.stop();
967+
} catch (Exception ex) {
968+
LOG.warn("Failed to stop metadataManager", e);
969+
}
970+
}
971+
throw e;
972+
}
959973
}
960974

961975
prefixManager = new PrefixManagerImpl(this, metadataManager, true);
@@ -1170,17 +1184,32 @@ private OzoneDelegationTokenSecretManager createDelegationTokenSecretManager(
11701184
conf.getTimeDuration(OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
11711185
OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT,
11721186
TimeUnit.MILLISECONDS);
1173-
long certificateGracePeriod = Duration.parse(
1174-
conf.get(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION,
1175-
HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT)).toMillis();
1176-
boolean tokenSanityChecksEnabled = conf.getBoolean(
1177-
HddsConfigKeys.HDDS_X509_GRACE_DURATION_TOKEN_CHECKS_ENABLED,
1178-
HddsConfigKeys.HDDS_X509_GRACE_DURATION_TOKEN_CHECKS_ENABLED_DEFAULT);
1179-
if (tokenSanityChecksEnabled && tokenMaxLifetime > certificateGracePeriod) {
1180-
throw new IllegalArgumentException("Certificate grace period " +
1181-
HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION +
1182-
" should be greater than maximum delegation token lifetime " +
1183-
OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY);
1187+
1188+
if (versionManager.isAllowed(OMLayoutFeature.DELEGATION_TOKEN_SYMMETRIC_SIGN)) {
1189+
// verify the configuration dependency between dt and secret key
1190+
SecretKeyConfig secretKeyConfig = new SecretKeyConfig(conf, SCM_CA_CERT_STORAGE_DIR);
1191+
long skExpiryDuration = secretKeyConfig.getExpiryDuration().toMillis();
1192+
long skRotationDuration = secretKeyConfig.getRotateDuration().toMillis();
1193+
if ((skRotationDuration + tokenMaxLifetime + tokenRemoverScanInterval) > skExpiryDuration) {
1194+
throw new IllegalArgumentException("Secret key expiry duration " +
1195+
HddsConfigKeys.HDDS_SECRET_KEY_EXPIRY_DURATION +
1196+
" should be greater than value of (" + OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY + " + " +
1197+
OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_KEY + " + " +
1198+
HddsConfigKeys.HDDS_SECRET_KEY_ROTATE_DURATION);
1199+
}
1200+
} else {
1201+
long certificateGracePeriod = Duration.parse(
1202+
conf.get(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION,
1203+
HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT)).toMillis();
1204+
boolean tokenSanityChecksEnabled = conf.getBoolean(
1205+
HddsConfigKeys.HDDS_X509_GRACE_DURATION_TOKEN_CHECKS_ENABLED,
1206+
HddsConfigKeys.HDDS_X509_GRACE_DURATION_TOKEN_CHECKS_ENABLED_DEFAULT);
1207+
if (tokenSanityChecksEnabled && tokenMaxLifetime > certificateGracePeriod) {
1208+
throw new IllegalArgumentException("Certificate grace period " +
1209+
HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION +
1210+
" should be greater than maximum delegation token lifetime " +
1211+
OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY);
1212+
}
11841213
}
11851214

11861215
return new OzoneDelegationTokenSecretManager.Builder()

0 commit comments

Comments
 (0)