Skip to content

Commit b0555bf

Browse files
Jiale Zhangjialez0
Jiale Zhang
authored and
jialez0
committed
Support private cloud Aliyun KMS endpoints for KBS
Allow the KBS Aliyun resource backend to use private cloud KMS endpoints with configurable TLS trust, while keeping the existing public cloud defaults. Recommend AccessKey credentials through environment variables and document the private cloud deployment and test flow. Signed-off-by: Jiale Zhang <xinjian.zjl@alibaba-inc.com>
1 parent b5aafbb commit b0555bf

13 files changed

Lines changed: 432 additions & 51 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎deps/kms/src/plugins/aliyun/client/client_key_client/mod.rs‎

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ use base64::{engine::general_purpose::STANDARD, Engine};
1010
use chrono::Utc;
1111
use log::{error, info};
1212
use prost::Message;
13-
use reqwest::{header::HeaderMap, Certificate, ClientBuilder};
13+
use reqwest::{header::HeaderMap, Certificate, Client, ClientBuilder};
1414
use serde::{Deserialize, Serialize};
1515
use serde_json::Value;
1616
use sha2::{Digest, Sha256};
@@ -53,27 +53,65 @@ impl ClientKeyClient {
5353
Ok(kms_instance_ca_cert)
5454
}
5555

56+
fn build_http_client(cert_pem: Option<&str>, insecure_skip_tls_verify: bool) -> Result<Client> {
57+
let mut builder = ClientBuilder::new().use_rustls_tls();
58+
59+
if let Some(cert_pem) = cert_pem.filter(|v| !v.is_empty()) {
60+
let cert = Self::read_kms_instance_cert(cert_pem.as_bytes())?;
61+
builder = builder.add_root_certificate(cert);
62+
} else if !insecure_skip_tls_verify {
63+
return Err(Error::AliyunKmsError(
64+
"kms instance ca cert is required unless insecure_skip_tls_verify is enabled"
65+
.to_string(),
66+
));
67+
}
68+
69+
if insecure_skip_tls_verify {
70+
builder = builder.danger_accept_invalid_certs(true);
71+
}
72+
73+
builder
74+
.build()
75+
.map_err(|e| Error::AliyunKmsError(format!("build http client failed: {e:?}")))
76+
}
77+
5678
pub fn new(
5779
client_key: &str,
5880
kms_instance_id: &str,
5981
password: &str,
6082
cert_pem: &str,
83+
) -> Result<Self> {
84+
Self::new_with_options(
85+
client_key,
86+
kms_instance_id,
87+
password,
88+
Some(cert_pem),
89+
None,
90+
false,
91+
)
92+
}
93+
94+
pub fn new_with_options(
95+
client_key: &str,
96+
kms_instance_id: &str,
97+
password: &str,
98+
cert_pem: Option<&str>,
99+
endpoint: Option<&str>,
100+
insecure_skip_tls_verify: bool,
61101
) -> Result<Self> {
62102
let credential = CredentialClientKey::new(client_key, password).map_err(|e| {
63103
Error::AliyunKmsError(format!(
64104
"create client_key credential of the kms instance failed: {e:?}"
65105
))
66106
})?;
67107

68-
let endpoint = format!("{kms_instance_id}.cryptoservice.kms.aliyuncs.com");
108+
let endpoint = endpoint
109+
.filter(|v| !v.is_empty())
110+
.map(ToOwned::to_owned)
111+
.unwrap_or_else(|| format!("{kms_instance_id}.cryptoservice.kms.aliyuncs.com"));
69112
let config = ConfigClientKey::new(kms_instance_id, &endpoint);
70113

71-
let cert = Self::read_kms_instance_cert(cert_pem.as_bytes())?;
72-
let http_client = ClientBuilder::new()
73-
.use_rustls_tls()
74-
.add_root_certificate(cert)
75-
.build()
76-
.map_err(|e| Error::AliyunKmsError(format!("build http client failed: {e:?}")))?;
114+
let http_client = Self::build_http_client(cert_pem, insecure_skip_tls_verify)?;
77115

78116
Ok(Self {
79117
credential,

‎deps/kms/src/plugins/aliyun/client/mod.rs‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,26 @@ impl AliyunKmsClient {
6363
Ok(Self::ClientKey { inner })
6464
}
6565

66+
pub fn new_client_key_client_with_options(
67+
client_key: &str,
68+
kms_instance_id: &str,
69+
password: &str,
70+
cert_pem: Option<&str>,
71+
endpoint: Option<&str>,
72+
insecure_skip_tls_verify: bool,
73+
) -> Result<Self> {
74+
let inner = ClientKeyClient::new_with_options(
75+
client_key,
76+
kms_instance_id,
77+
password,
78+
cert_pem,
79+
endpoint,
80+
insecure_skip_tls_verify,
81+
)?;
82+
83+
Ok(Self::ClientKey { inner })
84+
}
85+
6686
pub fn new_ecs_ram_role_client(ecs_ram_role_name: &str, region_id: &str) -> Self {
6787
let ecs_ram_role_client =
6888
EcsRamRoleClient::new(ecs_ram_role_name.to_string(), region_id.to_string());
@@ -77,12 +97,35 @@ impl AliyunKmsClient {
7797
access_key_secret: &str,
7898
region_id: &str,
7999
) -> Result<Self> {
80-
let endpoint = format!("kms.{region_id}.aliyuncs.com");
81-
let client = StsTokenClient::from_access_key(
100+
Self::new_access_key_client_with_options(
101+
access_key_id,
102+
access_key_secret,
103+
region_id,
104+
None,
105+
None,
106+
false,
107+
)
108+
}
109+
110+
pub fn new_access_key_client_with_options(
111+
access_key_id: &str,
112+
access_key_secret: &str,
113+
region_id: &str,
114+
endpoint: Option<&str>,
115+
ca_cert_pem: Option<&str>,
116+
insecure_skip_tls_verify: bool,
117+
) -> Result<Self> {
118+
let endpoint = endpoint
119+
.filter(|v| !v.is_empty())
120+
.map(ToOwned::to_owned)
121+
.unwrap_or_else(|| format!("kms.{region_id}.aliyuncs.com"));
122+
let client = StsTokenClient::from_access_key_with_options(
82123
access_key_id.to_string(),
83124
access_key_secret.to_string(),
84125
endpoint,
85126
region_id.to_string(),
127+
ca_cert_pem,
128+
insecure_skip_tls_verify,
86129
)?;
87130
Ok(Self::AccessKey { client })
88131
}

‎deps/kms/src/plugins/aliyun/client/sts_token_client/mod.rs‎

Lines changed: 42 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ use chrono::Utc;
1515
use credential::StsCredential;
1616
use log::error;
1717
use rand::{distributions::Alphanumeric, Rng};
18-
use reqwest::{header::HeaderMap, ClientBuilder};
18+
use reqwest::{header::HeaderMap, Certificate, Client, ClientBuilder};
1919
use serde::Deserialize;
2020
use serde_json::Value;
2121
use tokio::fs;
@@ -43,11 +43,29 @@ pub struct StsSettings {
4343
}
4444

4545
impl StsTokenClient {
46-
pub fn from_sts_token(sts: StsCredential, endpoint: String, region_id: String) -> Result<Self> {
47-
let http_client = ClientBuilder::new()
48-
.use_rustls_tls()
46+
fn build_http_client(
47+
ca_cert_pem: Option<&str>,
48+
insecure_skip_tls_verify: bool,
49+
) -> Result<Client> {
50+
let mut builder = ClientBuilder::new().use_rustls_tls();
51+
52+
if let Some(ca_cert_pem) = ca_cert_pem.filter(|v| !v.is_empty()) {
53+
let ca_cert = Certificate::from_pem(ca_cert_pem.as_bytes())
54+
.map_err(|e| Error::AliyunKmsError(format!("read kms ca cert failed: {e:?}")))?;
55+
builder = builder.add_root_certificate(ca_cert);
56+
}
57+
58+
if insecure_skip_tls_verify {
59+
builder = builder.danger_accept_invalid_certs(true);
60+
}
61+
62+
builder
4963
.build()
50-
.map_err(|e| Error::AliyunKmsError(format!("build http client failed: {e:?}")))?;
64+
.map_err(|e| Error::AliyunKmsError(format!("build http client failed: {e:?}")))
65+
}
66+
67+
pub fn from_sts_token(sts: StsCredential, endpoint: String, region_id: String) -> Result<Self> {
68+
let http_client = Self::build_http_client(None, false)?;
5169
Ok(Self {
5270
ak: sts.ak,
5371
sk: sts.sk,
@@ -64,10 +82,25 @@ impl StsTokenClient {
6482
endpoint: String,
6583
region_id: String,
6684
) -> Result<Self> {
67-
let http_client = ClientBuilder::new()
68-
.use_rustls_tls()
69-
.build()
70-
.map_err(|e| Error::AliyunKmsError(format!("build http client failed: {e:?}")))?;
85+
Self::from_access_key_with_options(
86+
access_key_id,
87+
access_key_secret,
88+
endpoint,
89+
region_id,
90+
None,
91+
false,
92+
)
93+
}
94+
95+
pub fn from_access_key_with_options(
96+
access_key_id: String,
97+
access_key_secret: String,
98+
endpoint: String,
99+
region_id: String,
100+
ca_cert_pem: Option<&str>,
101+
insecure_skip_tls_verify: bool,
102+
) -> Result<Self> {
103+
let http_client = Self::build_http_client(ca_cert_pem, insecure_skip_tls_verify)?;
71104
Ok(Self {
72105
ak: access_key_id,
73106
sk: access_key_secret,

‎helm-chart/README.md‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,49 @@
11
# Trustee Helm Chart
2+
3+
## Aliyun KMS Resource Backend
4+
5+
KBS can use Aliyun KMS generic secrets as the resource backend when the KBS image
6+
is built with the `aliyun` feature.
7+
8+
Set `kbs.aliyunKms.enabled=true` and choose one authentication mode:
9+
10+
- AAP client key: set `clientKey`, `kmsInstanceId`, `password`, and `certPem`.
11+
- AccessKey: set `regionId` and inject `ALIYUN_KMS_ACCESS_KEY_ID` and
12+
`ALIYUN_KMS_ACCESS_KEY_SECRET` through the pod environment. The recommended
13+
Helm path is to set `accessKeyExistingSecret` so the credentials come from an
14+
existing Kubernetes Secret.
15+
16+
For private cloud deployments, also set `endpoint` to the KMS intranet endpoint
17+
provided by the private cloud KMS owner. If the endpoint uses a private CA, set
18+
`certPem` to the CA certificate. `insecureSkipTlsVerify=true` is only intended
19+
for temporary test environments where the certificate chain cannot yet be
20+
trusted.
21+
22+
Example:
23+
24+
```yaml
25+
kbs:
26+
aliyunKms:
27+
enabled: true
28+
regionId: "cn-test"
29+
endpoint: "kms-intranet.cn-test.example.com"
30+
certPem: |
31+
-----BEGIN CERTIFICATE-----
32+
...
33+
-----END CERTIFICATE-----
34+
accessKeyExistingSecret: "aliyun-kms-credential"
35+
accessKeyIdSecretKey: "access_key_id"
36+
accessKeySecretSecretKey: "access_key_secret"
37+
insecureSkipTlsVerify: false
38+
```
39+
40+
Create the referenced secret before installing or upgrading the chart:
41+
42+
```shell
43+
kubectl create secret generic aliyun-kms-credential \
44+
--from-literal=access_key_id='LTAI...' \
45+
--from-literal=access_key_secret='secret...'
46+
```
47+
48+
The old `kmsIntanceId` value key is still accepted for compatibility, but new
49+
deployments should use `kmsInstanceId`.

‎helm-chart/trustee/templates/kbs-configmap.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@ data:
2222
{{- if .Values.kbs.aliyunKms.enabled }}
2323
type = "Aliyun"
2424
client_key = {{ .Values.kbs.aliyunKms.clientKey | quote }}
25-
kms_instance_id = "{{ .Values.kbs.aliyunKms.kmsIntanceId }}"
25+
kms_instance_id = "{{ default .Values.kbs.aliyunKms.kmsIntanceId .Values.kbs.aliyunKms.kmsInstanceId }}"
2626
password = "{{ .Values.kbs.aliyunKms.password }}"
2727
cert_pem = {{ .Values.kbs.aliyunKms.certPem | quote}}
28+
endpoint = {{ .Values.kbs.aliyunKms.endpoint | quote }}
29+
insecure_skip_tls_verify = {{ .Values.kbs.aliyunKms.insecureSkipTlsVerify }}
2830
{{- else }}
2931
type = "LocalFs"
3032
dir_path = "/opt/confidential-containers/kbs/repository"

‎helm-chart/trustee/templates/kbs-statefulset.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,33 @@ spec:
4343
env:
4444
- name: RUST_LOG
4545
value: {{ .Values.log_level }}
46+
{{- if .Values.kbs.aliyunKms.enabled }}
47+
{{- if .Values.kbs.aliyunKms.regionId }}
48+
- name: ALIYUN_KMS_REGION_ID
49+
value: {{ .Values.kbs.aliyunKms.regionId | quote }}
50+
{{- end }}
51+
{{- if .Values.kbs.aliyunKms.accessKeyExistingSecret }}
52+
- name: ALIYUN_KMS_ACCESS_KEY_ID
53+
valueFrom:
54+
secretKeyRef:
55+
name: {{ .Values.kbs.aliyunKms.accessKeyExistingSecret | quote }}
56+
key: {{ .Values.kbs.aliyunKms.accessKeyIdSecretKey | quote }}
57+
- name: ALIYUN_KMS_ACCESS_KEY_SECRET
58+
valueFrom:
59+
secretKeyRef:
60+
name: {{ .Values.kbs.aliyunKms.accessKeyExistingSecret | quote }}
61+
key: {{ .Values.kbs.aliyunKms.accessKeySecretSecretKey | quote }}
62+
{{- else }}
63+
{{- if .Values.kbs.aliyunKms.accessKeyId }}
64+
- name: ALIYUN_KMS_ACCESS_KEY_ID
65+
value: {{ .Values.kbs.aliyunKms.accessKeyId | quote }}
66+
{{- end }}
67+
{{- if .Values.kbs.aliyunKms.accessKeySecret }}
68+
- name: ALIYUN_KMS_ACCESS_KEY_SECRET
69+
value: {{ .Values.kbs.aliyunKms.accessKeySecret | quote }}
70+
{{- end }}
71+
{{- end }}
72+
{{- end }}
4673
ports:
4774
- name: http
4875
containerPort: {{ .Values.kbs.service.port }}

‎helm-chart/trustee/values.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,10 +70,25 @@ kbs:
7070

7171
aliyunKms:
7272
enabled: false
73+
# kmsIntanceId is kept for backward compatibility with older chart values.
7374
kmsIntanceId:
75+
kmsInstanceId:
7476
password:
7577
clientKey:
7678
certPem:
79+
# Recommended AccessKey configuration path: inject credentials through
80+
# ALIYUN_KMS_* environment variables. Prefer accessKeyExistingSecret for
81+
# production deployments.
82+
accessKeyId:
83+
accessKeySecret:
84+
accessKeyExistingSecret:
85+
accessKeyIdSecretKey: access_key_id
86+
accessKeySecretSecretKey: access_key_secret
87+
regionId:
88+
# Private cloud KMS intranet endpoint, for example kms-intranet.<region>.<domain>.
89+
endpoint:
90+
# Use only for test environments where the private cloud KMS certificate cannot be trusted.
91+
insecureSkipTlsVerify: false
7792

7893
as:
7994
pccsURL: ''

‎kbs/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ tempfile.workspace = true
9999
rstest.workspace = true
100100
reference-value-provider-service.path = "../rvps"
101101
serial_test = "3.0"
102+
toml.workspace = true
102103

103104
[build-dependencies]
104105
tonic-build = { workspace = true, optional = true }

‎kbs/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ If you want a self-signed cert for test cases, please refer to [the document](do
107107

108108
The KBS can use different backend storage. `LocalFs` will always be builtin.
109109
`ALIYUN` determines whether aliyun kms support will be built. The options
110-
are `true` or `false` (by defult). Please refer to [the document](docs/config.md#repository-configuration)
110+
are `true` or `false` (by default). Please refer to [the document](docs/config.md#resource-configuration)
111111
for more details.
112112

113113
## References

0 commit comments

Comments
 (0)