Skip to content

Commit b8f7a8b

Browse files
freeznetfreeznet
authored
feat(namespace): add offload policies support (#413)
1 parent dd20a7c commit b8f7a8b

16 files changed

Lines changed: 599 additions & 116 deletions

‎.licenserc.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ header:
2121
paths-ignore:
2222
- 'dist'
2323
- 'licenses'
24+
- 'node_modules/**'
25+
- '**/node_modules/**'
26+
- '.claude/worktrees/**'
2427
- '**/*.md'
2528
- 'LICENSE'
2629
- 'NOTICE'

‎api/v1alpha1/pulsarnamespace_types.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,11 @@ type PulsarNamespaceSpec struct {
229229
// +optional
230230
OffloadThresholdSize *resource.Quantity `json:"offloadThresholdSize,omitempty"`
231231

232+
// OffloadPolicies defines the namespace-level offload policies.
233+
// Use this to configure tiered storage driver, bucket, region, endpoint, and thresholds.
234+
// +optional
235+
OffloadPolicies *OffloadPolicies `json:"offloadPolicies,omitempty"`
236+
232237
// GeoReplicationRefs is a list of references to PulsarGeoReplication resources,
233238
// used to configure geo-replication for this namespace.
234239
// This is **ONLY** used when you are using PulsarGeoReplication for setting up geo-replication

‎api/v1alpha1/pulsartopic_types.go‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -268,19 +268,38 @@ type SchemaInfo struct {
268268
// This is a local type definition that mirrors the external library's OffloadPolicies
269269
// to ensure proper Kubernetes deep copy generation.
270270
type OffloadPolicies struct {
271+
OffloadersDirectory string `json:"offloadersDirectory,omitempty"`
271272
ManagedLedgerOffloadDriver string `json:"managedLedgerOffloadDriver,omitempty"`
272-
ManagedLedgerOffloadMaxThreads int `json:"managedLedgerOffloadMaxThreads,omitempty"`
273-
ManagedLedgerOffloadThresholdInBytes int64 `json:"managedLedgerOffloadThresholdInBytes,omitempty"`
274-
ManagedLedgerOffloadDeletionLagInMillis int64 `json:"managedLedgerOffloadDeletionLagInMillis,omitempty"`
275-
ManagedLedgerOffloadAutoTriggerSizeThresholdBytes int64 `json:"managedLedgerOffloadAutoTriggerSizeThresholdBytes,omitempty"`
276-
S3ManagedLedgerOffloadBucket string `json:"s3ManagedLedgerOffloadBucket,omitempty"`
273+
ManagedLedgerOffloadMaxThreads *int `json:"managedLedgerOffloadMaxThreads,omitempty"`
274+
ManagedLedgerOffloadReadThreads *int `json:"managedLedgerOffloadReadThreads,omitempty"`
275+
ManagedLedgerOffloadPrefetchRounds *int `json:"managedLedgerOffloadPrefetchRounds,omitempty"`
276+
ManagedLedgerOffloadThresholdInSeconds *int64 `json:"managedLedgerOffloadThresholdInSeconds,omitempty"`
277+
ManagedLedgerOffloadThresholdInBytes *int64 `json:"managedLedgerOffloadThresholdInBytes,omitempty"`
278+
ManagedLedgerOffloadDeletionLagInMillis *int64 `json:"managedLedgerOffloadDeletionLagInMillis,omitempty"`
279+
ManagedLedgerOffloadedReadPriority string `json:"managedLedgerOffloadedReadPriority,omitempty"`
280+
ManagedLedgerExtraConfigurations map[string]string `json:"managedLedgerExtraConfigurations,omitempty"`
281+
ManagedLedgerOffloadAutoTriggerSizeThresholdBytes *int64 `json:"managedLedgerOffloadAutoTriggerSizeThresholdBytes,omitempty"`
277282
S3ManagedLedgerOffloadRegion string `json:"s3ManagedLedgerOffloadRegion,omitempty"`
283+
S3ManagedLedgerOffloadBucket string `json:"s3ManagedLedgerOffloadBucket,omitempty"`
278284
S3ManagedLedgerOffloadServiceEndpoint string `json:"s3ManagedLedgerOffloadServiceEndpoint,omitempty"`
285+
S3ManagedLedgerOffloadMaxBlockSizeInBytes *int `json:"s3ManagedLedgerOffloadMaxBlockSizeInBytes,omitempty"`
286+
S3ManagedLedgerOffloadReadBufferSizeInBytes *int `json:"s3ManagedLedgerOffloadReadBufferSizeInBytes,omitempty"`
279287
S3ManagedLedgerOffloadCredentialID string `json:"s3ManagedLedgerOffloadCredentialId,omitempty"`
280288
S3ManagedLedgerOffloadCredentialSecret string `json:"s3ManagedLedgerOffloadCredentialSecret,omitempty"`
281289
S3ManagedLedgerOffloadRole string `json:"s3ManagedLedgerOffloadRole,omitempty"`
282290
S3ManagedLedgerOffloadRoleSessionName string `json:"s3ManagedLedgerOffloadRoleSessionName,omitempty"`
283-
OffloadersDirectory string `json:"offloadersDirectory,omitempty"`
291+
GCSManagedLedgerOffloadRegion string `json:"gcsManagedLedgerOffloadRegion,omitempty"`
292+
GCSManagedLedgerOffloadBucket string `json:"gcsManagedLedgerOffloadBucket,omitempty"`
293+
GCSManagedLedgerOffloadMaxBlockSizeInBytes *int `json:"gcsManagedLedgerOffloadMaxBlockSizeInBytes,omitempty"`
294+
GCSManagedLedgerOffloadReadBufferSizeInBytes *int `json:"gcsManagedLedgerOffloadReadBufferSizeInBytes,omitempty"`
295+
GCSManagedLedgerOffloadServiceAccountKeyFile string `json:"gcsManagedLedgerOffloadServiceAccountKeyFile,omitempty"`
296+
FileSystemProfilePath string `json:"fileSystemProfilePath,omitempty"`
297+
FileSystemURI string `json:"fileSystemURI,omitempty"`
298+
ManagedLedgerOffloadBucket string `json:"managedLedgerOffloadBucket,omitempty"`
299+
ManagedLedgerOffloadRegion string `json:"managedLedgerOffloadRegion,omitempty"`
300+
ManagedLedgerOffloadServiceEndpoint string `json:"managedLedgerOffloadServiceEndpoint,omitempty"`
301+
ManagedLedgerOffloadMaxBlockSizeInBytes *int `json:"managedLedgerOffloadMaxBlockSizeInBytes,omitempty"`
302+
ManagedLedgerOffloadReadBufferSizeInBytes *int `json:"managedLedgerOffloadReadBufferSizeInBytes,omitempty"`
284303
ManagedLedgerOffloadDriverMetadata map[string]string `json:"managedLedgerOffloadDriverMetadata,omitempty"`
285304
}
286305

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 77 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎charts/pulsar-resources-operator/crds/resource.streamnative.io_pulsarnamespaces.yaml‎

Lines changed: 85 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2025 StreamNative
1+
# Copyright 2026 StreamNative
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@ apiVersion: apiextensions.k8s.io/v1
1717
kind: CustomResourceDefinition
1818
metadata:
1919
annotations:
20-
controller-gen.kubebuilder.io/version: v0.15.0
20+
controller-gen.kubebuilder.io/version: v0.17.0
2121
name: pulsarnamespaces.resource.streamnative.io
2222
spec:
2323
group: resource.streamnative.io
@@ -147,9 +147,7 @@ spec:
147147
This field is effectively required, but due to backwards compatibility is
148148
allowed to be empty. Instances of this type with an empty value here are
149149
almost certainly wrong.
150-
TODO: Add other useful fields. apiVersion, kind, uid?
151150
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
152-
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
153151
type: string
154152
type: object
155153
x-kubernetes-map-type: atomic
@@ -206,9 +204,7 @@ spec:
206204
This field is effectively required, but due to backwards compatibility is
207205
allowed to be empty. Instances of this type with an empty value here are
208206
almost certainly wrong.
209-
TODO: Add other useful fields. apiVersion, kind, uid?
210207
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
211-
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
212208
type: string
213209
type: object
214210
x-kubernetes-map-type: atomic
@@ -274,6 +270,86 @@ spec:
274270
description: Name is the fully qualified namespace name in the format
275271
"tenant/namespace".
276272
type: string
273+
offloadPolicies:
274+
description: |-
275+
OffloadPolicies defines the namespace-level offload policies.
276+
Use this to configure tiered storage driver, bucket, region, endpoint, and thresholds.
277+
properties:
278+
fileSystemProfilePath:
279+
type: string
280+
fileSystemURI:
281+
type: string
282+
gcsManagedLedgerOffloadBucket:
283+
type: string
284+
gcsManagedLedgerOffloadMaxBlockSizeInBytes:
285+
type: integer
286+
gcsManagedLedgerOffloadReadBufferSizeInBytes:
287+
type: integer
288+
gcsManagedLedgerOffloadRegion:
289+
type: string
290+
gcsManagedLedgerOffloadServiceAccountKeyFile:
291+
type: string
292+
managedLedgerExtraConfigurations:
293+
additionalProperties:
294+
type: string
295+
type: object
296+
managedLedgerOffloadAutoTriggerSizeThresholdBytes:
297+
format: int64
298+
type: integer
299+
managedLedgerOffloadBucket:
300+
type: string
301+
managedLedgerOffloadDeletionLagInMillis:
302+
format: int64
303+
type: integer
304+
managedLedgerOffloadDriver:
305+
type: string
306+
managedLedgerOffloadDriverMetadata:
307+
additionalProperties:
308+
type: string
309+
type: object
310+
managedLedgerOffloadMaxBlockSizeInBytes:
311+
type: integer
312+
managedLedgerOffloadMaxThreads:
313+
type: integer
314+
managedLedgerOffloadPrefetchRounds:
315+
type: integer
316+
managedLedgerOffloadReadBufferSizeInBytes:
317+
type: integer
318+
managedLedgerOffloadReadThreads:
319+
type: integer
320+
managedLedgerOffloadRegion:
321+
type: string
322+
managedLedgerOffloadServiceEndpoint:
323+
type: string
324+
managedLedgerOffloadThresholdInBytes:
325+
format: int64
326+
type: integer
327+
managedLedgerOffloadThresholdInSeconds:
328+
format: int64
329+
type: integer
330+
managedLedgerOffloadedReadPriority:
331+
type: string
332+
offloadersDirectory:
333+
type: string
334+
s3ManagedLedgerOffloadBucket:
335+
type: string
336+
s3ManagedLedgerOffloadCredentialId:
337+
type: string
338+
s3ManagedLedgerOffloadCredentialSecret:
339+
type: string
340+
s3ManagedLedgerOffloadMaxBlockSizeInBytes:
341+
type: integer
342+
s3ManagedLedgerOffloadReadBufferSizeInBytes:
343+
type: integer
344+
s3ManagedLedgerOffloadRegion:
345+
type: string
346+
s3ManagedLedgerOffloadRole:
347+
type: string
348+
s3ManagedLedgerOffloadRoleSessionName:
349+
type: string
350+
s3ManagedLedgerOffloadServiceEndpoint:
351+
type: string
352+
type: object
277353
offloadThresholdSize:
278354
anyOf:
279355
- type: integer
@@ -528,16 +604,8 @@ spec:
528604
It follows the Kubernetes conventions for condition types and status.
529605
The "Ready" condition type is typically used to indicate the overall status of the namespace.
530606
items:
531-
description: "Condition contains details for one aspect of the current
532-
state of this API Resource.\n---\nThis struct is intended for
533-
direct use as an array at the field path .status.conditions. For
534-
example,\n\n\n\ttype FooStatus struct{\n\t // Represents the
535-
observations of a foo's current state.\n\t // Known .status.conditions.type
536-
are: \"Available\", \"Progressing\", and \"Degraded\"\n\t //
537-
+patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t
538-
\ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\"
539-
patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t
540-
\ // other fields\n\t}"
607+
description: Condition contains details for one aspect of the current
608+
state of this API Resource.
541609
properties:
542610
lastTransitionTime:
543611
description: |-
@@ -578,12 +646,7 @@ spec:
578646
- Unknown
579647
type: string
580648
type:
581-
description: |-
582-
type of condition in CamelCase or in foo.example.com/CamelCase.
583-
---
584-
Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be
585-
useful (see .node.status.conditions), the ability to deconflict is important.
586-
The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
649+
description: type of condition in CamelCase or in foo.example.com/CamelCase.
587650
maxLength: 316
588651
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
589652
type: string

0 commit comments

Comments
 (0)