Skip to content

[improve][broker] Add bookkeeperDeleted in topic internal stats. #20833

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dragonls wants to merge 11 commits into
base: master
Choose a base branch
from
Open

[improve][broker] Add bookkeeperDeleted in topic internal stats. #20833

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
03192b5
Add bookkeeperDeleted in topic internal stats.
Jul 18, 2023
bc8cfcd
Merge remote-tracking branch 'upstream/master' into add-bookkeeper-de…
Nov 6, 2023
45be5de
Merge remote-tracking branch 'upstream/master' into add-bookkeeper-de…
Apr 16, 2024
c6ca528
Add backward test cases.
Apr 17, 2024
5a1668e
Add backward test cases.
Apr 18, 2024
5411b3d
Add backward test cases.
Apr 23, 2024
1600975
Add backward test cases.
Apr 23, 2024
800af25
Add backward test cases.
Apr 23, 2024
84597d8
Add backward test cases.
Apr 24, 2024
52e3aca
Add backward test cases.
Apr 24, 2024
fe144e7
Add backward test cases.
Apr 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4398,6 +4398,7 @@ public CompletableFuture<ManagedLedgerInternalStats> getManagedLedgerInternalSta
info.entries = li.getEntries();
info.size = li.getSize();
info.offloaded = li.hasOffloadContext() && li.getOffloadContext().getComplete();
info.bookkeeperDeleted = li.hasOffloadContext() && li.getOffloadContext().getBookkeeperDeleted();
if (includeLedgerMetadata) {
// lookup metadata from the hashmap which contains completed async operations
info.metadata = ledgerMetadataFutures.get(li.getLedgerId()).getNow(null);
Expand Down
2 changes: 2 additions & 0 deletions pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2420,6 +2420,8 @@ public CompletableFuture<PersistentTopicInternalStats> getInternalStats(boolean
info.entries = li.getEntries();
info.size = li.getSize();
info.offloaded = li.hasOffloadContext() && li.getOffloadContext().getComplete();
info.bookkeeperDeleted =
li.hasOffloadContext() && li.getOffloadContext().getBookkeeperDeleted();
stats.ledgers.add(info);
if (includeLedgerMetadata) {
futures.add(ml.getLedgerMetadata(li.getLedgerId()).handle((lMetadata, ex) -> {
Expand Down
1 change: 1 addition & 0 deletions ...-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ public static class LedgerInfo {
public boolean offloaded;
public String metadata;
public boolean underReplicated;
public boolean bookkeeperDeleted;
Copy link
Copy Markdown
Contributor

@hangc0276 hangc0276 Nov 20, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We'd better add a compatibility test to cover the newly added field. Refer to: #21521 (comment)

}

/**
Expand Down
24 changes: 24 additions & 0 deletions ...ntegration/src/test/java/org/apache/pulsar/tests/integration/offload/TestBaseOffload.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats;
import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats;
import org.apache.pulsar.tests.integration.suites.PulsarTieredStorageTestSuite;
import org.awaitility.Awaitility;
Expand Down Expand Up @@ -304,12 +305,21 @@ protected void testPublishOffloadAndConsumeDeletionLag(String serviceUrl, String
"namespaces", "get-offload-deletion-lag", namespace).getStdout();
Assert.assertTrue(output.contains("Unset for namespace"));

PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(adminUrl).build();

long offloadedLedger = writeAndWaitForOffload(serviceUrl, adminUrl, topic);
// give it up to 5 seconds to delete, it shouldn't
// so we wait this every time
Thread.sleep(5000);
Assert.assertTrue(ledgerExistsInBookKeeper(offloadedLedger));

long finalOffloadedLedger1 = offloadedLedger;
ManagedLedgerInternalStats.LedgerInfo offloadedLedgerInfo =
admin.topics().getInternalStats(topic).ledgers.stream()
.filter((x) -> x.ledgerId == finalOffloadedLedger1).findFirst().get();
Assert.assertTrue(offloadedLedgerInfo.offloaded);
Assert.assertFalse(offloadedLedgerInfo.bookkeeperDeleted);

pulsarCluster.runAdminCommandOnAnyBroker("namespaces", "set-offload-deletion-lag", namespace,
"--lag", "0m");
output = pulsarCluster.runAdminCommandOnAnyBroker(
Expand All @@ -324,6 +334,12 @@ protected void testPublishOffloadAndConsumeDeletionLag(String serviceUrl, String
}
Assert.assertFalse(ledgerExistsInBookKeeper(offloadedLedger));

long finalOffloadedLedger2 = offloadedLedger;
offloadedLedgerInfo = admin.topics().getInternalStats(topic).ledgers.stream()
.filter((x) -> x.ledgerId == finalOffloadedLedger2).findFirst().get();
Assert.assertTrue(offloadedLedgerInfo.offloaded);
Assert.assertTrue(offloadedLedgerInfo.bookkeeperDeleted);

pulsarCluster.runAdminCommandOnAnyBroker("namespaces", "clear-offload-deletion-lag", namespace);

Thread.sleep(5); // wait 5 seconds to allow broker to see update
Expand All @@ -338,6 +354,14 @@ protected void testPublishOffloadAndConsumeDeletionLag(String serviceUrl, String
// so we wait this every time
Thread.sleep(5000);
Assert.assertTrue(ledgerExistsInBookKeeper(offloadedLedger));

long finalOffloadedLedger3 = offloadedLedger;
offloadedLedgerInfo = admin.topics().getInternalStats(topic).ledgers.stream()
.filter((x) -> x.ledgerId == finalOffloadedLedger3).findFirst().get();
Assert.assertTrue(offloadedLedgerInfo.offloaded);
Assert.assertFalse(offloadedLedgerInfo.bookkeeperDeleted);

admin.close();
}

protected void testDeleteOffloadedTopic(String serviceUrl, String adminUrl,
Expand Down