Skip to content

Commit 13c19b5

Browse files
BewareMyPowerBewareMyPower
authored
[improve][broker] Register the broker to metadata store without version id compare (#23298)
1 parent fc60ec0 commit 13c19b5

5 files changed

Lines changed: 96 additions & 31 deletions

File tree

‎pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryImpl.java‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,11 @@
2121
import static org.apache.pulsar.broker.loadbalance.LoadManager.LOADBALANCE_BROKERS_ROOT;
2222
import com.google.common.annotations.VisibleForTesting;
2323
import java.util.ArrayList;
24+
import java.util.EnumSet;
2425
import java.util.List;
2526
import java.util.Map;
2627
import java.util.Optional;
2728
import java.util.concurrent.CompletableFuture;
28-
import java.util.concurrent.CompletionException;
2929
import java.util.concurrent.ConcurrentHashMap;
3030
import java.util.concurrent.ExecutionException;
3131
import java.util.concurrent.RejectedExecutionException;
@@ -39,11 +39,11 @@
3939
import org.apache.pulsar.broker.ServiceConfiguration;
4040
import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData;
4141
import org.apache.pulsar.common.util.FutureUtil;
42+
import org.apache.pulsar.metadata.api.MetadataCache;
4243
import org.apache.pulsar.metadata.api.MetadataStoreException;
4344
import org.apache.pulsar.metadata.api.Notification;
4445
import org.apache.pulsar.metadata.api.NotificationType;
45-
import org.apache.pulsar.metadata.api.coordination.LockManager;
46-
import org.apache.pulsar.metadata.api.coordination.ResourceLock;
46+
import org.apache.pulsar.metadata.api.extended.CreateOption;
4747

4848
/**
4949
* The broker registry impl, base on the LockManager.
@@ -57,16 +57,14 @@ public class BrokerRegistryImpl implements BrokerRegistry {
5757

5858
private final BrokerLookupData brokerLookupData;
5959

60-
private final LockManager<BrokerLookupData> brokerLookupDataLockManager;
60+
private final MetadataCache<BrokerLookupData> brokerLookupDataMetadataCache;
6161

62-
private final String brokerId;
62+
private final String brokerIdKeyPath;
6363

6464
private final ScheduledExecutorService scheduler;
6565

6666
private final List<BiConsumer<String, NotificationType>> listeners;
6767

68-
private volatile ResourceLock<BrokerLookupData> brokerLookupDataLock;
69-
7068
protected enum State {
7169
Init,
7270
Started,
@@ -79,10 +77,10 @@ protected enum State {
7977
public BrokerRegistryImpl(PulsarService pulsar) {
8078
this.pulsar = pulsar;
8179
this.conf = pulsar.getConfiguration();
82-
this.brokerLookupDataLockManager = pulsar.getCoordinationService().getLockManager(BrokerLookupData.class);
80+
this.brokerLookupDataMetadataCache = pulsar.getLocalMetadataStore().getMetadataCache(BrokerLookupData.class);
8381
this.scheduler = pulsar.getLoadManagerExecutor();
8482
this.listeners = new ArrayList<>();
85-
this.brokerId = pulsar.getBrokerId();
83+
this.brokerIdKeyPath = keyPath(pulsar.getBrokerId());
8684
this.brokerLookupData = new BrokerLookupData(
8785
pulsar.getWebServiceAddress(),
8886
pulsar.getWebServiceAddressTls(),
@@ -122,7 +120,7 @@ public boolean isStarted() {
122120
public synchronized void register() throws MetadataStoreException {
123121
if (this.state == State.Started) {
124122
try {
125-
this.brokerLookupDataLock = brokerLookupDataLockManager.acquireLock(keyPath(brokerId), brokerLookupData)
123+
brokerLookupDataMetadataCache.put(brokerIdKeyPath, brokerLookupData, EnumSet.of(CreateOption.Ephemeral))
126124
.get(conf.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS);
127125
this.state = State.Registered;
128126
} catch (InterruptedException | ExecutionException | TimeoutException e) {
@@ -135,30 +133,37 @@ public synchronized void register() throws MetadataStoreException {
135133
public synchronized void unregister() throws MetadataStoreException {
136134
if (this.state == State.Registered) {
137135
try {
138-
this.brokerLookupDataLock.release()
136+
brokerLookupDataMetadataCache.delete(brokerIdKeyPath)
139137
.get(conf.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS);
140-
this.state = State.Started;
141-
} catch (CompletionException | InterruptedException | ExecutionException | TimeoutException e) {
138+
} catch (ExecutionException e) {
139+
if (e.getCause() instanceof MetadataStoreException.NotFoundException) {
140+
log.warn("{} has already been unregistered", brokerIdKeyPath);
141+
} else {
142+
throw MetadataStoreException.unwrap(e);
143+
}
144+
} catch (InterruptedException | TimeoutException e) {
142145
throw MetadataStoreException.unwrap(e);
146+
} finally {
147+
this.state = State.Started;
143148
}
144149
}
145150
}
146151

147152
@Override
148153
public String getBrokerId() {
149-
return this.brokerId;
154+
return pulsar.getBrokerId();
150155
}
151156

152157
@Override
153158
public CompletableFuture<List<String>> getAvailableBrokersAsync() {
154159
this.checkState();
155-
return brokerLookupDataLockManager.listLocks(LOADBALANCE_BROKERS_ROOT).thenApply(ArrayList::new);
160+
return brokerLookupDataMetadataCache.getChildren(LOADBALANCE_BROKERS_ROOT);
156161
}
157162

158163
@Override
159164
public CompletableFuture<Optional<BrokerLookupData>> lookupAsync(String broker) {
160165
this.checkState();
161-
return brokerLookupDataLockManager.readLock(keyPath(broker));
166+
return brokerLookupDataMetadataCache.get(keyPath(broker));
162167
}
163168

164169
public CompletableFuture<Map<String, BrokerLookupData>> getAvailableBrokerLookupDataAsync() {
@@ -192,13 +197,8 @@ public synchronized void close() throws PulsarServerException {
192197
try {
193198
this.listeners.clear();
194199
this.unregister();
195-
this.brokerLookupDataLockManager.close();
196200
} catch (Exception ex) {
197-
if (ex.getCause() instanceof MetadataStoreException.NotFoundException) {
198-
throw new PulsarServerException.NotFoundException(MetadataStoreException.unwrap(ex));
199-
} else {
200-
throw new PulsarServerException(MetadataStoreException.unwrap(ex));
201-
}
201+
log.error("Unexpected error when unregistering the broker registry", ex);
202202
} finally {
203203
this.state = State.Closed;
204204
}

‎pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/BrokerRegistryTest.java‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,7 @@ public void testRegisterAndLookup() throws Exception {
291291
}
292292

293293
@Test
294-
public void testRegisterFailWithSameBrokerId() throws Exception {
294+
public void testRegisterWithSameBrokerId() throws Exception {
295295
PulsarService pulsar1 = createPulsarService();
296296
PulsarService pulsar2 = createPulsarService();
297297
pulsar1.start();
@@ -301,14 +301,10 @@ public void testRegisterFailWithSameBrokerId() throws Exception {
301301
BrokerRegistryImpl brokerRegistry1 = createBrokerRegistryImpl(pulsar1);
302302
BrokerRegistryImpl brokerRegistry2 = createBrokerRegistryImpl(pulsar2);
303303
brokerRegistry1.start();
304-
try {
305-
brokerRegistry2.start();
306-
fail();
307-
} catch (Exception ex) {
308-
log.info("Broker registry start failed.", ex);
309-
assertTrue(ex instanceof PulsarServerException);
310-
assertTrue(ex.getMessage().contains("LockBusyException"));
311-
}
304+
brokerRegistry2.start();
305+
306+
pulsar1.close();
307+
pulsar2.close();
312308
}
313309

314310
@Test

‎pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,14 @@
1818
*/
1919
package org.apache.pulsar.metadata.api;
2020

21+
import java.util.EnumSet;
2122
import java.util.List;
2223
import java.util.Optional;
2324
import java.util.concurrent.CompletableFuture;
2425
import java.util.function.Function;
2526
import org.apache.pulsar.metadata.api.MetadataStoreException.AlreadyExistsException;
2627
import org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException;
28+
import org.apache.pulsar.metadata.api.extended.CreateOption;
2729

2830
/**
2931
* Represent the caching layer access for a specific type of objects.
@@ -128,6 +130,24 @@ public interface MetadataCache<T> {
128130
*/
129131
CompletableFuture<Void> create(String path, T value);
130132

133+
/**
134+
* Create or update the value of the given path in the metadata store without version comparison.
135+
* <p>
136+
* This method is equivalent to
137+
* {@link org.apache.pulsar.metadata.api.extended.MetadataStoreExtended#put(String, byte[], Optional, EnumSet)} or
138+
* {@link MetadataStore#put(String, byte[], Optional)} if the metadata store does not support this extended API,
139+
* with `Optional.empty()` as the 3rd argument. It means if the path does not exist, it will be created. If the path
140+
* already exists, the new value will override the old value.
141+
* </p>
142+
* @param path the path of the object in the metadata store
143+
* @param value the object to put in the metadata store
144+
* @param options the create options if the path does not in the metadata store
145+
* @return the future that indicates if this operation failed, it could fail with
146+
* {@link java.io.IOException} if the value failed to be serialized
147+
* {@link MetadataStoreException} if the metadata store operation failed
148+
*/
149+
CompletableFuture<Void> put(String path, T value, EnumSet<CreateOption> options);
150+
131151
/**
132152
* Delete an object from the metadata store.
133153
* <p>

‎pulsar-metadata/src/main/java/org/apache/pulsar/metadata/cache/impl/MetadataCacheImpl.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import com.github.benmanes.caffeine.cache.Caffeine;
2626
import com.google.common.annotations.VisibleForTesting;
2727
import java.io.IOException;
28+
import java.util.EnumSet;
2829
import java.util.List;
2930
import java.util.Optional;
3031
import java.util.concurrent.CompletableFuture;
@@ -47,12 +48,15 @@
4748
import org.apache.pulsar.metadata.api.MetadataStoreException.ContentDeserializationException;
4849
import org.apache.pulsar.metadata.api.MetadataStoreException.NotFoundException;
4950
import org.apache.pulsar.metadata.api.Notification;
51+
import org.apache.pulsar.metadata.api.extended.CreateOption;
52+
import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
5053
import org.apache.pulsar.metadata.impl.AbstractMetadataStore;
5154

5255
@Slf4j
5356
public class MetadataCacheImpl<T> implements MetadataCache<T>, Consumer<Notification> {
5457
@Getter
5558
private final MetadataStore store;
59+
private final MetadataStoreExtended storeExtended;
5660
private final MetadataSerde<T> serde;
5761

5862
private final AsyncLoadingCache<String, Optional<CacheGetResult<T>>> objCache;
@@ -67,6 +71,11 @@ public MetadataCacheImpl(MetadataStore store, JavaType type, MetadataCacheConfig
6771

6872
public MetadataCacheImpl(MetadataStore store, MetadataSerde<T> serde, MetadataCacheConfig cacheConfig) {
6973
this.store = store;
74+
if (store instanceof MetadataStoreExtended) {
75+
this.storeExtended = (MetadataStoreExtended) store;
76+
} else {
77+
this.storeExtended = null;
78+
}
7079
this.serde = serde;
7180

7281
Caffeine<Object, Object> cacheBuilder = Caffeine.newBuilder();
@@ -243,6 +252,21 @@ public CompletableFuture<Void> create(String path, T value) {
243252
return future;
244253
}
245254

255+
@Override
256+
public CompletableFuture<Void> put(String path, T value, EnumSet<CreateOption> options) {
257+
final byte[] bytes;
258+
try {
259+
bytes = serde.serialize(path, value);
260+
} catch (IOException e) {
261+
return CompletableFuture.failedFuture(e);
262+
}
263+
if (storeExtended != null) {
264+
return storeExtended.put(path, bytes, Optional.empty(), options).thenAccept(__ -> refresh(path));
265+
} else {
266+
return store.put(path, bytes, Optional.empty()).thenAccept(__ -> refresh(path));
267+
}
268+
}
269+
246270
@Override
247271
public CompletableFuture<Void> delete(String path) {
248272
return store.delete(path, Optional.empty());

‎pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataCacheTest.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import java.io.IOException;
3030
import java.nio.charset.StandardCharsets;
3131
import java.util.ArrayList;
32+
import java.util.EnumSet;
3233
import java.util.List;
3334
import java.util.Map;
3435
import java.util.Optional;
@@ -55,6 +56,7 @@
5556
import org.apache.pulsar.metadata.api.MetadataStoreFactory;
5657
import org.apache.pulsar.metadata.api.NotificationType;
5758
import org.apache.pulsar.metadata.api.Stat;
59+
import org.apache.pulsar.metadata.api.extended.CreateOption;
5860
import org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl;
5961
import org.awaitility.Awaitility;
6062
import org.testng.annotations.DataProvider;
@@ -597,4 +599,27 @@ public CustomClass deserialize(String path, byte[] content, Stat stat) throws IO
597599
assertEquals(res.getValue().b, 2);
598600
assertEquals(res.getValue().path, key1);
599601
}
602+
603+
@Test(dataProvider = "distributedImpl")
604+
public void testPut(String provider, Supplier<String> urlSupplier) throws Exception {
605+
@Cleanup final var store1 = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder()
606+
.build());
607+
final var cache1 = store1.getMetadataCache(Integer.class);
608+
@Cleanup final var store2 = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder()
609+
.build());
610+
final var cache2 = store2.getMetadataCache(Integer.class);
611+
final var key = "/testPut";
612+
613+
cache1.put(key, 1, EnumSet.of(CreateOption.Ephemeral)); // create
614+
Awaitility.await().untilAsserted(() -> {
615+
assertEquals(cache1.get(key).get().orElse(-1), 1);
616+
assertEquals(cache2.get(key).get().orElse(-1), 1);
617+
});
618+
619+
cache2.put(key, 2, EnumSet.of(CreateOption.Ephemeral)); // update
620+
Awaitility.await().untilAsserted(() -> {
621+
assertEquals(cache1.get(key).get().orElse(-1), 2);
622+
assertEquals(cache2.get(key).get().orElse(-1), 2);
623+
});
624+
}
600625
}

0 commit comments

Comments
 (0)