Skip to content

Commit fc30289

Browse files
stotystoty
committed
OMID-271 Support HBase 3
WIP
1 parent 7526263 commit fc30289

5 files changed

Lines changed: 146 additions & 22 deletions

File tree

‎hbase-client/src/main/java/org/apache/omid/transaction/TTable.java‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import org.apache.hadoop.hbase.Cell;
3232
import org.apache.hadoop.hbase.CellUtil;
3333
import org.apache.hadoop.hbase.HConstants;
34-
import org.apache.hadoop.hbase.HTableDescriptor;
3534
import org.apache.hadoop.hbase.KeyValue;
3635
import org.apache.hadoop.hbase.KeyValueUtil;
3736
import org.apache.hadoop.hbase.TableName;
@@ -517,16 +516,15 @@ public Configuration getConfiguration() {
517516
}
518517

519518
/**
520-
* Delegates to {@link Table#getTableDescriptor()}
519+
* Returns the backing table.
520+
* This is needed by Phoenix, so that it can implement getHTableDescriptor in its compatibility
521+
* module
521522
*
522-
* This deprecated method is implemented for backwards compatibility reasons.
523-
* use {@link TTable#getDescriptor()}
524-
*
525-
* @return HTableDescriptor an instance of HTableDescriptor
523+
* @return Table an instance of Table
526524
* @throws IOException if a remote or network exception occurs.
527525
*/
528-
public HTableDescriptor getTableDescriptor() throws IOException {
529-
return table.getTableDescriptor();
526+
public Table getHBaseTable() throws IOException {
527+
return table;
530528
}
531529

532530
/**

‎hbase-coprocessor/src/main/java/org/apache/omid/transaction/CellSkipFilterBase.java‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,11 @@
2222
import org.apache.hadoop.hbase.PrivateCellUtil;
2323
import org.apache.hadoop.hbase.filter.Filter;
2424
import org.apache.hadoop.hbase.filter.FilterBase;
25+
import org.slf4j.Logger;
26+
import org.slf4j.LoggerFactory;
2527

2628
import java.io.IOException;
29+
import java.lang.reflect.Method;
2730
import java.util.List;
2831

2932
/**
@@ -35,10 +38,26 @@
3538
*/
3639

3740
public class CellSkipFilterBase extends FilterBase {
41+
42+
private static final Logger LOG = LoggerFactory.getLogger(CellSkipFilterBase.class);
43+
3844
private final Filter filter;
3945
// remember the previous cell processed by filter when the return code was NEXT_COL or INCLUDE_AND_NEXT_COL
4046
private Cell skipColumn = null;
4147

48+
// Using reflection to avoid adding compatibility modules.
49+
private static final Method removedFilterRowKeyMethod;
50+
static {
51+
Method tmpFilterRowKeyMethod;
52+
try {
53+
tmpFilterRowKeyMethod = FilterBase.class.getMethod("filterRowKey", byte[].class, int.class, int.class);
54+
} catch (Exception e) {
55+
LOG.info("Could not get filterRowKey method handle by reflection. This is normal for HBase 3.x", e);
56+
tmpFilterRowKeyMethod = null;
57+
}
58+
removedFilterRowKeyMethod = tmpFilterRowKeyMethod;
59+
}
60+
4261
public CellSkipFilterBase(Filter filter) {
4362
this.filter = filter;
4463
}
@@ -60,8 +79,9 @@ private boolean skipCellVersion(Cell cell) {
6079
/**
6180
* This deprecated method is implemented for backwards compatibility reasons.
6281
* use {@link CellSkipFilterBase#filterKeyValue(Cell)}
82+
*
83+
* No @Override because HBase 3 completely removes this method
6384
*/
64-
@Override
6585
public ReturnCode filterKeyValue(Cell cell) throws IOException {
6686
return filterCell(cell);
6787
}
@@ -103,10 +123,23 @@ public void reset() throws IOException {
103123
/**
104124
* This deprecated method is implemented for backwards compatibility reasons.
105125
* use {@link CellSkipFilterBase#filterRowKey(Cell)}
126+
*
127+
* No @Override so that it compiles with HBase 3
106128
*/
107-
@Override
108129
public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
109-
return filter.filterRowKey(buffer, offset, length);
130+
// Even though this is deprecated, this is the actual implementation that filterRowKey(Cell)
131+
// calls in FilterBase in HBase 2, so we cannot call filterRowKey(Cell), because that would
132+
// cause infinite recursion
133+
if (removedFilterRowKeyMethod == null) {
134+
// This should never happen, this can only get called from HBase 2.x and
135+
// the reflected method is public.
136+
throw new UnsupportedOperationException();
137+
}
138+
try {
139+
return (boolean) (removedFilterRowKeyMethod.invoke(filter, buffer, offset, length));
140+
} catch (Exception e) {
141+
throw new UnsupportedOperationException(e);
142+
}
110143
}
111144

112145
@Override

‎hbase-coprocessor/src/main/java/org/apache/omid/transaction/TransactionVisibilityFilterBase.java‎

Lines changed: 65 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,14 @@
1818
package org.apache.omid.transaction;
1919

2020
import org.apache.phoenix.thirdparty.com.google.common.base.Optional;
21-
22-
21+
import org.slf4j.Logger;
22+
import org.slf4j.LoggerFactory;
2323
import org.apache.commons.collections4.map.LRUMap;
2424
import org.apache.hadoop.hbase.Cell;
2525
import org.apache.hadoop.hbase.CellUtil;
26+
import org.apache.hadoop.hbase.KeyValue;
27+
import org.apache.hadoop.hbase.KeyValueUtil;
28+
import org.apache.hadoop.hbase.Cell.Type;
2629
import org.apache.hadoop.hbase.client.Get;
2730
import org.apache.hadoop.hbase.client.Result;
2831
import org.apache.hadoop.hbase.filter.Filter;
@@ -31,8 +34,11 @@
3134

3235
import org.apache.hadoop.hbase.util.Bytes;
3336

34-
3537
import java.io.IOException;
38+
import java.lang.invoke.MethodHandle;
39+
import java.lang.invoke.MethodHandles;
40+
import java.lang.invoke.MethodType;
41+
import java.lang.reflect.Method;
3642
import java.util.HashMap;
3743

3844
import java.util.List;
@@ -41,6 +47,8 @@
4147

4248
public class TransactionVisibilityFilterBase extends FilterBase {
4349

50+
private static final Logger LOG = LoggerFactory.getLogger(TransactionVisibilityFilterBase.class);
51+
4452
// optional sub-filter to apply to visible cells
4553
private final Filter userFilter;
4654
private final SnapshotFilterImpl snapshotFilter;
@@ -51,6 +59,31 @@ public class TransactionVisibilityFilterBase extends FilterBase {
5159
// So no need to keep row name
5260
private final Map<ImmutableBytesWritable, Long> familyDeletionCache;
5361

62+
// Using reflection to avoid adding compatibility modules.
63+
private static final Method removedFilterRowKeyMethod;
64+
private static final MethodHandle removedSuperFilterRowKeyMethod;
65+
static {
66+
Method tmpFilterRowKeyMethod;
67+
try {
68+
tmpFilterRowKeyMethod = FilterBase.class.getMethod("filterRowKey", byte[].class, int.class, int.class);
69+
} catch (Exception e) {
70+
LOG.info("Could not get filterRowKey method handle by reflection. This is normal for HBase 3.x", e);
71+
tmpFilterRowKeyMethod = null;
72+
}
73+
removedFilterRowKeyMethod = tmpFilterRowKeyMethod;
74+
75+
MethodHandle tmpRemovedSuperFilterRowKeyMethod;
76+
try {
77+
tmpRemovedSuperFilterRowKeyMethod = MethodHandles.lookup().findSpecial(FilterBase.class, "filterRowKey",
78+
MethodType.methodType(boolean.class, byte[].class, int.class, int.class),
79+
TransactionVisibilityFilterBase.class);
80+
} catch (Exception e) {
81+
LOG.info("Could not get super.filterRowKey method handle by reflection. This is normal for HBase 3.x", e);
82+
tmpRemovedSuperFilterRowKeyMethod = null;
83+
}
84+
removedSuperFilterRowKeyMethod = tmpRemovedSuperFilterRowKeyMethod;
85+
}
86+
5487
public TransactionVisibilityFilterBase(Filter cellFilter,
5588
SnapshotFilterImpl snapshotFilter,
5689
HBaseTransaction hbaseTransaction) {
@@ -65,8 +98,9 @@ public TransactionVisibilityFilterBase(Filter cellFilter,
6598
/**
6699
* This deprecated method is implemented for backwards compatibility reasons.
67100
* use {@link TransactionVisibilityFilterBase#filterCell(Cell)}
101+
*
102+
* No @Override so that it compiles with HBase 3.x
68103
*/
69-
@Override
70104
public ReturnCode filterKeyValue(Cell cell) throws IOException {
71105
return filterCell(cell);
72106
}
@@ -202,16 +236,38 @@ public boolean filterRow() throws IOException {
202236
return super.filterRow();
203237
}
204238

205-
/**
239+
/**
206240
* This deprecated method is implemented for backwards compatibility reasons.
207241
* use {@link TransactionVisibilityFilterBase#filterRowKey(Cell)}
242+
*
243+
* No @Override so that it compiles with HBase 3.x
208244
*/
209-
@Override
210245
public boolean filterRowKey(byte[] buffer, int offset, int length) throws IOException {
246+
// Even though this is deprecated, this is the actual implementation that filterRowKey(Cell)
247+
// calls in FilterBase in HBase 2, so we cannot call filterRowKey(Cell), because that would
248+
// cause infinite recursion
249+
if (removedFilterRowKeyMethod == null) {
250+
// This should never happen, this can only get called from HBase 2.x and
251+
// the reflected method is public.
252+
throw new UnsupportedOperationException("Called from Hbase 2, but could net get Method handle");
253+
}
211254
if (userFilter != null) {
212-
return userFilter.filterRowKey(buffer, offset, length);
255+
try {
256+
return (boolean) (removedFilterRowKeyMethod.invoke(userFilter, buffer, offset, length));
257+
} catch (Exception e) {
258+
throw new UnsupportedOperationException(e);
259+
}
260+
}
261+
if (removedSuperFilterRowKeyMethod == null) {
262+
// This should never happen, this can only get called from HBase 2.x and
263+
// the reflected method is public.
264+
throw new UnsupportedOperationException("Called from Hbase 2, but could net get super method handle");
265+
}
266+
try {
267+
return (boolean) (removedSuperFilterRowKeyMethod.invoke(this, buffer, offset, length));
268+
} catch (Throwable e) {
269+
throw new UnsupportedOperationException(e);
213270
}
214-
return super.filterRowKey(buffer, offset, length);
215271
}
216272

217273
@Override
@@ -271,4 +327,5 @@ public byte[] toByteArray() throws IOException {
271327
public Filter getInnerFilter() {
272328
return userFilter;
273329
}
330+
274331
}

‎pom.xml‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -167,15 +167,17 @@
167167
<protobuf.plugin.version>0.6.1</protobuf.plugin.version>
168168
<junit.version>4.13.2</junit.version>
169169
<mockito.version>2.28.2</mockito.version>
170-
<disruptor.version>3.2.0</disruptor.version>
170+
<disruptor.version>3.3.6</disruptor.version>
171171
<metrics.version>3.0.1</metrics.version>
172172
<jcommander.version>1.82</jcommander.version>
173173
<commons.conf.version>1.10</commons.conf.version>
174174
<hamcrest.version>1.3</hamcrest.version>
175175
<guava.version>32.1.3-jre</guava.version>
176176
<!-- 2.12+ shades guava -->
177+
<!-- While 4.2 seems to support ZK 3.8 in general, it cannot start a ZK 3.8 test
178+
minicluster -->
179+
<!-- When using ZK 3.8, you need to use ZK 5.3-->
177180
<curator.version>4.2.0</curator.version>
178-
<zookeeper.version>3.5.9</zookeeper.version>
179181
<snakeyaml.version>2.0</snakeyaml.version>
180182
<beanutils.version>1.9.4</beanutils.version>
181183
<commons-io.version>2.11.0</commons-io.version>
@@ -482,6 +484,35 @@
482484

483485
<profiles>
484486

487+
<profile>
488+
<!-- When building with a HBase version that depends on ZK 3.8 (or later)
489+
the default Curator 4.2.0 TestServer cannot start the test mini cluster.
490+
491+
This profile is only necessary to run the tests.
492+
Omid artifacts built with curator 4.2.0 should work with HBase versions using ZK 3.8 -->
493+
<id>zk38</id>
494+
495+
<properties>
496+
<curator.version>5.3.0</curator.version>
497+
</properties>
498+
499+
<dependencyManagement>
500+
<dependencies>
501+
<dependency>
502+
<groupId>org.apache.curator</groupId>
503+
<artifactId>curator-test</artifactId>
504+
<version>${curator.version}</version>
505+
<exclusions>
506+
<exclusion>
507+
<groupId>org.junit.jupiter</groupId>
508+
<artifactId>junit-jupiter-api</artifactId>
509+
</exclusion>
510+
</exclusions>
511+
</dependency>
512+
</dependencies>
513+
</dependencyManagement>
514+
</profile>
515+
485516
<profile>
486517
<id>site-deploy</id>
487518
<pluginRepositories>
@@ -1031,6 +1062,11 @@
10311062
<groupId>org.apache.zookeeper</groupId>
10321063
<artifactId>zookeeper</artifactId>
10331064
</exclusion>
1065+
<exclusion>
1066+
<!-- from Curator 5 -->
1067+
<groupId>org.junit.jupiter</groupId>
1068+
<artifactId>junit-jupiter-api</artifactId>
1069+
</exclusion>
10341070
</exclusions>
10351071
<version>${curator.version}</version>
10361072
</dependency>

‎timestamp-storage/src/test/java/org/apache/omid/timestamp/storage/TestZKTimestampStorage.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ public void initStuff() throws Exception {
6969
try {
7070
zkServer = new TestingServer(ZK_PORT);
7171
} catch (BindException e) {
72-
System.err.println("Getting bind exception - retrying to allocate server");
72+
LOG.info("Getting bind exception - retrying to allocate server", e);
7373
zkServer = null;
7474
}
7575
}

0 commit comments

Comments
 (0)