deps: upgrade hbase2 to 2.3.6 (#3429)

mutianf · web-flow · commit 3ce4cbc6515c · 2022-02-01T11:07:09.000-05:00
* deps: upgrade hbase2 to 2.3.6

* clean up test

* fix comment

* Update async registery class

* fix test

* use BigtableConfiguration in BigtableEnv

* clean up code

* add more comments to BigtableAsyncRegistry

* use version properties in pom file

* use 1.12.7 for bytebuddy
diff --git a/bigtable-client-core-parent/bigtable-hbase/pom.xml b/bigtable-client-core-parent/bigtable-hbase/pom.xml
@@ -168,6 +168,12 @@ limitations under the License.
       </exclusions>
     </dependency>
 
+    <dependency>
+      <groupId>net.bytebuddy</groupId>
+      <artifactId>byte-buddy</artifactId>
+      <version>${byte.buddy.version}</version>
+    </dependency>
+
     <!-- Test -->
     <dependency>
       <groupId>com.google.cloud.bigtable</groupId>
diff --git a/bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableConfiguration.java b/bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableConfiguration.java
@@ -20,6 +20,7 @@
 import com.google.auth.Credentials;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.BigtableAsyncRegistry;
 import org.apache.hadoop.hbase.client.Connection;
 
 /** This class provides a simplified mechanism of creating a programmatic Bigtable Connection. */
@@ -104,7 +105,7 @@ private static Configuration injectBigtableImpls(Configuration configuration) {
       configuration.set(
           HBASE_CLIENT_ASYNC_CONNECTION_IMPL, BIGTABLE_HBASE_CLIENT_ASYNC_CONNECTION_CLASS);
       configuration.set(
-          HBASE_CLIENT_ASYNC_REGISTRY_IMPL, BIGTABLE_HBASE_CLIENT_ASYNC_REGISTRY_CLASS);
+          HBASE_CLIENT_ASYNC_REGISTRY_IMPL, BigtableAsyncRegistry.getSubClass().getName());
     } catch (ClassNotFoundException ignored) {
       // Skip if any of the async connection class doesn't exist
     }
@@ -193,7 +194,7 @@ public static Configuration withCredentials(Configuration conf, Credentials cred
   @Deprecated
   public static Configuration asyncConfigure(Configuration conf) {
     conf.set(HBASE_CLIENT_ASYNC_CONNECTION_IMPL, BIGTABLE_HBASE_CLIENT_ASYNC_CONNECTION_CLASS);
-    conf.set(HBASE_CLIENT_ASYNC_REGISTRY_IMPL, BIGTABLE_HBASE_CLIENT_ASYNC_REGISTRY_CLASS);
+    conf.set(HBASE_CLIENT_ASYNC_REGISTRY_IMPL, BigtableAsyncRegistry.getSubClass().getName());
     return conf;
   }
 
diff --git a/bigtable-client-core-parent/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/BigtableAsyncRegistry.java b/bigtable-client-core-parent/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/BigtableAsyncRegistry.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import com.google.api.core.InternalApi;
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.dynamic.DynamicType;
+import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
+import net.bytebuddy.dynamic.loading.InjectionClassLoader;
+import net.bytebuddy.implementation.FixedValue;
+import net.bytebuddy.implementation.InvocationHandlerAdapter;
+import net.bytebuddy.implementation.MethodCall;
+import net.bytebuddy.matcher.ElementMatchers;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Bigtable implementation of org.apache.hadoop.hbase.client.ConnectionRegistry or
+ * org.apache.hadoop.hbase.client.AsyncRegistry depending on the HBase 2 version. The default HBase
+ * 2 implementation provided by ZKAsyncRegistry / ZKConnectionRegistry assumes a ZooKeeper
+ * environment, which is not the case for Bigtable.
+ *
+ * <p>This class is injected via the system property: "hbase.client.registry.impl". For further
+ * details See HConstants#CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY,
+ * AsyncRegistry#createAsyncConnection() for hbase < 2.3 and
+ * ConnectionFactory#createAsyncConnection() for hbase >= 2.3.
+ *
+ * <p>For internal use only - public for technical reasons.
+ */
+@InternalApi("For internal usage only")
+public class BigtableAsyncRegistry {
+
+  private static Class<? extends BigtableAsyncRegistry> subClass = null;
+
+  /** Constructor for org.apache.hadoop.hbase.client.ZKConnectionRegistry for hbase >= 2.3. */
+  public BigtableAsyncRegistry() {}
+
+  /**
+   * Constructor for org.apache.hadoop.hbase.client.ZKAsyncRegistry for hbase < 2.3. Configuration
+   * is not used in BigtableAsyncRegistry.
+   */
+  public BigtableAsyncRegistry(Configuration conf) {}
+
+  /**
+   * Implements the async connection registry class depending on which class is present in the class
+   * path. #close and #getClusterId are implemented. #getClusterId is used in
+   * ConnectionFactory#getAsyncConnection, so a non null return value is required for successful
+   * creation of asyncConnection. Throws {@link UnsupportedOperationException} on all other methods.
+   */
+  private static Class<? extends BigtableAsyncRegistry> createSubClass()
+      throws NoSuchMethodException {
+    List<String> classNames =
+        ImmutableList.of(
+            "org.apache.hadoop.hbase.client.ConnectionRegistry",
+            "org.apache.hadoop.hbase.client.AsyncRegistry");
+
+    DynamicType.Builder<BigtableAsyncRegistry> subclassBuilder =
+        new ByteBuddy().subclass(BigtableAsyncRegistry.class);
+
+    for (String className : classNames) {
+      try {
+        subclassBuilder = subclassBuilder.implement(Class.forName(className));
+      } catch (ClassNotFoundException e) {
+        continue;
+      }
+    }
+
+    return subclassBuilder
+        .method(ElementMatchers.isAbstract())
+        .intercept(
+            InvocationHandlerAdapter.of(new AbstractBigtableAdmin.UnsupportedOperationsHandler()))
+        .method(ElementMatchers.named("close"))
+        .intercept(MethodCall.invoke(BigtableAsyncRegistry.class.getDeclaredMethod("closeNoop")))
+        .method(ElementMatchers.named("getClusterId"))
+        .intercept(FixedValue.value((CompletableFuture.completedFuture("NoopClusterId"))))
+        .make()
+        .load(InjectionClassLoader.getSystemClassLoader(), ClassLoadingStrategy.Default.INJECTION)
+        .getLoaded();
+  }
+
+  public static synchronized Class<? extends BigtableAsyncRegistry> getSubClass() {
+    if (subClass == null) {
+      try {
+        subClass = createSubClass();
+      } catch (NoSuchMethodException e) {
+        throw new RuntimeException(e);
+      }
+      if (subClass == null) {
+        throw new RuntimeException(
+            "Couldn't find org.apache.hadoop.hbase.client.ConnectionRegistry or org.apache.hadoop.hbase.client.AsyncRegistry in the classpath");
+      }
+    }
+    return subClass;
+  }
+
+  public void closeNoop() {}
+}
diff --git a/bigtable-hbase-1.x-parent/bigtable-hbase-1.x-hadoop/pom.xml b/bigtable-hbase-1.x-parent/bigtable-hbase-1.x-hadoop/pom.xml
@@ -213,6 +213,7 @@ limitations under the License.
                 <classEntry>org/apache/hadoop/hbase/client/AbstractBigtableAdmin</classEntry>
                 <classEntry>org/apache/hadoop/hbase/client/AbstractBigtableConnection</classEntry>
                 <classEntry>org/apache/hadoop/hbase/client/CommonConnection</classEntry>
+                <classEntry>org/apache/hadoop/hbase/client/BigtableAsyncRegistry</classEntry>
               </allowedJarClassEntries>
             </configuration>
           </execution>
diff --git a/bigtable-hbase-1.x-parent/bigtable-hbase-1.x-shaded/pom.xml b/bigtable-hbase-1.x-parent/bigtable-hbase-1.x-shaded/pom.xml
@@ -361,6 +361,7 @@ limitations under the License.
                 <classEntry>org/apache/hadoop/hbase/client/AbstractBigtableAdmin</classEntry>
                 <classEntry>org/apache/hadoop/hbase/client/AbstractBigtableConnection</classEntry>
                 <classEntry>org/apache/hadoop/hbase/client/CommonConnection</classEntry>
+                <classEntry>org/apache/hadoop/hbase/client/BigtableAsyncRegistry</classEntry>
               </allowedJarClassEntries>
             </configuration>
           </execution>
diff --git a/bigtable-hbase-1.x-parent/bigtable-hbase-1.x/pom.xml b/bigtable-hbase-1.x-parent/bigtable-hbase-1.x/pom.xml
@@ -149,7 +149,7 @@ limitations under the License.
     <dependency>
       <groupId>net.bytebuddy</groupId>
       <artifactId>byte-buddy</artifactId>
-      <version>1.12.7</version>
+      <version>${byte.buddy.version}</version>
     </dependency>
 
 
diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-hadoop/pom.xml b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-hadoop/pom.xml
@@ -32,7 +32,7 @@ limitations under the License.
 
   <properties>
     <!-- define a property that can be ignored by renovate -->
-    <hbase2-hadoop-slf4j.version>1.7.25</hbase2-hadoop-slf4j.version>
+    <hbase2-hadoop-slf4j.version>1.7.30</hbase2-hadoop-slf4j.version>
   </properties>
 
   <dependencies>
diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-integration-tests/pom.xml b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-integration-tests/pom.xml
@@ -35,7 +35,6 @@ limitations under the License.
     <hbase.version>${hbase2.version}</hbase.version>
     <google.bigtable.connection.impl>com.google.cloud.bigtable.hbase2_x.BigtableConnection</google.bigtable.connection.impl>
     <google.bigtable.async.connection.impl>org.apache.hadoop.hbase.client.BigtableAsyncConnection</google.bigtable.async.connection.impl>
-    <google.bigtable.registry.impl>org.apache.hadoop.hbase.client.BigtableAsyncRegistry</google.bigtable.registry.impl>
     <test.timeout>1800</test.timeout>
   </properties>
 
@@ -160,9 +159,9 @@ limitations under the License.
                   </includes>
                   <excludedGroups>KnownEmulatorGap,KnownGap</excludedGroups>
                   <systemPropertyVariables>
-                    <google.bigtable.test_env>bigtable</google.bigtable.test_env>
                     <google.bigtable.project.id>fake-project</google.bigtable.project.id>
                     <google.bigtable.instance.id>fake-instance</google.bigtable.instance.id>
+                    <google.bigtable.test_env>bigtable</google.bigtable.test_env>
                   </systemPropertyVariables>
                   <environmentVariables>
                     <BIGTABLE_EMULATOR_HOST>${bigtable.emulator.endpoint}</BIGTABLE_EMULATOR_HOST>
@@ -216,6 +215,10 @@ limitations under the License.
           <groupId>org.apache.hbase</groupId>
           <artifactId>hbase-shaded-client</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -262,6 +265,12 @@ limitations under the License.
       <artifactId>hbase-shaded-testing-util</artifactId>
       <version>${hbase2.version}</version>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <!-- Misc -->
@@ -285,6 +294,13 @@ limitations under the License.
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j.version}</version>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-log4j12</artifactId>
diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-shaded/pom.xml b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x-shaded/pom.xml
@@ -48,6 +48,16 @@ limitations under the License.
         <type>pom</type>
         <scope>import</scope>
       </dependency>
+      <dependency>
+        <groupId>org.slf4j</groupId>
+        <artifactId>slf4j-api</artifactId>
+        <version>${beam-slf4j.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.slf4j</groupId>
+        <artifactId>slf4j-log4j12</artifactId>
+        <version>${beam-slf4j.version}</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 
@@ -164,6 +174,7 @@ limitations under the License.
                   <!-- exclude hbase-shaded-client & all of its dependencies -->
                   <exclude>org.apache.hbase:hbase-shaded-client</exclude>
                   <exclude>org.slf4j:slf4j-api</exclude>
+                  <exclude>org.slf4j:slf4j-log4j12</exclude>
                   <exclude>log4j:log4j</exclude>
                   <exclude>org.apache.htrace:htrace-core4</exclude>
                   <exclude>org.apache.yetus:audience-annotations</exclude>
diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/pom.xml b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/pom.xml
@@ -84,6 +84,10 @@ limitations under the License.
           <groupId>org.slf4j</groupId>
           <artifactId>slf4j-log4j12</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -145,7 +149,7 @@ limitations under the License.
     <dependency>
       <groupId>net.bytebuddy</groupId>
       <artifactId>byte-buddy</artifactId>
-      <version>1.12.7</version>
+      <version>${byte.buddy.version}</version>
     </dependency>
 
     <!-- Test dependencies-->
diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableAdmin.java b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableAdmin.java
@@ -16,6 +16,7 @@
 package com.google.cloud.bigtable.hbase2_x;
 
 import com.google.api.core.InternalApi;
+import com.google.cloud.bigtable.hbase.util.FutureUtil;
 import com.google.cloud.bigtable.hbase.util.ModifyTableBuilder;
 import com.google.cloud.bigtable.hbase2_x.adapters.admin.TableAdapter2x;
 import com.google.common.base.Throwables;
@@ -34,7 +35,9 @@
 import java.util.concurrent.Future;
 import java.util.regex.Pattern;
 import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.description.type.TypeDescription;
 import net.bytebuddy.implementation.InvocationHandlerAdapter;
+import net.bytebuddy.implementation.MethodCall;
 import net.bytebuddy.matcher.ElementMatchers;
 import org.apache.hadoop.hbase.ClusterMetrics;
 import org.apache.hadoop.hbase.ClusterMetrics.Option;
@@ -207,16 +210,6 @@ public void snapshot(String snapshotId, TableName tableName, SnapshotType ignore
     snapshot(snapshotId, tableName);
   }
 
-  /** {@inheritDoc} */
-  @Override
-  public void snapshotAsync(SnapshotDescription snapshot)
-      throws IOException, SnapshotCreationException {
-    asyncAdmin.snapshot(snapshot);
-    LOG.warn(
-        "isSnapshotFinished() is not currently supported by BigtableAdmin.\n"
-            + "You may poll for existence of the snapshot with listSnapshots(snapshotName)");
-  }
-
   @Override
   public Future<Void> addColumnFamilyAsync(
       TableName tableName, ColumnFamilyDescriptor columnFamily) {
@@ -291,7 +284,7 @@ public void modifyTable(TableDescriptor tableDescriptor) throws IOException {
 
   @Override
   public void modifyTable(TableName tableName, TableDescriptor tableDescriptor) throws IOException {
-    super.modifyTable(tableName, new HTableDescriptor(tableDescriptor));
+    FutureUtil.unwrap(modifyTableAsync(tableName, tableDescriptor));
   }
 
   @Override
@@ -359,6 +352,32 @@ public ClusterStatus getClusterStatus() throws IOException {
         -1);
   }
 
+  /**
+   * HBase 2.x has different return type for snapshotAsync method in different versions. {@link
+   * #getSubclass} will generate the code to call the correct snapshotAsync method based on the
+   * hbase version.
+   */
+  protected void snapshotAsyncVoid(SnapshotDescription snapshot)
+      throws IOException, SnapshotCreationException {
+    asyncAdmin.snapshot(snapshot);
+    LOG.warn(
+        "isSnapshotFinished() is not currently supported by BigtableAdmin.\n"
+            + "You may poll for existence of the snapshot with listSnapshots(snapshotName)");
+  }
+
+  /**
+   * HBase 2.x has different return type for snapshotAsync method in different versions. {@link
+   * #getSubclass} will generate the code to call the correct snapshotAsync method based on the
+   * hbase version.
+   */
+  protected Future<Void> snapshotAsyncFuture(SnapshotDescription snapshot)
+      throws IOException, SnapshotCreationException {
+    LOG.warn(
+        "isSnapshotFinished() is not currently supported by BigtableAdmin.\n"
+            + "You may poll for existence of the snapshot with listSnapshots(snapshotName)");
+    return asyncAdmin.snapshot(snapshot);
+  }
+
   private static Class<? extends BigtableAdmin> adminClass = null;
 
   /**
@@ -367,7 +386,8 @@ public ClusterStatus getClusterStatus() throws IOException {
    * are called. If a method is implemented by BigtableAdmin, the generated class will invoke the
    * implementation in BigtableAdmin. Otherwise it'll throw {@link UnsupportedOperationException}.
    */
-  private static synchronized Class<? extends BigtableAdmin> getSubclass() {
+  private static synchronized Class<? extends BigtableAdmin> getSubclass()
+      throws NoSuchMethodException {
     if (adminClass == null) {
       adminClass =
           new ByteBuddy()
@@ -376,6 +396,21 @@ private static synchronized Class<? extends BigtableAdmin> getSubclass() {
               .intercept(
                   InvocationHandlerAdapter.of(
                       new AbstractBigtableAdmin.UnsupportedOperationsHandler()))
+              .method(
+                  ElementMatchers.named("snapshotAsync")
+                      .and(ElementMatchers.returns(TypeDescription.VOID)))
+              .intercept(
+                  MethodCall.invoke(
+                          BigtableAdmin.class.getDeclaredMethod(
+                              "snapshotAsyncVoid", SnapshotDescription.class))
+                      .withAllArguments())
+              .method(
+                  ElementMatchers.named("snapshotAsync").and(ElementMatchers.returns(Future.class)))
+              .intercept(
+                  MethodCall.invoke(
+                          BigtableAdmin.class.getDeclaredMethod(
+                              "snapshotAsyncFuture", SnapshotDescription.class))
+                      .withAllArguments())
               .make()
               .load(BigtableAdmin.class.getClassLoader())
               .getLoaded();
diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableAsyncTable.java b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableAsyncTable.java
diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableTable.java b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableTable.java
diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/org/apache/hadoop/hbase/client/BigtableAsyncConnection.java b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/org/apache/hadoop/hbase/client/BigtableAsyncConnection.java
diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/org/apache/hadoop/hbase/client/BigtableAsyncRegistry.java b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/org/apache/hadoop/hbase/client/BigtableAsyncRegistry.java
diff --git a/pom.xml b/pom.xml
