comment and file cleanup

Author: tianlei2

bigtable-dataflow-parent/bigtable-beam-import/src/main/java/com/google/cloud/bigtable/beam/hbasesnapshots/ImportJobFromHbaseSnapshot.java

@@ -333,8 +333,6 @@ static Pipeline buildPipelineWithMultipleSnapshots(
       return pipeline;
     }
     // Read records from hbase region files and write to Bigtable
-    //    PCollection<RegionConfig> hbaseRecords = restoredSnapshots
-    //            .apply("List Regions", new ListRegions());
     PCollection<KV<String, Iterable<Mutation>>> hbaseRecords =
         restoredSnapshots
             .apply("List Regions", new ListRegions())

bigtable-dataflow-parent/bigtable-beam-import/src/main/java/com/google/cloud/bigtable/beam/hbasesnapshots/SnapshotUtils.java

@@ -264,7 +264,6 @@ public static Map<String, String> getSnapshotsFromSnapshotPath(
     // Build GCS path from given string e.g:
     // gs://sym-bucket/snapshots/20220309230526/.hbase-snapshot
     GcsPath gcsPath = GcsPath.fromUri(importSnapshotpath);
-    // LOG.info("GCS Path:" + gcsPath + ";Object:" + gcsPath.getObject());
     Map<String, String> snapshots = new HashMap<>();
 
     List<StorageObject> objects =

bigtable-dataflow-parent/bigtable-beam-import/src/main/java/com/google/cloud/bigtable/beam/hbasesnapshots/conf/SnapshotConfig.java

@@ -38,7 +38,6 @@ public static Builder builder() {
 
   public abstract String getSourceLocation();
 
-  //  public abstract Path getSourcePath();
   @Memoized
   public Path getSourcePath() {
     return new Path(getSourceLocation());

bigtable-dataflow-parent/bigtable-beam-import/src/main/java/com/google/cloud/bigtable/beam/hbasesnapshots/transforms/ListRegions.java

@@ -67,7 +67,7 @@ private Map<Long, Long> computeRegionSize(SnapshotManifest snapshotManifest) {
         regionsSize.put(regionManifest.getRegionInfo().getRegionId(), totalSize);
       }
 
-      return regionsSize; // (int)Math.ceil((totalSize * 1.0)/GIGA_BYTE);
+      return regionsSize;
     }
 
     /**

bigtable-dataflow-parent/bigtable-beam-import/src/main/java/com/google/cloud/bigtable/beam/hbasesnapshots/transforms/ReadRegions.java

@@ -15,8 +15,6 @@
  */
 package com.google.cloud.bigtable.beam.hbasesnapshots.transforms;
 
-// import com.google.cloud.bigtable.beam.hbasesnapshots.ImportSnapshots;
-
 import com.google.api.core.InternalApi;
 import com.google.cloud.bigtable.beam.hbasesnapshots.conf.RegionConfig;
 import com.google.cloud.bigtable.beam.hbasesnapshots.conf.SnapshotConfig;
@@ -189,17 +187,14 @@ public void processElement(
       boolean hasSplit = false;
       try (HBaseRegionScanner scanner = newScanner(regionConfig, tracker.currentRestriction())) {
         for (Result result = scanner.next(); result != null; result = scanner.next()) {
-          //  if (flag==0 ) {
           if (tracker.tryClaim(ByteKey.copyFrom(result.getRow()))) {
             outputReceiver.output(KV.of(regionConfig.getSnapshotConfig(), result));
           } else {
             hasSplit = true;
             break;
           }
-          // }
         }
       }
-      // if (!hasSplit)
       tracker.tryClaim(ByteKey.EMPTY);
     }
 
@@ -357,21 +352,22 @@ public void processElement(
         throws IOException {
       if (element.getValue().listCells().isEmpty()) return;
       String targetTable = element.getKey().getTableName();
+      String snapshotName = element.getKey().getSnapshotName();
 
       // Limit the number of mutations per Put (server will reject >= 100k mutations per Put)
       byte[] rowKey = element.getValue().getRow();
       List<Mutation> mutations = new ArrayList<>();
 
       boolean logAndSkipIncompatibleRowMutations =
-          verifyRowMutationThresholds(rowKey, element.getValue().listCells(), mutations);
+          verifyRowMutationThresholds(rowKey, element.getValue().listCells(), mutations, snapshotName);
 
       if (!logAndSkipIncompatibleRowMutations && mutations.size() > 0) {
         outputReceiver.output(KV.of(targetTable, mutations));
       }
     }
 
     private boolean verifyRowMutationThresholds(
-        byte[] rowKey, List<Cell> cells, List<Mutation> mutations) throws IOException {
+        byte[] rowKey, List<Cell> cells, List<Mutation> mutations, String snapshotName) throws IOException {
       boolean logAndSkipIncompatibleRows = false;
 
       Put put = null;
@@ -384,9 +380,8 @@ private boolean verifyRowMutationThresholds(
 
         // handle large cells
         if (filterLargeCells && cell.getValueLength() > filterLargeCellThresholdBytes) {
-          // TODO add config name in log
           LOG.warn(
-              "Dropping mutation, cell value length, "
+              "For snapshot " + snapshotName + ": Dropping mutation, cell value length, "
                   + cell.getValueLength()
                   + ", exceeds filter length, "
                   + filterLargeCellThresholdBytes
@@ -407,23 +402,21 @@ private boolean verifyRowMutationThresholds(
         cellCount++;
       }
 
-      // TODO add config name in log
       if (filterLargeRows && totalByteSize > filterLargeRowThresholdBytes) {
         logAndSkipIncompatibleRows = true;
         LOG.warn(
-            "Dropping row, row length, "
+            "For snapshot " + snapshotName + ": Dropping row, row length, "
                 + totalByteSize
                 + ", exceeds filter length threshold, "
                 + filterLargeRowThresholdBytes
                 + ", row key: "
                 + Bytes.toStringBinary(rowKey));
       }
 
-      // TODO add config name in log
       if (filterLargeRowKeys && rowKey.length > filterLargeRowKeysThresholdBytes) {
         logAndSkipIncompatibleRows = true;
         LOG.warn(
-            "Dropping row, row key length, "
+            "For snapshot " + snapshotName + ": Dropping row, row key length, "
                 + rowKey.length
                 + ", exceeds filter length threshold, "
                 + filterLargeRowKeysThresholdBytes

bigtable-dataflow-parent/bigtable-beam-import/src/test/java/com/google/cloud/bigtable/beam/hbasesnapshots/SnapshotUtilsTest.java

@@ -225,7 +225,6 @@ public void testgetSubSetSnapshotsFromSnapshotPath() throws IOException {
         getMatchingSnapshotsFromSnapshotPath(snapshotList, ".*attachments.*");
     List<String> expectedResult =
         snapshotList.stream().filter(e -> e.contains("attachments")).collect(Collectors.toList());
-    // LOG.info("Matched:{} and expected:{}", snapshots.size(), expectedResult.size());
     assertThat(snapshots.size(), is(equalTo(expectedResult.size())));
     assertThat(snapshots.keySet(), containsInAnyOrder(expectedResult.toArray(new String[0])));
   }