Add option to drop extra columns from extracted spreadsheets

Author: stloyd

src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/GoogleSheetExtractor.php

@@ -14,6 +14,8 @@ final class GoogleSheetExtractor implements Extractor, LimitableExtractor
 {
     use Limitable;
 
+    private bool $dropExtraColumns = false;
+
     /**
      * @param array{dateTimeRenderOption?: string, majorDimension?: string, valueRenderOption?: string} $options
      */
@@ -57,24 +59,30 @@ public function extract(FlowContext $context) : \Generator
             $totalRows = 0;
         }
 
+        $headersCount = \count($headers);
+
         $shouldPutInputIntoRows = $context->config->shouldPutInputIntoRows();
 
         while (\count($values)) {
             $rows = \array_map(
-                function (array $rowData) use ($headers, $shouldPutInputIntoRows) {
-                    if (\count($headers) > \count($rowData)) {
+                function (array $rowData) use ($headers, $headersCount, $shouldPutInputIntoRows) {
+                    if ($headersCount > \count($rowData)) {
                         \array_push(
                             $rowData,
                             ...\array_map(
                                 static fn (int $i) => null,
-                                \range(1, \count($headers) - \count($rowData))
+                                \range(1, $headersCount - \count($rowData))
                             )
                         );
                     }
 
-                    if (\count($rowData) > \count($headers)) {
-                        /** @phpstan-ignore-next-line */
-                        $rowData = \array_chunk($rowData, \count($headers));
+                    if (\count($rowData) > $headersCount) {
+                        if ($this->dropExtraColumns) {
+                            $rowData = \array_slice($rowData, 0, $headersCount);
+                        } else {
+                            /** @phpstan-ignore-next-line */
+                            $rowData = \array_chunk($rowData, $headersCount);
+                        }
                     }
 
                     $row = \array_combine($headers, $rowData);
@@ -105,7 +113,7 @@ function (array $rowData) use ($headers, $shouldPutInputIntoRows) {
             }
 
             $cellsRange = $cellsRange->nextRows($this->rowsPerPage);
-            /** @var Sheets\ValueRange $response */
+
             $response = $this->service->spreadsheets_values->get($this->spreadsheetId, $cellsRange->toString(), $this->options);
             /**
              * @var array<array> $values
@@ -114,6 +122,13 @@ function (array $rowData) use ($headers, $shouldPutInputIntoRows) {
         }
     }
 
+    public function withDropExtraColumns(bool $dropExtraColumns) : self
+    {
+        $this->dropExtraColumns = $dropExtraColumns;
+
+        return $this;
+    }
+
     public function withHeader(bool $withHeader) : self
     {
         $this->withHeader = $withHeader;

src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/functions.php

@@ -5,7 +5,6 @@
 namespace Flow\ETL\Adapter\GoogleSheet;
 
 use Flow\ETL\Attribute\{DocumentationDSL, Module, Type};
-use Flow\ETL\Extractor;
 use Google\Client;
 use Google\Service\Sheets;
 
@@ -25,7 +24,7 @@ function from_google_sheet(
     bool $with_header = true,
     int $rows_per_page = 1000,
     array $options = [],
-) : Extractor {
+) : GoogleSheetExtractor {
     if ($auth_config instanceof Sheets) {
         $sheets = $auth_config;
     } else {
@@ -64,7 +63,7 @@ function from_google_sheet_columns(
     bool $with_header = true,
     int $rows_per_page = 1000,
     array $options = [],
-) : Extractor {
+) : GoogleSheetExtractor {
     if ($auth_config instanceof Sheets) {
         $sheets = $auth_config;
     } else {

src/adapter/etl-adapter-google-sheet/tests/Flow/ETL/Adapter/GoogleSheet/Tests/Unit/GoogleSheetExtractorTest.php

@@ -23,9 +23,8 @@ public function test_its_stop_fetching_data_if_processed_row_count_is_less_then_
             $sheetName = 'sheet',
             'A',
             'B',
-            true,
-            2,
-        );
+        )->withHeader(true)
+            ->withRowsPerPage(2);
         $spreadSheetIdEntry = string_entry('_spread_sheet_id', $spreadSheetId);
         $sheetNameEntry = string_entry('_sheet_name', $sheetName);
         $firstValueRangeMock = $this->createMock(ValueRange::class);
@@ -63,9 +62,7 @@ public function test_rows_in_batch_must_be_positive_integer() : void
             'sheet',
             'A',
             'B',
-            true,
-            0
-        );
+        )->withRowsPerPage(0);
     }
 
     public function test_works_for_no_data() : void
@@ -76,9 +73,8 @@ public function test_works_for_no_data() : void
             'sheet',
             'A',
             'B',
-            true,
-            20
-        );
+        )->withHeader(true)
+            ->withRowsPerPage(20);
         $ValueRangeMock = $this->createMock(ValueRange::class);
         $ValueRangeMock->method('getValues')->willReturn(null);