[Http] Add Http adapter functions

Avro & Excel added to DSL documentation

Author: stloyd

bin/docs.php

@@ -37,11 +37,14 @@ public function execute(InputInterface $input, OutputInterface $output) : int
 
         $paths = [
             __DIR__ . '/../src/core/etl/src/Flow/ETL/DSL/functions.php',
+            __DIR__ . '/../src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/functions.php',
             __DIR__ . '/../src/adapter/etl-adapter-chartjs/src/Flow/ETL/Adapter/ChartJS/functions.php',
             __DIR__ . '/../src/adapter/etl-adapter-csv/src/Flow/ETL/Adapter/CSV/functions.php',
             __DIR__ . '/../src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine/functions.php',
+            __DIR__ . '/../src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/DSL/functions.php',
             __DIR__ . '/../src/adapter/etl-adapter-elasticsearch/src/Flow/ETL/Adapter/Elasticsearch/functions.php',
             __DIR__ . '/../src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/functions.php',
+            __DIR__ . '/../src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/DSL/functions.php',
             __DIR__ . '/../src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/functions.php',
             __DIR__ . '/../src/adapter/etl-adapter-meilisearch/src/Flow/ETL/Adapter/Meilisearch/functions.php',
             __DIR__ . '/../src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/functions.php',

composer.json

@@ -146,6 +146,7 @@
             "src/adapter/etl-adapter-elasticsearch/src/Flow/ETL/Adapter/Elasticsearch/functions.php",
             "src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/DSL/functions.php",
             "src/adapter/etl-adapter-google-sheet/src/Flow/ETL/Adapter/GoogleSheet/functions.php",
+            "src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/DSL/functions.php",
             "src/adapter/etl-adapter-json/src/Flow/ETL/Adapter/JSON/functions.php",
             "src/adapter/etl-adapter-meilisearch/src/Flow/ETL/Adapter/Meilisearch/functions.php",
             "src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/functions.php",

src/adapter/etl-adapter-avro/src/Flow/ETL/Adapter/Avro/functions.php

@@ -6,9 +6,10 @@
 
 use function Flow\ETL\DSL\from_all;
 use Flow\ETL\Adapter\Avro\FlixTech\{AvroExtractor, AvroLoader};
-use Flow\ETL\{Extractor, Schema};
+use Flow\ETL\{Attribute\DocumentationDSL, Attribute\Module, Attribute\Type, Extractor, Schema};
 use Flow\Filesystem\Path;
 
+#[DocumentationDSL(module: Module::AVRO, type: Type::EXTRACTOR)]
 function from_avro(Path|string|array $path) : Extractor
 {
     if (\is_array($path)) {
@@ -29,6 +30,7 @@ function from_avro(Path|string|array $path) : Extractor
     );
 }
 
+#[DocumentationDSL(module: Module::AVRO, type: Type::LOADER)]
 function to_avro(Path|string $path, ?Schema $schema = null) : AvroLoader
 {
     return new AvroLoader(

src/adapter/etl-adapter-excel/src/Flow/ETL/Adapter/Excel/DSL/functions.php

@@ -21,6 +21,7 @@ function from_excel(
     return new ExcelExtractor(\is_string($path) ? Path::realpath($path) : $path);
 }
 
+#[DocumentationDSL(module: Module::EXCEL, type: DSLType::HELPER)]
 function is_valid_excel_sheet_name(string|ScalarFunction $sheet_name) : IsValidExcelSheetName
 {
     return new IsValidExcelSheetName($sheet_name);

src/adapter/etl-adapter-http/composer.json

@@ -29,7 +29,10 @@
             "Flow\\": [
                 "src/Flow"
             ]
-        }
+        },
+        "files": [
+          "src/Flow/ETL/Adapter/Http/DSL/functions.php"
+        ]
     },
     "autoload-dev": {
         "psr-4": {

src/adapter/etl-adapter-http/src/Flow/ETL/Adapter/Http/DSL/functions.php

@@ -0,0 +1,37 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\ETL\Adapter\Http;
+
+use Flow\ETL\{Adapter\Http\DynamicExtractor\NextRequestFactory,
+    Attribute\DocumentationDSL,
+    Attribute\Module,
+    Attribute\Type};
+use Psr\Http\Client\ClientInterface;
+use Psr\Http\Message\RequestInterface;
+
+#[DocumentationDSL(module: Module::HTTP, type: Type::EXTRACTOR)]
+function from_dynamic_http_requests(
+    ClientInterface $client,
+    NextRequestFactory $requestFactory,
+) : PsrHttpClientDynamicExtractor {
+    return new PsrHttpClientDynamicExtractor(
+        $client,
+        $requestFactory,
+    );
+}
+
+/**
+ * @param iterable<RequestInterface> $requests
+ */
+#[DocumentationDSL(module: Module::HTTP, type: Type::EXTRACTOR)]
+function from_static_http_requests(
+    ClientInterface $client,
+    iterable $requests,
+) : PsrHttpClientStaticExtractor {
+    return new PsrHttpClientStaticExtractor(
+        $client,
+        $requests,
+    );
+}

src/core/etl/src/Flow/ETL/Attribute/Module.php

@@ -6,18 +6,19 @@
 
 enum Module : string
 {
+    case AVRO = 'AVRO';
     case AZURE_FILESYSTEM = 'AZURE_FILESYSTEM';
     case AZURE_SDK = 'AZURE_SDK';
     case CHART_JS = 'CHART_JS';
     case CORE = 'CORE';
     case CSV = 'CSV';
-
     case DEPRECATED = 'DEPRECATED';
     case DOCTRINE = 'DOCTRINE';
     case ELASTIC_SEARCH = 'ELASTIC_SEARCH';
     case EXCEL = 'EXCEL';
     case FILESYSTEM = 'FILESYSTEM';
     case GOOGLE_SHEET = 'GOOGLE_SHEET';
+    case HTTP = 'HTTP';
     case JSON = 'JSON';
     case MEILI_SEARCH = 'MEILI_SEARCH';
     case PARQUET = 'PARQUET';

web/landing/resources/dsl.json

@@ -6,15 +6,18 @@
 
 enum Module : string
 {
+    case AVRO = 'Avro';
     case AZURE_FILESYSTEM = 'Azure Filesystem';
     case AZURE_SDK = 'Azure SDK';
     case CHART_JS = 'Chart.js';
     case CORE = 'Core';
     case CSV = 'CSV';
     case DOCTRINE = 'Doctrine';
     case ELASTIC_SEARCH = 'Elastic Search';
+    case EXCEL = 'Excel';
     case FILESYSTEM = 'Filesystem';
     case GOOGLE_SHEET = 'Google Sheet';
+    case HTTP = 'HTTP';
     case JSON = 'JSON';
     case MEILI_SEARCH = 'Meili Search';
     case PARQUET = 'Parquet';
@@ -48,6 +51,8 @@ public function priority() : int
             self::TYPES => 13,
             self::AZURE_FILESYSTEM => 14,
             self::AZURE_SDK => 15,
+            self::HTTP => 16,
+            self::EXCEL => 17,
             default => 99,
         };
     }

web/landing/src/Flow/Website/Model/Documentation/Module.php

@@ -32,7 +32,7 @@ public function test_documentation_dsl_page() : void
         self::assertResponseIsSuccessful();
         self::assertGreaterThan(0, $client->getCrawler()->filter('[data-dsl-function]')->count());
         self::assertGreaterThan(0, $client->getCrawler()->filter('[data-dsl-source-link]')->count());
-        self::assertEquals(17, $client->getCrawler()->filter('[data-dsl-module]')->count());
+        self::assertEquals(20, $client->getCrawler()->filter('[data-dsl-module]')->count());
         self::assertEquals(12, $client->getCrawler()->filter('[data-dsl-type]')->count());
     }