Add HTMLQuerySelectorAll & HTMLQuerySelector scalar functions

Author: stloyd

phpstan.neon

@@ -60,7 +60,15 @@ parameters:
     excludePaths:
         - src/cli/src/Flow/CLI/Command/*
         - src/core/etl/src/Flow/ETL/Formatter/ASCII/ASCIITable.php
+        - src/core/etl/src/Flow/ETL/Function/HTMLQuerySelectorAll.php
+        - src/core/etl/src/Flow/ETL/Function/HTMLQuerySelector.php
+        - src/core/etl/src/Flow/ETL/Row/Entry/HTMLEntry.php
         - src/core/etl/src/Flow/ETL/Sort/ExternalSort/RowsMinHeap.php
+        - src/core/etl/tests/Flow/ETL/Tests/Integration/Function/HTMLQuerySelectorAllTest.php
+        - src/core/etl/tests/Flow/ETL/Tests/Integration/Function/HTMLQuerySelectorTest.php
+        - src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HTMLQuerySelectorAllTest.php
+        - src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HTMLQuerySelectorTest.php
+        - src/core/etl/tests/Flow/ETL/Tests/Unit/Row/Entry/HTMLEntryTest.php
         - src/adapter/etl-adapter-avro/*
         - src/adapter/etl-adapter-elasticsearch/src/Flow/ETL/Adapter/Elasticsearch/ElasticsearchPHP/SearchResults.php
         - src/adapter/etl-adapter-elasticsearch/src/Flow/ETL/Adapter/Elasticsearch/ElasticsearchPHP/SearchParams.php
@@ -70,6 +78,7 @@ parameters:
         - src/lib/parquet/src/Flow/Parquet/ThriftModel/*
         - src/lib/parquet/src/Flow/Parquet/BinaryReader/*
         - src/lib/parquet/src/Flow/Parquet/Dremel/ColumnData/DefinitionConverter.php
+        - src/lib/types/src/Flow/Types/Type/Logical/HTMLType.php
 
     tmpDir: var/phpstan/cache
 

src/core/etl/src/Flow/ETL/DSL/functions.php

@@ -37,6 +37,7 @@
     type_xml_element as type_xml_element_new,
     types as types_new
 };
+use Dom\HTMLDocument;
 use Flow\Calculator\Rounding;
 use Flow\ETL\{Analyze,
     Attribute\DocumentationDSL,
@@ -205,7 +206,6 @@
     UnionType
 };
 use Flow\Types\Type\Types;
-use Flow\Types\Value\HTMLDocument;
 use UnitEnum;
 
 /**
@@ -632,7 +632,7 @@ function xml_element_entry(string $name, \DOMElement|string|null $value, ?Metada
  * @return Entry<?HTMLDocument>
  */
 #[DocumentationDSL(module: Module::CORE, type: DSLType::ENTRY)]
-function html_entry(string $name, HTMLDocument|string|null $value, ?Metadata $metadata = null) : Entry
+function html_entry(string $name, HTMLDocument|string|null $value, ?Metadata $metadata = null) : Entry // @phpstan-ignore class.notFound,class.notFound
 {
     return new HTMLEntry($name, $value, $metadata);
 }
@@ -1973,7 +1973,7 @@ function json_schema(string $name, bool $nullable = false, ?Metadata $metadata =
  * @return Definition<HTMLDocument>
  */
 #[DocumentationDSL(module: Module::CORE, type: DSLType::SCHEMA)]
-function html_schema(string $name, bool $nullable = false, ?Metadata $metadata = null) : Definition
+function html_schema(string $name, bool $nullable = false, ?Metadata $metadata = null) : Definition // @phpstan-ignore class.notFound
 {
     return Definition::html($name, $nullable, $metadata);
 }

src/core/etl/src/Flow/ETL/Function/HTMLQuerySelector.php

@@ -0,0 +1,33 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\ETL\Function;
+
+use Dom\{Element, HTMLDocument};
+use Flow\ETL\Row;
+
+final class HTMLQuerySelector extends ScalarFunctionChain
+{
+    public function __construct(
+        private readonly mixed $value,
+        private readonly ScalarFunction|string $selector,
+    ) {
+    }
+
+    public function eval(Row $row) : ?Element
+    {
+        if (\PHP_VERSION_ID < 80400) {
+            throw new \RuntimeException('This function requires \Dom\HTMLDocument extension available in PHP 8.4+.');
+        }
+
+        $value = (new Parameter($this->value))->asInstanceOf($row, HTMLDocument::class);
+        $selector = (new Parameter($this->selector))->asString($row);
+
+        if (null === $value || null === $selector) {
+            return null;
+        }
+
+        return $value->querySelector($selector);
+    }
+}

src/core/etl/src/Flow/ETL/Function/HTMLQuerySelectorAll.php

@@ -0,0 +1,52 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\ETL\Function;
+
+use DOM\{Element, HTMLDocument};
+use Flow\ETL\Row;
+
+final class HTMLQuerySelectorAll extends ScalarFunctionChain
+{
+    public function __construct(
+        private readonly mixed $value,
+        private readonly ScalarFunction|string $selector,
+    ) {
+    }
+
+    /**
+     * @return null|array<Element>
+     */
+    public function eval(Row $row) : ?array
+    {
+        if (\PHP_VERSION_ID < 80400) {
+            throw new \RuntimeException('This function requires \Dom\HTMLDocument extension available in PHP 8.4+.');
+        }
+
+        $value = (new Parameter($this->value))->asInstanceOf($row, HTMLDocument::class);
+        $selector = (new Parameter($this->selector))->asString($row);
+
+        if (null === $value || null === $selector) {
+            return null;
+        }
+
+        $result = $value->querySelectorAll($selector);
+
+        if (0 === $result->count()) {
+            return null;
+        }
+
+        $nodes = [];
+
+        foreach ($result as $node) {
+            if (!$node instanceof Element) {
+                continue;
+            }
+
+            $nodes[] = $node;
+        }
+
+        return $nodes;
+    }
+}

src/core/etl/src/Flow/ETL/Function/ScalarFunctionChain.php

@@ -311,6 +311,16 @@ public function hash(Algorithm $algorithm = new NativePHPHash()) : Hash
         return new Hash($this, $algorithm);
     }
 
+    public function htmlQuerySelector(ScalarFunction|string $path) : HTMLQuerySelector
+    {
+        return new HTMLQuerySelector($this, $path);
+    }
+
+    public function htmlQuerySelectorAll(ScalarFunction|string $path) : HTMLQuerySelectorAll
+    {
+        return new HTMLQuerySelectorAll($this, $path);
+    }
+
     /**
      * Returns the index of given $needle in string.
      */

src/core/etl/src/Flow/ETL/Row/Entry/HTMLEntry.php

@@ -5,10 +5,10 @@
 namespace Flow\ETL\Row\Entry;
 
 use function Flow\Types\DSL\{type_equals, type_html, type_optional};
+use Dom\HTMLDocument;
 use Flow\ETL\Row\{Entry, Reference};
 use Flow\ETL\Schema\{Definition, Metadata};
 use Flow\Types\Type;
-use Flow\Types\Value\HTMLDocument;
 
 /**
  * @implements Entry<?HTMLDocument>
@@ -32,7 +32,7 @@ public function __construct(
         ?Metadata $metadata = null,
     ) {
         if (\is_string($value)) {
-            $this->value = HTMLDocument::fromString($value);
+            $this->value = HTMLDocument::createFromString($value);
         } else {
             $this->value = $value;
         }
@@ -75,7 +75,7 @@ public function isEqual(Entry $entry) : bool
             return false;
         }
 
-        return $entry->value()?->toString() === $this->value?->toString();
+        return $entry->value()?->saveHtml() === $this->value?->saveHtml();
     }
 
     public function map(callable $mapper) : self
@@ -99,7 +99,7 @@ public function toString() : string
             return '';
         }
 
-        return $this->value->toString();
+        return $this->value->saveHtml();
     }
 
     public function type() : Type

src/core/etl/src/Flow/ETL/Schema/Definition.php

@@ -28,12 +28,13 @@
     type_xml,
     type_xml_element,
     types};
+use Dom\HTMLDocument;
 use Flow\ETL\Exception\{InvalidArgumentException, RuntimeException};
 use Flow\ETL\Row\{Entry, EntryReference, Reference};
 use Flow\Types\Type;
 use Flow\Types\Type\Logical\{ListType, MapType, OptionalType, StructureType};
 use Flow\Types\Type\{Native\FloatType, Native\IntegerType, Native\UnionType, TypeFactory};
-use Flow\Types\Value\{HTMLDocument, Uuid};
+use Flow\Types\Value\Uuid;
 
 /**
  * @template-covariant T
@@ -142,7 +143,7 @@ public static function fromArray(array $definition) : self
     /**
      * @return Definition<HTMLDocument>
      */
-    public static function html(string|Reference $entry, bool $nullable = false, ?Metadata $metadata = null) : self
+    public static function html(string|Reference $entry, bool $nullable = false, ?Metadata $metadata = null) : self // @phpstan-ignore class.notFound
     {
         return new self($entry, type_html(), $nullable, $metadata);
     }

src/core/etl/tests/Flow/ETL/Tests/Integration/Function/HTMLQuerySelectorAllTest.php

@@ -0,0 +1,51 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\ETL\Tests\Integration\Function;
+
+use function Flow\ETL\DSL\{df, from_rows, html_entry, ref, row, rows};
+use Dom\{Element, HTMLDocument};
+use PHPUnit\Framework\Attributes\RequiresPhp;
+use PHPUnit\Framework\TestCase;
+
+#[RequiresPhp('>= 8.4')]
+final class HTMLQuerySelectorAllTest extends TestCase
+{
+    private HTMLDocument $html;
+
+    protected function setUp() : void
+    {
+        $this->html = HTMLDocument::createFromString('<!DOCTYPE html><html lang="en"><head></head><body><div><span>foobar</span></div></body></html>');
+    }
+
+    public function test_invalid_query_all_on_html_document() : void
+    {
+        $rows = df()
+            ->read(from_rows(rows(row(html_entry('html_raw', $this->html)))))
+            ->withEntry('html', ref('html_raw')->htmlQuerySelectorAll('body div p'))
+            ->drop('html_raw')
+            ->fetch();
+
+        $results = $rows->toArray()[0]['html'] ?? [];
+
+        /* @phpstan-ignore-next-line */
+        self::assertCount(0, $results);
+    }
+
+    public function test_valid_query_all_on_html_document() : void
+    {
+        $rows = df()
+            ->read(from_rows(rows(row(html_entry('html_raw', $this->html)))))
+            ->withEntry('html', ref('html_raw')->htmlQuerySelectorAll('body div span'))
+            ->drop('html_raw')
+            ->fetch();
+
+        $results = $rows->toArray()[0]['html'] ?? [];
+
+        /* @phpstan-ignore-next-line */
+        self::assertCount(1, $results);
+        /* @phpstan-ignore-next-line */
+        self::assertContainsOnlyInstancesOf(Element::class, $results);
+    }
+}

src/core/etl/tests/Flow/ETL/Tests/Integration/Function/HTMLQuerySelectorTest.php

@@ -0,0 +1,35 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\ETL\Tests\Integration\Function;
+
+use function Flow\ETL\DSL\{df, from_rows, html_entry, ref, row, rows};
+use Dom\HTMLDocument;
+use PHPUnit\Framework\Attributes\RequiresPhp;
+use PHPUnit\Framework\TestCase;
+
+#[RequiresPhp('>= 8.4')]
+final class HTMLQuerySelectorTest extends TestCase
+{
+    private HTMLDocument $html;
+
+    protected function setUp() : void
+    {
+        $this->html = HTMLDocument::createFromString('<!DOCTYPE html><html lang="en"><head></head><body><div><span>foobar</span></div></body></html>');
+    }
+
+    public function test_invalid_query_on_html_document() : void
+    {
+        $rows = df()
+            ->read(from_rows(rows(row(html_entry('html_raw', $this->html)))))
+            ->withEntry('html', ref('html_raw')->htmlQuerySelector('body div p'))
+            ->drop('html_raw')
+            ->fetch();
+
+        $results = $rows->toArray()[0]['html'] ?? [];
+
+        /* @phpstan-ignore-next-line */
+        self::assertCount(0, $results);
+    }
+}

src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HTMLQuerySelectorAllTest.php

@@ -0,0 +1,53 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\ETL\Tests\Unit\Function;
+
+use function Flow\ETL\DSL\{config, flow_context, ref, row};
+use Dom\{Element, HTMLDocument};
+use PHPUnit\Framework\Attributes\RequiresPhp;
+use PHPUnit\Framework\TestCase;
+
+final class HTMLQuerySelectorAllTest extends TestCase
+{
+    #[RequiresPhp('>= 8.4')]
+    public function test_getting_elements_for_given_path() : void
+    {
+        $html = HTMLDocument::createFromString('<!DOCTYPE html><html><head></head><body><div><span>foobar</span></div></body></html>');
+
+        /** @var array<mixed> $result */
+        $result = ref('value')->htmlQuerySelectorAll('body div span')->eval(row(flow_context(config())->entryFactory()->create('value', $html)));
+
+        self::assertCount(1, $result);
+
+        /* @phpstan-ignore-next-line */
+        self::assertInstanceOf(Element::class, $result[0]);
+    }
+
+    #[RequiresPhp('< 8.4')]
+    public function test_getting_null_for_older_versions() : void
+    {
+        $this->expectException(\RuntimeException::class);
+
+        ref('value')->htmlQuerySelectorAll('body div p')->eval(row(flow_context(config())->entryFactory()->create('value', '')));
+    }
+
+    #[RequiresPhp('>= 8.4')]
+    public function test_getting_null_when_nothing_found() : void
+    {
+        $html = HTMLDocument::createFromString('<!DOCTYPE html><html><head></head><body><div><span>foobar</span></div></body></html>');
+
+        $result = ref('value')->htmlQuerySelectorAll('body div p')->eval(row(flow_context(config())->entryFactory()->create('value', $html)));
+
+        self::assertNull($result);
+    }
+
+    #[RequiresPhp('>= 8.4')]
+    public function test_invalid_value() : void
+    {
+        $result = ref('value')->htmlQuerySelectorAll('body div span')->eval(row(flow_context(config())->entryFactory()->create('value', '')));
+
+        self::assertNull($result);
+    }
+}