Skip to content

Commit 1f54d5c

Browse files
norberttechnorberttech
authored
feature: flow-php/arrow-ext (#2268)
* feature: flow-php/arrow-ext Introduce arrow-ext and expose parquet reader/writer. Introduce ParquetEngine abstraction into flow-php/parquet. Make flow-php/parquet use engines based on arrow extension availability. * fix: nix-shel arguments * fix: failing tests on ci/cd * fix: fragile website functional tests * chore: code review fixes * fix: another approach to fix fragiel website tests * chore: make arrow to report dynamic version * fix: added missing phpt test for arrow extension * fix: codecov configuration allow to upload partial coverages from different workflows using flags * fix: localstack version on ci/cd
1 parent 5b18a81 commit 1f54d5c

153 files changed

Lines changed: 12430 additions & 2039 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.codecov.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,14 @@
1+
codecov:
2+
notify:
3+
after_n_builds: 5
4+
15
comment:
26
layout: "condensed_header, diff, components"
7+
after_n_builds: 5
8+
9+
flag_management:
10+
default_rules:
11+
carryforward: true
312

413
component_management:
514
individual_components:

‎.github/workflows/job-arrow-extension.yml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
name: Arrow Extension
2+
3+
on:
4+
workflow_call:
5+
secrets:
6+
CODECOV_TOKEN:
7+
required: false
8+
9+
jobs:
10+
build:
11+
name: Build and Test Extension
12+
runs-on: ${{ matrix.os }}
13+
strategy:
14+
fail-fast: false
15+
matrix:
16+
os: ['ubuntu-latest', 'macos-latest']
17+
php: ['8.3', '8.4', '8.5']
18+
19+
steps:
20+
- uses: actions/checkout@v5
21+
22+
- name: Setup PHP Environment
23+
uses: ./.github/actions/setup-php-env
24+
with:
25+
php-version: ${{ matrix.php }}
26+
dependencies: locked
27+
extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib, zstd'
28+
tools: 'composer:v2, php-config'
29+
coverage: 'pcov'
30+
install-dependencies: 'false'
31+
32+
- name: Install Rust toolchain
33+
uses: dtolnay/rust-toolchain@stable
34+
35+
- name: Cache cargo registry and build
36+
uses: actions/cache@v4
37+
with:
38+
path: |
39+
~/.cargo/registry
40+
~/.cargo/git
41+
src/extension/arrow-ext/target
42+
key: ${{ runner.os }}-php${{ matrix.php }}-cargo-${{ hashFiles('src/extension/arrow-ext/Cargo.lock') }}
43+
restore-keys: |
44+
${{ runner.os }}-php${{ matrix.php }}-cargo-
45+
46+
- name: Install build dependencies (Ubuntu)
47+
if: runner.os == 'Linux'
48+
run: |
49+
sudo apt-get update
50+
sudo apt-get install -y build-essential clang libclang-dev
51+
52+
- name: Install build dependencies (macOS)
53+
if: runner.os == 'macOS'
54+
run: |
55+
brew install llvm
56+
echo "LIBCLANG_PATH=$(brew --prefix llvm)/lib" >> $GITHUB_ENV
57+
58+
- name: Set LIBCLANG_PATH (Ubuntu)
59+
if: runner.os == 'Linux'
60+
run: |
61+
LIBCLANG_SO=$(find /usr/lib -name 'libclang*.so' -o -name 'libclang*.so.*' 2>/dev/null | head -1)
62+
echo "LIBCLANG_PATH=$(dirname "$LIBCLANG_SO")" >> $GITHUB_ENV
63+
64+
- name: Build extension
65+
working-directory: src/extension/arrow-ext
66+
run: make build
67+
68+
- name: Run PHPT tests
69+
working-directory: src/extension/arrow-ext
70+
run: make test
71+
72+
- name: Install extension and verify
73+
working-directory: src/extension/arrow-ext
74+
run: |
75+
EXT_DIR=$(php -r 'echo ini_get("extension_dir");')
76+
sudo cp ext/modules/arrow.so "$EXT_DIR/"
77+
echo "extension=arrow.so" | sudo tee -a "$(php -r 'echo php_ini_loaded_file();')"
78+
php -m | grep arrow
79+
80+
- name: Install Composer Dependencies
81+
run: composer install --no-interaction --no-progress
82+
83+
- name: Run Parquet Integration Tests with Arrow
84+
run: composer test:lib:parquet
85+
86+
- name: Upload to Codecov
87+
if: ${{ !cancelled() && matrix.php == '8.3' && matrix.os == 'ubuntu-latest' }}
88+
uses: codecov/codecov-action@v5
89+
with:
90+
token: ${{ secrets.CODECOV_TOKEN }}
91+
directory: ./var/phpunit/coverage/clover
92+
flags: arrow-extension

‎.github/workflows/job-extension-tests.yml‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
on:
22
workflow_call:
3+
secrets:
4+
CODECOV_TOKEN:
5+
required: false
36

47
jobs:
58
extension-tests:
@@ -44,4 +47,12 @@ jobs:
4447
run: "composer test:lib:parquet -- --group zstd-extension"
4548

4649
- name: "Test Snappy"
47-
run: "composer test:lib:parquet -- --group snappy-extension"
50+
run: "composer test:lib:parquet -- --group snappy-extension"
51+
52+
- name: Upload to Codecov
53+
if: ${{ !cancelled() && matrix.php-version == '8.3' }}
54+
uses: codecov/codecov-action@v5
55+
with:
56+
token: ${{ secrets.CODECOV_TOKEN }}
57+
directory: ./var/phpunit/coverage/clover
58+
flags: extension-tests

‎.github/workflows/job-windows-tests.yml‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,6 @@ jobs:
6868
- name: "Test - Lib - Types"
6969
run: "composer test:lib:types"
7070

71-
- name: "Test - Lib - Parquet"
72-
run: "composer test:lib:parquet"
73-
74-
- name: "Test - Lib - Parquet Viewer"
75-
run: "composer test:lib:parquet-viewer"
76-
7771
- name: "Test - Lib - Snappy"
7872
run: "composer test:lib:snappy"
7973

@@ -98,9 +92,6 @@ jobs:
9892
- name: "Test - Adapter - Logger"
9993
run: "composer test:adapter:logger"
10094

101-
- name: "Test - Adapter - Parquet"
102-
run: "composer test:adapter:parquet"
103-
10495
- name: "Test - Adapter - Text"
10596
run: "composer test:adapter:text"
10697

‎.github/workflows/monorepo-split.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,8 @@ jobs:
103103

104104
- local_path: 'src/extension/pg-query-ext'
105105
split_repository: 'pg-query-ext'
106+
- local_path: 'src/extension/arrow-ext'
107+
split_repository: 'arrow-ext'
106108

107109
steps:
108110
- uses: actions/checkout@v5

‎.github/workflows/test-suite.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,16 @@ jobs:
4040

4141
extension-tests:
4242
uses: ./.github/workflows/job-extension-tests.yml
43+
secrets: inherit
4344

4445
pg-query-extension:
4546
uses: ./.github/workflows/job-pg-query-extension.yml
4647
secrets: inherit
4748

49+
arrow-extension:
50+
uses: ./.github/workflows/job-arrow-extension.yml
51+
secrets: inherit
52+
4853
windows-tests:
4954
uses: ./.github/workflows/job-windows-tests.yml
5055

‎.nix/pkgs/flow-php/package.nix‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@
55
php-brotli,
66
php-zstd,
77
php-pg-query-ext,
8+
php-arrow-ext,
89
with-pcov ? true,
910
with-xdebug ? false,
1011
with-blackfire ? false,
1112
with-pg-query-ext ? false,
13+
with-arrow-ext ? false,
1214
with-grpc ? false
1315
}:
1416

@@ -34,6 +36,7 @@ let
3436
++ (if with-pcov then [pcov] else [])
3537
++ (if with-blackfire then [blackfire] else [])
3638
++ (if with-pg-query-ext then [(php-pg-query-ext.override { inherit php; })] else [])
39+
++ (if with-arrow-ext then [(php-arrow-ext.override { inherit php; })] else [])
3740
++ (if with-grpc then [grpc] else [])
3841
);
3942
in

‎.nix/pkgs/php-arrow-ext/package.nix‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{
2+
php,
3+
lib,
4+
stdenv,
5+
rustPlatform,
6+
clang,
7+
llvmPackages,
8+
arrow-ext-version ? "dev",
9+
}:
10+
11+
let
12+
extSrc = builtins.path {
13+
path = ../../../src/extension/arrow-ext;
14+
name = "arrow-ext-src";
15+
filter = path: type:
16+
let baseName = baseNameOf path;
17+
in !(
18+
baseName == "target" ||
19+
baseName == "vendor" ||
20+
baseName == "ext" ||
21+
baseName == ".gitignore" ||
22+
baseName == ".gitattributes" ||
23+
baseName == "composer.json" ||
24+
baseName == "composer.lock"
25+
);
26+
};
27+
pkg = rustPlatform.buildRustPackage {
28+
pname = "php-arrow-ext";
29+
version = arrow-ext-version;
30+
31+
src = extSrc;
32+
33+
cargoLock.lockFile = ../../../src/extension/arrow-ext/Cargo.lock;
34+
35+
nativeBuildInputs = [
36+
clang
37+
llvmPackages.libclang
38+
php.unwrapped
39+
php.unwrapped.dev
40+
];
41+
42+
env = {
43+
LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";
44+
PHP_CONFIG = "${php.unwrapped.dev}/bin/php-config";
45+
PHP = "${php.unwrapped}/bin/php";
46+
ARROW_VERSION = arrow-ext-version;
47+
};
48+
49+
installPhase = let
50+
targetDir = "target/${stdenv.hostPlatform.rust.rustcTargetSpec}/release";
51+
in ''
52+
runHook preInstall
53+
mkdir -p $out/lib/php/extensions
54+
cp ${targetDir}/libarrow${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/php/extensions/arrow.so
55+
runHook postInstall
56+
'';
57+
58+
doCheck = false;
59+
60+
meta = with lib; {
61+
description = "Apache Arrow PHP extension powered by Rust";
62+
license = licenses.mit;
63+
};
64+
};
65+
in
66+
pkg // { extensionName = "arrow"; }

‎composer.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,8 @@
160160
"src/lib/snappy/src/Flow",
161161
"src/lib/telemetry/src/Flow",
162162
"src/lib/types/src/Flow",
163-
"src/tools/documentation/src/Flow"
163+
"src/tools/documentation/src/Flow",
164+
"src/extension/arrow-ext/php/Flow"
164165
],
165166
"Flow\\Doctrine\\Bulk\\": [
166167
"src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk"

0 commit comments

Comments
 (0)