Skip to content

Commit 044b5f8

Browse files
chenghuichenchenghuichen
committed
Initialize daft-paimon connector module
1 parent 4e65076 commit 044b5f8

3 files changed

Lines changed: 221 additions & 77 deletions

File tree

‎.github/workflows/paimon-daft-checks.yml‎

Lines changed: 4 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,8 @@ concurrency:
3636
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
3737
cancel-in-progress: true
3838

39-
env:
40-
DAFT_VERSION: "0.7.11"
41-
4239
jobs:
43-
lint:
40+
lint-and-test:
4441
runs-on: ubuntu-latest
4542
steps:
4643
- name: Checkout code
@@ -51,52 +48,7 @@ jobs:
5148
with:
5249
python-version: '3.10'
5350

54-
- name: Install flake8
55-
run: python -m pip install flake8==4.0.1
56-
57-
- name: Run flake8
58-
run: |
59-
cd paimon-daft
60-
python -m flake8 --config=dev/cfg.ini daft_paimon/ tests/
61-
62-
test:
63-
runs-on: ubuntu-latest
64-
needs: lint
65-
strategy:
66-
fail-fast: false
67-
matrix:
68-
python-version: ['3.10', '3.11']
69-
70-
steps:
71-
- name: Checkout code
72-
uses: actions/checkout@v6
73-
74-
- name: Set up Python ${{ matrix.python-version }}
75-
uses: actions/setup-python@v5
76-
with:
77-
python-version: ${{ matrix.python-version }}
78-
79-
- name: Install pypaimon from local paimon-python
80-
run: |
81-
cd paimon-python
82-
python -m pip install --upgrade pip
83-
python -m pip install -e .
84-
85-
- name: Install pinned dependencies
86-
run: |
87-
python -m pip install \
88-
daft==${{ env.DAFT_VERSION }} \
89-
pyarrow==16.0.0 \
90-
numpy==1.24.3 \
91-
pandas==2.0.3 \
92-
"pytest~=7.0"
93-
94-
- name: Install daft-paimon
95-
run: |
96-
cd paimon-daft
97-
python -m pip install --no-deps -e .
98-
99-
- name: Run tests
51+
- name: Run checks
10052
run: |
101-
cd paimon-daft
102-
python -m pytest tests/ -v --tb=short
53+
chmod +x paimon-daft/dev/lint-python.sh
54+
./paimon-daft/dev/lint-python.sh

‎paimon-daft/dev/lint-python.sh‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
#!/usr/bin/env bash
2+
################################################################################
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
################################################################################
19+
20+
# lint-python.sh
21+
# Modeled after paimon-python/dev/lint-python.sh.
22+
# Supports -e (exclude) and -i (include) to select check stages.
23+
24+
set -eo pipefail
25+
26+
# ── helpers ──────────────────────────────────────────────────────────────────
27+
28+
function print_function() {
29+
local STAGE_LENGTH=48
30+
local left_edge_len right_edge_len str
31+
case "$1" in
32+
"STAGE")
33+
left_edge_len=$(((STAGE_LENGTH-${#2})/2))
34+
right_edge_len=$((STAGE_LENGTH-${#2}-left_edge_len))
35+
str="$(seq -s "=" $left_edge_len | tr -d "[:digit:]")""$2""$(seq -s "=" $right_edge_len | tr -d "[:digit:]")"
36+
;;
37+
*)
38+
str="$(seq -s "=" $STAGE_LENGTH | tr -d "[:digit:]")"
39+
;;
40+
esac
41+
echo "$str" | tee -a "$LOG_FILE"
42+
}
43+
44+
function contains_element() {
45+
local arr=($1)
46+
if echo "${arr[@]}" | grep -w "$2" &>/dev/null; then
47+
echo true
48+
else
49+
echo false
50+
fi
51+
}
52+
53+
# ── check definitions ────────────────────────────────────────────────────────
54+
55+
function flake8_check() {
56+
print_function "STAGE" "flake8 checks"
57+
58+
local FLAKE8_PATH
59+
FLAKE8_PATH="$(which flake8)"
60+
if [ ! -f "$FLAKE8_PATH" ]; then
61+
echo "flake8 not found. Please install: pip install flake8"
62+
exit 1
63+
fi
64+
65+
local PYTHON_SOURCE
66+
PYTHON_SOURCE="$(find . \( -path ./dev -o -path ./.tox -o -path ./.venv -o -path ./build -o -path ./dist \) -prune -o -type f -name "*.py" -print)"
67+
68+
if [[ ! "$PYTHON_SOURCE" ]]; then
69+
echo "No python files found!"
70+
exit 1
71+
fi
72+
73+
($FLAKE8_PATH --config=./dev/cfg.ini $PYTHON_SOURCE) 2>&1 | tee -a "$LOG_FILE"
74+
75+
if [ $? -ne 0 ]; then
76+
print_function "STAGE" "flake8 checks... [FAILED]"
77+
exit 1
78+
else
79+
print_function "STAGE" "flake8 checks... [SUCCESS]"
80+
fi
81+
}
82+
83+
function pytest_check() {
84+
print_function "STAGE" "pytest checks"
85+
86+
local PYTEST_PATH
87+
PYTEST_PATH="$(which pytest)"
88+
if [ ! -f "$PYTEST_PATH" ]; then
89+
echo "pytest not found. Please install: pip install pytest"
90+
exit 1
91+
fi
92+
93+
($PYTEST_PATH tests/ -v --tb=short) 2>&1 | tee -a "$LOG_FILE"
94+
95+
if [ $? -ne 0 ]; then
96+
print_function "STAGE" "pytest checks... [FAILED]"
97+
exit 1
98+
else
99+
print_function "STAGE" "pytest checks... [SUCCESS]"
100+
fi
101+
}
102+
103+
# ── ordered list of all checks ───────────────────────────────────────────────
104+
105+
ALL_CHECKS=("flake8_check" "pytest_check")
106+
107+
# ── option parsing ───────────────────────────────────────────────────────────
108+
109+
EXCLUDE_CHECKS=()
110+
INCLUDE_CHECKS=()
111+
112+
USAGE="
113+
usage: $0 [options]
114+
-h print this help message and exit
115+
-e checks exclude checks (comma-separated)
116+
-i checks include only these checks (comma-separated)
117+
-l list all supported checks
118+
Examples:
119+
./lint-python.sh => run all checks
120+
./lint-python.sh -e pytest => run all except pytest
121+
./lint-python.sh -i flake8 => run only flake8
122+
"
123+
124+
while getopts "he:i:l" arg; do
125+
case "$arg" in
126+
h) printf "%s\\n" "$USAGE"; exit 0 ;;
127+
e) IFS=',' read -ra EXCLUDE_CHECKS <<< "$OPTARG" ;;
128+
i) IFS=',' read -ra INCLUDE_CHECKS <<< "$OPTARG" ;;
129+
l)
130+
echo "Supported checks:"
131+
for c in "${ALL_CHECKS[@]}"; do echo " ${c%%_check}"; done
132+
exit 0
133+
;;
134+
?) printf "ERROR: unknown option, try -h\\n"; exit 1 ;;
135+
esac
136+
done
137+
138+
# ── resolve which checks to run ──────────────────────────────────────────────
139+
140+
CHECKS_TO_RUN=("${ALL_CHECKS[@]}")
141+
142+
if [ ${#EXCLUDE_CHECKS[@]} -gt 0 ] && [ ${#INCLUDE_CHECKS[@]} -gt 0 ]; then
143+
echo "Cannot use -e and -i simultaneously."
144+
exit 1
145+
fi
146+
147+
if [ ${#EXCLUDE_CHECKS[@]} -gt 0 ]; then
148+
for ex in "${EXCLUDE_CHECKS[@]}"; do
149+
CHECKS_TO_RUN=("${CHECKS_TO_RUN[@]/${ex}_check}")
150+
done
151+
fi
152+
153+
if [ ${#INCLUDE_CHECKS[@]} -gt 0 ]; then
154+
CHECKS_TO_RUN=()
155+
for inc in "${INCLUDE_CHECKS[@]}"; do
156+
if [[ $(contains_element "${ALL_CHECKS[*]}" "${inc}_check") == true ]]; then
157+
CHECKS_TO_RUN+=("${inc}_check")
158+
else
159+
echo "Unknown check: $inc"; exit 1
160+
fi
161+
done
162+
fi
163+
164+
# ── setup ────────────────────────────────────────────────────────────────────
165+
166+
SCRIPT_PATH="$(cd "$(dirname "$0")" && pwd)"
167+
cd "$SCRIPT_PATH/.." || exit
168+
CURRENT_DIR="$(pwd)"
169+
REPO_ROOT="$(cd "$CURRENT_DIR/.." && pwd)"
170+
171+
LOG_DIR="$CURRENT_DIR/dev/log"
172+
mkdir -p "$LOG_DIR"
173+
LOG_FILE="$LOG_DIR/daft-paimon-lint.log"
174+
echo "" > "$LOG_FILE"
175+
176+
# ── pinned versions (only what we own) ───────────────────────────────────────
177+
178+
DAFT_VERSION="0.7.11"
179+
FLAKE8_VERSION="4.0.1"
180+
181+
# ── install dependencies ────────────────────────────────────────────────────
182+
183+
function install_deps() {
184+
print_function "STAGE" "installing dependencies"
185+
186+
python -m pip install --upgrade pip -q
187+
188+
# pypaimon from local source — brings its own pyarrow/pandas/numpy/etc.
189+
if [ -d "$REPO_ROOT/paimon-python" ]; then
190+
(cd "$REPO_ROOT/paimon-python" && python -m pip install -q -e .)
191+
else
192+
echo "WARNING: paimon-python not found at $REPO_ROOT/paimon-python, falling back to PyPI"
193+
python -m pip install -q pypaimon
194+
fi
195+
196+
# daft and dev tools — only versions we control
197+
python -m pip install -q \
198+
"daft==${DAFT_VERSION}" \
199+
"flake8==${FLAKE8_VERSION}" \
200+
"pytest~=7.0"
201+
202+
# daft-paimon itself (no-deps, everything is already installed)
203+
python -m pip install --no-deps -q -e "$CURRENT_DIR"
204+
205+
print_function "STAGE" "dependencies installed"
206+
}
207+
208+
install_deps
209+
210+
# ── run ──────────────────────────────────────────────────────────────────────
211+
212+
print_function "STAGE" "checks starting"
213+
for check in "${CHECKS_TO_RUN[@]}"; do
214+
[ -z "$check" ] && continue
215+
$check
216+
done
217+
echo "All checks finished. Log: $LOG_FILE"

‎paimon-daft/dev/requirements-dev.txt‎

Lines changed: 0 additions & 25 deletions
This file was deleted.

0 commit comments

Comments
 (0)