diff --git a/.idea/misc.xml b/.idea/misc.xml index e0fd5ae..abefd3e 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -8,5 +8,5 @@ - + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6cae65e..e4d8ebd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,75 +14,122 @@ # Pull base image # --------------- ARG MAVEN_VERSION=3.9.9 +ARG NODE_VERSION=20 ARG BASE_REGISTRY=registry.access.redhat.com/ubi8 ARG BASE_IMAGE=ubi-minimal ARG JAVA_OPT="-XX:UseSVE=0" -FROM docker.io/library/maven:${MAVEN_VERSION} AS builder +############################################# +# Stage 1: Build Java Application +############################################# +FROM docker.io/library/maven:${MAVEN_VERSION} AS java-builder LABEL stage=pgcomparebuilder -ARG JAVA_OPT - -ENV _JAVA_OPTIONS=${JAVA_OPT} WORKDIR /app -COPY . ./ +COPY pom.xml ./ +COPY src ./src + +RUN mvn clean install -DskipTests + +############################################# +# Stage 2: Build Next.js UI +############################################# +FROM docker.io/library/node:${NODE_VERSION}-alpine AS ui-builder -RUN mvn clean install +WORKDIR /app/ui +COPY ui/package*.json ./ +RUN npm ci +COPY ui/ ./ +RUN npm run build -FROM ${BASE_REGISTRY}/${BASE_IMAGE} as multi-stage +############################################# +# Stage 3: Multi-stage Production Image +############################################# +FROM ${BASE_REGISTRY}/${BASE_IMAGE} AS multi-stage ARG JAVA_OPT +ARG NODE_VERSION +ARG TARGETARCH -RUN microdnf install java-21-openjdk -y +RUN microdnf install java-21-openjdk tar xz -y && microdnf clean all + +RUN NODE_ARCH=$([ "$TARGETARCH" = "arm64" ] && echo "arm64" || echo "x64") && \ + curl -fsSL https://nodejs.org/dist/v${NODE_VERSION}.9.0/node-v${NODE_VERSION}.9.0-linux-${NODE_ARCH}.tar.xz | tar -xJ -C /usr/local --strip-components=1 USER 0 -RUN mkdir /opt/pgcompare \ +RUN mkdir -p /opt/pgcompare/ui /opt/pgcompare/lib \ && chown -R 1001:1001 /opt/pgcompare -COPY --from=builder /app/docker/start.sh /opt/pgcompare - -COPY --from=builder /app/docker/pgcompare.properties /etc/pgcompare/ +COPY docker/start.sh /opt/pgcompare/ +COPY docker/pgcompare.properties /etc/pgcompare/ +COPY --from=java-builder /app/target/*.jar /opt/pgcompare/ +COPY --from=java-builder /app/target/lib/ /opt/pgcompare/lib/ -COPY --from=builder /app/target/* /opt/pgcompare/ +COPY --from=ui-builder /app/ui/.next/standalone/ /opt/pgcompare/ui/ +COPY --from=ui-builder /app/ui/.next/static /opt/pgcompare/ui/.next/static +COPY --from=ui-builder /app/ui/public /opt/pgcompare/ui/public -RUN chmod 770 /opt/pgcompare/start.sh +RUN chmod 770 /opt/pgcompare/start.sh \ + && chown -R 1001:1001 /opt/pgcompare USER 1001 ENV PGCOMPARE_HOME=/opt/pgcompare \ PGCOMPARE_CONFIG=/etc/pgcompare/pgcompare.properties \ + PGCOMPARE_MODE=standard \ PATH=/opt/pgcompare:$PATH \ - _JAVA_OPTIONS=${JAVA_OPT} + _JAVA_OPTIONS=${JAVA_OPT} \ + PORT=3000 \ + HOSTNAME=0.0.0.0 + +EXPOSE 3000 CMD ["start.sh"] WORKDIR "/opt/pgcompare" -## Local Platform Build -FROM ${BASE_REGISTRY}/${BASE_IMAGE} as local +############################################# +# Stage 4: Local Platform Build +############################################# +FROM ${BASE_REGISTRY}/${BASE_IMAGE} AS local +ARG JAVA_OPT +ARG NODE_VERSION +ARG TARGETARCH + +RUN microdnf install java-21-openjdk tar xz -y && microdnf clean all -RUN microdnf install java-21-openjdk -y +RUN NODE_ARCH=$([ "$TARGETARCH" = "arm64" ] && echo "arm64" || echo "x64") && \ + curl -fsSL https://nodejs.org/dist/v${NODE_VERSION}.9.0/node-v${NODE_VERSION}.9.0-linux-${NODE_ARCH}.tar.xz | tar -xJ -C /usr/local --strip-components=1 USER 0 -RUN mkdir /opt/pgcompare \ +RUN mkdir -p /opt/pgcompare/ui /opt/pgcompare/lib \ && chown -R 1001:1001 /opt/pgcompare COPY docker/start.sh /opt/pgcompare/ - COPY docker/pgcompare.properties /etc/pgcompare/ +COPY target/*.jar /opt/pgcompare/ +COPY target/lib/ /opt/pgcompare/lib/ -COPY target/* /opt/pgcompare/ +COPY ui/.next/standalone/ /opt/pgcompare/ui/ +COPY ui/.next/static /opt/pgcompare/ui/.next/static +COPY ui/public /opt/pgcompare/ui/public -RUN chmod 770 /opt/pgcompare/start.sh +RUN chmod 770 /opt/pgcompare/start.sh \ + && chown -R 1001:1001 /opt/pgcompare USER 1001 ENV PGCOMPARE_HOME=/opt/pgcompare \ PGCOMPARE_CONFIG=/etc/pgcompare/pgcompare.properties \ + PGCOMPARE_MODE=standard \ PATH=/opt/pgcompare:$PATH \ - _JAVA_OPTIONS=${JAVA_OPT} + _JAVA_OPTIONS=${JAVA_OPT} \ + PORT=3000 \ + HOSTNAME=0.0.0.0 + +EXPOSE 3000 CMD ["start.sh"] diff --git a/README.md b/README.md index 0a5ddcb..3425326 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ Before initiating the build and installation process, ensure the following prere # Getting Started +> **Note:** The `main` branch contains active development and may be unstable. For production use, we recommend checking out a stable release tag (e.g., `git checkout v0.6.0`). See [Releases](https://github.com/CrunchyData/pgCompare/releases) for available versions. + ## 1. Fork the repository ## 2. Clone and Build @@ -58,6 +60,7 @@ Before initiating the build and installation process, ensure the following prere ```shell git clone --depth 1 git@github.com:/pgCompare.git cd pgCompare +git checkout v0.6.0 # Optional: checkout a stable release mvn clean install ``` @@ -68,6 +71,15 @@ By default, the application looks for the properties file in the execution direc At a minimal the `repo-xxxxx` parameters are required in the properties file (or specified by environment parameters). Besides the properties file and environment variables, another alternative is to store the property settings in the `dc_project` table. Settings can be stored in the `project_config` column in JSON format ({"parameter": "value"}). Certain system parameters like log-destination can only be specified via the properties file or environment variables. +You can also import/export configuration using the CLI: +```shell +# Export project configuration to properties file +java -jar pgcompare.jar export-config -p 1 --file project-config.properties + +# Import properties file to project configuration +java -jar pgcompare.jar import-config -p 1 --file my-config.properties +``` + ## 4. Initialize Repository Run the script or use the command below to set up the PostgreSQL repository: @@ -96,18 +108,30 @@ Actions: - **check**: Recompare the out of sync rows from previous compare - **compare**: Perform database compare - **copy-table**: Copy pgCompare metadata for table. Must specify table alias to copy using --table option -- **discover**: Disocver tables and columns +- **discover**: Discover tables and columns +- **export-config**: Export project configuration to properties file +- **export-mapping**: Export table/column mappings to YAML file +- **import-config**: Import properties file to project configuration +- **import-mapping**: Import table/column mappings from YAML file - **init**: Initialize the repository database +- **server**: Run in server mode (daemon that polls work queue for jobs) +- **test-connection**: Test database connections and report status Options: -b|--batch {batch nbr} + -o|--file {path} File for export/import operations + + --overwrite Overwrite existing mappings during import + -p|--project Project ID -r|--report {file} Create html report of compare - -t|--table {target table} + -t|--table {target table} (supports wildcards for export/import) + + -n|--name {server name} Server name for server mode (default: pgcompare-server) --help @@ -143,7 +167,19 @@ java -jar pgcompare.jar check --batch 0 # Upgrading -## Version 0.5.0 Enhacements +## Version 0.6.0 Enhancements + +- **Server Mode** - Run pgCompare as a daemon that polls a work queue for jobs +- **Web UI Enhancements** - Job scheduling, real-time progress tracking, job control (pause/resume/stop) +- **Signal Handling** - Graceful shutdown (SIGINT), immediate termination (SIGTERM), config reload (SIGHUP) +- **Fix SQL Improvements** - Enhanced data type handling, proper column mapping for UPDATE statements +- **Database Schema** - New tables for server mode (dc_server, dc_job, dc_job_control, dc_job_progress, dc_job_log) + +**Note:** Drop and recreate the repository to upgrade to 0.6.0. + +For more details review the [v0.6.0 Release Notes](RELEASE_NOTES_v0.6.0.md) + +## Version 0.5.0 Enhancements - Snowflake Support - Full integration for Snowflake as source/target - SQL Fix Generation - Automatic generation of INSERT/UPDATE/DELETE statements (Preview, limited ability) @@ -153,7 +189,7 @@ java -jar pgcompare.jar check --batch 0 **Note:** Drop and recreate the repository to upgrade to 0.5.0. -For more details review the [v0.5.0 Release Noes](RELEASE_NOTES_v0.5.0.md) +For more details review the [v0.5.0 Release Notes](RELEASE_NOTES_v0.5.0.md) ## Version 0.4.0 Enhancements @@ -201,6 +237,8 @@ Examples: Projects allow for the repository to maintain different mappings for different compare objectives. This allows a central pgCompare repository to be used for multiple compare projects. Each table has a `pid` column which is the project id. If no project is specified, the default project (pid = 1) is used. +> **Note:** Project 1 (pid=1) is reserved as the default project and is created automatically during repository initialization. Do not delete or reassign this project. + # Viewing Results ## Summary from Last Run @@ -272,6 +310,12 @@ Properties are categorized into four sections: system, repository, source, and t Default: 3 +#### job-logging-enabled + + When set to true, log messages are written to the `dc_job_log` table in addition to the standard log destination. This enables viewing job logs through the pgCompare UI. The job must have an associated job ID (either from server mode or standalone job tracking). + + Default: false + #### loader-threads Sets the number of threads to load data into the temporary tables. Set to 0 to disable loader threads. diff --git a/RELEASE_NOTES_v0.6.0.md b/RELEASE_NOTES_v0.6.0.md new file mode 100644 index 0000000..b0db428 --- /dev/null +++ b/RELEASE_NOTES_v0.6.0.md @@ -0,0 +1,118 @@ +# pgCompare v0.6.0 Release Notes + +## New Features + +### Server Mode +pgCompare can now run as a daemon server that polls a work queue for jobs. This enables: + +- **Multi-server deployment**: Run multiple pgCompare instances that automatically pick up work +- **Work queue scheduling**: Submit compare, check, or discover jobs via the UI or external tools +- **Job control signals**: Pause, resume, stop, or terminate running jobs gracefully +- **Real-time progress tracking**: Monitor job progress at the table level + +#### Usage +```shell +# Start pgCompare in server mode +java -jar pgcompare.jar server --name my-server-01 + +# Server mode only requires repository connection info in properties file +# Source/target database connections are loaded from project configuration +``` + +#### Command Line Options +- `-n|--name `: Set the server name (default: pgcompare-server) + +### UI Enhancements + +#### Dashboard Overview +- Real-time server status monitoring with heartbeat tracking +- Running, pending, and completed job overview +- Quick access to job details and progress + +#### Job Scheduling & Management +- Schedule compare, check, or discover jobs from the UI +- Target specific servers or let any available server pick up work +- Set job priority (1-10) for queue ordering +- Optional scheduled start time for deferred execution +- Real-time job progress with per-table status + +#### Job Control +- **Pause**: Temporarily halt a running job (preserves progress) +- **Resume**: Continue a paused job +- **Stop**: Gracefully stop a job (completes current table) +- **Terminate**: Immediately stop a job + +#### Navigation & Search +- Search/filter projects and tables in the navigation tree +- Breadcrumb navigation component +- Connection status indicator with auto-refresh + +#### Data Management +- Bulk operations for enabling/disabling multiple tables +- Export data to CSV or JSON format +- Pagination for large result sets + +#### User Experience +- Toast notifications replace browser alerts +- Loading skeletons for better perceived performance +- Keyboard shortcuts support + +### Signal Handling & Graceful Shutdown +pgCompare now properly handles OS signals for clean shutdown and query cancellation: + +- **SIGINT (Ctrl+C)**: Graceful shutdown - completes the current table comparison before exiting +- **SIGTERM**: Immediate termination - cancels all running database queries and exits +- **SIGHUP**: Reload configuration from properties file without restart + +Active database statements are tracked and can be cancelled on demand, preventing orphaned queries on the source/target databases during forced shutdowns. + +## Database Schema Changes + +**Note:** Drop and recreate the repository to upgrade to 0.6.0. + +New tables for server mode: +- `dc_server`: Server registration and heartbeat tracking +- `dc_work_queue`: Job queue with priority scheduling +- `dc_job_control`: Control signals for running jobs +- `dc_job_progress`: Per-table progress tracking + +## API Endpoints + +New REST API endpoints: +- `GET /api/servers`: List registered servers +- `GET /api/jobs`: List jobs with filtering +- `POST /api/jobs`: Submit a new job +- `GET /api/jobs/{id}`: Get job details +- `DELETE /api/jobs/{id}`: Delete a completed/failed job +- `POST /api/jobs/{id}/control`: Send control signal (pause/resume/stop/terminate) +- `GET /api/jobs/{id}/progress`: Get job progress with per-table status +- `GET /api/health`: Check database connection status + +## Configuration + +### Server Mode Properties +Server mode only requires repository connection properties: +```properties +repo-host=localhost +repo-port=5432 +repo-dbname=pgcompare +repo-schema=pgcompare +repo-user=postgres +repo-password=secret +``` + +Project-specific source/target database settings are loaded from the `dc_project.project_config` column. + +## Upgrade Guide + +1. Stop all running pgCompare instances +2. Backup your current repository database +3. Drop the existing pgCompare schema: `DROP SCHEMA pgcompare CASCADE;` +4. Run `java -jar pgcompare.jar init` to create the new schema +5. Reconfigure your projects and table mappings + +## Known Limitations + +- Server mode requires PostgreSQL 14+ for `SKIP LOCKED` support +- Job control signals may have up to 5 second delay before processing +- Servers are marked offline after 2 minutes without heartbeat diff --git a/database/pgCompare.sql b/database/pgCompare.sql index 87f24cf..3404aca 100644 --- a/database/pgCompare.sql +++ b/database/pgCompare.sql @@ -40,7 +40,8 @@ CREATE TABLE dc_source ( pk_hash varchar(100) NULL, column_hash varchar(100) NULL, compare_result bpchar(1) NULL, - thread_nbr int4 NULL + thread_nbr int4 NULL, + fix_sql text NULL ); -- DROP TABLE dc_table; @@ -122,7 +123,95 @@ CREATE TABLE dc_target ( pk_hash varchar(100) NULL, column_hash varchar(100) NULL, compare_result bpchar(1) NULL, - thread_nbr int4 NULL + thread_nbr int4 NULL, + fix_sql text NULL +); + + +-- DROP TABLE dc_server; + +CREATE TABLE dc_server ( + server_id uuid DEFAULT gen_random_uuid() NOT NULL, + server_name text NOT NULL, + server_host text NOT NULL, + server_pid int8 NOT NULL, + status varchar(20) DEFAULT 'active' NOT NULL, + registered_at timestamptz DEFAULT current_timestamp NOT NULL, + last_heartbeat timestamptz DEFAULT current_timestamp NOT NULL, + current_job_id uuid NULL, + server_config jsonb NULL, + CONSTRAINT dc_server_pk PRIMARY KEY (server_id), + CONSTRAINT dc_server_status_check CHECK (status IN ('active', 'idle', 'busy', 'offline', 'terminated')) +); + +-- DROP TABLE dc_job; + +CREATE TABLE dc_job ( + job_id uuid DEFAULT gen_random_uuid() NOT NULL, + pid int8 NOT NULL, + rid int8 NULL, + job_type varchar(20) DEFAULT 'compare' NOT NULL, + status varchar(20) DEFAULT 'pending' NOT NULL, + priority int4 DEFAULT 5 NOT NULL, + batch_nbr int4 DEFAULT 0 NOT NULL, + table_filter text NULL, + target_server_id uuid NULL, + assigned_server_id uuid NULL, + created_at timestamptz DEFAULT current_timestamp NOT NULL, + scheduled_at timestamptz NULL, + started_at timestamptz NULL, + completed_at timestamptz NULL, + created_by text NULL, + job_config jsonb NULL, + result_summary jsonb NULL, + error_message text NULL, + source varchar(20) DEFAULT 'server' NOT NULL, + CONSTRAINT dc_job_pk PRIMARY KEY (job_id), + CONSTRAINT dc_job_type_check CHECK (job_type IN ('compare', 'check', 'discover', 'test-connection')), + CONSTRAINT dc_job_status_check CHECK (status IN ('pending', 'scheduled', 'running', 'paused', 'completed', 'error', 'failed', 'cancelled')), + CONSTRAINT dc_job_priority_check CHECK (priority BETWEEN 1 AND 10), + CONSTRAINT dc_job_source_check CHECK (source IN ('server', 'standalone', 'api')) +); + +-- DROP TABLE dc_job_control; + +CREATE TABLE dc_job_control ( + control_id serial NOT NULL, + job_id uuid NOT NULL, + signal varchar(20) NOT NULL, + requested_at timestamptz DEFAULT current_timestamp NOT NULL, + processed_at timestamptz NULL, + requested_by text NULL, + CONSTRAINT dc_job_control_pk PRIMARY KEY (control_id), + CONSTRAINT dc_job_control_signal_check CHECK (signal IN ('pause', 'resume', 'stop', 'terminate')) +); + +-- DROP TABLE dc_job_progress; + +CREATE TABLE dc_job_progress ( + job_id uuid NOT NULL, + tid int8 NOT NULL, + table_name text NOT NULL, + status varchar(20) DEFAULT 'pending' NOT NULL, + started_at timestamptz NULL, + completed_at timestamptz NULL, + error_message text NULL, + cid int4 NULL, + CONSTRAINT dc_job_progress_pk PRIMARY KEY (job_id, tid), + CONSTRAINT dc_job_progress_status_check CHECK (status IN ('pending', 'running', 'completed', 'failed', 'skipped')) +); + +-- DROP TABLE dc_job_log; + +CREATE TABLE dc_job_log ( + log_id serial NOT NULL, + job_id uuid NOT NULL, + log_ts timestamptz DEFAULT current_timestamp NOT NULL, + log_level varchar(10) NOT NULL, + thread_name varchar(50) NULL, + message text NOT NULL, + context jsonb NULL, + CONSTRAINT dc_job_log_pk PRIMARY KEY (log_id) ); @@ -134,6 +223,10 @@ CREATE INDEX dc_result_idx1 ON dc_result USING btree (table_name, compare_start) CREATE INDEX dc_table_history_idx1 ON dc_table_history USING btree (tid, start_dt); CREATE INDEX dc_table_idx1 ON dc_table USING btree (table_alias); CREATE INDEX dc_table_column_idx1 ON dc_table_column USING btree (column_alias, tid, column_id); +CREATE INDEX dc_server_idx1 ON dc_server USING btree (status, last_heartbeat); +CREATE INDEX dc_job_idx1 ON dc_job USING btree (status, priority DESC, created_at); +CREATE INDEX dc_job_idx2 ON dc_job USING btree (pid, status); +CREATE INDEX dc_job_log_idx1 ON dc_job_log USING btree (job_id, log_ts); -- -- Foreign Keys @@ -141,6 +234,10 @@ CREATE INDEX dc_table_column_idx1 ON dc_table_column USING btree (column_alias, ALTER TABLE dc_table_column ADD CONSTRAINT dc_table_column_fk FOREIGN KEY (tid) REFERENCES dc_table(tid) ON DELETE CASCADE; ALTER TABLE dc_table_column_map ADD CONSTRAINT dc_table_column_map_fk FOREIGN KEY (column_id) REFERENCES dc_table_column(column_id) ON DELETE CASCADE; ALTER TABLE dc_table_map ADD CONSTRAINT dc_table_map_fk FOREIGN KEY (tid) REFERENCES dc_table(tid) ON DELETE CASCADE; +ALTER TABLE dc_job ADD CONSTRAINT dc_job_fk1 FOREIGN KEY (pid) REFERENCES dc_project(pid) ON DELETE CASCADE; +ALTER TABLE dc_job_control ADD CONSTRAINT dc_job_control_fk1 FOREIGN KEY (job_id) REFERENCES dc_job(job_id) ON DELETE CASCADE; +ALTER TABLE dc_job_progress ADD CONSTRAINT dc_job_progress_fk1 FOREIGN KEY (job_id) REFERENCES dc_job(job_id) ON DELETE CASCADE; +ALTER TABLE dc_job_log ADD CONSTRAINT dc_job_log_fk1 FOREIGN KEY (job_id) REFERENCES dc_job(job_id) ON DELETE CASCADE; -- -- Data diff --git a/docker/README.md b/docker/README.md index 387c1fe..5ec0886 100644 --- a/docker/README.md +++ b/docker/README.md @@ -4,20 +4,114 @@ For building instructions, see the comments in the `Dockerfile`. -## Using Container +### Multi-stage Build (recommended) +```shell +docker buildx build --load --platform linux/amd64,linux/arm64 --target multi-stage -t pgcompare:latest -t pgcompare:v0.6.0 . +``` + +### Local Build (requires pre-built artifacts) +```shell +# First build the Java and UI artifacts +mvn clean install +cd ui && npm ci && npm run build && cd .. + +# Then build the container +docker build --target local -t pgcompare:latest . +``` + +## Container Modes + +The container supports multiple operational modes controlled by the `PGCOMPARE_MODE` environment variable: + +| Mode | Description | +|------|-------------| +| `standard` | (Default) Runs the Java application with `PGCOMPARE_OPTIONS` | +| `server` | **[PREVIEW]** Runs the Java application in server mode for distributed processing | +| `ui` | **[PREVIEW]** Runs only the Next.js web UI | +| `all` | **[PREVIEW]** Runs both the Java server and the Next.js UI | + +> **Note:** The `server`, `ui`, and `all` modes are currently in preview and may change in future releases. + +## Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `PGCOMPARE_MODE` | Container operation mode | `standard` | +| `PGCOMPARE_OPTIONS` | CLI options for standard mode | `--batch 0 --project 1` | +| `PGCOMPARE_SERVER_NAME` | Server name for server/all modes | Container hostname | +| `PGCOMPARE_CONFIG` | Path to properties file | `/etc/pgcompare/pgcompare.properties` | +| `PORT` | Port for the UI (ui/all modes) | `3000` | +| `DATABASE_URL` | PostgreSQL connection for UI | Required for ui/all modes | -Create a `pgcompare.properties` file as outlined in the project README file. The properties file will be mounted in the /etc/pgcompare directory of the container as seen below. It can be placed in an alternate location by setting the PGCOMPARE_CONFIG environment variable. +## Usage Examples +### Standard Mode (default) +Run a comparison batch: ```shell docker run --name pgcompare \ - -v /Users/bpace/app/gitecto/db-projects/pgCompare-Test/pgcompare.properties:/etc/pgcompare/pgcompare.properties \ - -e PGCOMPARE_OPTIONS="--batch 0 --project 1" \ - cbrianpace/pgcompare:v0.3.1 + -v /path/to/pgcompare.properties:/etc/pgcompare/pgcompare.properties \ + -e PGCOMPARE_OPTIONS="--batch 0 --project 1" \ + pgcompare:latest +``` + +### Server Mode +Run as a worker server for distributed comparisons: +```shell +docker run --name pgcompare-server \ + -v /path/to/pgcompare.properties:/etc/pgcompare/pgcompare.properties \ + -e PGCOMPARE_MODE=server \ + -e PGCOMPARE_SERVER_NAME=worker-1 \ + pgcompare:latest +``` -podman run --name pgcompare \ - -v /Users/bpace/app/gitecto/db-projects/pgCompare-Test/pgcompare.properties:/etc/pgcompare/pgcompare.properties \ - -e PGCOMPARE_OPTIONS="--batch 0 --project 1" \ - cbrianpace/pgcompare:v0.3.1 +### UI Mode +Run only the web interface: +```shell +docker run --name pgcompare-ui \ + -p 3000:3000 \ + -e PGCOMPARE_MODE=ui \ + -e DATABASE_URL="postgresql://user:pass@host:5432/pgcompare" \ + pgcompare:latest +``` +### All Mode +Run both server and UI together: +```shell +docker run --name pgcompare-all \ + -p 3000:3000 \ + -v /path/to/pgcompare.properties:/etc/pgcompare/pgcompare.properties \ + -e PGCOMPARE_MODE=all \ + -e DATABASE_URL="postgresql://user:pass@host:5432/pgcompare" \ + pgcompare:latest ``` +## Docker Compose Example + +```yaml +version: '3.8' + +services: + pgcompare-db: + image: postgres:16 + environment: + POSTGRES_USER: pgcompare + POSTGRES_PASSWORD: pgcompare + POSTGRES_DB: pgcompare + volumes: + - pgcompare-data:/var/lib/postgresql/data + + pgcompare: + image: pgcompare:latest + ports: + - "3000:3000" + environment: + PGCOMPARE_MODE: all + DATABASE_URL: postgresql://pgcompare:pgcompare@pgcompare-db:5432/pgcompare + volumes: + - ./pgcompare.properties:/etc/pgcompare/pgcompare.properties + depends_on: + - pgcompare-db + +volumes: + pgcompare-data: +``` diff --git a/docker/start.sh b/docker/start.sh index bfd29d5..15c13ae 100644 --- a/docker/start.sh +++ b/docker/start.sh @@ -4,26 +4,87 @@ function _int() { echo "Stopping container." echo "SIGINT received, shutting down!" + cleanup } ########### SIGTERM handler ############ function _term() { echo "Stopping container." echo "SIGTERM received, shutting down!" + cleanup } ########### SIGKILL handler ############ function _kill() { echo "SIGKILL received, shutting down!" + cleanup } +function cleanup() { + if [ -n "$JAVA_PID" ]; then + kill $JAVA_PID 2>/dev/null + fi + if [ -n "$UI_PID" ]; then + kill $UI_PID 2>/dev/null + fi + exit 0 +} + +trap _int SIGINT +trap _term SIGTERM +trap _kill SIGKILL + ################################### ############# MAIN ################ ################################### -if [ "$PGCOMPARE_OPTIONS" == "" ]; -then - export PGCOMPARE_OPTIONS="--batch 0 --project 1" -fi +PGCOMPARE_MODE=${PGCOMPARE_MODE:-standard} + +echo "pgCompare starting in '${PGCOMPARE_MODE}' mode..." -java -jar /opt/pgcompare/pgcompare.jar $PGCOMPARE_OPTIONS \ No newline at end of file +case "$PGCOMPARE_MODE" in + standard) + if [ -z "$PGCOMPARE_OPTIONS" ]; then + export PGCOMPARE_OPTIONS="--batch 0 --project 1" + fi + echo "Running: java -jar /opt/pgcompare/pgcompare.jar $PGCOMPARE_OPTIONS" + java -jar /opt/pgcompare/pgcompare.jar $PGCOMPARE_OPTIONS + ;; + + server) + SERVER_NAME=${PGCOMPARE_SERVER_NAME:-$(hostname -s)} + echo "Running: java -jar /opt/pgcompare/pgcompare.jar server --name $SERVER_NAME" + java -jar /opt/pgcompare/pgcompare.jar server --name "$SERVER_NAME" + ;; + + ui) + echo "Starting Next.js UI on port ${PORT:-3000}..." + cd /opt/pgcompare/ui + node server.js & + UI_PID=$! + wait $UI_PID + ;; + + all) + SERVER_NAME=${PGCOMPARE_SERVER_NAME:-$(hostname -s)} + + echo "Starting Java server mode..." + java -jar /opt/pgcompare/pgcompare.jar server --name "$SERVER_NAME" & + JAVA_PID=$! + + sleep 2 + + echo "Starting Next.js UI on port ${PORT:-3000}..." + cd /opt/pgcompare/ui + node server.js & + UI_PID=$! + + wait $JAVA_PID $UI_PID + ;; + + *) + echo "ERROR: Unknown PGCOMPARE_MODE '$PGCOMPARE_MODE'" + echo "Valid modes: standard, server, ui, all" + exit 1 + ;; +esac diff --git a/docs/advanced-tuning-guide.md b/docs/advanced-tuning-guide.md new file mode 100644 index 0000000..da31bd0 --- /dev/null +++ b/docs/advanced-tuning-guide.md @@ -0,0 +1,273 @@ +# Advanced Tuning Guide + +This guide covers advanced configuration options for pgCompare that are typically not required for most workloads. These settings should only be used after confirming a specific bottleneck through monitoring and testing. + +## Table of Contents + +1. [Loader Threads](#loader-threads) +2. [Message Queue Configuration](#message-queue-configuration) +3. [When to Use Advanced Threading](#when-to-use-advanced-threading) +4. [Diagnosing Bottlenecks](#diagnosing-bottlenecks) +5. [Advanced Thread Configuration Examples](#advanced-thread-configuration-examples) + +--- + +## Loader Threads + +> **Important:** In most circumstances, loader threads (`loader-threads > 0`) do not provide significant performance benefits and add complexity. The default value of `0` is recommended for the vast majority of workloads. Only consider enabling loader threads after confirming a bottleneck in staging table inserts. + +### What Are Loader Threads? + +Loader threads decouple the data fetching process from repository loading by introducing an intermediate queue: + +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Compare Thread │ ──► │ Message Queue │ ──► │ Loader Thread │ +│ (Fetch & Hash) │ │ (Blocking) │ │ (Insert to DB) │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ +``` + +Without loader threads (default, `loader-threads=0`): +- Compare threads fetch data, compute hashes, and insert directly into staging tables +- Simpler architecture with lower memory overhead +- Sufficient for most workloads + +With loader threads (`loader-threads > 0`): +- Compare threads place hash data into blocking queues +- Multiple loader threads consume from queues in parallel +- Loader threads batch-insert records into staging tables +- Adds complexity and memory overhead + +### Enabling Loader Threads + +```properties +loader-threads=8 +``` + +Or via environment variable: +```bash +export PGCOMPARE_LOADER_THREADS=8 +``` + +### Configuration Options + +| Property | Default | Description | +|----------|---------|-------------| +| `loader-threads` | 0 | Number of loader threads per side (source/target) | +| `message-queue-size` | 1000 | Size of blocking queue between compare and loader threads | + +### Impact Analysis + +| loader-threads | Benefits | Costs | +|---------------|----------|-------| +| 0 | Simpler, lower memory, recommended default | N/A | +| 2-4 | May help if staging inserts are bottleneck | Moderate memory, added complexity | +| 8 | Faster loading (rare cases) | Higher memory, more connections | +| 16+ | Maximum throughput (very rare cases) | High resource usage | + +--- + +## Message Queue Configuration + +Message queues are only used when `loader-threads > 0`. They buffer data between compare threads and loader threads. + +### message-queue-size + +```properties +message-queue-size=1000 +``` + +### Queue Space Warning + +If you see log messages indicating threads are **waiting for queue space**, you need to increase `message-queue-size`. This blocking condition slows down comparison as producer threads (compare threads) wait for consumer threads (loader threads) to free up queue slots. + +```properties +# If seeing "waiting for queue space" messages +message-queue-size=2000 +``` + +### Memory Usage Warning + +Each queued message holds row data in memory. Larger queue sizes consume significantly more heap memory. + +**Calculate memory impact:** +``` +Queue Memory = message-queue-size × average_row_size × number_of_parallel_threads +``` + +**Example:** +- message-queue-size = 2000 +- average_row_size = 500 bytes +- parallel_degree = 4 + +Memory per side = 2000 × 500 × 4 = 4MB per side (8MB total for source + target) + +For very large queues (10,000+) with many threads, memory can grow significantly. Monitor JVM heap usage and increase `-Xmx` if needed. + +### Tuning Guidelines + +```properties +# For memory-constrained environments +message-queue-size=500 + +# Default +message-queue-size=1000 + +# If seeing "waiting for queue space" messages +message-queue-size=2000 + +# For extreme throughput (with adequate memory) +message-queue-size=5000 +``` + +--- + +## When to Use Advanced Threading + +### You Probably DON'T Need Loader Threads If: + +- Table sizes are under 100 million rows +- Comparison completes in acceptable time with default settings +- Repository database is not showing high CPU/IO during staging inserts +- Network latency to source/target is the primary bottleneck + +### You MIGHT Benefit from Loader Threads If: + +- Tables exceed 100+ million rows AND comparison is slow +- Monitoring shows compare threads spending significant time waiting on staging inserts +- Repository database is the confirmed bottleneck (not source/target databases) +- You have tested with `loader-threads=0` first and confirmed it's not sufficient + +### Decision Flowchart + +``` +Is comparison slow? + │ + ├─► No ──► Keep loader-threads=0 + │ + └─► Yes ──► Where is the bottleneck? + │ + ├─► Source/Target DB ──► Tune queries, add indexes + │ + ├─► Network ──► Increase batch-fetch-size + │ + ├─► pgCompare CPU ──► Increase parallel_degree + │ + └─► Repository inserts ──► Consider loader-threads > 0 +``` + +--- + +## Diagnosing Bottlenecks + +Before enabling loader threads, identify where time is being spent. + +### Monitor Repository Database + +```sql +-- Check for waiting queries during comparison +SELECT + pid, + state, + wait_event_type, + wait_event, + query +FROM pg_stat_activity +WHERE application_name LIKE 'pgCompare%' +AND state != 'idle'; + +-- Check staging table insert rate +SELECT + relname, + n_tup_ins, + n_tup_upd, + n_tup_del +FROM pg_stat_user_tables +WHERE relname LIKE 'dc_%_stg_%'; +``` + +### Monitor JVM Thread Activity + +```bash +# Get thread dump during comparison +jstack $(pgrep -f pgcompare) | grep -A 5 "compare\|loader" +``` + +### Check Log Output + +Enable debug logging to see timing information: + +```properties +log-level=DEBUG +``` + +Look for patterns indicating where time is spent: +- Long fetch times → Source/target database bottleneck +- Long insert times → Repository bottleneck (may benefit from loader threads) +- Queue full messages → Increase message-queue-size + +--- + +## Advanced Thread Configuration Examples + +### Example: Confirmed Repository Bottleneck + +After monitoring confirms staging table inserts are the bottleneck: + +```properties +# Enable loader threads +loader-threads=8 +message-queue-size=2000 + +# Other settings +batch-fetch-size=10000 +batch-commit-size=10000 +observer-throttle=true +observer-throttle-size=2000000 +``` + +```bash +java -Xms4g -Xmx8g -jar pgcompare.jar compare --batch 0 +``` + +### Example: Very Large Tables (500M+ rows) + +Only if default settings are insufficient: + +```properties +loader-threads=16 +message-queue-size=4000 +batch-fetch-size=20000 +batch-commit-size=20000 +observer-throttle=true +observer-throttle-size=5000000 +observer-vacuum=true +``` + +```sql +-- Set high parallel degree with mod_column +UPDATE dc_table SET parallel_degree = 8 WHERE table_alias = 'huge_table'; +UPDATE dc_table_map SET mod_column = 'id' WHERE tid = (SELECT tid FROM dc_table WHERE table_alias = 'huge_table'); +``` + +```bash +java -Xms8g -Xmx16g -XX:+UseG1GC -jar pgcompare.jar compare --table huge_table +``` + +### Thread Configuration Matrix + +| Scenario | parallel_degree | loader-threads | message-queue-size | +|----------|-----------------|----------------|-------------------| +| Default (recommended) | 1-8 | 0 | N/A | +| Repository bottleneck confirmed | 4-8 | 4-8 | 1000-2000 | +| Extreme (very rare) | 8-16 | 8-16 | 2000-4000 | + +--- + +## Summary + +- **Start simple**: Use `loader-threads=0` (default) for all workloads initially +- **Monitor first**: Identify the actual bottleneck before adding complexity +- **Test incrementally**: If you enable loader threads, start with `loader-threads=4` and increase gradually +- **Watch memory**: Larger queues and more threads require more heap memory +- **Document findings**: Record what settings work for your specific workload diff --git a/docs/large-tables-guide.md b/docs/large-tables-guide.md new file mode 100644 index 0000000..37594d6 --- /dev/null +++ b/docs/large-tables-guide.md @@ -0,0 +1,731 @@ +# Handling Large Tables with Parallel Processing + +This guide covers strategies for efficiently comparing large tables using pgCompare's parallel threading capabilities. + +## Table of Contents + +1. [Understanding Parallel Processing](#understanding-parallel-processing) +2. [Thread Architecture](#thread-architecture) +3. [Configuring Parallel Degree](#configuring-parallel-degree) +4. [Observer Thread Configuration](#observer-thread-configuration) +5. [Memory Management](#memory-management) +6. [Staging Table Optimization](#staging-table-optimization) +7. [Initial and Delta Comparisons](#initial-and-delta-comparisons) +8. [Examples and Scenarios](#examples-and-scenarios) +9. [Troubleshooting](#troubleshooting) + +--- + +## Understanding Parallel Processing + +pgCompare uses a multi-threaded architecture to efficiently compare large datasets. When comparing tables with millions of rows, parallel processing can significantly reduce comparison time. + +### Key Concepts + +- **Parallel Degree**: Number of concurrent comparison threads per table +- **Observer Thread**: Thread that reconciles matches and manages staging tables +- **Staging Tables**: Temporary PostgreSQL tables that store hash values during comparison + +### When to Use Parallel Processing + +| Table Size | Recommended Parallel Degree | +|-----------|---------------------------| +| < 100,000 rows | 1 | +| 100,000 - 1M rows | 2 | +| 1M - 10M rows | 4 | +| 10M - 100M rows | 8 | +| > 100M rows | 8-16 | + +--- + +## Thread Architecture + +### Overview + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ Thread Manager │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ Compare │ │ Compare │ │ Compare │ │ +│ │ Thread 0 │ │ Thread 1 │ │ Thread N │ ... │ +│ │ (Source) │ │ (Source) │ │ (Source) │ │ +│ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │ +│ │ │ │ │ +│ ▼ ▼ ▼ │ +│ ┌─────────────────────────────────────────────────────┐ │ +│ │ Staging Table (Source) │ │ +│ └─────────────────────────┬───────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────┐ │ +│ │ Observer Thread │ │ +│ │ (Reconciliation) │ │ +│ └─────────────────────────┬───────────────────────────┘ │ +│ │ │ +│ ┌─────────────────────────────────────────────────────┐ │ +│ │ Staging Table (Target) │ │ +│ └─────────────────────────┬───────────────────────────┘ │ +│ │ │ +│ (Same structure for Target threads) │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +### Thread Types + +1. **Compare Threads (DataComparisonThread)** + - Execute queries against source/target databases + - Compute hash values for primary keys and columns + - Write directly to staging tables + +2. **Observer Thread (ObserverThread)** + - Monitor staging tables for matching hash pairs + - Remove matched rows from staging tables + - Perform periodic vacuuming + - Track reconciliation progress + +> **Note:** Loader threads are an advanced feature covered in the [Advanced Tuning Guide](advanced-tuning-guide.md). They are not required for most workloads. + +--- + +## Configuring Parallel Degree + +### Per-Table Configuration + +Set parallel degree for individual tables in the repository: + +```sql +-- Set parallel degree to 4 for a specific table +UPDATE dc_table +SET parallel_degree = 4 +WHERE table_alias = 'large_orders'; +``` + +### Via YAML Import + +```yaml +tables: + - alias: "large_orders" + enabled: true + batchNumber: 1 + parallelDegree: 4 + source: + schema: "SALES" + table: "LARGE_ORDERS" + target: + schema: "sales" + table: "large_orders" +``` + +### Requirements for Parallel Processing + +When `parallel_degree > 1`, you **must** specify a `mod_column` in the table mapping. The `mod_column` must be a **numeric column** (INTEGER, BIGINT, NUMERIC without decimals). + +```sql +-- Set mod_column for parallel processing (both source and target) +UPDATE dc_table_map +SET mod_column = 'order_id' +WHERE tid = 123; +``` + +#### How mod_column Works + +pgCompare uses the mod_column to distribute rows across threads using modulo arithmetic: + +```sql +-- With parallel_degree = 4: +-- Thread 0 processes: WHERE MOD(order_id, 4) = 0 +-- Thread 1 processes: WHERE MOD(order_id, 4) = 1 +-- Thread 2 processes: WHERE MOD(order_id, 4) = 2 +-- Thread 3 processes: WHERE MOD(order_id, 4) = 3 +``` + +This ensures each thread processes a distinct subset of rows without overlap. + +### mod_column Requirements + +| Requirement | Description | +|-------------|-------------| +| **Data Type** | Must be numeric (INTEGER, BIGINT, NUMERIC without decimals) | +| **NOT NULL** | Column should not contain NULL values | +| **Distribution** | Values should be evenly distributed for balanced workload | +| **Indexed** | Indexing the column improves query performance | + +### Choosing a Mod Column + +**Good candidates:** +- Primary key columns (id, order_id, customer_id) +- Auto-increment/sequence columns +- Surrogate keys + +**Poor candidates:** +- Columns with many NULLs +- Columns with skewed distributions (e.g., status codes) +- Decimal/float columns +- Non-numeric columns + +#### Finding a Suitable mod_column + +```sql +-- Check available numeric columns for a table +SELECT tc.column_alias, tcm.data_type, tcm.column_name +FROM dc_table t +JOIN dc_table_column tc ON t.tid = tc.tid +JOIN dc_table_column_map tcm ON tc.tid = tcm.tid AND tc.column_id = tcm.column_id +WHERE t.table_alias = 'orders' + AND tcm.data_type IN ('integer', 'bigint', 'int', 'number', 'numeric') + AND tcm.column_origin = 'source'; +``` + +#### Complete Parallel Setup Example + +```sql +-- 1. Get the table ID +SELECT tid FROM dc_table WHERE table_alias = 'orders'; +-- Returns: 42 + +-- 2. Set parallel degree +UPDATE dc_table SET parallel_degree = 4 WHERE tid = 42; + +-- 3. Set mod_column on BOTH source and target mappings +UPDATE dc_table_map SET mod_column = 'order_id' WHERE tid = 42; + +-- 4. Verify configuration +SELECT + t.table_alias, + t.parallel_degree, + tm.dest_type, + tm.mod_column +FROM dc_table t +JOIN dc_table_map tm ON t.tid = tm.tid +WHERE t.tid = 42; +``` + +--- + +## Observer Thread Configuration + +The observer thread manages the reconciliation process and staging table maintenance. + +### Throttling + +Throttling prevents staging tables from growing unbounded: + +```properties +# Enable throttling (recommended) +observer-throttle=true + +# Pause loading after 2M rows in staging +observer-throttle-size=2000000 +``` + +**How it works:** +1. Compare threads insert data into staging tables +2. When row count exceeds `observer-throttle-size`, compare threads pause +3. Observer thread reconciles matches and removes them +4. When staging table shrinks, compare threads resume + +### Vacuum Settings + +```properties +# Enable vacuum during reconciliation +observer-vacuum=true +``` + +Benefits of observer vacuum: +- Reclaims space from deleted rows +- Maintains staging table performance +- Prevents table bloat + +### Optimizing Observer Performance + +For very large tables, tune these PostgreSQL settings in the observer connection: + +```sql +-- Applied automatically by pgCompare +SET enable_nestloop='off'; +SET work_mem='512MB'; +SET maintenance_work_mem='1024MB'; +``` + +--- + +## Memory Management + +### Java Heap Configuration + +For large tables, increase Java heap size: + +```bash +# Minimum 512MB, Maximum 4GB +java -Xms512m -Xmx4g -jar pgcompare.jar compare --batch 0 + +# For very large comparisons +java -Xms2g -Xmx8g -jar pgcompare.jar compare --batch 0 +``` + +### Memory Usage Factors + +| Factor | Impact | +|--------|--------| +| parallel_degree | Linear increase per thread | +| batch-fetch-size | Memory for fetched result sets | + +### Recommended Heap Sizes + +| Table Size | Parallel Degree | Recommended Heap | +|-----------|-----------------|------------------| +| < 1M rows | 1-2 | 512MB - 1GB | +| 1M - 10M | 2-4 | 1GB - 2GB | +| 10M - 50M | 4-8 | 2GB - 4GB | +| 50M - 100M | 8 | 4GB - 8GB | +| > 100M | 8-16 | 8GB+ | + +--- + +## Staging Table Optimization + +### Staging Table Parallel Degree + +Control PostgreSQL parallelism for staging tables: + +```properties +# Use 4 parallel workers for staging table operations +stage-table-parallel=4 +``` + +### Repository PostgreSQL Tuning + +For optimal staging table performance, configure the repository database: + +```sql +-- Recommended settings for large comparisons +ALTER SYSTEM SET shared_buffers = '2048MB'; +ALTER SYSTEM SET work_mem = '256MB'; +ALTER SYSTEM SET maintenance_work_mem = '512MB'; +ALTER SYSTEM SET max_parallel_workers = 16; +ALTER SYSTEM SET max_parallel_workers_per_gather = 4; +ALTER SYSTEM SET effective_cache_size = '6GB'; + +SELECT pg_reload_conf(); +``` + +### Monitoring Staging Tables + +During comparison, you can monitor staging table size: + +```sql +-- Check staging table sizes during comparison +SELECT + schemaname, + relname, + n_live_tup, + n_dead_tup, + pg_size_pretty(pg_total_relation_size(relid)) as size +FROM pg_stat_user_tables +WHERE relname LIKE 'dc_%_stg_%' +ORDER BY n_live_tup DESC; +``` + +--- + +## Initial and Delta Comparisons + +For very large tables, a common strategy is to perform an initial full comparison, then use row filters to compare only new or modified data in subsequent runs. This dramatically reduces comparison time for ongoing validation. + +### Strategy Overview + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Day 1: Initial Full Compare │ +│ - Compare all rows (may take hours for large tables) │ +│ - Record timestamp of comparison completion │ +└─────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ Day 2+: Delta Compare │ +│ - Set table_filter to only include rows modified since │ +│ the last comparison │ +│ - Compare runs in minutes instead of hours │ +└─────────────────────────────────────────────────────────────┘ +``` + +### Step 1: Initial Full Comparison + +Run the first comparison without any row filter to validate all data: + +```sql +-- Ensure no filter is set for initial compare +UPDATE dc_table_map +SET table_filter = NULL +WHERE tid = (SELECT tid FROM dc_table WHERE table_alias = 'orders'); +``` + +```bash +# Run full comparison +java -Xms4g -Xmx8g -jar pgcompare.jar compare --table orders + +# Record the timestamp when comparison completes +# Example: 2024-01-15 14:30:00 +``` + +### Step 2: Set Up Delta Comparison + +After the initial compare, configure a filter to only compare rows modified since the last run: + +```sql +-- PostgreSQL source/target +UPDATE dc_table_map +SET table_filter = 'modified_date > ''2024-01-15 14:30:00''' +WHERE tid = (SELECT tid FROM dc_table WHERE table_alias = 'orders'); + +-- Oracle source +UPDATE dc_table_map +SET table_filter = 'modified_date > TO_DATE(''2024-01-15 14:30:00'', ''YYYY-MM-DD HH24:MI:SS'')' +WHERE tid = (SELECT tid FROM dc_table WHERE table_alias = 'orders') +AND dest_type = 'source'; + +-- Snowflake source/target +UPDATE dc_table_map +SET table_filter = 'modified_date > ''2024-01-15 14:30:00''::TIMESTAMP' +WHERE tid = (SELECT tid FROM dc_table WHERE table_alias = 'orders'); +``` + +### Step 3: Run Delta Comparison + +```bash +# Delta compare runs much faster - only new/modified rows +java -jar pgcompare.jar compare --table orders +``` + +### Automating Delta Updates + +Create a simple workflow to update the filter after each successful comparison: + +```sql +-- Before running comparison, update filter to current timestamp +-- This captures rows modified since last run + +-- For PostgreSQL +UPDATE dc_table_map +SET table_filter = 'modified_date > ''' || + (CURRENT_TIMESTAMP - INTERVAL '1 day')::text || '''' +WHERE tid = (SELECT tid FROM dc_table WHERE table_alias = 'orders'); + +-- For rolling 7-day window +UPDATE dc_table_map +SET table_filter = 'modified_date > CURRENT_DATE - INTERVAL ''7 days''' +WHERE tid = (SELECT tid FROM dc_table WHERE table_alias = 'orders'); +``` + +### Splitting Large Tables Across Batches + +For extremely large tables, you can register the same table multiple times with different filters, allowing you to split the comparison across batches or run them in parallel on different machines. + +#### Example: Split by Date Range + +```sql +-- Register table for historical data (batch 1) +INSERT INTO dc_table (pid, table_alias, enabled, batch_nbr, parallel_degree) +VALUES (1, 'orders_historical', true, 1, 4) +RETURNING tid; +-- Returns tid: 100 + +-- Register same table for recent data (batch 2) +INSERT INTO dc_table (pid, table_alias, enabled, batch_nbr, parallel_degree) +VALUES (1, 'orders_recent', true, 2, 4) +RETURNING tid; +-- Returns tid: 101 + +-- Set up mappings for historical (orders older than 1 year) +INSERT INTO dc_table_map (tid, dest_type, schema_name, table_name, table_filter) +VALUES +(100, 'source', 'SALES', 'ORDERS', 'order_date < ADD_MONTHS(SYSDATE, -12)'), +(100, 'target', 'sales', 'orders', 'order_date < CURRENT_DATE - INTERVAL ''1 year'''); + +-- Set up mappings for recent (orders within last year) +INSERT INTO dc_table_map (tid, dest_type, schema_name, table_name, table_filter) +VALUES +(101, 'source', 'SALES', 'ORDERS', 'order_date >= ADD_MONTHS(SYSDATE, -12)'), +(101, 'target', 'sales', 'orders', 'order_date >= CURRENT_DATE - INTERVAL ''1 year'''); +``` + +#### Example: Split by Primary Key Range + +```sql +-- Register table for first half of data +INSERT INTO dc_table (pid, table_alias, enabled, batch_nbr, parallel_degree) +VALUES (1, 'orders_part1', true, 1, 8) +RETURNING tid; +-- Returns tid: 200 + +-- Register table for second half +INSERT INTO dc_table (pid, table_alias, enabled, batch_nbr, parallel_degree) +VALUES (1, 'orders_part2', true, 2, 8) +RETURNING tid; +-- Returns tid: 201 + +-- Set filters by ID range +UPDATE dc_table_map SET table_filter = 'order_id <= 50000000' WHERE tid = 200; +UPDATE dc_table_map SET table_filter = 'order_id > 50000000' WHERE tid = 201; +``` + +#### Running Split Comparisons + +```bash +# Run batches sequentially +java -jar pgcompare.jar compare --batch 1 +java -jar pgcompare.jar compare --batch 2 + +# Or run on separate machines simultaneously +# Machine 1: +java -jar pgcompare.jar compare --table orders_historical + +# Machine 2: +java -jar pgcompare.jar compare --table orders_recent +``` + +### Combining Strategies + +For maximum efficiency on very large tables, combine parallel processing with delta comparisons: + +```sql +-- Configure table with parallel degree and mod_column +UPDATE dc_table +SET parallel_degree = 8 +WHERE table_alias = 'orders_recent'; + +UPDATE dc_table_map +SET mod_column = 'order_id', + table_filter = 'modified_date > CURRENT_DATE - INTERVAL ''1 day''' +WHERE tid = (SELECT tid FROM dc_table WHERE table_alias = 'orders_recent'); +``` + +### Best Practices for Delta Comparisons + +1. **Ensure timestamp columns are indexed** on both source and target for filter performance +2. **Use consistent timestamp formats** across source and target filters +3. **Add buffer time** to filters (e.g., subtract 1 hour) to catch rows being modified during comparison +4. **Periodically run full comparisons** to catch any data drift not captured by delta filters +5. **Document your filter logic** so team members understand the comparison scope + +--- + +## Examples and Scenarios + +### Example 1: 10 Million Row Table + +**Configuration:** + +```properties +# pgcompare.properties +batch-fetch-size=5000 +batch-commit-size=5000 +observer-throttle=true +observer-throttle-size=1000000 +``` + +**Table Setup:** + +```sql +-- Set parallel degree and mod column +UPDATE dc_table +SET parallel_degree = 4 +WHERE table_alias = 'transactions'; + +UPDATE dc_table_map +SET mod_column = 'transaction_id' +WHERE tid = (SELECT tid FROM dc_table WHERE table_alias = 'transactions'); +``` + +**Run:** + +```bash +java -Xms1g -Xmx4g -jar pgcompare.jar compare --table transactions +``` + +### Example 2: 100+ Million Row Table + +**Configuration:** + +```properties +# pgcompare.properties +batch-fetch-size=10000 +batch-commit-size=10000 +observer-throttle=true +observer-throttle-size=5000000 +observer-vacuum=true +``` + +**Table Setup:** + +```sql +-- Set parallel degree and mod column +UPDATE dc_table +SET parallel_degree = 8 +WHERE table_alias = 'event_log'; + +UPDATE dc_table_map +SET mod_column = 'event_id' +WHERE tid = (SELECT tid FROM dc_table WHERE table_alias = 'event_log'); +``` + +**Run:** + +```bash +java -Xms4g -Xmx16g -jar pgcompare.jar compare --table event_log +``` + +### Example 3: Multiple Large Tables + +**Configuration for batch processing:** + +```sql +-- Assign tables to batches by size +UPDATE dc_table SET batch_nbr = 1, parallel_degree = 8 +WHERE table_alias IN ('huge_table_1', 'huge_table_2'); + +UPDATE dc_table SET batch_nbr = 2, parallel_degree = 4 +WHERE table_alias IN ('medium_table_1', 'medium_table_2', 'medium_table_3'); + +UPDATE dc_table SET batch_nbr = 3, parallel_degree = 1 +WHERE table_alias IN ('small_table_1', 'small_table_2'); +``` + +**Run batches sequentially:** + +```bash +# Run huge tables first (batch 1) +java -Xms4g -Xmx16g -jar pgcompare.jar compare --batch 1 + +# Run medium tables (batch 2) +java -Xms2g -Xmx8g -jar pgcompare.jar compare --batch 2 + +# Run small tables (batch 3) +java -Xms512m -Xmx2g -jar pgcompare.jar compare --batch 3 +``` + +--- + +## Troubleshooting + +### Problem: "Parallel degree > 1 but no mod_column specified" + +**Cause:** Missing mod_column configuration for parallel processing. + +**Solution:** +```sql +-- mod_column must be a numeric column +UPDATE dc_table_map +SET mod_column = 'your_numeric_pk_column' +WHERE tid = your_tid; +``` + +### Problem: mod_column set but parallel processing not working + +**Cause:** mod_column is not a numeric data type. + +**Solution:** Choose a numeric column (INTEGER, BIGINT, etc.): +```sql +-- Check column data types +SELECT column_name, data_type +FROM dc_table_column_map +WHERE tid = your_tid AND column_origin = 'source'; + +-- Set to a numeric column +UPDATE dc_table_map SET mod_column = 'id' WHERE tid = your_tid; +``` + +### Problem: Out of Memory Error + +**Cause:** Insufficient heap for data volume and thread count. + +**Solution:** +1. Increase Java heap: `-Xmx8g` +2. Reduce `batch-fetch-size` +3. Reduce `parallel_degree` + +### Problem: Staging Tables Growing Too Large + +**Cause:** Observer thread can't keep up with data loading. + +**Solution:** +```properties +# Enable throttling +observer-throttle=true +observer-throttle-size=1000000 + +# Enable vacuum +observer-vacuum=true +``` + +### Problem: Comparison Running Slowly + +**Possible causes and solutions:** + +1. **Database query performance** + ```sql + -- Add indexes on source/target tables + CREATE INDEX ON source_table (pk_column); + ``` + +2. **Repository performance** + ```sql + -- Tune PostgreSQL for repository + ALTER SYSTEM SET work_mem = '256MB'; + ALTER SYSTEM SET max_parallel_workers = 8; + ``` + +3. **Network latency** + - Use databases closer to pgCompare server + - Consider running pgCompare on the same network segment + +4. **Parallelism** + ```properties + # Increase batch size + batch-fetch-size=10000 + ``` + ```sql + -- Increase parallel degree (requires numeric mod_column) + UPDATE dc_table SET parallel_degree = 8 WHERE table_alias = 'my_table'; + UPDATE dc_table_map SET mod_column = 'id' WHERE tid = ; + ``` + +### Problem: Lock Contention in Repository + +**Cause:** Multiple threads competing for staging table access. + +**Solution:** +```properties +# Use separate staging tables per thread +# (This is done automatically with parallel_degree) + +# Reduce commit frequency +batch-commit-size=5000 +``` + +### Monitoring Thread Activity + +Check thread status during comparison: + +```bash +# Monitor Java threads +jps -l | grep pgcompare +jstack | grep -E "(compare|observer)" +``` + +--- + +## Best Practices Summary + +1. **Start conservative**: Begin with `parallel_degree=2` and increase gradually +2. **Monitor resources**: Watch memory, CPU, and database connections +3. **Use throttling**: Always enable `observer-throttle=true` for large tables +4. **Choose good mod_column**: Use numeric primary keys for even distribution +5. **Tune batch sizes**: Match `batch-fetch-size` to network latency +6. **Size heap appropriately**: Allocate based on table size and parallelism +7. **Enable vacuum**: Use `observer-vacuum=true` for long-running comparisons +8. **Test incrementally**: Run on subset first with `--table` option diff --git a/docs/mapping-export-import.md b/docs/mapping-export-import.md new file mode 100644 index 0000000..551caf0 --- /dev/null +++ b/docs/mapping-export-import.md @@ -0,0 +1,254 @@ +# pgCompare Mapping Export/Import + +This document describes the YAML-based export and import functionality for table and column mappings in pgCompare. + +## Overview + +The mapping export/import feature allows you to: +- Export existing table and column mappings to a human-readable YAML file +- Edit mappings in your favorite text editor +- Import mappings to add new tables or update existing ones +- Filter exports/imports by table name patterns + +This is particularly useful for: +- Bulk editing of column mappings +- Backing up mapping configurations +- Copying mappings between projects or environments +- Version controlling your mapping definitions + +## Commands + +### Export Mappings + +```shell +java -jar pgcompare.jar export-mapping [options] +``` + +**Options:** +- `-p|--project ` - Project ID (default: 1) +- `-o|--file ` - Output file path (default: `pgcompare-mappings-.yaml`) +- `-t|--table ` - Filter tables by name pattern (supports `*` wildcard) + +**Examples:** + +```shell +# Export all mappings for project 1 +java -jar pgcompare.jar export-mapping + +# Export to a specific file +java -jar pgcompare.jar export-mapping --file my-mappings.yaml + +# Export only tables starting with "customer" +java -jar pgcompare.jar export-mapping --table "customer*" + +# Export tables matching a pattern for project 2 +java -jar pgcompare.jar export-mapping -p 2 -t "*_staging" +``` + +### Import Mappings + +```shell +java -jar pgcompare.jar import-mapping --file [options] +``` + +**Options:** +- `-p|--project ` - Project ID to import into (default: 1) +- `-o|--file ` - Input YAML file path (required) +- `--overwrite` - Replace existing mappings (without this flag, existing tables are skipped) +- `-t|--table ` - Filter which tables to import (supports `*` wildcard) + +**Examples:** + +```shell +# Import mappings, adding only new tables +java -jar pgcompare.jar import-mapping --file my-mappings.yaml + +# Import and overwrite existing mappings +java -jar pgcompare.jar import-mapping --file my-mappings.yaml --overwrite + +# Import only specific tables +java -jar pgcompare.jar import-mapping --file mappings.yaml --table "orders*" --overwrite +``` + +## YAML File Format + +The exported YAML file has a hierarchical structure that mirrors the database schema: + +```yaml +version: "1.0" +exportDate: "2025-02-10T12:00:00" +projectId: 1 +projectName: "default" +tables: + - alias: "customer" # Unique identifier linking source and target + enabled: true # Set to false to skip this table in comparisons + batchNumber: 1 # Batch grouping for parallel processing + parallelDegree: 1 # Degree of parallelism for this table + source: # Source database table location + schema: "sales" + table: "customers" + schemaPreserveCase: false # Preserve case for schema name + tablePreserveCase: false # Preserve case for table name + modColumn: null # (Reserved for future use) + tableFilter: null # (Reserved for future use) + target: # Target database table location + schema: "dwh" + table: "dim_customer" + schemaPreserveCase: false + tablePreserveCase: false + columns: + - alias: "customer_id" # Column alias linking source/target columns + enabled: true # Set to false to exclude from comparison + source: + columnName: "id" + dataType: "integer" + dataClass: "integer" + dataLength: null + numberPrecision: 10 + numberScale: 0 + nullable: false + primaryKey: true # Mark primary key columns + mapExpression: null # Custom SQL expression (see below) + supported: true + preserveCase: false + mapType: "column" + target: + columnName: "customer_id" + dataType: "bigint" + dataClass: "integer" + # ... similar fields +``` + +### Field Descriptions + +#### Table Level + +| Field | Description | +|-------|-------------| +| `alias` | Unique identifier that links source and target tables. Used internally by pgCompare. | +| `enabled` | When `false`, the table is excluded from comparison operations. | +| `batchNumber` | Groups tables for batch processing. Tables in the same batch are processed together. | +| `parallelDegree` | Number of parallel threads to use when processing this table. | + +#### Table Location (source/target) + +| Field | Description | +|-------|-------------| +| `schema` | Database schema name | +| `table` | Table name within the schema | +| `schemaPreserveCase` | If `true`, schema name is quoted to preserve case | +| `tablePreserveCase` | If `true`, table name is quoted to preserve case | + +#### Column Level + +| Field | Description | +|-------|-------------| +| `alias` | Unique identifier linking source and target columns | +| `enabled` | When `false`, column is excluded from hash comparison | + +#### Column Mapping (source/target) + +| Field | Description | +|-------|-------------| +| `columnName` | Actual column name in the database | +| `dataType` | Database-specific data type (e.g., "varchar", "integer") | +| `dataClass` | Normalized data class: "string", "integer", "numeric", "timestamp", etc. | +| `dataLength` | Character/byte length for string types | +| `numberPrecision` | Total digits for numeric types | +| `numberScale` | Decimal places for numeric types | +| `nullable` | Whether the column allows NULL values | +| `primaryKey` | `true` for columns that form the primary key | +| `mapExpression` | Custom SQL expression for value transformation | +| `supported` | `false` if data type is not supported for comparison | +| `preserveCase` | If `true`, column name is quoted to preserve case | + +## Using Map Expressions + +The `mapExpression` field allows you to apply SQL transformations when comparing column values. This is useful when: + +- Column names differ between source and target +- Data types need conversion +- Formatting differs between databases + +**Examples:** + +```yaml +# Convert timestamp to date for comparison +mapExpression: "DATE(created_at)" + +# Trim whitespace +mapExpression: "TRIM(customer_name)" + +# Handle NULL values +mapExpression: "COALESCE(status, 'unknown')" + +# Case-insensitive comparison +mapExpression: "LOWER(email)" +``` + +## Unused Fields (Reserved) + +The following fields exist in the repository schema but are not currently used. They are included in exports for completeness and future compatibility: + +| Table | Field | Status | +|-------|-------|--------| +| `dc_table_column` | `enabled` | Implemented but may not be fully utilized in all code paths | +| `dc_table_map` | `mod_column` | Reserved for modification tracking | +| `dc_table_map` | `table_filter` | Reserved for row filtering during comparison | +| `dc_table_column_map` | `map_type` | Defaults to "column"; other values reserved for future use | + +## Workflow Examples + +### 1. Initial Setup with Discovery, then Fine-tuning + +```shell +# First, discover tables automatically +java -jar pgcompare.jar discover + +# Export the discovered mappings +java -jar pgcompare.jar export-mapping --file mappings.yaml + +# Edit mappings.yaml in your editor to: +# - Disable tables you don't want to compare +# - Add mapExpressions for columns that need transformation +# - Mark additional columns as primary keys + +# Import your changes +java -jar pgcompare.jar import-mapping --file mappings.yaml --overwrite +``` + +### 2. Copying Mappings Between Projects + +```shell +# Export from project 1 +java -jar pgcompare.jar export-mapping -p 1 --file prod-mappings.yaml + +# Edit the file if needed, then import to project 2 +java -jar pgcompare.jar import-mapping -p 2 --file prod-mappings.yaml +``` + +### 3. Selective Updates + +```shell +# Export only order-related tables +java -jar pgcompare.jar export-mapping --table "order*" --file order-mappings.yaml + +# Make changes, then import only those tables +java -jar pgcompare.jar import-mapping --file order-mappings.yaml --table "order*" --overwrite +``` + +## Tips + +1. **Always back up before overwriting**: The `--overwrite` flag will delete existing table mappings before importing. + +2. **Use table filters**: For large projects, export/import subsets of tables to make files more manageable. + +3. **Version control your mappings**: YAML files work well with Git and other version control systems. + +4. **Validate before importing**: Review the YAML file structure before importing to avoid errors. + +5. **Test with a small subset**: When making significant changes, test with a few tables first before applying to all. + +## Note on Null Values + +The YAML export omits fields that have null values to keep the file clean and readable. When importing, any omitted fields will use their default values. This is valid YAML/JSON behavior - `null` is a valid value, but omitting null fields produces cleaner output. diff --git a/docs/performance-tuning-guide.md b/docs/performance-tuning-guide.md new file mode 100644 index 0000000..8b7c2d8 --- /dev/null +++ b/docs/performance-tuning-guide.md @@ -0,0 +1,721 @@ +# Performance Tuning Guide + +This guide covers parameter tuning and optimization strategies for pgCompare across different workloads and environments. + +## Table of Contents + +1. [Performance Fundamentals](#performance-fundamentals) +2. [Batch Size Tuning](#batch-size-tuning) +3. [Thread Configuration](#thread-configuration) +4. [Hash Method Selection](#hash-method-selection) +5. [Database Sorting Options](#database-sorting-options) +6. [Repository Database Tuning](#repository-database-tuning) +7. [Java Virtual Machine Tuning](#java-virtual-machine-tuning) +8. [Network Optimization](#network-optimization) +9. [Monitoring and Diagnostics](#monitoring-and-diagnostics) +10. [Tuning Profiles](#tuning-profiles) + +--- + +## Performance Fundamentals + +### Understanding the Comparison Pipeline + +``` +Source DB → [Fetch] → [Hash] → [Queue] → [Load] → Repository → [Reconcile] → Results +Target DB → [Fetch] → [Hash] → [Queue] → [Load] → Repository → [Reconcile] → Results +``` + +### Key Performance Factors + +| Factor | Impact | Configuration | +|--------|--------|---------------| +| Fetch Size | Database round-trips | `batch-fetch-size` | +| Commit Size | Repository transactions | `batch-commit-size` | +| Parallelism | CPU utilization | `parallel_degree`, `loader-threads` | +| Hash Method | Database vs Java load | `column-hash-method` | +| Sort Location | Memory vs I/O | `database-sort` | + +### Bottleneck Identification + +1. **Source/Target Database** - Query execution time +2. **Network** - Data transfer latency +3. **pgCompare Application** - Hash computation, queue processing +4. **Repository Database** - Insert/reconcile operations + +--- + +## Batch Size Tuning + +### batch-fetch-size + +Controls rows fetched per database round-trip from source/target. + +| Value | Use Case | +|-------|----------| +| 1000 | High-latency networks, limited memory | +| 2000 | Default, balanced performance | +| 5000 | Low-latency networks, adequate memory | +| 10000+ | Local/same-datacenter, high memory | + +**Configuration:** +```properties +batch-fetch-size=5000 +``` + +**Tuning Guidelines:** + +``` +Optimal batch-fetch-size ≈ (Available Memory) / (Avg Row Size × Concurrent Threads × 3) +``` + +**Example:** +- 4GB available memory +- 500 bytes average row size +- 8 concurrent threads +- Optimal: 4GB / (500 × 8 × 3) ≈ 330,000 → Use 10,000 (practical limit) + +### batch-commit-size + +Controls rows committed per transaction to repository. + +| Value | Use Case | +|-------|----------| +| 1000 | High transaction rate, small tables | +| 2000 | Default, balanced | +| 5000 | Large tables, lower transaction overhead | +| 10000 | Very large tables, minimal transactions | + +**Configuration:** +```properties +batch-commit-size=5000 +``` + +**Relationship with batch-fetch-size:** + +```properties +# Optimal: Match or align values +batch-fetch-size=5000 +batch-commit-size=5000 + +# Alternative: Commit less frequently +batch-fetch-size=5000 +batch-commit-size=10000 +``` + +### batch-progress-report-size + +Controls progress reporting frequency. + +```properties +# Report every 1 million rows (default) +batch-progress-report-size=1000000 + +# More frequent updates for monitoring +batch-progress-report-size=500000 + +# Less overhead for very large tables +batch-progress-report-size=5000000 +``` + +--- + +## Thread Configuration + +### parallel_degree (Per-Table) + +Number of concurrent comparison threads for a single table. + +**IMPORTANT:** To use `parallel_degree > 1`, you **must** also specify a `mod_column` value in `dc_table_map`. The `mod_column` must be a **numeric column** (integer, bigint, etc.) that pgCompare uses to partition the data across threads using modulo arithmetic. Be sure that the column is part of the primary key, allows for equal partitioning of the data, and has an index for best performance. + +#### Setting Up Parallel Degree + +**Step 1: Set parallel_degree on the table** +```sql +UPDATE dc_table SET parallel_degree = 4 WHERE table_alias = 'large_table'; +``` + +**Step 2: Set mod_column on both source and target mappings** +```sql +-- mod_column must be a numeric column (typically the primary key) +UPDATE dc_table_map +SET mod_column = 'id' +WHERE tid = (SELECT tid FROM dc_table WHERE table_alias = 'large_table'); +``` + +#### How mod_column Works + +pgCompare uses the `mod_column` to distribute rows across parallel threads: + +```sql +-- Thread 0 processes: WHERE MOD(id, 4) = 0 +-- Thread 1 processes: WHERE MOD(id, 4) = 1 +-- Thread 2 processes: WHERE MOD(id, 4) = 2 +-- Thread 3 processes: WHERE MOD(id, 4) = 3 +``` + +This ensures each thread processes a distinct subset of rows without overlap. + +#### mod_column Requirements + +| Requirement | Description | +|-------------|-------------| +| **Data Type** | Must be numeric (INTEGER, BIGINT, NUMERIC without decimals) | +| **NOT NULL** | Column should not contain NULL values | +| **Distribution** | Values should be evenly distributed for balanced workload | +| **Indexed** | Indexing the column improves query performance | + +#### Finding a Suitable mod_column + +```sql +-- Check available numeric columns +SELECT tc.column_alias, tcm.data_type, tcm.column_name +FROM dc_table t +JOIN dc_table_column tc ON t.tid = tc.tid +JOIN dc_table_column_map tcm ON tc.tid = tcm.tid AND tc.column_id = tcm.column_id +WHERE t.table_alias = 'large_table' + AND tcm.data_type IN ('integer', 'bigint', 'int', 'number', 'numeric') + AND tcm.column_origin = 'source'; +``` + +**Good candidates:** +- Primary key columns (id, order_id, customer_id) +- Surrogate keys +- Sequence-generated columns + +**Poor candidates:** +- Columns with many NULLs +- Columns with skewed distributions (e.g., status codes) +- Decimal/float columns + +#### Complete Parallel Setup Example + +```sql +-- 1. Get the table ID +SELECT tid FROM dc_table WHERE table_alias = 'orders'; +-- Returns: 42 + +-- 2. Set parallel degree +UPDATE dc_table SET parallel_degree = 8 WHERE tid = 42; + +-- 3. Set mod_column on BOTH source and target mappings +UPDATE dc_table_map SET mod_column = 'order_id' WHERE tid = 42; + +-- 4. Verify configuration +SELECT + t.table_alias, + t.parallel_degree, + tm.dest_type, + tm.mod_column +FROM dc_table t +JOIN dc_table_map tm ON t.tid = tm.tid +WHERE t.tid = 42; +``` + +#### Recommendations + +| Table Rows | parallel_degree | +|-----------|-----------------| +| < 100K | 1 | +| 100K - 1M | 2 | +| 1M - 10M | 4 | +| 10M - 100M | 8 | +| > 100M | 8-16 | + +### loader-threads (Advanced) + +> **Note:** Loader threads are an advanced feature that is not required for most workloads. The default value of `0` is recommended. See the [Advanced Tuning Guide](advanced-tuning-guide.md) for detailed information on when and how to use loader threads. + +```properties +# Default (recommended for most workloads) +loader-threads=0 +``` + +--- + +## Hash Method Selection + +### column-hash-method Options + +| Method | Description | Best For | +|--------|-------------|----------| +| `database` | Hash computed in source/target DB | Powerful databases, low network bandwidth | +| `hybrid` | Hash computed in pgCompare | Weak databases, high bandwidth | + +### Database Method + +```properties +column-hash-method=database +``` + +**Advantages:** +- Reduces data transferred (only hashes) +- Leverages database compute resources + +**SQL Generated:** +```sql +-- PostgreSQL +SELECT MD5(col1::text || col2::text) AS hash, pk FROM table; + +-- Oracle +SELECT DBMS_CRYPTO.HASH(col1 || col2, 2) AS hash, pk FROM table; +``` + +### Hybrid Method + +```properties +column-hash-method=hybrid +``` + +**Advantages:** +- Consistent hashing across platforms +- Reduces database load +- Better for cross-platform comparisons + +**When to Use Hybrid:** +- Source and target are different database types +- Database servers are resource-constrained +- Network bandwidth is not a bottleneck + +### Performance Comparison + +| Scenario | database | hybrid | +|----------|----------|--------| +| Same DB type | ✓ Faster | Slower | +| Cross-platform | May differ | ✓ Consistent | +| Weak DB server | High load | ✓ Lower load | +| Slow network | ✓ Less data | More data | + +--- + +## Database Sorting Options + +### database-sort + +Determines whether row sorting is performed on the source/target databases or in the repository. Only disable database-sort if the overhead is too much on the source/target database resources. It will be faster and easier on the compare repository to have the source and target databases perform the sorting so data is loaded in order from both sides. + +```properties +# Sort on source/target databases (default) +database-sort=true + +# Sort in repository +database-sort=false +``` + +### database-sort=true (Default) + +**Advantages:** +- Utilizes source/target database indexes +- Reduces repository memory pressure +- Better for large, indexed tables + +**SQL Impact:** +```sql +SELECT hash, pk FROM table ORDER BY pk; +``` + +### database-sort=false + +**Advantages:** +- Reduces source/target database load +- Better when source/target lack indexes +- Useful for remote/cloud databases with high query costs + +**Repository Impact:** +- Higher memory usage +- Sorting happens during reconciliation +- Requires adequate repository resources +- Slower compare speeds + +### Choosing Sort Location + +| Condition | Recommendation | +|-----------|----------------| +| PK indexed on source/target | `database-sort=true` | +| No PK index | `database-sort=false` | +| Source/target CPU constrained | `database-sort=false` | +| Repository memory limited | `database-sort=true` | +| Cloud DB (pay per query) | `database-sort=false` | + +--- + +## Repository Database Tuning + +### PostgreSQL Configuration + +**Memory Settings:** + +```sql +-- For dedicated repository server +ALTER SYSTEM SET shared_buffers = '2GB'; +ALTER SYSTEM SET effective_cache_size = '6GB'; +ALTER SYSTEM SET work_mem = '256MB'; +ALTER SYSTEM SET maintenance_work_mem = '512MB'; +``` + +**Parallelism:** + +```sql +ALTER SYSTEM SET max_parallel_workers = 16; +ALTER SYSTEM SET max_parallel_workers_per_gather = 4; +ALTER SYSTEM SET parallel_tuple_cost = 0.001; +ALTER SYSTEM SET parallel_setup_cost = 100; +``` + +**Write Performance:** + +```sql +ALTER SYSTEM SET wal_level = 'minimal'; +ALTER SYSTEM SET max_wal_senders = 0; +ALTER SYSTEM SET wal_buffers = '64MB'; +ALTER SYSTEM SET checkpoint_completion_target = 0.9; +ALTER SYSTEM SET checkpoint_timeout = 600; +``` + +**Apply Changes:** +```sql +SELECT pg_reload_conf(); +-- Some settings require restart +``` + +### Connection Settings + +```sql +-- Increase connections for parallel operations +ALTER SYSTEM SET max_connections = 200; + +-- Connection pooling recommended for production +``` + +### Staging Table Optimization + +pgCompare creates temporary staging tables. Optimize with: + +```properties +# Set parallel degree for staging table operations +stage-table-parallel=4 +``` + +### Repository Sizing + +| Comparison Size | Recommended Repository | +|-----------------|----------------------| +| < 10M rows | 2 vCPU, 4GB RAM | +| 10M - 100M rows | 4 vCPU, 8GB RAM | +| 100M - 1B rows | 8 vCPU, 16GB RAM | +| > 1B rows | 16+ vCPU, 32GB+ RAM | + +--- + +## Java Virtual Machine Tuning + +### Heap Size Configuration + +```bash +# Minimum and maximum heap +java -Xms2g -Xmx8g -jar pgcompare.jar compare + +# Metaspace (for class metadata) +java -Xms2g -Xmx8g -XX:MaxMetaspaceSize=256m -jar pgcompare.jar compare +``` + +### Heap Size Guidelines + +| Total Data Size | Recommended Heap | +|-----------------|------------------| +| < 10M rows | 512MB - 1GB | +| 10M - 50M rows | 1GB - 2GB | +| 50M - 200M rows | 2GB - 4GB | +| 200M - 500M rows | 4GB - 8GB | +| > 500M rows | 8GB - 16GB | + +### Garbage Collection Tuning + +**For throughput (large batches):** +```bash +java -Xms4g -Xmx8g \ + -XX:+UseG1GC \ + -XX:MaxGCPauseMillis=200 \ + -jar pgcompare.jar compare +``` + +**For low latency:** +```bash +java -Xms4g -Xmx8g \ + -XX:+UseZGC \ + -jar pgcompare.jar compare +``` + +### Thread Stack Size + +For many concurrent threads: +```bash +java -Xss512k -Xms4g -Xmx8g -jar pgcompare.jar compare +``` + +### Complete JVM Configuration Example + +```bash +java \ + -Xms4g \ + -Xmx8g \ + -XX:+UseG1GC \ + -XX:MaxGCPauseMillis=200 \ + -XX:+UseStringDeduplication \ + -XX:MaxMetaspaceSize=256m \ + -Djava.net.preferIPv4Stack=true \ + -jar pgcompare.jar compare --batch 0 +``` + +--- + +## Network Optimization + +### Reduce Network Round-Trips + +```properties +# Fetch more data per round-trip +batch-fetch-size=10000 + +# Use database-side hashing +column-hash-method=database +``` + +### Connection Configuration + +**JDBC connection pooling** - pgCompare manages connections internally, but ensure: + +```properties +# Use direct connections (not pgBouncer) +repo-host=postgres-direct.example.com +``` + +### Network Latency Impact + +| Latency | Recommended batch-fetch-size | +|---------|------------------------------| +| < 1ms (same datacenter) | 10000+ | +| 1-10ms (same region) | 5000 | +| 10-50ms (cross-region) | 2000 | +| > 50ms (intercontinental) | 1000 | + +### SSL Considerations + +```properties +# Disable SSL for internal networks +repo-sslmode=disable +source-sslmode=disable +target-sslmode=disable + +# Enable SSL for external networks +repo-sslmode=require +source-sslmode=require +target-sslmode=require +``` + +--- + +## Monitoring and Diagnostics + +### Enable Debug Logging + +```properties +log-level=DEBUG +log-destination=/var/log/pgcompare/debug.log +``` + +### Progress Monitoring + +Watch progress during comparison: + +```bash +# Monitor log output +tail -f /var/log/pgcompare/debug.log | grep -E "(Progress|Matched|Complete)" +``` + +### Repository Monitoring + +```sql +-- Active connections +SELECT count(*) FROM pg_stat_activity WHERE application_name LIKE 'pgCompare%'; + +-- Staging table sizes +SELECT + relname, + n_live_tup, + n_dead_tup, + pg_size_pretty(pg_total_relation_size(relid)) as size +FROM pg_stat_user_tables +WHERE relname LIKE 'dc_%_stg_%'; + +-- Lock monitoring +SELECT + locktype, + relation::regclass, + mode, + granted +FROM pg_locks +WHERE relation::regclass::text LIKE 'dc_%'; +``` + +### JVM Monitoring + +```bash +# Monitor heap usage +jstat -gc $(pgrep -f pgcompare) 5000 + +# Thread dump +jstack $(pgrep -f pgcompare) > thread-dump.txt + +# Heap dump (if OOM suspected) +jmap -dump:format=b,file=heap.hprof $(pgrep -f pgcompare) +``` + +### Performance Metrics + +Track these during comparison: + +| Metric | Healthy Range | Action if Outside | +|--------|---------------|-------------------| +| Rows/second | 10K-100K+ | Check network, increase parallelism | +| Memory usage | 50-80% of heap | Adjust heap size | +| CPU usage | 60-90% | Adjust thread count | +| Repository I/O | Sustained writes | Check disk, tune PostgreSQL | + +--- + +## Tuning Profiles + +### Profile: Small Tables (< 1M rows) + +```properties +# pgcompare-small.properties +batch-fetch-size=2000 +batch-commit-size=2000 +column-hash-method=database +database-sort=true +observer-throttle=false +``` + +```bash +java -Xms512m -Xmx1g -jar pgcompare.jar compare +``` + +### Profile: Medium Tables (1M - 50M rows) + +```properties +# pgcompare-medium.properties +batch-fetch-size=5000 +batch-commit-size=5000 +column-hash-method=database +database-sort=true +observer-throttle=true +observer-throttle-size=1000000 +``` + +```sql +UPDATE dc_table SET parallel_degree = 2 WHERE enabled = true; +UPDATE dc_table_map SET mod_column = 'id' WHERE mod_column IS NULL; +``` + +```bash +java -Xms1g -Xmx4g -jar pgcompare.jar compare +``` + +### Profile: Large Tables (50M - 500M rows) + +```properties +# pgcompare-large.properties +batch-fetch-size=10000 +batch-commit-size=10000 +column-hash-method=database +database-sort=true +observer-throttle=true +observer-throttle-size=2000000 +observer-vacuum=true +stage-table-parallel=4 +``` + +```sql +UPDATE dc_table SET parallel_degree = 4 WHERE enabled = true; +UPDATE dc_table_map SET mod_column = 'id' WHERE mod_column IS NULL; +``` + +```bash +java -Xms4g -Xmx8g -XX:+UseG1GC -jar pgcompare.jar compare +``` + +### Profile: Very Large Tables (> 500M rows) + +```properties +# pgcompare-xlarge.properties +batch-fetch-size=20000 +batch-commit-size=20000 +batch-progress-report-size=5000000 +column-hash-method=database +database-sort=true +observer-throttle=true +observer-throttle-size=5000000 +observer-vacuum=true +stage-table-parallel=8 +``` + +```sql +UPDATE dc_table SET parallel_degree = 8 WHERE enabled = true; +UPDATE dc_table_map SET mod_column = 'id' WHERE mod_column IS NULL; +``` + +```bash +java -Xms8g -Xmx16g -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -jar pgcompare.jar compare +``` + +### Profile: Cross-Platform (Oracle to PostgreSQL) + +```properties +# pgcompare-crossplatform.properties +batch-fetch-size=5000 +batch-commit-size=5000 +column-hash-method=hybrid # Consistent hashing across platforms +database-sort=true +number-cast=standard # Avoid notation differences +float-scale=3 +observer-throttle=true +``` + +### Profile: Cloud Database (High Latency) + +```properties +# pgcompare-cloud.properties +batch-fetch-size=10000 # Maximize data per round-trip +batch-commit-size=10000 +column-hash-method=database +database-sort=false # Reduce cloud DB query cost +observer-throttle=true +observer-throttle-size=1000000 +``` + +--- + +## Quick Reference + +### Essential Parameters + +| Parameter | Default | Tuning Direction | +|-----------|---------|------------------| +| `batch-fetch-size` | 2000 | ↑ for low latency, ↓ for low memory | +| `batch-commit-size` | 2000 | ↑ for throughput, ↓ for safety | +| `parallel_degree` | 1 | ↑ for large tables (per-table, requires mod_column) | +| `column-hash-method` | database | hybrid for cross-platform | +| `database-sort` | true | false if DB constrained | + +### Performance Checklist + +- [ ] Set appropriate `batch-fetch-size` for network latency +- [ ] Set `parallel_degree` per table for large tables +- [ ] Set `mod_column` to a numeric column when `parallel_degree > 1` (required) +- [ ] Configure JVM heap for data volume +- [ ] Tune repository PostgreSQL settings +- [ ] Enable `observer-throttle` for large tables +- [ ] Monitor progress and adjust as needed + +> **Note:** For advanced threading options (loader-threads, message-queue-size), see the [Advanced Tuning Guide](advanced-tuning-guide.md). diff --git a/docs/reference-guide.md b/docs/reference-guide.md new file mode 100644 index 0000000..3f00923 --- /dev/null +++ b/docs/reference-guide.md @@ -0,0 +1,419 @@ +# pgCompare Reference Guide + +## Database Schema + +pgCompare uses a PostgreSQL repository database to store project configurations, table mappings, comparison results, and server mode job management. + +### Entity Relationship Diagram + +``` +┌─────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ pgCompare Schema ERD │ +└─────────────────────────────────────────────────────────────────────────────────────────────────┘ + +┌──────────────────┐ ┌──────────────────┐ ┌──────────────────────────┐ +│ dc_project │ │ dc_server │ │ dc_job │ +├──────────────────┤ ├──────────────────┤ ├──────────────────────────┤ +│ *pid │◄─────────│ server_id │ │ *job_id │ +│ project_name │ │ │ server_name │ ┌───►│ pid │◄───┐ +│ project_config │ │ │ server_host │ │ │ job_type │ │ +└────────┬─────────┘ │ │ server_pid │ │ │ status │ │ + │ │ │ status │ │ │ priority │ │ + │ │ │ registered_at │ │ │ batch_nbr │ │ + │1 │ │ last_heartbeat │ │ │ table_filter │ │ + │ │ │ current_job_id │ │ │ target_server_id ───────┼────┤ + │ │ │ server_config │ │ │ assigned_server_id ─────┼────┤ + │ │ └──────────────────┘ │ │ created_at │ │ + │ │ │ │ scheduled_at │ │ + │ └─────────────────────────────┼────│ started_at │ │ + │ │ │ completed_at │ │ + ▼N │ │ created_by │ │ +┌──────────────────┐ │ │ job_config │ │ +│ dc_table │ │ │ result_summary │ │ +├──────────────────┤ │ │ error_message │ │ +│ *tid │◄──────────────────────────────────┘ └────────────┬─────────────┘ │ +│ pid │ │ │ +│ table_alias │ │1 │ +│ enabled │ │ │ +│ batch_nbr │ ▼N │ +│ parallel_degree │ ┌──────────────────────────┐ ┌──────────────────────────┐ │ +└────────┬─────────┘ │ dc_job_control │ │ dc_job_progress │ │ + │ ├──────────────────────────┤ ├──────────────────────────┤ │ + │1 │ *control_id │ │ *job_id ─────────────────┼─────┘ + │ │ job_id ─────────────────┼────►│ *tid │ + │ │ signal │ │ table_name │ + ├───────┬───────│ requested_at │ │ status │ + │ │ │ processed_at │ │ started_at │ + │ │ │ requested_by │ │ completed_at │ + ▼N │ └──────────────────────────┘ │ source_cnt │ +┌────────────────┐│ │ target_cnt │ +│ dc_table_map ││ │ equal_cnt │ +├────────────────┤│ │ not_equal_cnt │ +│ *tid ││ │ missing_source_cnt │ +│ *dest_type ││ │ missing_target_cnt │ +│ *schema_name ││ │ error_message │ +│ *table_name ││ └──────────────────────────┘ +│ mod_column ││ +│ table_filter ││ +│ schema_pres...││ +│ table_pres... ││ +└────────────────┘│ + │ + ▼N + ┌──────────────────────┐ + │ dc_table_column │ + ├──────────────────────┤ + │ *column_id │◄───────────┐ + │ tid │ │ + │ column_alias │ │1 + │ enabled │ │ + └──────────┬───────────┘ │ + │ │ + │1 │ + │ │ + ▼N │ + ┌──────────────────────┐ │ + │ dc_table_column_map │ │ + ├──────────────────────┤ │ + │ tid │ │ + │ *column_id ──────────┼────────────┘ + │ *column_origin │ + │ *column_name │ + │ data_type │ + │ data_class │ + │ data_length │ + │ number_precision │ ┌──────────────────────┐ + │ number_scale │ │ dc_table_history │ + │ column_nullable │ ├──────────────────────┤ + │ column_primarykey │ │ tid │ + │ map_expression │ │ batch_nbr │ + │ supported │ │ start_dt │ + │ preserve_case │ │ end_dt │ + │ map_type │ │ action_result │ + └──────────────────────┘ │ row_count │ + └──────────────────────┘ + + ┌──────────────────────┐ ┌──────────────────────┐ + │ dc_source │ │ dc_target │ + ├──────────────────────┤ ├──────────────────────┤ + │ tid │ │ tid │ + │ table_name │ │ table_name │ + │ batch_nbr │ │ batch_nbr │ + │ pk │ │ pk │ + │ pk_hash │ │ pk_hash │ + │ column_hash │ │ column_hash │ + │ compare_result │ │ compare_result │ + │ thread_nbr │ │ thread_nbr │ + └──────────────────────┘ └──────────────────────┘ + + ┌──────────────────────┐ + │ dc_result │ + ├──────────────────────┤ + │ *cid │ + │ rid │ + │ tid │ + │ table_name │ + │ status │ + │ compare_start │ + │ equal_cnt │ + │ missing_source_cnt │ + │ missing_target_cnt │ + │ not_equal_cnt │ + │ source_cnt │ + │ target_cnt │ + │ compare_end │ + └──────────────────────┘ + +Legend: + * = Primary Key column(s) + ─► = Foreign Key relationship + 1 = One side of relationship + N = Many side of relationship +``` + +--- + +## Table Reference + +### Core Tables + +#### dc_project +Stores project configurations for comparison jobs. + +| Column | Type | Nullable | Default | Description | +|--------|------|----------|---------|-------------| +| pid | bigint | NO | auto-generated | Project ID (Primary Key) | +| project_name | text | NO | 'default' | Name of the project | +| project_config | jsonb | YES | NULL | Project configuration in JSON format | + +#### dc_table +Defines tables to be compared within a project. + +| Column | Type | Nullable | Default | Description | +|--------|------|----------|---------|-------------| +| tid | bigint | NO | auto-generated | Table ID (Primary Key) | +| pid | bigint | NO | 1 | Project ID (Foreign Key to dc_project) | +| table_alias | text | YES | NULL | Alias name for the table | +| enabled | boolean | YES | true | Whether table comparison is enabled | +| batch_nbr | integer | YES | 1 | Batch number for grouping | +| parallel_degree | integer | YES | 1 | Degree of parallelism for comparison | + +#### dc_table_map +Maps source/target schema and table names for each table definition. + +| Column | Type | Nullable | Default | Description | +|--------|------|----------|---------|-------------| +| tid | bigint | NO | - | Table ID (Primary Key, FK to dc_table) | +| dest_type | varchar(20) | NO | 'target' | Destination type: 'source' or 'target' (Primary Key) | +| schema_name | text | NO | - | Schema name (Primary Key) | +| table_name | text | NO | - | Table name (Primary Key) | +| mod_column | varchar(200) | YES | NULL | Modification tracking column | +| table_filter | varchar(200) | YES | NULL | WHERE clause filter for the table | +| schema_preserve_case | boolean | YES | false | Preserve schema name case | +| table_preserve_case | boolean | YES | false | Preserve table name case | + +#### dc_table_column +Defines column aliases for table comparisons. + +| Column | Type | Nullable | Default | Description | +|--------|------|----------|---------|-------------| +| column_id | bigint | NO | auto-generated | Column ID (Primary Key) | +| tid | bigint | NO | - | Table ID (Foreign Key to dc_table) | +| column_alias | text | NO | - | Alias name for the column | +| enabled | boolean | YES | true | Whether column is included in comparison | + +#### dc_table_column_map +Maps column details between source and target systems. + +| Column | Type | Nullable | Default | Description | +|--------|------|----------|---------|-------------| +| column_id | bigint | NO | - | Column ID (Primary Key, FK to dc_table_column) | +| column_origin | varchar(10) | NO | 'source' | Origin: 'source' or 'target' (Primary Key) | +| column_name | text | NO | - | Column name (Primary Key) | +| tid | bigint | NO | - | Table ID | +| data_type | text | NO | - | Data type | +| data_class | varchar(20) | YES | 'string' | Data classification | +| data_length | integer | YES | NULL | Data length | +| number_precision | integer | YES | NULL | Numeric precision | +| number_scale | integer | YES | NULL | Numeric scale | +| column_nullable | boolean | YES | true | Whether column allows NULL | +| column_primarykey | boolean | YES | false | Whether column is part of primary key | +| map_expression | text | YES | NULL | Custom mapping expression | +| supported | boolean | YES | true | Whether data type is supported | +| preserve_case | boolean | YES | false | Preserve column name case | +| map_type | varchar(15) | NO | 'column' | Mapping type | + +--- + +### Comparison Tables + +#### dc_source +Temporary storage for source database row hashes during comparison. + +| Column | Type | Nullable | Default | Description | +|--------|------|----------|---------|-------------| +| tid | bigint | YES | NULL | Table ID | +| table_name | text | YES | NULL | Table name | +| batch_nbr | integer | YES | NULL | Batch number | +| pk | jsonb | YES | NULL | Primary key values as JSON | +| pk_hash | varchar(100) | YES | NULL | Hash of primary key | +| column_hash | varchar(100) | YES | NULL | Hash of row columns | +| compare_result | char(1) | YES | NULL | Comparison result code | +| thread_nbr | integer | YES | NULL | Thread number for parallel processing | + +#### dc_target +Temporary storage for target database row hashes during comparison. + +| Column | Type | Nullable | Default | Description | +|--------|------|----------|---------|-------------| +| tid | bigint | YES | NULL | Table ID | +| table_name | text | YES | NULL | Table name | +| batch_nbr | integer | YES | NULL | Batch number | +| pk | jsonb | YES | NULL | Primary key values as JSON | +| pk_hash | varchar(100) | YES | NULL | Hash of primary key | +| column_hash | varchar(100) | YES | NULL | Hash of row columns | +| compare_result | char(1) | YES | NULL | Comparison result code | +| thread_nbr | integer | YES | NULL | Thread number for parallel processing | + +#### dc_result +Stores comparison results summary for each table. + +| Column | Type | Nullable | Default | Description | +|--------|------|----------|---------|-------------| +| cid | serial | NO | auto-generated | Comparison ID (Primary Key) | +| rid | numeric | YES | NULL | Run ID | +| tid | bigint | YES | NULL | Table ID | +| table_name | text | YES | NULL | Table name | +| status | varchar | YES | NULL | Comparison status | +| compare_start | timestamptz | YES | NULL | Comparison start time | +| compare_end | timestamptz | YES | NULL | Comparison end time | +| equal_cnt | integer | YES | NULL | Count of equal rows | +| missing_source_cnt | integer | YES | NULL | Count of rows missing in source | +| missing_target_cnt | integer | YES | NULL | Count of rows missing in target | +| not_equal_cnt | integer | YES | NULL | Count of rows that differ | +| source_cnt | integer | YES | NULL | Total source row count | +| target_cnt | integer | YES | NULL | Total target row count | + +#### dc_table_history +Historical record of table comparison operations. + +| Column | Type | Nullable | Default | Description | +|--------|------|----------|---------|-------------| +| tid | bigint | NO | - | Table ID | +| batch_nbr | integer | NO | - | Batch number | +| start_dt | timestamptz | NO | - | Operation start time | +| end_dt | timestamptz | YES | NULL | Operation end time | +| action_result | jsonb | YES | NULL | Result details as JSON | +| row_count | bigint | YES | NULL | Number of rows processed | + +--- + +### Server Mode Tables + +#### dc_server +Registers pgCompare server instances running in daemon mode. + +| Column | Type | Nullable | Default | Description | +|--------|------|----------|---------|-------------| +| server_id | uuid | NO | gen_random_uuid() | Server ID (Primary Key) | +| server_name | text | NO | - | Human-readable server name | +| server_host | text | NO | - | Hostname where server is running | +| server_pid | bigint | NO | - | Process ID of the server | +| status | varchar(20) | NO | 'active' | Server status | +| registered_at | timestamptz | NO | current_timestamp | When server registered | +| last_heartbeat | timestamptz | NO | current_timestamp | Last heartbeat timestamp | +| current_job_id | uuid | YES | NULL | Currently executing job ID | +| server_config | jsonb | YES | NULL | Server configuration | + +**Status Values:** +- `active` - Server is active and ready +- `idle` - Server is idle, waiting for work +- `busy` - Server is processing a job +- `offline` - Server has not sent heartbeat recently +- `terminated` - Server has been shut down + +#### dc_job +Queue of comparison jobs to be executed by servers. + +| Column | Type | Nullable | Default | Description | +|--------|------|----------|---------|-------------| +| job_id | uuid | NO | gen_random_uuid() | Job ID (Primary Key) | +| pid | bigint | NO | - | Project ID (Foreign Key to dc_project) | +| job_type | varchar(20) | NO | 'compare' | Type of job | +| status | varchar(20) | NO | 'pending' | Job status | +| priority | integer | NO | 5 | Priority (1-10, higher = more urgent) | +| batch_nbr | integer | NO | 0 | Batch number to process | +| table_filter | text | YES | NULL | Filter to limit tables | +| target_server_id | uuid | YES | NULL | Specific server to run on (NULL = any) | +| assigned_server_id | uuid | YES | NULL | Server that claimed the job | +| created_at | timestamptz | NO | current_timestamp | When job was created | +| scheduled_at | timestamptz | YES | NULL | When job should start | +| started_at | timestamptz | YES | NULL | When job actually started | +| completed_at | timestamptz | YES | NULL | When job completed | +| created_by | text | YES | NULL | Who created the job | +| job_config | jsonb | YES | NULL | Additional job configuration | +| result_summary | jsonb | YES | NULL | Summary of results | +| error_message | text | YES | NULL | Error message if failed | + +**Job Types:** +- `compare` - Full comparison of tables +- `check` - Quick row count check +- `discover` - Discover tables and columns + +**Status Values:** +- `pending` - Waiting to be claimed +- `scheduled` - Scheduled for future execution +- `running` - Currently executing +- `paused` - Temporarily paused +- `completed` - Successfully completed +- `failed` - Failed with error +- `cancelled` - Cancelled by user + +#### dc_job_control +Control signals for managing running jobs. + +| Column | Type | Nullable | Default | Description | +|--------|------|----------|---------|-------------| +| control_id | serial | NO | auto-generated | Control ID (Primary Key) | +| job_id | uuid | NO | - | Job ID (Foreign Key to dc_job) | +| signal | varchar(20) | NO | - | Control signal | +| requested_at | timestamptz | NO | current_timestamp | When signal was sent | +| processed_at | timestamptz | YES | NULL | When signal was processed | +| requested_by | text | YES | NULL | Who sent the signal | + +**Signal Values:** +- `pause` - Pause the job after current table +- `resume` - Resume a paused job +- `stop` - Stop gracefully after current table +- `terminate` - Stop immediately + +#### dc_job_progress +Tracks progress of running jobs at the table level. + +| Column | Type | Nullable | Default | Description | +|--------|------|----------|---------|-------------| +| job_id | uuid | NO | - | Job ID (Primary Key, FK to dc_job) | +| tid | bigint | NO | - | Table ID (Primary Key) | +| table_name | text | NO | - | Table name | +| status | varchar(20) | NO | 'pending' | Table comparison status | +| started_at | timestamptz | YES | NULL | When table comparison started | +| completed_at | timestamptz | YES | NULL | When table comparison completed | +| source_cnt | bigint | YES | 0 | Source row count | +| target_cnt | bigint | YES | 0 | Target row count | +| equal_cnt | bigint | YES | 0 | Count of equal rows | +| not_equal_cnt | bigint | YES | 0 | Count of differing rows | +| missing_source_cnt | bigint | YES | 0 | Rows missing in source | +| missing_target_cnt | bigint | YES | 0 | Rows missing in target | +| error_message | text | YES | NULL | Error message if failed | + +**Status Values:** +- `pending` - Waiting to be processed +- `running` - Currently being compared +- `completed` - Successfully completed +- `failed` - Failed with error +- `skipped` - Skipped (e.g., disabled) + +--- + +## Indexes + +| Index Name | Table | Columns | Purpose | +|------------|-------|---------|---------| +| dc_result_idx1 | dc_result | table_name, compare_start | Query results by table and time | +| dc_table_history_idx1 | dc_table_history | tid, start_dt | Query history by table and time | +| dc_table_idx1 | dc_table | table_alias | Lookup tables by alias | +| dc_table_column_idx1 | dc_table_column | column_alias, tid, column_id | Lookup columns by alias | +| dc_server_idx1 | dc_server | status, last_heartbeat | Find active servers | +| dc_job_idx1 | dc_job | status, priority DESC, created_at | Claim next job by priority | +| dc_job_idx2 | dc_job | pid, status | Query jobs by project | + +--- + +## Foreign Keys + +| Constraint | Table | Column(s) | References | On Delete | +|------------|-------|-----------|------------|-----------| +| dc_table_column_fk | dc_table_column | tid | dc_table(tid) | CASCADE | +| dc_table_column_map_fk | dc_table_column_map | column_id | dc_table_column(column_id) | CASCADE | +| dc_table_map_fk | dc_table_map | tid | dc_table(tid) | CASCADE | +| dc_job_fk1 | dc_job | pid | dc_project(pid) | CASCADE | +| dc_job_control_fk1 | dc_job_control | job_id | dc_job(job_id) | CASCADE | +| dc_job_progress_fk1 | dc_job_progress | job_id | dc_job(job_id) | CASCADE | + +--- + +## Functions + +### dc_copy_table(p_pid integer, p_tid integer) +Duplicates a table configuration including all mappings and column definitions. + +**Parameters:** +- `p_pid` - Project ID +- `p_tid` - Table ID to copy + +**Returns:** The new table ID (bigint) + +**Usage:** +```sql +SELECT pgcompare.dc_copy_table(1, 5); +``` diff --git a/docs/table-filtering-guide.md b/docs/table-filtering-guide.md new file mode 100644 index 0000000..593fe59 --- /dev/null +++ b/docs/table-filtering-guide.md @@ -0,0 +1,1016 @@ +# Table Filtering Guide + +This guide covers techniques for filtering which tables are processed during discovery, comparison, and mapping operations. + +## Table of Contents + +1. [Overview](#overview) +2. [Command Line Filtering](#command-line-filtering) +3. [Batch Number Filtering](#batch-number-filtering) +4. [Enabling/Disabling Tables](#enablingdisabling-tables) +5. [Schema-Level Filtering](#schema-level-filtering) +6. [Row-Level Filtering (table_filter)](#row-level-filtering-table_filter) +7. [Column Filtering](#column-filtering) +8. [Mapping Import/Export Filters](#mapping-importexport-filters) +9. [Advanced Filtering Strategies](#advanced-filtering-strategies) + +--- + +## Overview + +pgCompare provides multiple levels of filtering: + +| Level | Method | Use Case | +|-------|--------|----------| +| Table | `--table` option | Process specific table (exact match) | +| Batch | `--batch` option | Group tables for processing | +| Enable/Disable | `enabled` column | Temporarily skip tables | +| Schema | Properties file | Limit schema scope | +| Row | `table_filter` column | Compare subset of rows (delta compares) | +| Column | `enabled` column | Exclude columns from comparison | + +--- + +## Command Line Filtering + +### Filter by Table Name + +The `--table` option performs **exact match** filtering for compare, check, and discover operations: + +```bash +# Compare a single table (exact match) +java -jar pgcompare.jar compare --table orders + +# Discover a single table +java -jar pgcompare.jar discover --table customers + +# Recheck a single table +java -jar pgcompare.jar check --table orders +``` + +### Case Sensitivity + +Table filtering is case-insensitive by default: + +```bash +# These are equivalent +java -jar pgcompare.jar compare --table "ORDERS" +java -jar pgcompare.jar compare --table "orders" +java -jar pgcompare.jar compare --table "Orders" +``` + +### Wildcards (Export/Import Only) + +**Important:** Wildcard patterns using `*` are **only supported** for `export-mapping` and `import-mapping` operations: + +```bash +# Export tables starting with "customer" (wildcards supported) +java -jar pgcompare.jar export-mapping --file customer-tables.yaml --table "customer*" + +# Import tables ending with "_archive" (wildcards supported) +java -jar pgcompare.jar import-mapping --file mappings.yaml --table "*_archive" +``` + +**Wildcards are NOT supported for:** +- `compare` - Use batch numbers or run multiple commands +- `check` - Use batch numbers or run multiple commands +- `discover` - Use batch numbers or run multiple commands + +### Comparing Multiple Tables + +To compare multiple related tables, use batch numbers or run separate commands: + +```bash +# Option 1: Run multiple commands +java -jar pgcompare.jar compare --table orders +java -jar pgcompare.jar compare --table order_items +java -jar pgcompare.jar compare --table order_history + +# Option 2: Assign tables to a batch and run by batch +java -jar pgcompare.jar compare --batch 2 +``` + +--- + +## Batch Number Filtering + +### Understanding Batches + +Batch numbers group tables for organized processing: + +| Batch Number | Behavior | +|-------------|----------| +| 0 | Process all tables regardless of batch assignment | +| 1-N | Process only tables assigned to that batch | + +### Assigning Batch Numbers + +**During Discovery:** + +Tables are assigned to batch 1 by default. Modify assignments in the repository: + +```sql +-- Assign high-priority tables to batch 1 +UPDATE dc_table +SET batch_nbr = 1 +WHERE table_alias IN ('orders', 'customers', 'products'); + +-- Assign archive tables to batch 2 +UPDATE dc_table +SET batch_nbr = 2 +WHERE table_alias LIKE '%_archive'; + +-- Assign remaining tables to batch 3 +UPDATE dc_table +SET batch_nbr = 3 +WHERE batch_nbr = 1 +AND table_alias NOT IN ('orders', 'customers', 'products'); +``` + +**Via YAML Import:** + +```yaml +tables: + - alias: "orders" + batchNumber: 1 + enabled: true + - alias: "orders_archive" + batchNumber: 2 + enabled: true +``` + +### Running by Batch + +```bash +# Process batch 1 only +java -jar pgcompare.jar compare --batch 1 + +# Process batch 2 only +java -jar pgcompare.jar compare --batch 2 + +# Process all batches +java -jar pgcompare.jar compare --batch 0 +``` + +### Using Environment Variable + +```bash +export PGCOMPARE_BATCH=1 +java -jar pgcompare.jar compare + +# Override environment variable with command line +java -jar pgcompare.jar compare --batch 2 +``` + +### Batch Strategy Examples + +**By Table Size:** +```sql +-- Large tables: batch 1 (run separately with more resources) +UPDATE dc_table SET batch_nbr = 1 WHERE table_alias IN ('event_log', 'audit_trail'); + +-- Medium tables: batch 2 +UPDATE dc_table SET batch_nbr = 2 WHERE table_alias IN ('orders', 'order_items'); + +-- Small reference tables: batch 3 +UPDATE dc_table SET batch_nbr = 3 WHERE table_alias IN ('countries', 'currencies'); +``` + +**By Priority:** +```sql +-- Critical business tables: batch 1 +UPDATE dc_table SET batch_nbr = 1 WHERE table_alias IN ('accounts', 'transactions'); + +-- Secondary tables: batch 2 +UPDATE dc_table SET batch_nbr = 2 WHERE batch_nbr != 1; +``` + +--- + +## Enabling/Disabling Tables + +### Disable Tables Temporarily + +Skip tables without removing their configuration: + +```sql +-- Disable a specific table +UPDATE dc_table +SET enabled = false +WHERE table_alias = 'temp_staging'; + +-- Disable multiple tables +UPDATE dc_table +SET enabled = false +WHERE table_alias LIKE '%_temp'; +``` + +### Re-Enable Tables + +```sql +UPDATE dc_table +SET enabled = true +WHERE table_alias = 'temp_staging'; +``` + +### Via YAML + +```yaml +tables: + - alias: "active_orders" + enabled: true + - alias: "archived_orders" + enabled: false # Will be skipped during comparison +``` + +### Check Disabled Tables + +```sql +SELECT table_alias, enabled, batch_nbr +FROM dc_table +WHERE enabled = false +ORDER BY table_alias; +``` + +--- + +## Schema-Level Filtering + +### Configure in Properties File + +Limit discovery to specific schemas: + +```properties +# Source schema +source-schema=SALES + +# Target schema +target-schema=sales +``` + +### Multi-Schema Scenarios + +For comparing across different schemas: + +**Same schema names:** +```properties +source-schema=PRODUCTION +target-schema=production +``` + +**Different schema names:** +```properties +source-schema=ORACLE_SALES +target-schema=pg_sales +``` + +### Discover from Multiple Schemas + +Run discovery multiple times with different configurations: + +```bash +# Discover SALES schema +export PGCOMPARE_SOURCE_SCHEMA=SALES +export PGCOMPARE_TARGET_SCHEMA=sales +java -jar pgcompare.jar discover + +# Discover HR schema (different project) +export PGCOMPARE_SOURCE_SCHEMA=HR +export PGCOMPARE_TARGET_SCHEMA=hr +java -jar pgcompare.jar discover --project 2 +``` + +--- + +## Row-Level Filtering (table_filter) + +The `table_filter` column in `dc_table_map` allows you to specify a SQL WHERE clause condition to limit which rows are compared. This is one of the most powerful features for optimizing comparisons and enabling delta/incremental comparisons. + +### Why Use Row-Level Filtering? + +| Use Case | Benefit | +|----------|---------| +| **Delta Comparisons** | Compare only recently modified rows instead of entire table | +| **Incremental Validation** | Validate data in time-based slices | +| **Subset Testing** | Test comparison on a sample before full run | +| **Performance** | Dramatically reduce comparison time for large tables | +| **Active Record Focus** | Skip soft-deleted or archived records | + +### Basic Syntax + +The `table_filter` value should be a SQL condition (without the WHERE keyword - it's added automatically with AND): + +```sql +-- Set filter on a table mapping +UPDATE dc_table_map +SET table_filter = 'status = ''ACTIVE''' +WHERE tid = (SELECT tid FROM dc_table WHERE table_alias = 'customers') + AND dest_type = 'source'; +``` + +### Setting Filters on Both Source and Target + +**Important:** You typically need to set the filter on BOTH source and target mappings: + +```sql +-- Get the tid for the table +SELECT tid FROM dc_table WHERE table_alias = 'orders'; +-- Returns: 123 + +-- Set filter on SOURCE mapping +UPDATE dc_table_map +SET table_filter = 'order_date >= ''2024-01-01''' +WHERE tid = 123 AND dest_type = 'source'; + +-- Set filter on TARGET mapping +UPDATE dc_table_map +SET table_filter = 'order_date >= ''2024-01-01''' +WHERE tid = 123 AND dest_type = 'target'; +``` + +### Delta Comparison Examples + +Delta comparisons compare only rows that have changed since a specific point in time. This is invaluable for ongoing replication validation. + +**Example 1: Compare Last 7 Days of Changes** + +```sql +-- For Oracle source +UPDATE dc_table_map +SET table_filter = 'modified_date > SYSDATE - 7' +WHERE tid = 123 AND dest_type = 'source'; + +-- For PostgreSQL target +UPDATE dc_table_map +SET table_filter = 'modified_date > CURRENT_DATE - INTERVAL ''7 days''' +WHERE tid = 123 AND dest_type = 'target'; +``` + +**Example 2: Compare Since Last Successful Run** + +```sql +-- Store last run timestamp +-- After successful compare, record: 2024-01-15 10:30:00 + +-- Next run - Oracle source +UPDATE dc_table_map +SET table_filter = 'modified_date > TO_TIMESTAMP(''2024-01-15 10:30:00'', ''YYYY-MM-DD HH24:MI:SS'')' +WHERE tid = 123 AND dest_type = 'source'; + +-- PostgreSQL target +UPDATE dc_table_map +SET table_filter = 'modified_date > ''2024-01-15 10:30:00''::timestamp' +WHERE tid = 123 AND dest_type = 'target'; +``` + +**Example 3: Daily Incremental Comparison** + +```sql +-- Compare only today's data (useful for daily validation jobs) + +-- Oracle source +UPDATE dc_table_map +SET table_filter = 'TRUNC(created_date) = TRUNC(SYSDATE)' +WHERE tid = 123 AND dest_type = 'source'; + +-- PostgreSQL target +UPDATE dc_table_map +SET table_filter = 'created_date::date = CURRENT_DATE' +WHERE tid = 123 AND dest_type = 'target'; +``` + +### Filtering by Record Status + +**Compare Only Active Records:** + +```sql +UPDATE dc_table_map +SET table_filter = 'status = ''ACTIVE'' AND deleted_flag = ''N''' +WHERE tid = 123; -- Updates both source and target if same filter applies +``` + +**Compare Specific Regions:** + +```sql +-- Source (Oracle) +UPDATE dc_table_map +SET table_filter = 'region_code IN (''US'', ''CA'', ''MX'')' +WHERE tid = 123 AND dest_type = 'source'; + +-- Target (PostgreSQL) +UPDATE dc_table_map +SET table_filter = 'region_code IN (''US'', ''CA'', ''MX'')' +WHERE tid = 123 AND dest_type = 'target'; +``` + +### Filtering by Primary Key Range + +Useful for comparing large tables in chunks: + +```sql +-- Compare first million records +UPDATE dc_table_map +SET table_filter = 'id BETWEEN 1 AND 1000000' +WHERE tid = 123; + +-- Run comparison +java -jar pgcompare.jar compare --table orders + +-- Compare next million +UPDATE dc_table_map +SET table_filter = 'id BETWEEN 1000001 AND 2000000' +WHERE tid = 123; + +-- Run comparison again +java -jar pgcompare.jar compare --table orders +``` + +### Database-Specific Syntax + +Different databases require different SQL syntax. Set appropriate filters for each side: + +| Database | Date Example | +|----------|-------------| +| Oracle | `modified_date > SYSDATE - 7` | +| PostgreSQL | `modified_date > CURRENT_DATE - INTERVAL '7 days'` | +| MySQL/MariaDB | `modified_date > DATE_SUB(NOW(), INTERVAL 7 DAY)` | +| SQL Server | `modified_date > DATEADD(day, -7, GETDATE())` | +| DB2 | `modified_date > CURRENT DATE - 7 DAYS` | +| Snowflake | `modified_date > DATEADD(day, -7, CURRENT_DATE())` | + +### Automating Delta Filters + +**Script to Update Filters Daily:** + +```bash +#!/bin/bash +# update_delta_filters.sh + +# Calculate yesterday's date +YESTERDAY=$(date -d "yesterday" +%Y-%m-%d) + +# Update source filters (Oracle) +psql -d pgcompare -c " +UPDATE dc_table_map +SET table_filter = 'modified_date >= TO_DATE(''$YESTERDAY'', ''YYYY-MM-DD'')' +WHERE dest_type = 'source' + AND tid IN (SELECT tid FROM dc_table WHERE batch_nbr = 1); +" + +# Update target filters (PostgreSQL) +psql -d pgcompare -c " +UPDATE dc_table_map +SET table_filter = 'modified_date >= ''$YESTERDAY''::date' +WHERE dest_type = 'target' + AND tid IN (SELECT tid FROM dc_table WHERE batch_nbr = 1); +" + +# Run comparison +java -jar pgcompare.jar compare --batch 1 +``` + +**Using Repository View for Dynamic Filters:** + +```sql +-- Create a function to generate filter for each table +CREATE OR REPLACE FUNCTION get_delta_filter(p_dest_type text, p_days int) +RETURNS text AS $$ +BEGIN + IF p_dest_type = 'source' THEN + -- Oracle syntax + RETURN 'modified_date > SYSDATE - ' || p_days; + ELSE + -- PostgreSQL syntax + RETURN 'modified_date > CURRENT_DATE - INTERVAL ''' || p_days || ' days'''; + END IF; +END; +$$ LANGUAGE plpgsql; + +-- Apply delta filters to all tables in batch 1 +UPDATE dc_table_map tm +SET table_filter = get_delta_filter(tm.dest_type, 7) +WHERE tid IN (SELECT tid FROM dc_table WHERE batch_nbr = 1); +``` + +### Clearing Filters + +To remove filters and compare full tables: + +```sql +-- Clear filter for specific table +UPDATE dc_table_map +SET table_filter = NULL +WHERE tid = 123; + +-- Clear all filters for a project +UPDATE dc_table_map tm +SET table_filter = NULL +FROM dc_table t +WHERE tm.tid = t.tid AND t.pid = 1; +``` + +### Performance Considerations + +1. **Index Support**: Ensure filter columns are indexed on source/target databases +2. **Selectivity**: Highly selective filters dramatically improve performance +3. **Statistics**: Keep database statistics up-to-date for optimal query plans + +```sql +-- Example: Ensure modified_date is indexed +-- On Oracle source +CREATE INDEX idx_orders_modified ON orders(modified_date); + +-- On PostgreSQL target +CREATE INDEX idx_orders_modified ON orders(modified_date); +``` + +### Viewing Current Filters + +```sql +-- View all table filters +SELECT + t.table_alias, + tm.dest_type, + tm.table_filter +FROM dc_table t +JOIN dc_table_map tm ON t.tid = tm.tid +WHERE tm.table_filter IS NOT NULL +ORDER BY t.table_alias, tm.dest_type; +``` + +--- + +## Column Filtering + +### Disable Specific Columns + +Exclude columns from comparison: + +```sql +-- Disable audit columns +UPDATE dc_table_column +SET enabled = false +WHERE column_alias IN ('created_by', 'modified_by', 'created_date', 'modified_date'); + +-- Disable for specific table +UPDATE dc_table_column +SET enabled = false +WHERE tid = (SELECT tid FROM dc_table WHERE table_alias = 'orders') +AND column_alias = 'internal_notes'; +``` + +### Via YAML + +```yaml +columns: + - alias: "customer_id" + enabled: true + - alias: "created_timestamp" + enabled: false # Excluded from comparison + - alias: "modified_by" + enabled: false # Excluded from comparison +``` + +### Column Support Status + +The `supported` flag indicates if pgCompare can compare the column data type: + +```sql +-- Check unsupported columns +SELECT t.table_alias, tc.column_alias, tcm.data_type, tcm.supported +FROM dc_table t +JOIN dc_table_column tc ON t.tid = tc.tid +JOIN dc_table_column_map tcm ON tc.tid = tcm.tid AND tc.column_id = tcm.column_id +WHERE tcm.supported = false; +``` + +### Custom Column Expressions (map_expression) + +The `map_expression` column in `dc_table_column_map` allows you to define custom SQL expressions that transform column values before comparison. This is essential when: + +- Source and target databases store data in different formats +- You need to normalize data for accurate comparison +- Cross-platform data type differences cause false mismatches +- Columns contain data that needs transformation before hashing + +#### How map_expression Works + +When pgCompare builds the comparison SQL, it uses `map_expression` instead of the raw column name. The expression is executed on the respective database (source or target) and the result is hashed for comparison. + +```sql +-- Without map_expression: +SELECT MD5(order_date::text) FROM orders; + +-- With map_expression = 'TO_CHAR(order_date, ''YYYYMMDD'')': +SELECT MD5(TO_CHAR(order_date, 'YYYYMMDD')) FROM orders; +``` + +#### Setting map_expression + +```sql +-- Basic syntax +UPDATE dc_table_column_map +SET map_expression = '' +WHERE tid = + AND column_id = + AND column_origin = ''; +``` + +#### Common Use Cases + +**1. Date/Time Formatting** + +Normalize date formats across platforms (pgCompare compares dates to the second): + +```sql +-- Oracle source: Convert to standard format +UPDATE dc_table_column_map +SET map_expression = 'TO_CHAR(created_date, ''YYYYMMDDHH24MISS'')' +WHERE column_name = 'CREATED_DATE' AND column_origin = 'source'; + +-- PostgreSQL target: Match the format +UPDATE dc_table_column_map +SET map_expression = 'TO_CHAR(created_date, ''YYYYMMDDHH24MISS'')' +WHERE column_name = 'created_date' AND column_origin = 'target'; +``` + +**2. Handling NULL Values** + +Replace NULLs with a consistent value: + +```sql +-- Replace NULL with empty string for comparison +UPDATE dc_table_column_map +SET map_expression = 'COALESCE(description, '''')' +WHERE column_name = 'DESCRIPTION'; + +-- Replace NULL with specific value +UPDATE dc_table_column_map +SET map_expression = 'COALESCE(status, ''UNKNOWN'')' +WHERE column_name = 'STATUS'; +``` + +**3. String Normalization** + +Handle whitespace and case differences: + +```sql +-- Trim leading/trailing whitespace +UPDATE dc_table_column_map +SET map_expression = 'TRIM(customer_name)' +WHERE column_name = 'CUSTOMER_NAME'; + +-- Normalize to uppercase for case-insensitive comparison +UPDATE dc_table_column_map +SET map_expression = 'UPPER(email_address)' +WHERE column_name = 'EMAIL_ADDRESS'; + +-- Trim and uppercase combined +UPDATE dc_table_column_map +SET map_expression = 'UPPER(TRIM(company_name))' +WHERE column_name = 'COMPANY_NAME'; + +-- Remove all whitespace (useful for phone numbers, etc.) +-- PostgreSQL +UPDATE dc_table_column_map +SET map_expression = 'REGEXP_REPLACE(phone_number, ''\s+'', '''', ''g'')' +WHERE column_name = 'phone_number' AND column_origin = 'target'; + +-- Oracle +UPDATE dc_table_column_map +SET map_expression = 'REGEXP_REPLACE(phone_number, ''\s+'', '''')' +WHERE column_name = 'PHONE_NUMBER' AND column_origin = 'source'; +``` + +**4. Numeric Precision Handling** + +Address floating-point precision differences: + +```sql +-- Round to 2 decimal places +UPDATE dc_table_column_map +SET map_expression = 'ROUND(unit_price, 2)' +WHERE column_name = 'UNIT_PRICE'; + +-- Truncate to avoid rounding differences +UPDATE dc_table_column_map +SET map_expression = 'TRUNC(amount, 2)' +WHERE column_name = 'AMOUNT' AND column_origin = 'source'; + +-- Cast to specific numeric format +UPDATE dc_table_column_map +SET map_expression = 'CAST(quantity AS DECIMAL(10,2))' +WHERE column_name = 'QUANTITY'; +``` + +**5. Boolean Handling** + +Normalize boolean representations across platforms: + +```sql +-- Oracle (using 'Y'/'N') +UPDATE dc_table_column_map +SET map_expression = 'CASE WHEN is_active = ''Y'' THEN ''true'' ELSE ''false'' END' +WHERE column_name = 'IS_ACTIVE' AND column_origin = 'source'; + +-- PostgreSQL (using native boolean) +UPDATE dc_table_column_map +SET map_expression = 'CASE WHEN is_active THEN ''true'' ELSE ''false'' END' +WHERE column_name = 'is_active' AND column_origin = 'target'; +``` + +**6. JSON/Complex Data Types** + +Extract specific values from JSON columns: + +```sql +-- PostgreSQL: Extract JSON field +UPDATE dc_table_column_map +SET map_expression = 'metadata->>''version''' +WHERE column_name = 'metadata' AND column_origin = 'target'; + +-- Sort JSON keys for consistent comparison +UPDATE dc_table_column_map +SET map_expression = 'jsonb_sort_keys(config_data)' +WHERE column_name = 'config_data' AND column_origin = 'target'; +``` + +**7. Substring/Partial Comparison** + +Compare only portions of columns: + +```sql +-- Compare first 10 characters only +UPDATE dc_table_column_map +SET map_expression = 'SUBSTR(long_description, 1, 10)' +WHERE column_name = 'LONG_DESCRIPTION' AND column_origin = 'source'; + +UPDATE dc_table_column_map +SET map_expression = 'SUBSTRING(long_description, 1, 10)' +WHERE column_name = 'long_description' AND column_origin = 'target'; +``` + +**8. Concatenation for Composite Comparisons** + +Combine multiple columns into one comparison value: + +```sql +-- Combine first and last name +UPDATE dc_table_column_map +SET map_expression = 'first_name || '' '' || last_name' +WHERE column_name = 'FULL_NAME' AND column_origin = 'source'; +``` + +#### Database-Specific Expressions + +Different databases require different SQL syntax: + +| Function | Oracle | PostgreSQL | MySQL | SQL Server | +|----------|--------|------------|-------|------------| +| Trim | `TRIM(col)` | `TRIM(col)` | `TRIM(col)` | `LTRIM(RTRIM(col))` | +| Upper | `UPPER(col)` | `UPPER(col)` | `UPPER(col)` | `UPPER(col)` | +| Substring | `SUBSTR(col,1,10)` | `SUBSTRING(col,1,10)` | `SUBSTRING(col,1,10)` | `SUBSTRING(col,1,10)` | +| NVL/Coalesce | `NVL(col,'')` | `COALESCE(col,'')` | `IFNULL(col,'')` | `ISNULL(col,'')` | +| Round | `ROUND(col,2)` | `ROUND(col,2)` | `ROUND(col,2)` | `ROUND(col,2)` | +| Date Format | `TO_CHAR(col,'YYYYMMDD')` | `TO_CHAR(col,'YYYYMMDD')` | `DATE_FORMAT(col,'%Y%m%d')` | `FORMAT(col,'yyyyMMdd')` | + +#### Viewing Current Expressions + +```sql +-- View all map_expressions +SELECT + t.table_alias, + tc.column_alias, + tcm.column_origin, + tcm.column_name, + tcm.map_expression +FROM dc_table t +JOIN dc_table_column tc ON t.tid = tc.tid +JOIN dc_table_column_map tcm ON tc.tid = tcm.tid AND tc.column_id = tcm.column_id +WHERE tcm.map_expression IS NOT NULL +ORDER BY t.table_alias, tc.column_alias, tcm.column_origin; +``` + +#### Clearing Expressions + +```sql +-- Remove expression (use raw column value) +UPDATE dc_table_column_map +SET map_expression = NULL +WHERE tid = 123 AND column_id = 5; +``` + +#### Best Practices + +1. **Test expressions independently**: Run the SQL expression directly on each database to verify results before setting map_expression +2. **Match output formats**: Ensure source and target expressions produce identical output formats +3. **Consider performance**: Complex expressions add processing overhead +4. **Document changes**: Keep track of custom expressions for maintenance +5. **Handle NULLs consistently**: Ensure both sides handle NULL the same way + +#### Troubleshooting Expression Issues + +**Problem: Comparison still shows differences after setting expression** + +```sql +-- Verify expressions are set correctly +SELECT column_origin, column_name, map_expression +FROM dc_table_column_map +WHERE tid = 123 AND column_id = 5; + +-- Test expressions directly on databases +-- Oracle +SELECT TO_CHAR(created_date, 'YYYYMMDDHH24MISS') FROM orders WHERE id = 1; + +-- PostgreSQL +SELECT TO_CHAR(created_date, 'YYYYMMDDHH24MISS') FROM orders WHERE id = 1; +``` + +**Problem: Expression syntax error** + +The expression is executed as-is in the database query. Check: +- Proper escaping of single quotes (`''` in SQL) +- Database-specific function names +- Column name case sensitivity + +--- + +## Mapping Import/Export Filters + +### Export with Wildcard Filters + +Export specific tables to YAML (wildcards ARE supported here): + +```bash +# Export tables starting with "customer" +java -jar pgcompare.jar export-mapping --file customer-tables.yaml --table "customer*" + +# Export tables ending with "_archive" +java -jar pgcompare.jar export-mapping --file archive-tables.yaml --table "*_archive" + +# Export single table +java -jar pgcompare.jar export-mapping --file orders.yaml --table "orders" + +# Export all tables in project 2 +java -jar pgcompare.jar export-mapping --file project2.yaml --project 2 +``` + +### Import with Wildcard Filters + +Import specific tables from YAML (wildcards ARE supported here): + +```bash +# Import only customer tables from a full export +java -jar pgcompare.jar import-mapping --file full-export.yaml --table "customer*" + +# Import with overwrite +java -jar pgcompare.jar import-mapping --file updated-mappings.yaml --table "orders" --overwrite +``` + +--- + +## Advanced Filtering Strategies + +### Strategy 1: Development vs Production + +Maintain separate configurations for different environments: + +```bash +# Development - compare sample of data using row filters +export PGCOMPARE_CONFIG=dev-config.properties +java -jar pgcompare.jar compare --table orders + +# Production - full comparison +export PGCOMPARE_CONFIG=prod-config.properties +java -jar pgcompare.jar compare --batch 0 +``` + +### Strategy 2: Incremental Daily Comparisons + +```sql +-- Create a function to update all delta filters +CREATE OR REPLACE FUNCTION update_daily_filters() RETURNS void AS $$ +BEGIN + -- Update source filters (Oracle) + UPDATE dc_table_map + SET table_filter = 'modified_date >= TRUNC(SYSDATE) - 1' + WHERE dest_type = 'source'; + + -- Update target filters (PostgreSQL) + UPDATE dc_table_map + SET table_filter = 'modified_date >= CURRENT_DATE - 1' + WHERE dest_type = 'target'; +END; +$$ LANGUAGE plpgsql; + +-- Run before daily comparison +SELECT update_daily_filters(); +``` + +Run daily: +```bash +java -jar pgcompare.jar compare --batch 0 +``` + +### Strategy 3: Prioritized Comparison Pipeline + +```bash +#!/bin/bash + +# Step 1: Compare critical tables with full validation +echo "Comparing critical tables..." +java -Xmx4g -jar pgcompare.jar compare --batch 1 --report critical-report.html + +# Step 2: Compare secondary tables +echo "Comparing secondary tables..." +java -Xmx2g -jar pgcompare.jar compare --batch 2 --report secondary-report.html + +# Step 3: Compare archive tables (can fail without blocking) +echo "Comparing archive tables..." +java -Xmx1g -jar pgcompare.jar compare --batch 3 --report archive-report.html || true + +echo "All comparisons complete" +``` + +### Strategy 4: Table-by-Table Testing + +Before running full comparison, test individual tables: + +```bash +# Test with smallest tables first +java -jar pgcompare.jar compare --table countries +java -jar pgcompare.jar compare --table currencies + +# If successful, run medium tables +java -jar pgcompare.jar compare --table products +java -jar pgcompare.jar compare --table customers + +# Finally, run large tables +java -jar pgcompare.jar compare --table orders +java -jar pgcompare.jar compare --table order_items +``` + +### Strategy 5: Filter by Table Characteristics + +```sql +-- Find tables without primary keys (may need special handling) +SELECT t.table_alias +FROM dc_table t +WHERE NOT EXISTS ( + SELECT 1 FROM dc_table_column_map tcm + JOIN dc_table_column tc ON tcm.tid = tc.tid AND tcm.column_id = tc.column_id + WHERE tc.tid = t.tid AND tcm.column_primarykey = true +); + +-- Disable tables without PKs +UPDATE dc_table t +SET enabled = false +WHERE NOT EXISTS ( + SELECT 1 FROM dc_table_column_map tcm + JOIN dc_table_column tc ON tcm.tid = tc.tid AND tcm.column_id = tc.column_id + WHERE tc.tid = t.tid AND tcm.column_primarykey = true +); +``` + +--- + +## Quick Reference + +### Command Line Options + +| Option | Wildcards | Description | +|--------|-----------|-------------| +| `compare --table "name"` | No | Exact table name only | +| `check --table "name"` | No | Exact table name only | +| `discover --table "name"` | No | Exact table name only | +| `export-mapping --table "pattern*"` | Yes | Supports wildcards | +| `import-mapping --table "pattern*"` | Yes | Supports wildcards | +| `--batch N` | N/A | Filter by batch number | +| `--batch 0` | N/A | All batches | + +### SQL Filter Examples + +| Purpose | SQL | +|---------|-----| +| Disable table | `UPDATE dc_table SET enabled = false WHERE table_alias = 'x'` | +| Set batch | `UPDATE dc_table SET batch_nbr = 2 WHERE table_alias LIKE '%_archive'` | +| Row filter | `UPDATE dc_table_map SET table_filter = 'status = ''A''' WHERE tid = 123` | +| Delta filter | `UPDATE dc_table_map SET table_filter = 'modified_date > SYSDATE - 7' WHERE tid = 123` | +| Clear filter | `UPDATE dc_table_map SET table_filter = NULL WHERE tid = 123` | +| Disable column | `UPDATE dc_table_column SET enabled = false WHERE column_alias = 'x'` | + +### Common Patterns + +```bash +# Compare single table (exact match) +java -jar pgcompare.jar compare --table orders + +# Compare by batch +java -jar pgcompare.jar compare --batch 1 + +# Export with wildcards +java -jar pgcompare.jar export-mapping --file export.yaml --table "customer*" + +# Import with wildcards and overwrite +java -jar pgcompare.jar import-mapping --file import.yaml --table "*_archive" --overwrite +``` diff --git a/docs/user-guide.md b/docs/user-guide.md new file mode 100644 index 0000000..9f9d0c7 --- /dev/null +++ b/docs/user-guide.md @@ -0,0 +1,855 @@ +# pgCompare User Guide + +## Table of Contents + +1. [Introduction](#introduction) +2. [Installation](#installation) +3. [Quick Start](#quick-start) +4. [Command Reference](#command-reference) +5. [Configuration](#configuration) +6. [Table Discovery](#table-discovery) +7. [Running Comparisons](#running-comparisons) +8. [Rechecking Discrepancies](#rechecking-discrepancies) +9. [Mapping Import/Export](#mapping-importexport) +10. [HTML Reports](#html-reports) +11. [Projects](#projects) +12. [Viewing Results](#viewing-results) +13. [SQL Fix Generation](#sql-fix-generation) +14. [Supported Databases](#supported-databases) +15. [Signal Handling](#signal-handling) + +--- + +## Introduction + +pgCompare is a Java-based data comparison tool designed to validate data consistency between source and target databases. It uses hash-based comparisons to efficiently identify discrepancies in large datasets with minimal database overhead. + +### Use Cases + +- **Data Migration Validation**: Compare data during or after migrating from Oracle, DB2, MariaDB, MySQL, MSSQL, or Snowflake to PostgreSQL +- **Logical Replication Verification**: Validate data consistency across replicated databases +- **Active-Active Configuration Testing**: Regularly verify data synchronization between database nodes + +### How It Works + +1. pgCompare computes hash values for primary keys and non-key columns +2. Hashes are stored in a PostgreSQL repository database +3. Comparisons are performed using parallel threads for optimal performance +4. Discrepancies are identified and stored for review or remediation + +--- + +## Installation + +### Prerequisites + +- Java 21 or later +- Maven 3.9 or later +- PostgreSQL 15 or later (for the repository database) +- JDBC drivers for your source/target databases + +### Build from Source + +```bash +git clone --depth 1 git@github.com:CrunchyData/pgCompare.git +cd pgCompare +mvn clean install +``` + +### Verify Installation + +```bash +java -jar target/pgcompare.jar --version +``` + +Expected output: +``` +Version: 0.5.0.0 +``` + +--- + +## Quick Start + +### Step 1: Create Configuration File + +Create `pgcompare.properties` in your working directory: + +```properties +# Repository Database +repo-host=localhost +repo-port=5432 +repo-dbname=pgcompare +repo-user=pgcompare +repo-password=your_password +repo-schema=pgcompare +repo-sslmode=prefer + +# Source Database (Oracle example) +source-type=oracle +source-host=oracle-server.example.com +source-port=1521 +source-dbname=ORCL +source-user=source_user +source-password=source_password +source-schema=HR + +# Target Database (PostgreSQL) +target-type=postgres +target-host=postgres-server.example.com +target-port=5432 +target-dbname=mydb +target-user=target_user +target-password=target_password +target-schema=hr +``` + +### Step 2: Initialize Repository + +```bash +java -jar pgcompare.jar init +``` + +### Step 3: Discover Tables + +```bash +java -jar pgcompare.jar discover +``` + +### Step 4: Run Comparison + +```bash +java -jar pgcompare.jar compare --batch 0 +``` + +### Step 5: Review Results + +```bash +java -jar pgcompare.jar check --batch 0 --report comparison-report.html +``` + +--- + +## Command Reference + +### Syntax + +```bash +java -jar pgcompare.jar [options] +``` + +### Actions + +| Action | Description | +|--------|-------------| +| `init` | Initialize the repository database schema | +| `discover` | Discover and map tables from source and target schemas | +| `compare` | Perform data comparison between source and target | +| `check` | Recompare out-of-sync rows from previous comparison | +| `copy-table` | Copy pgCompare metadata for a table | +| `export-mapping` | Export table/column mappings to YAML file | +| `import-mapping` | Import table/column mappings from YAML file | + +### Options + +| Option | Short | Description | +|--------|-------|-------------| +| `--batch ` | `-b` | Specify batch number (0 = all batches) | +| `--file ` | `-o` | File path for export/import operations | +| `--overwrite` | | Overwrite existing mappings during import | +| `--project ` | `-p` | Project ID for multi-project environments | +| `--report ` | `-r` | Generate HTML report to specified file | +| `--table ` | `-t` | Limit operation to specified table (wildcards supported for export/import only) | +| `--fix` | `-f` | Generate SQL statements to fix discrepancies (experimental) | +| `--help` | `-h` | Display help information | +| `--version` | `-v` | Display version information | + +--- + +## Configuration + +### Configuration Sources + +pgCompare supports three configuration sources with the following precedence (highest to lowest): + +1. **dc_project table** - Settings stored in the repository database +2. **Environment variables** - Prefixed with `PGCOMPARE_` +3. **Properties file** - Default: `pgcompare.properties` in current directory + +### Environment Variable Format + +Convert property names to environment variables: +- Replace `-` with `_` +- Convert to uppercase +- Prefix with `PGCOMPARE_` + +**Examples:** +```bash +# Property: batch-fetch-size=5000 +export PGCOMPARE_BATCH_FETCH_SIZE=5000 +``` + +### Custom Configuration File Location + +```bash +export PGCOMPARE_CONFIG=/path/to/custom-config.properties +java -jar pgcompare.jar compare +``` + +### System Properties + +| Property | Default | Description | +|----------|---------|-------------| +| `batch-fetch-size` | 2000 | Number of rows fetched per database round-trip | +| `batch-commit-size` | 2000 | Number of rows committed per batch | +| `batch-progress-report-size` | 1000000 | Rows between progress reports | +| `column-hash-method` | database | Hash computation location: `database` or `hybrid` | +| `database-sort` | true | Sort rows on source/target database | +| `float-scale` | 3 | Scale for low-precision number casting | +| `job-logging-enabled` | false | Write log messages to `dc_job_log` table for UI viewing | +| `log-level` | INFO | Logging verbosity: DEBUG, INFO, WARNING, SEVERE | +| `log-destination` | stdout | Log output location | +| `number-cast` | notation | Number format: `notation` (scientific) or `standard` | +| `observer-throttle` | true | Enable throttling to prevent staging table overflow | +| `observer-throttle-size` | 2000000 | Rows before throttle activates | +| `observer-vacuum` | true | Vacuum staging tables during checkpoints | +| `stage-table-parallel` | 0 | Parallel degree for staging tables | + +> **Note:** For advanced threading options (loader-threads, message-queue-size), see the [Advanced Tuning Guide](advanced-tuning-guide.md). + +### Repository Properties + +| Property | Description | +|----------|-------------| +| `repo-host` | Repository PostgreSQL server hostname | +| `repo-port` | Repository PostgreSQL server port | +| `repo-dbname` | Repository database name | +| `repo-user` | Repository database username | +| `repo-password` | Repository database password | +| `repo-schema` | Repository schema name | +| `repo-sslmode` | SSL mode: `disable`, `prefer`, `require` | + +### Source/Target Properties + +| Property | Description | +|----------|-------------| +| `source-type` | Database type: `postgres`, `oracle`, `db2`, `mariadb`, `mysql`, `mssql`, `snowflake` | +| `source-host` | Database server hostname | +| `source-port` | Database server port | +| `source-dbname` | Database name (or service name for Oracle) | +| `source-user` | Database username | +| `source-password` | Database password | +| `source-schema` | Schema containing tables to compare | +| `source-sslmode` | SSL mode for connection | +| `source-warehouse` | Snowflake virtual warehouse (Snowflake only) | + +Replace `source-` with `target-` for target database properties. + +--- + +## Table Discovery + +### Automatic Discovery + +The discover action scans source and target schemas to create table and column mappings: + +```bash +java -jar pgcompare.jar discover +``` + +This will: +1. Query metadata from both databases +2. Match tables by name +3. Match columns by name +4. Create mappings in the repository + +### Discover Specific Tables + +```bash +# Discover a single table +java -jar pgcompare.jar discover --table "orders" +``` + +### Manual Table Registration + +For complex scenarios, insert mappings directly into repository tables. Note that `tid` and `column_id` are auto-generated, so use `RETURNING` to capture them for subsequent inserts: + +**Step 1: dc_table** - Create table definition and capture generated tid +```sql +-- Insert table and get the auto-generated tid +INSERT INTO dc_table (pid, table_alias, enabled, batch_nbr, parallel_degree) +VALUES (1, 'customer_orders', true, 1, 4) +RETURNING tid; +-- Returns: tid = 123 (example) +``` + +**Step 2: dc_table_map** - Create source/target table locations using the returned tid +```sql +-- Source mapping (use tid from Step 1) +INSERT INTO dc_table_map (tid, dest_type, schema_name, table_name) +VALUES (123, 'source', 'SALES', 'CUSTOMER_ORDERS'); + +-- Target mapping +INSERT INTO dc_table_map (tid, dest_type, schema_name, table_name) +VALUES (123, 'target', 'sales', 'customer_orders'); +``` + +**Step 3: dc_table_column** - Create column definition and capture generated column_id +```sql +-- Insert column and get the auto-generated column_id +INSERT INTO dc_table_column (tid, column_alias, enabled) +VALUES (123, 'customer_id', true) +RETURNING column_id; +-- Returns: column_id = 1 (example) +``` + +**Step 4: dc_table_column_map** - Create column mappings for source and target +```sql +-- Source column mapping (use tid and column_id from previous steps) +INSERT INTO dc_table_column_map ( + tid, column_id, column_origin, column_name, + data_type, data_class, column_primarykey +) +VALUES (123, 1, 'source', 'CUSTOMER_ID', 'NUMBER', 'integer', true); + +-- Target column mapping +INSERT INTO dc_table_column_map ( + tid, column_id, column_origin, column_name, + data_type, data_class, column_primarykey +) +VALUES (123, 1, 'target', 'customer_id', 'bigint', 'integer', true); +``` + +**Complete Example with Variables (psql)** +```sql +-- Using psql variables to chain inserts +\set pid 1 + +INSERT INTO dc_table (pid, table_alias, enabled, batch_nbr, parallel_degree) +VALUES (:pid, 'customer_orders', true, 1, 4) +RETURNING tid AS new_tid \gset + +INSERT INTO dc_table_map (tid, dest_type, schema_name, table_name) +VALUES (:new_tid, 'source', 'SALES', 'CUSTOMER_ORDERS'); + +INSERT INTO dc_table_map (tid, dest_type, schema_name, table_name) +VALUES (:new_tid, 'target', 'sales', 'customer_orders'); + +INSERT INTO dc_table_column (tid, column_alias, enabled) +VALUES (:new_tid, 'customer_id', true) +RETURNING column_id AS new_col_id \gset + +INSERT INTO dc_table_column_map (tid, column_id, column_origin, column_name, data_type, data_class, column_primarykey) +VALUES (:new_tid, :new_col_id, 'source', 'CUSTOMER_ID', 'NUMBER', 'integer', true); + +INSERT INTO dc_table_column_map (tid, column_id, column_origin, column_name, data_type, data_class, column_primarykey) +VALUES (:new_tid, :new_col_id, 'target', 'customer_id', 'bigint', 'integer', true); +``` + +--- + +## Running Comparisons + +### Basic Comparison + +Compare all discovered tables: + +```bash +java -jar pgcompare.jar compare --batch 0 +``` + +### Compare Specific Batch + +```bash +# Compare only batch 1 +java -jar pgcompare.jar compare --batch 1 + +# Compare batch 2 +java -jar pgcompare.jar compare --batch 2 +``` + +### Compare Specific Table + +```bash +java -jar pgcompare.jar compare --table "orders" +``` + +### Generate Report During Comparison + +```bash +java -jar pgcompare.jar compare --batch 0 --report results.html +``` + +### Using Environment Variable for Batch + +```bash +export PGCOMPARE_BATCH=1 +java -jar pgcompare.jar compare +``` + +### Understanding Batch Numbers + +Batch numbers allow grouping tables for organized comparisons: + +| Batch | Purpose | +|-------|---------| +| 0 | Process all tables regardless of batch assignment | +| 1-N | Process only tables assigned to that batch number | + +Assign batch numbers during discovery or via the `dc_table.batch_nbr` column. + +--- + +## Rechecking Discrepancies + +### Recheck All Out-of-Sync Rows + +```bash +java -jar pgcompare.jar check --batch 0 +``` + +### Recheck with Report Generation + +```bash +java -jar pgcompare.jar check --batch 0 --report recheck-results.html +``` + +### Recheck Specific Table + +```bash +java -jar pgcompare.jar check --table "orders" --report orders-recheck.html +``` + +### Generate Fix SQL Statements (Experimental) + +```bash +java -jar pgcompare.jar check --batch 0 --fix --report fix-statements.html +``` + +The `--fix` option generates INSERT, UPDATE, and DELETE statements to synchronize the target with the source. + +--- + +## Mapping Import/Export + +### Export Mappings to YAML + +Export all table/column mappings: + +```bash +java -jar pgcompare.jar export-mapping --file mappings.yaml +``` + +Export specific tables: + +```bash +# Export tables starting with "customer" +java -jar pgcompare.jar export-mapping --file customer-mappings.yaml --table "customer*" + +# Export single table +java -jar pgcompare.jar export-mapping --file orders.yaml --table "orders" +``` + +### YAML Format Example + +```yaml +version: "1.0" +exportDate: "2025-01-15T10:30:00" +projectId: 1 +projectName: "Migration Project" +tables: + - alias: "customer_orders" + enabled: true + batchNumber: 1 + parallelDegree: 4 + source: + schema: "SALES" + table: "CUSTOMER_ORDERS" + schemaPreserveCase: false + tablePreserveCase: false + target: + schema: "sales" + table: "customer_orders" + schemaPreserveCase: false + tablePreserveCase: false + columns: + - alias: "customer_id" + enabled: true + source: + columnName: "CUSTOMER_ID" + dataType: "NUMBER" + dataClass: "integer" + primaryKey: true + nullable: false + preserveCase: false + target: + columnName: "customer_id" + dataType: "bigint" + dataClass: "integer" + primaryKey: true + nullable: false + preserveCase: false + - alias: "order_date" + enabled: true + source: + columnName: "ORDER_DATE" + dataType: "DATE" + dataClass: "date" + primaryKey: false + mapExpression: "TO_CHAR(ORDER_DATE, 'YYYYMMDDHH24MISS')" + target: + columnName: "order_date" + dataType: "timestamp" + dataClass: "date" +``` + +### Import Mappings from YAML + +Import mappings (skip existing): + +```bash +java -jar pgcompare.jar import-mapping --file mappings.yaml +``` + +Import with overwrite: + +```bash +java -jar pgcompare.jar import-mapping --file mappings.yaml --overwrite +``` + +Import specific tables: + +```bash +java -jar pgcompare.jar import-mapping --file mappings.yaml --table "customer*" +``` + +### Use Cases for Import/Export + +1. **Version Control**: Export mappings to YAML and track changes in Git +2. **Environment Promotion**: Export from dev, import to test/production +3. **Backup/Restore**: Save mappings before schema changes +4. **Manual Customization**: Export, edit YAML, import with `--overwrite` + +--- + +## HTML Reports + +### Generate Report During Comparison + +```bash +java -jar pgcompare.jar compare --batch 0 --report comparison-report.html +``` + +### Generate Report During Recheck + +```bash +java -jar pgcompare.jar check --batch 0 --report recheck-report.html +``` + +### Report Contents + +The HTML report includes: + +1. **Job Summary** + - Tables processed + - Total elapsed time + - Rows per second throughput + - Total rows compared + - Out-of-sync row count + +2. **Table Summary** + - Per-table comparison status + - Row counts (equal, not equal, missing source, missing target) + - Elapsed time per table + +3. **Check Results** (for recheck operations) + - Primary key values + - Comparison status + - Detailed results + +4. **Fix SQL** (when `--fix` is enabled) + - Generated INSERT/UPDATE/DELETE statements + - Primary key for each fix + +--- + +## Projects + +Projects allow maintaining multiple comparison configurations in a single repository. + +> **Note:** Project 1 (pid=1) is reserved as the default project and is created automatically during repository initialization. Do not delete or reassign this project. + +### Create a New Project + +```sql +INSERT INTO dc_project (project_name, project_config) +VALUES ('Migration Project', '{"batch-fetch-size": "5000", "observer-throttle": "true"}'); +``` + +### Use a Specific Project + +```bash +java -jar pgcompare.jar discover --project 2 +java -jar pgcompare.jar compare --project 2 --batch 0 +``` + +### Store Project-Specific Configuration + +Configuration can be stored in the `project_config` column as JSON: + +```sql +UPDATE dc_project +SET project_config = '{"batch-fetch-size": "5000", "batch-commit-size": "5000", "observer-throttle": "true"}' +WHERE pid = 2; +``` + +### List Projects + +```sql +SELECT pid, project_name, project_config +FROM dc_project +ORDER BY pid; +``` + +--- + +## Viewing Results + +### Summary from Last Run + +```sql +WITH mr AS (SELECT max(rid) rid FROM dc_result) +SELECT + compare_start, + table_name, + status, + equal_cnt + not_equal_cnt + missing_source_cnt + missing_target_cnt AS total_cnt, + equal_cnt, + not_equal_cnt, + missing_source_cnt + missing_target_cnt AS missing_cnt +FROM dc_result r +JOIN mr ON (mr.rid = r.rid) +ORDER BY table_name; +``` + +### Out-of-Sync Rows + +```sql +SELECT + COALESCE(s.table_name, t.table_name) AS table_name, + CASE + WHEN s.compare_result = 'n' THEN 'out-of-sync' + WHEN s.compare_result = 'm' THEN 'missing target' + WHEN t.compare_result = 'm' THEN 'missing source' + END AS compare_result, + COALESCE(s.pk, t.pk) AS primary_key +FROM dc_source s +FULL OUTER JOIN dc_target t ON s.pk = t.pk AND s.tid = t.tid; +``` + +### Detailed Results by Table + +```sql +SELECT + t.table_alias, + r.compare_start, + r.compare_end, + r.status, + r.source_cnt, + r.target_cnt, + r.equal_cnt, + r.not_equal_cnt, + r.missing_source_cnt, + r.missing_target_cnt +FROM dc_result r +JOIN dc_table t ON r.tid = t.tid +WHERE t.table_alias = 'orders' +ORDER BY r.compare_start DESC +LIMIT 10; +``` + +### View Primary Keys for Out-of-Sync Rows + +```sql +SELECT + table_name, + pk, + compare_result, + batch_nbr +FROM dc_source +WHERE compare_result IN ('n', 'm') +ORDER BY table_name, pk; +``` + +--- + +## SQL Fix Generation + +### Overview + +The SQL fix generation feature (experimental) creates INSERT, UPDATE, and DELETE statements to synchronize the target database with the source. + +### Enable Fix Generation + +```bash +java -jar pgcompare.jar check --batch 0 --fix +``` + +### Generated Statement Types + +| Source State | Target State | Generated SQL | +|-------------|-------------|---------------| +| Row exists | Row missing | INSERT INTO target | +| Row missing | Row exists | DELETE FROM target | +| Row exists | Row exists (different) | UPDATE target | + +### Example Output + +``` +Fix SQL Statements: +=================== + +Table: orders (3 statements) + PK: {"order_id": 12345} + INSERT INTO sales.orders (order_id, customer_id, amount) VALUES (12345, 100, 250.00); + + PK: {"order_id": 12346} + UPDATE sales.orders SET amount = 300.00, status = 'shipped' WHERE order_id = 12346; + + PK: {"order_id": 12347} + DELETE FROM sales.orders WHERE order_id = 12347; + +Total Fix SQL Statements: 3 +``` + +### Limitations + +- Feature is experimental - review generated SQL before execution +- Complex data types may not be handled correctly +- Large-scale fixes should be batched manually +- Always test fix statements in non-production first + +--- + +## Supported Databases + +### Database Types + +| Database | Source | Target | Type Value | +|----------|--------|--------|------------| +| PostgreSQL | Yes | Yes | `postgres` | +| Oracle | Yes | Yes | `oracle` | +| IBM DB2 | Yes | Yes | `db2` | +| MariaDB | Yes | Yes | `mariadb` | +| MySQL | Yes | Yes | `mysql` | +| Microsoft SQL Server | Yes | Yes | `mssql` | +| Snowflake | Yes | Yes | `snowflake` | + +### Connection Examples + +**Oracle:** +```properties +source-type=oracle +source-host=oracle-server.example.com +source-port=1521 +source-dbname=ORCL +source-user=hr +source-password=password +source-schema=HR +``` + +**PostgreSQL:** +```properties +target-type=postgres +target-host=postgres-server.example.com +target-port=5432 +target-dbname=mydb +target-user=postgres +target-password=password +target-schema=public +target-sslmode=prefer +``` + +**Snowflake:** +```properties +source-type=snowflake +source-host=account.snowflakecomputing.com +source-dbname=MYDB +source-user=snowflake_user +source-password=password +source-schema=PUBLIC +source-warehouse=COMPUTE_WH +``` + +**DB2:** +```properties +source-type=db2 +source-host=db2-server.example.com +source-port=50000 +source-dbname=SAMPLE +source-user=db2inst1 +source-password=password +source-schema=DB2INST1 +``` + +### Known Limitations + +1. **Date/Timestamps**: Compared only to the second (format: DDMMYYYYHH24MISS) +2. **Unsupported Types**: blob, long, longraw, bytea +3. **Boolean**: Cross-platform comparison limitations +4. **Floating Point**: Low-precision types (float, real) cannot be compared to high-precision types (double) +5. **Float Scale**: All low-precision types cast using scale of 3 (1 for Snowflake) +6. **Float Casting**: Use `number-cast` option to switch between `standard` and `notation` formats + +--- + +## Next Steps + +- [Handling Large Tables](large-tables-guide.md) - Parallel processing and optimization +- [Table Filtering Guide](table-filtering-guide.md) - Advanced filtering techniques +- [Performance Tuning](performance-tuning-guide.md) - Optimizing for your workload +- [Quick Reference](quick-reference.md) - Command cheat sheet + +--- + +## Signal Handling + +pgCompare supports Unix signals for shutdown and dynamic configuration reload: + +### Graceful Shutdown (SIGINT/Ctrl+C) +Send SIGINT (Ctrl+C) to gracefully stop a running comparison. The current table comparison will complete before the application exits. + +```shell +# Use Ctrl+C if running in foreground +# Or send SIGINT +kill -INT +``` + +### Immediate Termination (SIGTERM) +Send SIGTERM to immediately cancel all running queries and terminate. Use this when you need to stop immediately without waiting for the current table to complete. + +```shell +# Find the pgCompare process +ps aux | grep pgcompare + +# Send SIGTERM for immediate termination +kill -TERM +# or simply +kill +``` + +### Configuration Reload (SIGHUP) +Send SIGHUP to reload the properties file without stopping the application. This allows dynamic adjustment of certain parameters during long-running comparisons. + +```shell +# Reload configuration +kill -HUP +``` + +**Dynamically reloadable properties:** +- `batch-fetch-size`, `batch-commit-size`, `batch-progress-report-size` +- `loader-threads`, `message-queue-size` +- `observer-throttle`, `observer-throttle-size`, `observer-vacuum` +- `log-level`, `float-scale`, `database-sort` + +**Note:** Connection properties (repo-*, source-*, target-*) cannot be reloaded at runtime. diff --git a/pgcompare.properties.sample b/pgcompare.properties.sample index 46cc81f..53749cc 100644 --- a/pgcompare.properties.sample +++ b/pgcompare.properties.sample @@ -54,6 +54,11 @@ log-destination = stdout # default: INFO log-level = INFO +# Enable logging to job log table for server mode and standalone runs +# When enabled, log entries are stored in dc_job_log table for viewing in UI +# default: false +job-logging-enabled = false + # Determines whether to presort the rows on the source or target database (append ORDER BY to SELECT statement). # default: true database-sort = true diff --git a/pom.xml b/pom.xml index 3c72783..46392f1 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ 21 21 UTF-8 - 1.18.38 + 1.18.42 @@ -21,19 +21,19 @@ com.ibm.db2 jcc - 12.1.2.0 + 12.1.3.0 org.postgresql postgresql - 42.7.8 + 42.7.10 commons-cli commons-cli - 1.10.0 + 1.11.0 @@ -45,19 +45,19 @@ com.oracle.database.jdbc ojdbc11 - 23.9.0.25.07 + 23.26.1.0.0 org.json json - 20250517 + 20251224 com.mysql mysql-connector-j - 9.4.0 + 9.6.0 @@ -69,19 +69,19 @@ org.apache.commons commons-lang3 - 3.19.0 + 3.20.0 org.mariadb.jdbc mariadb-java-client - 3.5.3 + 3.5.7 net.snowflake snowflake-jdbc - 3.27.0 + 4.0.1 @@ -137,7 +137,12 @@ net.snowflake snowflake-jdbc - 3.24.2 + 4.0.1 + + + org.yaml + snakeyaml + 2.6 @@ -251,6 +256,27 @@ + + org.codehaus.mojo + exec-maven-plugin + 3.5.0 + + + set-executable-permissions + package + + exec + + + chmod + + +x + ${project.build.directory}/pgcompare + + + + + pgcompare diff --git a/src/main/java/com/crunchydata/config/ApplicationContext.java b/src/main/java/com/crunchydata/config/ApplicationContext.java index d4bade5..af5f030 100644 --- a/src/main/java/com/crunchydata/config/ApplicationContext.java +++ b/src/main/java/com/crunchydata/config/ApplicationContext.java @@ -16,12 +16,17 @@ package com.crunchydata.config; +import java.net.InetAddress; import java.sql.Connection; import static com.crunchydata.service.DatabaseConnectionService.getConnection; import static com.crunchydata.config.Settings.*; +import com.crunchydata.service.ConnectionTestService; import com.crunchydata.service.RepositoryInitializationService; +import com.crunchydata.service.ServerModeService; +import com.crunchydata.service.SignalHandlerService; +import com.crunchydata.service.StandaloneJobService; import com.crunchydata.util.LoggingUtils; import com.crunchydata.util.ValidationUtils; @@ -41,6 +46,12 @@ public class ApplicationContext { private static final String THREAD_NAME = "main"; private static final String ACTION_CHECK = "check"; private static final String ACTION_INIT = "init"; + private static final String ACTION_EXPORT_CONFIG = "export-config"; + private static final String ACTION_EXPORT_MAPPING = "export-mapping"; + private static final String ACTION_IMPORT_CONFIG = "import-config"; + private static final String ACTION_IMPORT_MAPPING = "import-mapping"; + private static final String ACTION_SERVER = "server"; + private static final String ACTION_TEST_CONNECTION = "test-connection"; private static final String CONN_TYPE_POSTGRES = "postgres"; private static final String CONN_TYPE_REPO = "repo"; private static final String CONN_TYPE_SOURCE = "source"; @@ -100,14 +111,17 @@ public void initialize() throws Exception { Runtime.getRuntime().addShutdownHook(new Thread(() -> LoggingUtils.write("info", THREAD_NAME, "Shutting down"))); + // Register signal handlers for graceful shutdown and config reload + SignalHandlerService.initialize(); + // Log startup information logStartupInfo(); // Connect to repository database connectToRepository(); - // Load project configuration (skip for init action) - if (!action.equals(ACTION_INIT)) { + // Load project configuration (skip for init, server, and test-connection actions) + if (!action.equals(ACTION_INIT) && !action.equals(ACTION_SERVER) && !action.equals(ACTION_TEST_CONNECTION)) { setProjectConfig(connRepo, pid); } @@ -130,8 +144,9 @@ public void initialize() throws Exception { * */ public void executeAction() { - // Connect to source and target databases (skip for init action) - if (!action.equals(ACTION_INIT)) { + // Connect to source and target databases (skip for init, server, test-connection, and config/mapping actions) + if (!action.equals(ACTION_INIT) && !action.equals(ACTION_SERVER) && !action.equals(ACTION_TEST_CONNECTION) && !action.equals(ACTION_EXPORT_MAPPING) && !action.equals(ACTION_IMPORT_MAPPING) + && !action.equals(ACTION_EXPORT_CONFIG) && !action.equals(ACTION_IMPORT_CONFIG)) { connectToSourceAndTarget(); } @@ -141,12 +156,32 @@ public void executeAction() { performDiscovery(); break; case "check": + performCheck(); + break; case "compare": performCompare(); break; case "copy-table": performCopyTable(); break; + case "export-config": + performExportConfig(); + break; + case "export-mapping": + performExportMapping(); + break; + case "import-config": + performImportConfig(); + break; + case "import-mapping": + performImportMapping(); + break; + case "server": + performServerMode(); + break; + case "test-connection": + performTestConnection(); + break; default: throw new IllegalArgumentException("Invalid action specified: " + action); } @@ -213,18 +248,69 @@ private void performDiscovery() { LoggingUtils.write("info", THREAD_NAME, "Performing table discovery"); String table = (cmd.hasOption("table")) ? cmd.getOptionValue("table").toLowerCase() : ""; - // Discover Tables - com.crunchydata.controller.DiscoverController.performTableDiscovery(Props, pid, table, connRepo, connSource, connTarget); + StandaloneJobService jobService = null; + if (StandaloneJobService.isJobTrackingAvailable(connRepo) && StandaloneJobService.ensureProjectExists(connRepo, pid)) { + jobService = new StandaloneJobService(connRepo); + jobService.startJob(pid, "discover", batchParameter, table, startStopWatch); + } + + try { + // Discover Tables + com.crunchydata.controller.DiscoverController.performTableDiscovery(Props, pid, table, connRepo, connSource, connTarget); + + // Discover Columns + com.crunchydata.controller.DiscoverController.performColumnDiscovery(Props, pid, table, connRepo, connSource, connTarget); - // Discover Columns - com.crunchydata.controller.DiscoverController.performColumnDiscovery(Props, pid, table, connRepo, connSource, connTarget); + if (jobService != null) { + jobService.completeJob(null); + } + } catch (Exception e) { + if (jobService != null) { + jobService.failJob(e.getMessage()); + } + throw e; + } } /** * Perform comparison operation. */ private void performCompare() { - com.crunchydata.controller.CompareController.performCompare(this); + performCompareWithJobTracking(false); + } + + /** + * Perform check (recheck) operation. + */ + private void performCheck() { + performCompareWithJobTracking(true); + } + + /** + * Perform comparison or check operation with job tracking. + */ + private void performCompareWithJobTracking(boolean isCheck) { + String table = (cmd.hasOption("table")) ? cmd.getOptionValue("table").toLowerCase() : ""; + String jobType = isCheck ? "check" : "compare"; + + StandaloneJobService jobService = null; + if (StandaloneJobService.isJobTrackingAvailable(connRepo) && StandaloneJobService.ensureProjectExists(connRepo, pid)) { + jobService = new StandaloneJobService(connRepo); + jobService.startJob(pid, jobType, batchParameter, table, startStopWatch); + } + + try { + com.crunchydata.controller.CompareController.performCompare(this, jobService); + + if (jobService != null) { + jobService.completeJob(null); + } + } catch (Exception e) { + if (jobService != null) { + jobService.failJob(e.getMessage()); + } + throw e; + } } /** @@ -233,4 +319,86 @@ private void performCompare() { private void performCopyTable() { com.crunchydata.controller.TableController.performCopyTable(this); } + + private void performExportMapping() { + String tableFilter = Props.getProperty("table", ""); + String outputFile = Props.getProperty("mappingFile", ""); + com.crunchydata.controller.MappingController.performExport(connRepo, pid, tableFilter, outputFile); + } + + private void performImportMapping() { + String tableFilter = Props.getProperty("table", ""); + String inputFile = Props.getProperty("mappingFile", ""); + boolean overwrite = Boolean.parseBoolean(Props.getProperty("overwrite", "false")); + com.crunchydata.controller.MappingController.performImport(connRepo, pid, tableFilter, inputFile, overwrite); + } + + private void performExportConfig() { + String outputFile = Props.getProperty("mappingFile", ""); + try { + com.crunchydata.service.ConfigExportService.exportToProperties(connRepo, pid, outputFile); + } catch (Exception e) { + LoggingUtils.write("severe", THREAD_NAME, String.format("Export config failed: %s", e.getMessage())); + throw new RuntimeException("Failed to export configuration", e); + } + } + + private void performImportConfig() { + String inputFile = Props.getProperty("mappingFile", ""); + if (inputFile == null || inputFile.isEmpty()) { + throw new IllegalArgumentException("Input file path is required for import-config. Use --file parameter."); + } + try { + com.crunchydata.service.ConfigImportService.importFromProperties(connRepo, pid, inputFile); + } catch (Exception e) { + LoggingUtils.write("severe", THREAD_NAME, String.format("Import config failed: %s", e.getMessage())); + throw new RuntimeException("Failed to import configuration", e); + } + } + + private void performServerMode() { + LoggingUtils.write("info", THREAD_NAME, "Starting pgCompare in server mode"); + String serverName = Props.getProperty("serverName", getDefaultServerName()); + + ServerModeService serverService = new ServerModeService(connRepo, serverName); + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + LoggingUtils.write("info", THREAD_NAME, "Shutdown hook triggered - stopping server"); + serverService.stop(); + })); + + serverService.start(); + } + + private String getDefaultServerName() { + try { + String hostname = InetAddress.getLocalHost().getHostName(); + int dotIndex = hostname.indexOf('.'); + return dotIndex > 0 ? hostname.substring(0, dotIndex) : hostname; + } catch (Exception e) { + return "pgcompare-server"; + } + } + + private void performTestConnection() { + LoggingUtils.write("info", THREAD_NAME, "Testing database connections"); + + // Load project configuration for the specified project + setProjectConfig(connRepo, pid); + + // Test all connections + var results = ConnectionTestService.testAllConnections(); + + // Print results as JSON for easier parsing + ConnectionTestService.printResultsAsJson(results); + + // Also print human-readable format + ConnectionTestService.printResults(results); + + // Exit with appropriate code + boolean allSuccess = results.values().stream().allMatch(r -> r.success); + if (!allSuccess) { + System.exit(1); + } + } } diff --git a/src/main/java/com/crunchydata/config/ApplicationState.java b/src/main/java/com/crunchydata/config/ApplicationState.java new file mode 100644 index 0000000..7038223 --- /dev/null +++ b/src/main/java/com/crunchydata/config/ApplicationState.java @@ -0,0 +1,197 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.config; + +import java.sql.Statement; +import java.util.Collections; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Singleton class that manages global application state for signal handling. + * Provides thread-safe flags for shutdown requests and configuration reload. + * + * Signal handling: + * - SIGINT (Ctrl+C): Graceful shutdown - complete current table comparison + * - SIGTERM: Immediate termination - cancel all queries and stop + * - SIGHUP: Reload configuration file + * + * @author Brian Pace + */ +public class ApplicationState { + + private static final ApplicationState INSTANCE = new ApplicationState(); + + private final AtomicBoolean gracefulShutdownRequested = new AtomicBoolean(false); + private final AtomicBoolean immediateTerminationRequested = new AtomicBoolean(false); + private final AtomicBoolean reloadRequested = new AtomicBoolean(false); + private final AtomicBoolean shutdownComplete = new AtomicBoolean(false); + + private final Set activeStatements = Collections.newSetFromMap(new ConcurrentHashMap<>()); + + private ApplicationState() { + } + + public static ApplicationState getInstance() { + return INSTANCE; + } + + /** + * Request a graceful shutdown of the application. + * Current table comparison will complete before exit. + */ + public void requestGracefulShutdown() { + gracefulShutdownRequested.set(true); + } + + /** + * Request immediate termination of the application. + * All queries will be cancelled and application will exit. + */ + public void requestImmediateTermination() { + immediateTerminationRequested.set(true); + gracefulShutdownRequested.set(true); + cancelAllStatements(); + } + + /** + * Check if any shutdown has been requested (graceful or immediate). + * + * @return true if shutdown was requested + */ + public boolean isShutdownRequested() { + return gracefulShutdownRequested.get() || immediateTerminationRequested.get(); + } + + /** + * Check if graceful shutdown has been requested. + * + * @return true if graceful shutdown was requested + */ + public boolean isGracefulShutdownRequested() { + return gracefulShutdownRequested.get() && !immediateTerminationRequested.get(); + } + + /** + * Check if immediate termination has been requested. + * + * @return true if immediate termination was requested + */ + public boolean isImmediateTerminationRequested() { + return immediateTerminationRequested.get(); + } + + /** + * Mark shutdown as complete. + */ + public void markShutdownComplete() { + shutdownComplete.set(true); + } + + /** + * Check if shutdown is complete. + * + * @return true if shutdown is complete + */ + public boolean isShutdownComplete() { + return shutdownComplete.get(); + } + + /** + * Request a configuration reload. + */ + public void requestReload() { + reloadRequested.set(true); + } + + /** + * Check and clear the reload request flag. + * + * @return true if reload was requested (and clears the flag) + */ + public boolean checkAndClearReloadRequest() { + return reloadRequested.getAndSet(false); + } + + /** + * Check if reload has been requested without clearing the flag. + * + * @return true if reload was requested + */ + public boolean isReloadRequested() { + return reloadRequested.get(); + } + + /** + * Register an active statement for potential cancellation. + * + * @param stmt the statement to register + */ + public void registerStatement(Statement stmt) { + if (stmt != null) { + activeStatements.add(stmt); + } + } + + /** + * Unregister a statement when it completes. + * + * @param stmt the statement to unregister + */ + public void unregisterStatement(Statement stmt) { + if (stmt != null) { + activeStatements.remove(stmt); + } + } + + /** + * Cancel all registered active statements. + */ + public void cancelAllStatements() { + for (Statement stmt : activeStatements) { + try { + if (stmt != null && !stmt.isClosed()) { + stmt.cancel(); + } + } catch (Exception e) { + // Ignore cancellation errors + } + } + activeStatements.clear(); + } + + /** + * Get the number of active statements. + * + * @return count of active statements + */ + public int getActiveStatementCount() { + return activeStatements.size(); + } + + /** + * Reset all state flags. Useful for testing. + */ + public void reset() { + gracefulShutdownRequested.set(false); + immediateTerminationRequested.set(false); + reloadRequested.set(false); + shutdownComplete.set(false); + activeStatements.clear(); + } +} diff --git a/src/main/java/com/crunchydata/config/CommandLineParser.java b/src/main/java/com/crunchydata/config/CommandLineParser.java index 7e09a5e..35fc5d6 100644 --- a/src/main/java/com/crunchydata/config/CommandLineParser.java +++ b/src/main/java/com/crunchydata/config/CommandLineParser.java @@ -85,6 +85,11 @@ public static CommandLine parse(String[] args) { } Props.setProperty("fix", String.valueOf(cmd.hasOption("fix"))); + Props.setProperty("overwrite", String.valueOf(cmd.hasOption("overwrite"))); + Props.setProperty("mappingFile", cmd.getOptionValue("file", "")); + if (cmd.hasOption("name")) { + Props.setProperty("serverName", cmd.getOptionValue("name")); + } Integer batchParameter = (cmd.hasOption("batch")) ? Integer.parseInt(cmd.getOptionValue("batch")) : @@ -114,10 +119,13 @@ private static Options createOptions() { options.addOption(new Option("b", "batch", true, "Batch Number")); options.addOption(new Option("h", "help", false, "Usage and help")); options.addOption(new Option("f", "fix", false, "Generate SQL to fix out of sync issue (experimental, use with caution)")); + options.addOption(new Option("o", "file", true, "File path for export/import operations")); + options.addOption(new Option(null, "overwrite", false, "Overwrite existing mappings during import")); options.addOption(new Option("p", "project", true, "Project ID")); options.addOption(new Option("r", "report", true, "Generate report")); - options.addOption(new Option("t", "table", true, "Limit to specified table")); + options.addOption(new Option("t", "table", true, "Limit to specified table (supports wildcards for export/import)")); options.addOption(new Option("v", "version", false, "Version")); + options.addOption(new Option("n", "name", true, "Server name (for server mode)")); return options; } @@ -130,16 +138,25 @@ public static void showHelp() { System.out.println("pgcompare "); System.out.println(); System.out.println("Actions:"); - System.out.println(" check Recompare the out of sync rows from previous compare"); - System.out.println(" compare Perform database compare"); - System.out.println(" copy-table Copy pgCompare metadata for table. Must specify table alias to copy using --table option"); - System.out.println(" discover Discover tables and columns"); - System.out.println(" init Initialize the repository database"); + System.out.println(" check Recompare the out of sync rows from previous compare"); + System.out.println(" compare Perform database compare"); + System.out.println(" copy-table Copy pgCompare metadata for table. Must specify table alias to copy using --table option"); + System.out.println(" discover Discover tables and columns"); + System.out.println(" export-config Export project configuration to properties file (requires --file)"); + System.out.println(" export-mapping Export table/column mappings to YAML file"); + System.out.println(" import-config Import properties file to project configuration"); + System.out.println(" import-mapping Import table/column mappings from YAML file"); + System.out.println(" init Initialize the repository database"); + System.out.println(" server Run in server mode (daemon that polls work queue)"); + System.out.println(" test-connection Test database connections and report status"); System.out.println("Options:"); System.out.println(" -b|--batch "); - System.out.println(" -p|--project Project ID"); - System.out.println(" -r|--report Create html report of compare"); - System.out.println(" -t|--table "); + System.out.println(" -n|--name Server name for server mode (default: pgcompare-server)"); + System.out.println(" -o|--file File path for export/import (default: pgcompare-mappings-.yaml)"); + System.out.println(" --overwrite Overwrite existing mappings during import"); + System.out.println(" -p|--project Project ID"); + System.out.println(" -r|--report Create html report of compare"); + System.out.println(" -t|--table Limit to specified table (supports wildcards * for export/import)"); System.out.println(" --help"); System.out.println(); } diff --git a/src/main/java/com/crunchydata/config/Settings.java b/src/main/java/com/crunchydata/config/Settings.java index aec2a80..1f5a747 100644 --- a/src/main/java/com/crunchydata/config/Settings.java +++ b/src/main/java/com/crunchydata/config/Settings.java @@ -65,8 +65,9 @@ public class Settings { public static Properties Props; - public static final String VERSION = "0.5.0.0"; + public static final String VERSION = "0.6.0.0"; private static final String paramFile = (System.getenv("PGCOMPARE_CONFIG") == null) ? "pgcompare.properties" : System.getenv("PGCOMPARE_CONFIG"); + private static final Object reloadLock = new Object(); public static Map> validPropertyValues = Map.of( "column-hash-method", Set.of("database", "hybrid", "raw"), @@ -187,7 +188,12 @@ public static Properties setEnvironment (Properties prop) { */ public static void setProjectConfig (Connection conn, Integer pid) { - JSONObject projectConfig = new JSONObject(RepoController.getProjectConfig(conn, pid)); + String configString = RepoController.getProjectConfig(conn, pid); + if (configString == null || configString.isEmpty()) { + return; + } + + JSONObject projectConfig = new JSONObject(configString); if ( ! projectConfig.isEmpty() ) { @@ -205,4 +211,93 @@ public static void setProjectConfig (Connection conn, Integer pid) { } + /** + * Reloads properties from the configuration file. + * This method is called in response to a SIGHUP signal. + * Thread-safe: uses synchronization to prevent concurrent reloads. + * + * Note: Only certain properties can be dynamically changed: + * - batch-fetch-size, batch-commit-size, batch-progress-report-size + * - loader-threads, message-queue-size + * - observer-throttle, observer-throttle-size, observer-vacuum + * - log-level + * + * Connection properties (repo-*, source-*, target-*) are NOT reloaded + * as changing them mid-execution would cause issues. + */ + public static void reloadProperties() { + synchronized (reloadLock) { + LoggingUtils.write("info", "Settings", "Reloading configuration from " + paramFile); + + if (!FileSystemUtils.FileExistsCheck(paramFile)) { + LoggingUtils.write("warning", "Settings", "Configuration file not found: " + paramFile); + return; + } + + Properties newProps = setDefaults(); + + try (InputStream stream = new FileInputStream(paramFile)) { + newProps.load(stream); + } catch (Exception e) { + LoggingUtils.write("severe", "Settings", "Failed to reload configuration: " + e.getMessage()); + return; + } + + for (Object key : newProps.keySet()) { + Object value = newProps.get(key); + if (value instanceof String) { + newProps.setProperty((String) key, ((String) value).trim()); + } + } + + Properties finalProps = setEnvironment(newProps); + + updateDynamicProperties(finalProps); + + LoggingUtils.write("info", "Settings", "Configuration reload complete"); + } + } + + /** + * Updates only the dynamic properties that can be safely changed at runtime. + * + * @param newProps the newly loaded properties + */ + private static void updateDynamicProperties(Properties newProps) { + String[] dynamicProps = { + "batch-fetch-size", + "batch-commit-size", + "batch-progress-report-size", + "loader-threads", + "message-queue-size", + "observer-throttle", + "observer-throttle-size", + "observer-vacuum", + "log-level", + "float-scale", + "database-sort" + }; + + for (String prop : dynamicProps) { + String oldValue = Props.getProperty(prop); + String newValue = newProps.getProperty(prop); + + if (newValue != null && !newValue.equals(oldValue)) { + Props.setProperty(prop, newValue); + LoggingUtils.write("info", "Settings", + String.format("Property '%s' changed: %s -> %s", prop, oldValue, newValue)); + } + } + } + + /** + * Gets the current configuration file path. + * + * @return the path to the configuration file + */ + public static String getConfigFilePath() { + return paramFile; + } + + } diff --git a/src/main/java/com/crunchydata/config/sql/RepoSQLConstants.java b/src/main/java/com/crunchydata/config/sql/RepoSQLConstants.java index 66467a5..a0a616f 100644 --- a/src/main/java/com/crunchydata/config/sql/RepoSQLConstants.java +++ b/src/main/java/com/crunchydata/config/sql/RepoSQLConstants.java @@ -71,7 +71,8 @@ CREATE TABLE dc_source ( pk_hash varchar(100) NULL, column_hash varchar(100) NULL, compare_result bpchar(1) NULL, - thread_nbr int4 NULL + thread_nbr int4 NULL, + fix_sql text NULL ) """; @@ -184,7 +185,8 @@ CREATE TABLE dc_target ( pk_hash varchar(100) NULL, column_hash varchar(100) NULL, compare_result bpchar(1) NULL, - thread_nbr int4 NULL + thread_nbr int4 NULL, + fix_sql text NULL ) """; @@ -300,19 +302,19 @@ AND EXISTS (SELECT 1 FROM dc_source s WHERE s.tid=? AND t.pk_hash=s.pk_hash AND """; String SQL_REPO_SELECT_OUTOFSYNC_ROWS = """ - SELECT DISTINCT tid, pk_hash, pk + SELECT DISTINCT ON (pk_hash) tid, pk_hash, pk FROM (SELECT tid, pk_hash, pk FROM dc_source WHERE tid = ? AND compare_result is not null AND compare_result != 'e' - UNION + UNION ALL SELECT tid, pk_hash, pk FROM dc_target WHERE tid = ? AND compare_result is not null AND compare_result != 'e') x - ORDER BY tid + ORDER BY pk_hash, tid """; @@ -333,7 +335,7 @@ AND EXISTS (SELECT 1 FROM dc_source s WHERE s.tid=? AND t.pk_hash=s.pk_hash AND String SQL_REPO_DCRESULT_UPDATE_STATUSANDCOUNT = """ UPDATE dc_result SET missing_source_cnt=?, missing_target_cnt=?, not_equal_cnt=?, status=?, compare_end=current_timestamp WHERE cid=? - RETURNING equal_cnt, missing_source_cnt, missing_target_cnt, not_equal_cnt, status + RETURNING equal_cnt, missing_source_cnt, missing_target_cnt, not_equal_cnt, status, source_cnt, target_cnt """; String SQL_REPO_DCRESULT_CLEAN = """ @@ -520,4 +522,27 @@ JOIN dc_table_map m ON (t.tid=m.tid) AND t.table_alias = ? """; + // + // Repository SQL - Fix SQL Updates + // + String SQL_REPO_DCSOURCE_UPDATE_FIXSQL = """ + UPDATE dc_source SET fix_sql = ? WHERE tid = ? AND pk_hash = ? + """; + + String SQL_REPO_DCTARGET_UPDATE_FIXSQL = """ + UPDATE dc_target SET fix_sql = ? WHERE tid = ? AND pk_hash = ? + """; + + String SQL_REPO_DCSOURCE_SELECT_FIXSQL_BYTID = """ + SELECT tid, pk, pk_hash, compare_result, fix_sql + FROM dc_source + WHERE tid = ? AND fix_sql IS NOT NULL + """; + + String SQL_REPO_DCTARGET_SELECT_FIXSQL_BYTID = """ + SELECT tid, pk, pk_hash, compare_result, fix_sql + FROM dc_target + WHERE tid = ? AND fix_sql IS NOT NULL + """; + } diff --git a/src/main/java/com/crunchydata/config/sql/ServerModeSQLConstants.java b/src/main/java/com/crunchydata/config/sql/ServerModeSQLConstants.java new file mode 100644 index 0000000..05989f8 --- /dev/null +++ b/src/main/java/com/crunchydata/config/sql/ServerModeSQLConstants.java @@ -0,0 +1,382 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.config.sql; + +public interface ServerModeSQLConstants { + + // DC_SERVER - Server registration table + String REPO_DDL_DC_SERVER = """ + CREATE TABLE dc_server ( + server_id uuid DEFAULT gen_random_uuid() NOT NULL, + server_name text NOT NULL, + server_host text NOT NULL, + server_pid int8 NOT NULL, + status varchar(20) DEFAULT 'active' NOT NULL, + registered_at timestamptz DEFAULT current_timestamp NOT NULL, + last_heartbeat timestamptz DEFAULT current_timestamp NOT NULL, + current_job_id uuid NULL, + server_config jsonb NULL, + CONSTRAINT dc_server_pk PRIMARY KEY (server_id), + CONSTRAINT dc_server_status_check CHECK (status IN ('active', 'idle', 'busy', 'offline', 'terminated')) + ) + """; + + String REPO_DDL_DC_SERVER_IDX1 = """ + CREATE INDEX dc_server_idx1 ON dc_server USING btree (status, last_heartbeat) + """; + + // DC_JOB - Job queue for scheduled jobs + String REPO_DDL_DC_JOB = """ + CREATE TABLE dc_job ( + job_id uuid DEFAULT gen_random_uuid() NOT NULL, + pid int8 NOT NULL, + rid int8 NULL, + job_type varchar(20) DEFAULT 'compare' NOT NULL, + status varchar(20) DEFAULT 'pending' NOT NULL, + priority int4 DEFAULT 5 NOT NULL, + batch_nbr int4 DEFAULT 0 NOT NULL, + table_filter text NULL, + target_server_id uuid NULL, + assigned_server_id uuid NULL, + created_at timestamptz DEFAULT current_timestamp NOT NULL, + scheduled_at timestamptz NULL, + started_at timestamptz NULL, + completed_at timestamptz NULL, + created_by text NULL, + job_config jsonb NULL, + result_summary jsonb NULL, + error_message text NULL, + source varchar(20) DEFAULT 'server' NOT NULL, + CONSTRAINT dc_job_pk PRIMARY KEY (job_id), + CONSTRAINT dc_job_type_check CHECK (job_type IN ('compare', 'check', 'discover', 'test-connection')), + CONSTRAINT dc_job_status_check CHECK (status IN ('pending', 'scheduled', 'running', 'paused', 'completed', 'error', 'failed', 'cancelled')), + CONSTRAINT dc_job_priority_check CHECK (priority BETWEEN 1 AND 10), + CONSTRAINT dc_job_source_check CHECK (source IN ('server', 'standalone', 'api')) + ) + """; + + String REPO_DDL_DC_JOB_IDX1 = """ + CREATE INDEX dc_job_idx1 ON dc_job USING btree (status, priority DESC, created_at) + """; + + String REPO_DDL_DC_JOB_IDX2 = """ + CREATE INDEX dc_job_idx2 ON dc_job USING btree (pid, status) + """; + + String REPO_DDL_DC_JOB_FK1 = """ + ALTER TABLE dc_job ADD CONSTRAINT dc_job_fk1 FOREIGN KEY (pid) REFERENCES dc_project(pid) ON DELETE CASCADE + """; + + // DC_JOB_CONTROL - Job control signals table + String REPO_DDL_DC_JOB_CONTROL = """ + CREATE TABLE dc_job_control ( + control_id serial NOT NULL, + job_id uuid NOT NULL, + signal varchar(20) NOT NULL, + requested_at timestamptz DEFAULT current_timestamp NOT NULL, + processed_at timestamptz NULL, + requested_by text NULL, + CONSTRAINT dc_job_control_pk PRIMARY KEY (control_id), + CONSTRAINT dc_job_control_signal_check CHECK (signal IN ('pause', 'resume', 'stop', 'terminate')) + ) + """; + + String REPO_DDL_DC_JOB_CONTROL_FK1 = """ + ALTER TABLE dc_job_control ADD CONSTRAINT dc_job_control_fk1 FOREIGN KEY (job_id) REFERENCES dc_job(job_id) ON DELETE CASCADE + """; + + // DC_JOB_PROGRESS - Track progress of running jobs (status only, counts come from dc_result) + String REPO_DDL_DC_JOB_PROGRESS = """ + CREATE TABLE dc_job_progress ( + job_id uuid NOT NULL, + tid int8 NOT NULL, + table_name text NOT NULL, + status varchar(20) DEFAULT 'pending' NOT NULL, + started_at timestamptz NULL, + completed_at timestamptz NULL, + error_message text NULL, + cid int4 NULL, + CONSTRAINT dc_job_progress_pk PRIMARY KEY (job_id, tid), + CONSTRAINT dc_job_progress_status_check CHECK (status IN ('pending', 'running', 'completed', 'failed', 'skipped')) + ) + """; + + String REPO_DDL_DC_JOB_PROGRESS_FK1 = """ + ALTER TABLE dc_job_progress ADD CONSTRAINT dc_job_progress_fk1 FOREIGN KEY (job_id) REFERENCES dc_job(job_id) ON DELETE CASCADE + """; + + // DC_JOB_LOG - Store log entries for jobs + String REPO_DDL_DC_JOB_LOG = """ + CREATE TABLE dc_job_log ( + log_id serial NOT NULL, + job_id uuid NOT NULL, + log_ts timestamptz DEFAULT current_timestamp NOT NULL, + log_level varchar(10) NOT NULL, + thread_name varchar(50) NULL, + message text NOT NULL, + context jsonb NULL, + CONSTRAINT dc_job_log_pk PRIMARY KEY (log_id) + ) + """; + + String REPO_DDL_DC_JOB_LOG_IDX1 = """ + CREATE INDEX dc_job_log_idx1 ON dc_job_log USING btree (job_id, log_ts) + """; + + String REPO_DDL_DC_JOB_LOG_FK1 = """ + ALTER TABLE dc_job_log ADD CONSTRAINT dc_job_log_fk1 FOREIGN KEY (job_id) REFERENCES dc_job(job_id) ON DELETE CASCADE + """; + + // Server registration and heartbeat queries + String SQL_SERVER_REGISTER = """ + INSERT INTO dc_server (server_name, server_host, server_pid, status, server_config) + VALUES (?, ?, ?, 'idle', ?::jsonb) + RETURNING server_id + """; + + String SQL_SERVER_DELETE_BY_NAME = """ + DELETE FROM dc_server + WHERE server_name = ? + RETURNING server_id, server_name, status + """; + + String SQL_SERVER_HEARTBEAT = """ + UPDATE dc_server + SET last_heartbeat = current_timestamp, status = ? + WHERE server_id = ?::uuid + """; + + String SQL_SERVER_UNREGISTER = """ + UPDATE dc_server + SET status = 'terminated', last_heartbeat = current_timestamp + WHERE server_id = ?::uuid + """; + + String SQL_SERVER_DELETE = """ + DELETE FROM dc_server WHERE server_id = ?::uuid + """; + + String SQL_SERVER_SELECT_ACTIVE = """ + SELECT server_id, server_name, server_host, server_pid, status, + registered_at, last_heartbeat, current_job_id, server_config + FROM dc_server + WHERE status != 'terminated' + AND last_heartbeat > current_timestamp - interval '5 minutes' + ORDER BY status, last_heartbeat DESC + """; + + String SQL_SERVER_MARK_STALE = """ + UPDATE dc_server + SET status = 'offline' + WHERE status NOT IN ('terminated', 'offline') + AND last_heartbeat < current_timestamp - interval '2 minutes' + """; + + String SQL_SERVER_SELECT_STALE_TO_MARK = """ + SELECT server_id, server_name, server_host + FROM dc_server + WHERE status NOT IN ('terminated', 'offline') + AND last_heartbeat < current_timestamp - interval '2 minutes' + """; + + String SQL_SERVER_DELETE_STALE = """ + DELETE FROM dc_server + WHERE status != 'terminated' + AND last_heartbeat < current_timestamp - interval '5 minutes' + """; + + String SQL_SERVER_SELECT_STALE_TO_DELETE = """ + SELECT server_id, server_name, server_host + FROM dc_server + WHERE status != 'terminated' + AND last_heartbeat < current_timestamp - interval '5 minutes' + """; + + // Orphaned job detection - jobs that are running but assigned server is offline/terminated/missing + String SQL_JOB_SELECT_ORPHANED = """ + SELECT j.job_id, j.job_type, j.assigned_server_id, s.server_name, s.status as server_status + FROM dc_job j + LEFT JOIN dc_server s ON j.assigned_server_id = s.server_id + WHERE j.status = 'running' + AND j.source != 'standalone' + AND (s.server_id IS NULL + OR s.status IN ('offline', 'terminated') + OR s.last_heartbeat < current_timestamp - interval '5 minutes') + """; + + String SQL_JOB_MARK_ORPHANED_FAILED = """ + UPDATE dc_job + SET status = 'failed', + completed_at = current_timestamp, + error_message = 'Job orphaned: assigned server is no longer available' + WHERE job_id = ?::uuid AND status = 'running' + """; + + // Job queue management queries + String SQL_JOB_INSERT = """ + INSERT INTO dc_job (pid, job_type, priority, batch_nbr, table_filter, + target_server_id, scheduled_at, created_by, job_config) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb) + RETURNING job_id + """; + + String SQL_JOB_CLAIM_NEXT = """ + UPDATE dc_job + SET status = 'running', assigned_server_id = ?::uuid, started_at = current_timestamp + WHERE job_id = ( + SELECT job_id FROM dc_job + WHERE status = 'pending' + AND (scheduled_at IS NULL OR scheduled_at <= current_timestamp) + AND (target_server_id IS NULL OR target_server_id = ?::uuid) + ORDER BY priority DESC, created_at ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED + ) + RETURNING job_id, pid, job_type, batch_nbr, table_filter, job_config + """; + + String SQL_JOB_UPDATE_STATUS = """ + UPDATE dc_job + SET status = ?, completed_at = CASE WHEN ? IN ('completed', 'error', 'failed', 'cancelled') THEN current_timestamp ELSE completed_at END, + result_summary = COALESCE(?::jsonb, result_summary), + error_message = COALESCE(?, error_message) + WHERE job_id = ?::uuid + """; + + String SQL_JOB_SELECT_BY_STATUS = """ + SELECT job_id, pid, job_type, status, priority, batch_nbr, table_filter, + target_server_id, assigned_server_id, created_at, scheduled_at, + started_at, completed_at, created_by, job_config, result_summary, error_message + FROM dc_job + WHERE status = ANY(?) + ORDER BY priority DESC, created_at ASC + """; + + String SQL_JOB_SELECT_BY_PROJECT = """ + SELECT job_id, pid, job_type, status, priority, batch_nbr, table_filter, + target_server_id, assigned_server_id, created_at, scheduled_at, + started_at, completed_at, created_by, job_config, result_summary, error_message + FROM dc_job + WHERE pid = ? + ORDER BY created_at DESC + LIMIT ? + """; + + String SQL_JOB_SELECT_RUNNING = """ + SELECT w.job_id, w.pid, w.job_type, w.status, w.batch_nbr, w.started_at, + s.server_name, s.server_host + FROM dc_job w + LEFT JOIN dc_server s ON w.assigned_server_id = s.server_id + WHERE w.status = 'running' + """; + + // Job control queries + String SQL_JOBCONTROL_INSERT = """ + INSERT INTO dc_job_control (job_id, signal, requested_by) + VALUES (?::uuid, ?, ?) + RETURNING control_id + """; + + String SQL_JOBCONTROL_CHECK_PENDING = """ + SELECT control_id, signal + FROM dc_job_control + WHERE job_id = ?::uuid AND processed_at IS NULL + ORDER BY requested_at ASC + LIMIT 1 + """; + + String SQL_JOBCONTROL_MARK_PROCESSED = """ + UPDATE dc_job_control + SET processed_at = current_timestamp + WHERE control_id = ? + """; + + // Job progress queries + String SQL_JOBPROGRESS_INSERT = """ + INSERT INTO dc_job_progress (job_id, tid, table_name, status) + VALUES (?::uuid, ?, ?, 'pending') + ON CONFLICT (job_id, tid) DO UPDATE SET status = 'pending' + """; + + String SQL_JOBPROGRESS_UPDATE = """ + UPDATE dc_job_progress + SET status = ?, started_at = COALESCE(started_at, current_timestamp), + completed_at = CASE WHEN ? IN ('completed', 'failed') THEN current_timestamp ELSE completed_at END, + error_message = COALESCE(?, error_message), + cid = COALESCE(?, cid) + WHERE job_id = ?::uuid AND tid = ? + """; + + String SQL_JOBPROGRESS_SELECT_BY_JOB = """ + SELECT job_id, tid, table_name, status, started_at, completed_at, + error_message, cid + FROM dc_job_progress + WHERE job_id = ?::uuid + ORDER BY table_name + """; + + String SQL_JOBPROGRESS_SUMMARY = """ + SELECT + COUNT(*) as total_tables, + COUNT(*) FILTER (WHERE jp.status = 'completed') as completed_tables, + COUNT(*) FILTER (WHERE jp.status = 'running') as running_tables, + COUNT(*) FILTER (WHERE jp.status = 'failed') as failed_tables, + COALESCE(SUM(r.source_cnt), 0) as total_source, + COALESCE(SUM(r.equal_cnt), 0) as total_equal, + COALESCE(SUM(r.not_equal_cnt), 0) as total_not_equal, + COALESCE(SUM(r.missing_source_cnt), 0) + COALESCE(SUM(r.missing_target_cnt), 0) as total_missing + FROM dc_job_progress jp + LEFT JOIN dc_result r ON jp.cid = r.cid + WHERE jp.job_id = ?::uuid + """; + + String SQL_JOB_SET_RID = """ + UPDATE dc_job SET rid = ? WHERE job_id = ?::uuid + """; + + // Job log queries + String SQL_JOBLOG_INSERT = """ + INSERT INTO dc_job_log (job_id, log_level, thread_name, message, context) + VALUES (?::uuid, ?, ?, ?, ?::jsonb) + """; + + String SQL_JOBLOG_SELECT_BY_JOB = """ + SELECT log_id, job_id, log_ts, log_level, thread_name, message, context + FROM dc_job_log + WHERE job_id = ?::uuid + ORDER BY log_ts, log_id + """; + + String SQL_JOBLOG_SELECT_BY_JOB_PAGED = """ + SELECT log_id, job_id, log_ts, log_level, thread_name, message, context + FROM dc_job_log + WHERE job_id = ?::uuid + ORDER BY log_ts, log_id + LIMIT ? OFFSET ? + """; + + String SQL_JOBLOG_COUNT_BY_JOB = """ + SELECT COUNT(*) FROM dc_job_log WHERE job_id = ?::uuid + """; + + // Standalone job creation + String SQL_JOB_CREATE_STANDALONE = """ + INSERT INTO dc_job (pid, job_type, status, batch_nbr, table_filter, started_at, source, rid) + VALUES (?, ?, 'running', ?, ?, current_timestamp, 'standalone', ?) + RETURNING job_id + """; +} diff --git a/src/main/java/com/crunchydata/controller/CompareController.java b/src/main/java/com/crunchydata/controller/CompareController.java index d86c05b..a45eaff 100644 --- a/src/main/java/com/crunchydata/controller/CompareController.java +++ b/src/main/java/com/crunchydata/controller/CompareController.java @@ -24,6 +24,7 @@ import com.crunchydata.model.DataComparisonTable; import com.crunchydata.model.DataComparisonTableMap; import com.crunchydata.core.database.SQLExecutionHelper; +import com.crunchydata.service.StandaloneJobService; import com.crunchydata.util.LoggingUtils; import javax.sql.rowset.CachedRowSet; @@ -57,6 +58,16 @@ public class CompareController { * @param context Application context */ public static void performCompare(ApplicationContext context) { + performCompare(context, null); + } + + /** + * Perform database comparison operation with optional job progress tracking. + * + * @param context Application context + * @param jobService Optional job service for progress tracking (can be null) + */ + public static void performCompare(ApplicationContext context, StandaloneJobService jobService) { boolean isCheck = Props.getProperty("isCheck").equals("true"); String tableFilter = context.getCmd().hasOption("table") ? context.getCmd().getOptionValue("table").toLowerCase() : ""; @@ -68,8 +79,8 @@ public static void performCompare(ApplicationContext context) { RepoController repoController = new RepoController(); CachedRowSet tablesResultSet = getTables(context.getPid(), context.getConnRepo(), context.getBatchParameter(), tableFilter, isCheck); - // Process tables and collect results - TableController.ComparisonResults results = TableController.reconcileTables(tablesResultSet, isCheck, repoController, context); + // Process tables and collect results (with optional progress tracking) + TableController.ComparisonResults results = TableController.reconcileTables(tablesResultSet, isCheck, repoController, context, jobService); // Close result set if (tablesResultSet != null) { @@ -125,7 +136,7 @@ public static JSONObject reconcileData(Connection connRepo, Connection connSourc ColumnMetadata ciTarget = getColumnInfo(columnMap, "target", Props.getProperty("target-type"), dctmTarget.getSchemaName(), dctmTarget.getTableName(), - !check && "database".equals(Props.getProperty("column-hash-method"))); + "database".equals(Props.getProperty("column-hash-method"))); logColumnMetadata(ciSource, ciTarget); diff --git a/src/main/java/com/crunchydata/controller/MappingController.java b/src/main/java/com/crunchydata/controller/MappingController.java new file mode 100644 index 0000000..88c36ea --- /dev/null +++ b/src/main/java/com/crunchydata/controller/MappingController.java @@ -0,0 +1,73 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.controller; + +import com.crunchydata.service.MappingExportService; +import com.crunchydata.service.MappingImportService; +import com.crunchydata.util.LoggingUtils; + +import java.sql.Connection; + +public class MappingController { + + private static final String THREAD_NAME = "mapping-ctrl"; + + public static void performExport(Connection connRepo, Integer pid, String tableFilter, String outputFile) { + try { + LoggingUtils.write("info", THREAD_NAME, "Starting mapping export"); + + String file = outputFile; + if (file == null || file.isEmpty()) { + file = "pgcompare-mappings-" + pid + ".yaml"; + } + + MappingExportService.exportToYaml(connRepo, pid, tableFilter, file); + + LoggingUtils.write("info", THREAD_NAME, String.format("Export completed successfully to: %s", file)); + System.out.println("Export completed: " + file); + + } catch (Exception e) { + LoggingUtils.write("severe", THREAD_NAME, String.format("Export failed: %s", e.getMessage())); + throw new RuntimeException("Mapping export failed", e); + } + } + + public static void performImport(Connection connRepo, Integer pid, String tableFilter, + String inputFile, boolean overwrite) { + try { + LoggingUtils.write("info", THREAD_NAME, "Starting mapping import"); + + if (inputFile == null || inputFile.isEmpty()) { + throw new IllegalArgumentException("Input file is required for import. Use --file option."); + } + + MappingImportService.ImportResult result = MappingImportService.importFromYaml( + connRepo, pid, inputFile, overwrite, tableFilter); + + LoggingUtils.write("info", THREAD_NAME, "Import completed successfully"); + System.out.println("Import completed:"); + System.out.println(" Tables added: " + result.tablesAdded()); + System.out.println(" Tables updated: " + result.tablesUpdated()); + System.out.println(" Tables skipped: " + result.tablesSkipped()); + System.out.println(" Columns processed: " + result.columnsProcessed()); + + } catch (Exception e) { + LoggingUtils.write("severe", THREAD_NAME, String.format("Import failed: %s", e.getMessage())); + throw new RuntimeException("Mapping import failed", e); + } + } +} diff --git a/src/main/java/com/crunchydata/controller/RepoController.java b/src/main/java/com/crunchydata/controller/RepoController.java index 4f13d8d..339c992 100644 --- a/src/main/java/com/crunchydata/controller/RepoController.java +++ b/src/main/java/com/crunchydata/controller/RepoController.java @@ -93,13 +93,14 @@ public void deleteDataCompare(Connection conn, Integer tid, Integer batchNbr) { try { ArrayList binds = new ArrayList<>(); binds.add(tid); - binds.add(batchNbr); - SQLExecutionHelper.simpleUpdate(conn, "DELETE FROM dc_source WHERE tid=? AND batch_nbr=?", binds, true); - SQLExecutionHelper.simpleUpdate(conn, "DELETE FROM dc_target WHERE tid=? AND batch_nbr=?", binds, true); + // Delete all rows for this tid (not filtered by batch_nbr) to ensure clean slate + // The MARK queries in ResultProcessor use tid only, so we must purge all data for tid + SQLExecutionHelper.simpleUpdate(conn, "DELETE FROM dc_source WHERE tid=?", binds, true); + SQLExecutionHelper.simpleUpdate(conn, "DELETE FROM dc_target WHERE tid=?", binds, true); LoggingUtils.write("info", THREAD_NAME, - String.format("Data comparison results deleted for table %d, batch %d", tid, batchNbr)); + String.format("Data comparison results deleted for table %d", tid)); } catch (Exception e) { LoggingUtils.write("severe", THREAD_NAME, @@ -121,6 +122,21 @@ public static String getProjectConfig(Connection conn, Integer pid) { return SQLExecutionHelper.simpleSelectReturnString(conn, SQL_REPO_DCPROJECT_GETBYPID, binds); } + /** + * Saves the project settings to dc_project table. + * + * @param conn Database connection + * @param pid Project ID + * @param configJson JSON string containing project configuration + */ + public static void saveProjectConfig(Connection conn, Integer pid, String configJson) { + ArrayList binds = new ArrayList<>(); + binds.add(configJson); + binds.add(pid); + SQLExecutionHelper.simpleUpdate(conn, "UPDATE dc_project SET project_config = ?::jsonb WHERE pid = ?", binds, true); + LoggingUtils.write("info", THREAD_NAME, String.format("Project config saved for project %d", pid)); + } + /** * Loads findings from the staging table into the main table using the optimized StagingOperationsService. * diff --git a/src/main/java/com/crunchydata/controller/TableController.java b/src/main/java/com/crunchydata/controller/TableController.java index b931726..4eab86b 100644 --- a/src/main/java/com/crunchydata/controller/TableController.java +++ b/src/main/java/com/crunchydata/controller/TableController.java @@ -20,6 +20,7 @@ import com.crunchydata.core.database.SQLExecutionHelper; import com.crunchydata.model.DataComparisonTable; import com.crunchydata.model.DataComparisonTableMap; +import com.crunchydata.service.StandaloneJobService; import com.crunchydata.util.LoggingUtils; import org.json.JSONArray; import org.json.JSONObject; @@ -115,10 +116,35 @@ public static int performCopyTable(ApplicationContext context) { * @throws SQLException if database operations fail */ public static ComparisonResults reconcileTables(CachedRowSet tablesResultSet, boolean isCheck, RepoController repoController, ApplicationContext context) throws SQLException { + return reconcileTables(tablesResultSet, isCheck, repoController, context, null); + } + + /** + * Process all tables in the result set with optional job progress tracking. + * + * @param tablesResultSet Result set containing tables to process + * @param isCheck Whether this is a recheck operation + * @param repoController Repository controller instance + * @param context Application context + * @param jobService Optional job service for progress tracking (can be null) + * @return ComparisonResults containing processed tables and results + * @throws SQLException if database operations fail + */ + public static ComparisonResults reconcileTables(CachedRowSet tablesResultSet, boolean isCheck, RepoController repoController, ApplicationContext context, StandaloneJobService jobService) throws SQLException { JSONArray runResults = new JSONArray(); int tablesProcessed = 0; + // Pre-populate progress for all tables if job service is available + if (jobService != null) { + while (tablesResultSet.next()) { + long tid = tablesResultSet.getLong("tid"); + String tableAlias = tablesResultSet.getString("table_alias"); + jobService.initializeTableProgress(tid, tableAlias); + } + tablesResultSet.beforeFirst(); + } + while (tablesResultSet.next()) { tablesProcessed++; @@ -127,11 +153,33 @@ public static ComparisonResults reconcileTables(CachedRowSet tablesResultSet, bo JSONObject actionResult; + // Mark table as running + if (jobService != null) { + jobService.updateTableProgress(table.getTid(), "running", null, null); + } + if (table.getEnabled()) { actionResult = reconcileEnabledTable(table, isCheck, repoController, context); } else { actionResult = createSkippedTableResult(table); } + + // Update table progress based on result + if (jobService != null) { + String status = "completed"; + String errorMessage = null; + Integer cid = actionResult.optInt("cid", 0); + + String compareStatus = actionResult.optString("compareStatus", ""); + if (STATUS_FAILED.equals(compareStatus) || STATUS_ERROR.equals(actionResult.optString("status"))) { + status = "failed"; + errorMessage = actionResult.optString("error", null); + } else if (STATUS_SKIPPED.equals(actionResult.optString("status")) || STATUS_DISABLED.equals(compareStatus)) { + status = "skipped"; + } + + jobService.updateTableProgress(table.getTid(), status, errorMessage, cid > 0 ? cid : null); + } runResults.put(actionResult); } diff --git a/src/main/java/com/crunchydata/core/comparison/ResultProcessor.java b/src/main/java/com/crunchydata/core/comparison/ResultProcessor.java index 89cc864..696f141 100644 --- a/src/main/java/com/crunchydata/core/comparison/ResultProcessor.java +++ b/src/main/java/com/crunchydata/core/comparison/ResultProcessor.java @@ -93,16 +93,20 @@ private static ReconciliationStats calculateReconciliationStats(Connection connR binds.add(tid); binds.add(tid); - // Calculate missing source records + // Calculate missing source records (rows in target that don't exist in source) int missingSource = SQLExecutionHelper.simpleUpdate(connRepo, SQL_REPO_DCSOURCE_MARKMISSING, binds, true); - // Calculate missing target records + // Calculate missing target records (rows in source that don't exist in target) int missingTarget = SQLExecutionHelper.simpleUpdate(connRepo, SQL_REPO_DCTARGET_MARKMISSING, binds, true); // Calculate not equal records int notEqual = SQLExecutionHelper.simpleUpdate(connRepo, SQL_REPO_DCSOURCE_MARKNOTEQUAL, binds, true); SQLExecutionHelper.simpleUpdate(connRepo, SQL_REPO_DCTARGET_MARKNOTEQUAL, binds, true); + LoggingUtils.write("info", THREAD_NAME, + String.format("Reconciliation stats for tid=%d: missingSource=%d, missingTarget=%d, notEqual=%d", + tid, missingSource, missingTarget, notEqual)); + return new ReconciliationStats(missingSource, missingTarget, notEqual); } @@ -118,7 +122,8 @@ private static void updateResultWithStats(JSONObject result, ReconciliationStats result.put("notEqual", stats.notEqual()); // Determine final status - if ("processing".equals(result.getString("compareStatus"))) { + String currentStatus = result.optString("compareStatus", "processing"); + if ("processing".equals(currentStatus)) { boolean hasOutOfSyncRecords = stats.missingSource() + stats.missingTarget() + stats.notEqual() > 0; result.put("compareStatus", hasOutOfSyncRecords ? "out-of-sync" : "in-sync"); } @@ -143,8 +148,17 @@ private static void updateDatabaseResults(Connection connRepo, JSONObject result try (var crs = SQLExecutionHelper.simpleUpdateReturning(connRepo, SQL_REPO_DCRESULT_UPDATE_STATUSANDCOUNT, binds)) { if (crs.next()) { int equal = crs.getInt(1); + int sourceCnt = crs.getInt(6); + int targetCnt = crs.getInt(7); result.put("equal", equal); + result.put("sourceCount", sourceCnt); + result.put("targetCount", targetCnt); result.put("totalRows", equal + result.getInt("missingSource") + result.getInt("missingTarget") + result.getInt("notEqual")); + + LoggingUtils.write("info", THREAD_NAME, + String.format("Database results for cid=%d: equal=%d, sourceCount=%d, targetCount=%d, missingSource=%d, missingTarget=%d, notEqual=%d", + cid, equal, sourceCnt, targetCnt, + result.getInt("missingSource"), result.getInt("missingTarget"), result.getInt("notEqual"))); } } } diff --git a/src/main/java/com/crunchydata/core/threading/DataComparisonThread.java b/src/main/java/com/crunchydata/core/threading/DataComparisonThread.java index 5848b57..6400042 100644 --- a/src/main/java/com/crunchydata/core/threading/DataComparisonThread.java +++ b/src/main/java/com/crunchydata/core/threading/DataComparisonThread.java @@ -23,6 +23,7 @@ import java.text.DecimalFormat; import java.util.concurrent.BlockingQueue; +import com.crunchydata.config.ApplicationState; import com.crunchydata.controller.RepoController; import com.crunchydata.model.ColumnMetadata; import com.crunchydata.model.DataComparisonTable; @@ -122,6 +123,8 @@ public void run() { //conn.setAutoCommit(false); stmt = conn.prepareStatement(sql); stmt.setFetchSize(fetchSize); + + ApplicationState.getInstance().registerStatement(stmt); rs = stmt.executeQuery(); StringBuilder columnValue = new StringBuilder(); @@ -135,6 +138,15 @@ public void run() { DataComparisonResult[] dc = new DataComparisonResult[batchCommitSize]; while (rs.next()) { + if (ApplicationState.getInstance().isImmediateTerminationRequested()) { + LoggingUtils.write("info", threadName, String.format("(%s) Immediate termination requested - stopping now", targetType)); + break; + } + if (ts.isShutdownRequested()) { + LoggingUtils.write("info", threadName, String.format("(%s) Shutdown requested - stopping gracefully", targetType)); + break; + } + columnValue.setLength(0); if (! useDatabaseHash) { @@ -350,6 +362,8 @@ private void signalThreadCompletion() { */ private void cleanupResources(String threadName, ResultSet rs, PreparedStatement stmt, PreparedStatement stmtLoad, Connection connRepo, Connection conn) { + ApplicationState.getInstance().unregisterStatement(stmt); + try { if (rs != null) { rs.close(); diff --git a/src/main/java/com/crunchydata/core/threading/DataValidationThread.java b/src/main/java/com/crunchydata/core/threading/DataValidationThread.java index 72e5d77..7467120 100644 --- a/src/main/java/com/crunchydata/core/threading/DataValidationThread.java +++ b/src/main/java/com/crunchydata/core/threading/DataValidationThread.java @@ -84,6 +84,13 @@ public static JSONObject checkRows (Connection repoConn, Connection sourceConn, PreparedStatement stmt = null; ResultSet rs = null; + // Get job_id from logging context if available + String jobId = null; + LoggingUtils.JobLogContext ctx = LoggingUtils.getJobContext(); + if (ctx != null && ctx.getJobId() != null) { + jobId = ctx.getJobId().toString(); + } + try { stmt = repoConn.prepareStatement(SQL_REPO_SELECT_OUTOFSYNC_ROWS); stmt.setObject(1, dct.getTid()); @@ -150,10 +157,20 @@ public static JSONObject checkRows (Connection repoConn, Connection sourceConn, JSONObject row = rows.getJSONObject(i); if (row.has("fixSQL")) { JSONObject fixSQLEntry = new JSONObject(); - fixSQLEntry.put("pk", row.get("pk")); - fixSQLEntry.put("sql", row.getString("fixSQL")); + String pkValue = row.getString("pk"); + String pkHash = row.getString("pkHash"); + String fixSQL = row.getString("fixSQL"); + fixSQLEntry.put("pk", pkValue); + fixSQLEntry.put("sql", fixSQL); fixSQLStatements.put(fixSQLEntry); fixSQLCount++; + + // Store fix SQL to database (dc_source or dc_target) + storeFixSQL(repoConn, dct.getTid(), pkHash, fixSQL); + + // Log each fix SQL statement + LoggingUtils.write("info", THREAD_NAME, + String.format("Fix SQL [%s] PK=%s: %s", dct.getTableAlias(), pkValue, fixSQL)); } } @@ -219,6 +236,7 @@ public static JSONObject compareRowforCheck (Connection repoConn, Connection sou try { rowResult.put("pk", dcRow.getPk()); + rowResult.put("pkHash", dcRow.getPkHash()); if (sourceRow.size() > 0 && targetRow.size() == 0) { rowResult.put("compareStatus", OUT_OF_SYNC_STATUS); @@ -349,5 +367,50 @@ private static void updateResultCounts(Connection repoConn, JSONObject rowResult SQLExecutionHelper.simpleUpdate(repoConn, SQL_REPO_DCRESULT_UPDATE_ALLCOUNTS, binds, true); } + + /** + * Stores fix SQL to dc_source or dc_target based on fix type. + * INSERT/UPDATE → dc_source (row exists in source) + * DELETE → dc_target (row only exists in target) + */ + private static void storeFixSQL(Connection repoConn, long tid, String pkHash, String fixSQL) { + try { + String fixType = determineFixType(fixSQL); + String sql; + + if ("delete".equals(fixType)) { + sql = SQL_REPO_DCTARGET_UPDATE_FIXSQL; + } else { + sql = SQL_REPO_DCSOURCE_UPDATE_FIXSQL; + } + + ArrayList binds = new ArrayList<>(); + binds.add(fixSQL); + binds.add(tid); + binds.add(pkHash); + + SQLExecutionHelper.simpleUpdate(repoConn, sql, binds, true); + LoggingUtils.write("debug", THREAD_NAME, + String.format("Stored fix SQL (type=%s) for tid=%d, pk_hash=%s", fixType, tid, pkHash)); + } catch (Exception e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Failed to store fix SQL to database: %s", e.getMessage())); + } + } + + /** + * Determines the fix type from SQL statement. + */ + private static String determineFixType(String sql) { + String trimmedSQL = sql.trim().toUpperCase(); + if (trimmedSQL.startsWith("INSERT")) { + return "insert"; + } else if (trimmedSQL.startsWith("UPDATE")) { + return "update"; + } else if (trimmedSQL.startsWith("DELETE")) { + return "delete"; + } + return "update"; // default fallback + } } diff --git a/src/main/java/com/crunchydata/core/threading/ObserverThread.java b/src/main/java/com/crunchydata/core/threading/ObserverThread.java index a6bd273..31bc25e 100644 --- a/src/main/java/com/crunchydata/core/threading/ObserverThread.java +++ b/src/main/java/com/crunchydata/core/threading/ObserverThread.java @@ -22,6 +22,7 @@ import java.text.DecimalFormat; import java.util.ArrayList; +import com.crunchydata.config.ApplicationState; import com.crunchydata.controller.RepoController; import com.crunchydata.model.DataComparisonTable; import com.crunchydata.core.database.SQLExecutionHelper; @@ -178,6 +179,15 @@ private void executeReconciliationObserver(String threadName, Connection repoCon int tmpRowCount; while (lastRun <= MAX_LAST_RUN_COUNT) { + if (ApplicationState.getInstance().isImmediateTerminationRequested()) { + LoggingUtils.write("info", threadName, "Immediate termination requested - stopping now"); + break; + } + if (ApplicationState.getInstance().isShutdownRequested()) { + LoggingUtils.write("info", threadName, "Shutdown requested - completing current reconciliation"); + break; + } + // Remove matching rows tmpRowCount = stmtSU.executeUpdate(); cntEqual += tmpRowCount; diff --git a/src/main/java/com/crunchydata/core/threading/ThreadManager.java b/src/main/java/com/crunchydata/core/threading/ThreadManager.java index 378b8cc..963fd38 100644 --- a/src/main/java/com/crunchydata/core/threading/ThreadManager.java +++ b/src/main/java/com/crunchydata/core/threading/ThreadManager.java @@ -16,6 +16,7 @@ package com.crunchydata.core.threading; +import com.crunchydata.config.ApplicationState; import com.crunchydata.controller.RepoController; import com.crunchydata.model.ColumnMetadata; import com.crunchydata.model.DataComparisonTable; @@ -201,6 +202,11 @@ private static void waitForThreadCompletion() throws InterruptedException { LoggingUtils.write("info", THREAD_NAME, "Waiting for reconcile threads to complete"); joinThreads(observerList); + if (ApplicationState.getInstance().isShutdownRequested()) { + LoggingUtils.write("info", THREAD_NAME, "Graceful shutdown completed"); + ApplicationState.getInstance().markShutdownComplete(); + } + LoggingUtils.write("info", THREAD_NAME, "All reconciliation threads completed"); } diff --git a/src/main/java/com/crunchydata/core/threading/ThreadSync.java b/src/main/java/com/crunchydata/core/threading/ThreadSync.java index 2fe97c4..ad1423a 100644 --- a/src/main/java/com/crunchydata/core/threading/ThreadSync.java +++ b/src/main/java/com/crunchydata/core/threading/ThreadSync.java @@ -16,6 +16,8 @@ package com.crunchydata.core.threading; +import com.crunchydata.config.ApplicationState; + /** * Utility class for thread synchronization. * @@ -84,4 +86,13 @@ public synchronized void observerNotify() { } } + /** + * Check if a graceful shutdown has been requested. + * + * @return true if shutdown was requested via signal + */ + public boolean isShutdownRequested() { + return ApplicationState.getInstance().isShutdownRequested(); + } + } diff --git a/src/main/java/com/crunchydata/model/yaml/ColumnDefinition.java b/src/main/java/com/crunchydata/model/yaml/ColumnDefinition.java new file mode 100644 index 0000000..f578a68 --- /dev/null +++ b/src/main/java/com/crunchydata/model/yaml/ColumnDefinition.java @@ -0,0 +1,31 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.model.yaml; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ColumnDefinition { + private String alias; + private Boolean enabled; + private ColumnMapping source; + private ColumnMapping target; +} diff --git a/src/main/java/com/crunchydata/model/yaml/ColumnMapping.java b/src/main/java/com/crunchydata/model/yaml/ColumnMapping.java new file mode 100644 index 0000000..6ef506d --- /dev/null +++ b/src/main/java/com/crunchydata/model/yaml/ColumnMapping.java @@ -0,0 +1,38 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.model.yaml; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ColumnMapping { + private String columnName; + private String dataType; + private String dataClass; + private Integer dataLength; + private Integer numberPrecision; + private Integer numberScale; + private Boolean nullable; + private Boolean primaryKey; + private String mapExpression; + private Boolean supported; + private Boolean preserveCase; +} diff --git a/src/main/java/com/crunchydata/model/yaml/MappingExport.java b/src/main/java/com/crunchydata/model/yaml/MappingExport.java new file mode 100644 index 0000000..fdd5cc0 --- /dev/null +++ b/src/main/java/com/crunchydata/model/yaml/MappingExport.java @@ -0,0 +1,34 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.model.yaml; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class MappingExport { + private String version; + private String exportDate; + private Integer projectId; + private String projectName; + private List tables; +} diff --git a/src/main/java/com/crunchydata/model/yaml/TableDefinition.java b/src/main/java/com/crunchydata/model/yaml/TableDefinition.java new file mode 100644 index 0000000..85c1aba --- /dev/null +++ b/src/main/java/com/crunchydata/model/yaml/TableDefinition.java @@ -0,0 +1,36 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.model.yaml; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class TableDefinition { + private String alias; + private Boolean enabled; + private Integer batchNumber; + private Integer parallelDegree; + private TableLocation source; + private TableLocation target; + private List columns; +} diff --git a/src/main/java/com/crunchydata/model/yaml/TableLocation.java b/src/main/java/com/crunchydata/model/yaml/TableLocation.java new file mode 100644 index 0000000..15a3d76 --- /dev/null +++ b/src/main/java/com/crunchydata/model/yaml/TableLocation.java @@ -0,0 +1,31 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.model.yaml; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class TableLocation { + private String schema; + private String table; + private Boolean schemaPreserveCase; + private Boolean tablePreserveCase; +} diff --git a/src/main/java/com/crunchydata/service/ConfigExportService.java b/src/main/java/com/crunchydata/service/ConfigExportService.java new file mode 100644 index 0000000..33e2cba --- /dev/null +++ b/src/main/java/com/crunchydata/service/ConfigExportService.java @@ -0,0 +1,114 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.service; + +import com.crunchydata.config.Settings; +import com.crunchydata.controller.RepoController; +import com.crunchydata.util.LoggingUtils; +import org.json.JSONObject; + +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.sql.Connection; +import java.util.Iterator; +import java.util.Properties; +import java.util.TreeSet; + +public class ConfigExportService { + + private static final String THREAD_NAME = "config-export"; + + public static ExportResult exportToProperties(Connection conn, Integer pid, String outputFile) + throws IOException { + + if (outputFile == null || outputFile.isEmpty()) { + throw new IllegalArgumentException("Output file path is required for export-config. Use --file parameter."); + } + + LoggingUtils.write("info", THREAD_NAME, + String.format("Exporting configuration from project %d to %s", pid, outputFile)); + + String configJsonStr = RepoController.getProjectConfig(conn, pid); + JSONObject configJson = new JSONObject(configJsonStr); + + if (configJson.isEmpty()) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("No configuration found for project %d", pid)); + return new ExportResult(0, outputFile); + } + + Properties defaultProps = Settings.setDefaults(); + int exportedCount = 0; + + try (PrintWriter writer = new PrintWriter(new FileWriter(outputFile))) { + writer.println("# pgCompare Configuration"); + writer.println("# Exported from project " + pid); + writer.println("# Only non-default values are included"); + writer.println(); + + TreeSet sortedKeys = new TreeSet<>(); + Iterator keys = configJson.keys(); + while (keys.hasNext()) { + sortedKeys.add(keys.next()); + } + + String currentSection = ""; + for (String key : sortedKeys) { + String value = configJson.get(key).toString(); + String defaultValue = defaultProps.getProperty(key); + + if (defaultValue != null && value.equals(defaultValue)) { + continue; + } + + String section = getSectionForKey(key); + if (!section.equals(currentSection)) { + if (!currentSection.isEmpty()) { + writer.println(); + } + writer.println("# " + section); + currentSection = section; + } + + writer.println(key + "=" + value); + exportedCount++; + LoggingUtils.write("info", THREAD_NAME, + String.format("Exported property: %s = %s", key, value)); + } + } + + LoggingUtils.write("info", THREAD_NAME, + String.format("Export complete: %d properties exported to %s", exportedCount, outputFile)); + + return new ExportResult(exportedCount, outputFile); + } + + private static String getSectionForKey(String key) { + if (key.startsWith("repo-")) { + return "Repository Settings"; + } else if (key.startsWith("source-")) { + return "Source Database Settings"; + } else if (key.startsWith("target-")) { + return "Target Database Settings"; + } else { + return "System Settings"; + } + } + + public record ExportResult(int propertiesExported, String outputFile) {} +} diff --git a/src/main/java/com/crunchydata/service/ConfigImportService.java b/src/main/java/com/crunchydata/service/ConfigImportService.java new file mode 100644 index 0000000..c9cf44f --- /dev/null +++ b/src/main/java/com/crunchydata/service/ConfigImportService.java @@ -0,0 +1,134 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.service; + +import com.crunchydata.config.Settings; +import com.crunchydata.core.database.SQLExecutionHelper; +import com.crunchydata.util.LoggingUtils; +import org.json.JSONObject; + +import java.io.FileInputStream; +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Properties; + +public class ConfigImportService { + + private static final String THREAD_NAME = "config-import"; + + private static final String SQL_CHECK_PROJECT_EXISTS = """ + SELECT pid FROM dc_project WHERE pid = ? + """; + + private static final String SQL_INSERT_PROJECT = """ + INSERT INTO dc_project (project_name, project_config) VALUES (?, ?::jsonb) + RETURNING pid + """; + + private static final String SQL_UPDATE_PROJECT_CONFIG = """ + UPDATE dc_project SET project_config = ?::jsonb WHERE pid = ? + """; + + public static ImportResult importFromProperties(Connection conn, Integer pid, String inputFile) + throws IOException, SQLException { + + LoggingUtils.write("info", THREAD_NAME, + String.format("Importing configuration from %s to project %d", inputFile, pid)); + + Properties fileProps = new Properties(); + try (FileInputStream fis = new FileInputStream(inputFile)) { + fileProps.load(fis); + } + + Properties defaultProps = Settings.setDefaults(); + JSONObject configJson = new JSONObject(); + int importedCount = 0; + int skippedCount = 0; + + for (String key : fileProps.stringPropertyNames()) { + String rawValue = fileProps.getProperty(key); + if (rawValue == null) { + skippedCount++; + continue; + } + String value = rawValue.trim(); + String defaultValue = defaultProps.getProperty(key); + + if (key.contains("password") || key.equals("config-file")) { + skippedCount++; + LoggingUtils.write("info", THREAD_NAME, + String.format("Skipping sensitive/system property: %s", key)); + continue; + } + + if (defaultValue == null || !value.equals(defaultValue)) { + configJson.put(key, value); + importedCount++; + LoggingUtils.write("info", THREAD_NAME, + String.format("Importing property: %s = %s", key, value)); + } else { + skippedCount++; + } + } + + boolean projectExists = checkProjectExists(conn, pid); + + if (projectExists) { + updateProjectConfig(conn, pid, configJson.toString()); + LoggingUtils.write("info", THREAD_NAME, + String.format("Updated existing project %d with configuration", pid)); + } else { + Integer newPid = createProject(conn, "project-" + pid, configJson.toString()); + LoggingUtils.write("info", THREAD_NAME, + String.format("Created new project with pid %d", newPid)); + } + + conn.commit(); + + ImportResult result = new ImportResult(importedCount, skippedCount, projectExists); + LoggingUtils.write("info", THREAD_NAME, + String.format("Import complete: %d properties imported, %d skipped, project %s", + importedCount, skippedCount, projectExists ? "updated" : "created")); + + return result; + } + + private static boolean checkProjectExists(Connection conn, Integer pid) { + ArrayList binds = new ArrayList<>(); + binds.add(pid); + Integer existingPid = SQLExecutionHelper.simpleSelectReturnInteger(conn, SQL_CHECK_PROJECT_EXISTS, binds); + return existingPid != null; + } + + private static void updateProjectConfig(Connection conn, Integer pid, String configJson) { + ArrayList binds = new ArrayList<>(); + binds.add(configJson); + binds.add(pid); + SQLExecutionHelper.simpleUpdate(conn, SQL_UPDATE_PROJECT_CONFIG, binds, true); + } + + private static Integer createProject(Connection conn, String projectName, String configJson) { + ArrayList binds = new ArrayList<>(); + binds.add(projectName); + binds.add(configJson); + return SQLExecutionHelper.simpleUpdateReturningInteger(conn, SQL_INSERT_PROJECT, binds); + } + + public record ImportResult(int propertiesImported, int propertiesSkipped, boolean projectUpdated) {} +} diff --git a/src/main/java/com/crunchydata/service/ConnectionTestService.java b/src/main/java/com/crunchydata/service/ConnectionTestService.java new file mode 100644 index 0000000..0e8f04a --- /dev/null +++ b/src/main/java/com/crunchydata/service/ConnectionTestService.java @@ -0,0 +1,287 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.service; + +import com.crunchydata.util.LoggingUtils; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; + +import static com.crunchydata.config.Settings.Props; + +/** + * Service for testing database connections and returning detailed results. + * + * @author Brian Pace + */ +public class ConnectionTestService { + + private static final String THREAD_NAME = "connection-test"; + + /** + * Result of a connection test. + */ + public static class ConnectionTestResult { + public boolean success; + public String connectionType; + public String databaseType; + public String host; + public int port; + public String database; + public String schema; + public String user; + public String databaseProductName; + public String databaseProductVersion; + public String driverName; + public String driverVersion; + public String errorMessage; + public String errorDetail; + public long responseTimeMs; + + public Map toMap() { + Map result = new HashMap<>(); + result.put("success", success); + result.put("connectionType", connectionType); + result.put("databaseType", databaseType); + result.put("host", host); + result.put("port", port); + result.put("database", database); + result.put("schema", schema); + result.put("user", user); + result.put("databaseProductName", databaseProductName); + result.put("databaseProductVersion", databaseProductVersion); + result.put("driverName", driverName); + result.put("driverVersion", driverVersion); + result.put("errorMessage", errorMessage); + result.put("errorDetail", errorDetail); + result.put("responseTimeMs", responseTimeMs); + return result; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(String.format("Connection Type: %s%n", connectionType)); + sb.append(String.format("Status: %s%n", success ? "SUCCESS" : "FAILED")); + sb.append(String.format("Host: %s%n", host)); + sb.append(String.format("Port: %d%n", port)); + sb.append(String.format("Database: %s%n", database)); + sb.append(String.format("Schema: %s%n", schema)); + sb.append(String.format("User: %s%n", user)); + sb.append(String.format("Response Time: %d ms%n", responseTimeMs)); + + if (success) { + sb.append(String.format("Database Product: %s %s%n", databaseProductName, databaseProductVersion)); + sb.append(String.format("Driver: %s %s%n", driverName, driverVersion)); + } else { + sb.append(String.format("Error: %s%n", errorMessage)); + if (errorDetail != null && !errorDetail.isEmpty()) { + sb.append(String.format("Detail: %s%n", errorDetail)); + } + } + + return sb.toString(); + } + } + + /** + * Test a connection by type (repository, source, or target). + * + * @param connectionType The type of connection (repo, source, target) + * @return ConnectionTestResult with the test results + */ + public static ConnectionTestResult testConnection(String connectionType) { + ConnectionTestResult result = new ConnectionTestResult(); + result.connectionType = connectionType; + + String prefix = connectionType.equals("repo") ? "repo" : connectionType; + String platform = connectionType.equals("repo") ? "postgres" : Props.getProperty(connectionType + "-type"); + + result.databaseType = platform; + result.host = Props.getProperty(prefix + "-host"); + result.port = Integer.parseInt(Props.getProperty(prefix + "-port", "0")); + result.database = Props.getProperty(prefix + "-dbname"); + result.schema = Props.getProperty(prefix + "-schema"); + result.user = Props.getProperty(prefix + "-user"); + + long startTime = System.currentTimeMillis(); + Connection conn = null; + + try { + LoggingUtils.write("info", THREAD_NAME, + String.format("Testing %s connection to %s:%d/%s", connectionType, result.host, result.port, result.database)); + + conn = DatabaseConnectionService.getConnection(platform, prefix); + + if (conn == null) { + result.success = false; + result.errorMessage = "Failed to establish connection"; + result.errorDetail = "Connection returned null. Check credentials and network connectivity."; + } else { + result.success = true; + + DatabaseMetaData metaData = conn.getMetaData(); + result.databaseProductName = metaData.getDatabaseProductName(); + result.databaseProductVersion = metaData.getDatabaseProductVersion(); + result.driverName = metaData.getDriverName(); + result.driverVersion = metaData.getDriverVersion(); + + LoggingUtils.write("info", THREAD_NAME, + String.format("%s connection successful: %s %s", + connectionType, result.databaseProductName, result.databaseProductVersion)); + } + } catch (SQLException e) { + result.success = false; + result.errorMessage = e.getMessage(); + result.errorDetail = formatSQLException(e); + LoggingUtils.write("warning", THREAD_NAME, + String.format("%s connection failed: %s", connectionType, e.getMessage())); + } catch (Exception e) { + result.success = false; + result.errorMessage = e.getMessage(); + result.errorDetail = e.getClass().getName() + ": " + e.getMessage(); + LoggingUtils.write("warning", THREAD_NAME, + String.format("%s connection failed: %s", connectionType, e.getMessage())); + } finally { + result.responseTimeMs = System.currentTimeMillis() - startTime; + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + // Ignore close errors + } + } + } + + return result; + } + + /** + * Test all connections (repository, source, and target). + * + * @return Map containing results for each connection type + */ + public static Map testAllConnections() { + Map results = new HashMap<>(); + + results.put("repository", testConnection("repo")); + results.put("source", testConnection("source")); + results.put("target", testConnection("target")); + + return results; + } + + /** + * Format a SQLException with all chained exceptions. + */ + private static String formatSQLException(SQLException e) { + StringBuilder sb = new StringBuilder(); + SQLException current = e; + int count = 0; + + while (current != null && count < 5) { + if (count > 0) { + sb.append("\nCaused by: "); + } + sb.append(String.format("SQLState: %s, ErrorCode: %d, Message: %s", + current.getSQLState(), current.getErrorCode(), current.getMessage())); + current = current.getNextException(); + count++; + } + + return sb.toString(); + } + + /** + * Print test results to stdout in a formatted way. + */ + public static void printResults(Map results) { + System.out.println(); + System.out.println("=".repeat(60)); + System.out.println("CONNECTION TEST RESULTS"); + System.out.println("=".repeat(60)); + + for (Map.Entry entry : results.entrySet()) { + System.out.println(); + System.out.println("-".repeat(40)); + System.out.println(entry.getKey().toUpperCase()); + System.out.println("-".repeat(40)); + System.out.println(entry.getValue().toString()); + } + + System.out.println("=".repeat(60)); + + boolean allSuccess = results.values().stream().allMatch(r -> r.success); + System.out.println(String.format("Overall Status: %s", allSuccess ? "ALL CONNECTIONS SUCCESSFUL" : "SOME CONNECTIONS FAILED")); + System.out.println("=".repeat(60)); + System.out.println(); + } + + /** + * Print test results as JSON to stdout. + */ + public static void printResultsAsJson(Map results) { + StringBuilder json = new StringBuilder(); + json.append("{\n"); + + int i = 0; + for (Map.Entry entry : results.entrySet()) { + if (i > 0) { + json.append(",\n"); + } + json.append(String.format(" \"%s\": %s", entry.getKey(), toJson(entry.getValue()))); + i++; + } + + json.append("\n}"); + System.out.println(json.toString()); + } + + private static String toJson(ConnectionTestResult result) { + StringBuilder json = new StringBuilder(); + json.append("{\n"); + json.append(String.format(" \"success\": %s,\n", result.success)); + json.append(String.format(" \"connectionType\": \"%s\",\n", escape(result.connectionType))); + json.append(String.format(" \"databaseType\": \"%s\",\n", escape(result.databaseType))); + json.append(String.format(" \"host\": \"%s\",\n", escape(result.host))); + json.append(String.format(" \"port\": %d,\n", result.port)); + json.append(String.format(" \"database\": \"%s\",\n", escape(result.database))); + json.append(String.format(" \"schema\": \"%s\",\n", escape(result.schema))); + json.append(String.format(" \"user\": \"%s\",\n", escape(result.user))); + json.append(String.format(" \"databaseProductName\": \"%s\",\n", escape(result.databaseProductName))); + json.append(String.format(" \"databaseProductVersion\": \"%s\",\n", escape(result.databaseProductVersion))); + json.append(String.format(" \"driverName\": \"%s\",\n", escape(result.driverName))); + json.append(String.format(" \"driverVersion\": \"%s\",\n", escape(result.driverVersion))); + json.append(String.format(" \"errorMessage\": \"%s\",\n", escape(result.errorMessage))); + json.append(String.format(" \"errorDetail\": \"%s\",\n", escape(result.errorDetail))); + json.append(String.format(" \"responseTimeMs\": %d\n", result.responseTimeMs)); + json.append(" }"); + return json.toString(); + } + + private static String escape(String str) { + if (str == null) return ""; + return str.replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t"); + } +} diff --git a/src/main/java/com/crunchydata/service/JobAwareCompareController.java b/src/main/java/com/crunchydata/service/JobAwareCompareController.java new file mode 100644 index 0000000..aba5f1a --- /dev/null +++ b/src/main/java/com/crunchydata/service/JobAwareCompareController.java @@ -0,0 +1,334 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.service; + +import com.crunchydata.config.ApplicationState; +import com.crunchydata.controller.ColumnController; +import com.crunchydata.controller.RepoController; +import com.crunchydata.controller.TableController; +import com.crunchydata.core.comparison.ResultProcessor; +import com.crunchydata.core.database.SQLExecutionHelper; +import com.crunchydata.core.threading.DataValidationThread; +import com.crunchydata.core.threading.ThreadManager; +import com.crunchydata.model.ColumnMetadata; +import com.crunchydata.model.DataComparisonTable; +import com.crunchydata.model.DataComparisonTableMap; +import com.crunchydata.util.LoggingUtils; + +import javax.sql.rowset.CachedRowSet; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.UUID; + +import static com.crunchydata.config.Settings.Props; +import static com.crunchydata.config.sql.RepoSQLConstants.SQL_REPO_DCTABLECOLUMNMAP_FULLBYTID; +import static com.crunchydata.controller.ColumnController.getColumnInfo; +import static com.crunchydata.controller.RepoController.createCompareId; +import static com.crunchydata.service.SQLSyntaxService.buildGetTablesSQL; +import static com.crunchydata.service.SQLSyntaxService.generateCompareSQL; + +import org.json.JSONObject; + +/** + * Job-aware compare controller that integrates with ServerModeService + * for progress tracking and control signal handling. + * + * @author Brian Pace + */ +public class JobAwareCompareController { + + private static final String THREAD_NAME = "job-compare"; + + /** + * Perform comparison with job progress tracking. + * + * @param jobId Job ID for progress tracking + * @param connRepo Repository connection + * @param connSource Source database connection + * @param connTarget Target database connection + * @param pid Project ID + * @param batchNbr Batch number + * @param serverService Server service for progress updates + */ + public static void performCompare(UUID jobId, Connection connRepo, Connection connSource, + Connection connTarget, int pid, int batchNbr, + ServerModeService serverService) throws Exception { + + boolean isCheck = Props.getProperty("isCheck", "false").equals("true"); + String tableFilter = Props.getProperty("table", ""); + + LoggingUtils.write("info", THREAD_NAME, + String.format("Starting job %s: pid=%d, batch=%d, check=%s", jobId, pid, batchNbr, isCheck)); + + // Get tables to process + CachedRowSet tablesResultSet = getTables(pid, connRepo, batchNbr, tableFilter, isCheck); + + if (tablesResultSet == null) { + throw new RuntimeException("Failed to retrieve tables for comparison"); + } + + try { + // Pre-populate progress for all tables before processing + LoggingUtils.write("info", THREAD_NAME, "Pre-populating job progress for all tables"); + int tableCount = 0; + while (tablesResultSet.next()) { + long tid = tablesResultSet.getLong("tid"); + String tableAlias = tablesResultSet.getString("table_alias"); + serverService.initializeJobProgress(jobId, tid, tableAlias); + tableCount++; + } + LoggingUtils.write("info", THREAD_NAME, + String.format("Found %d table(s) to process", tableCount)); + + // Reset cursor to beginning + tablesResultSet.beforeFirst(); + + RepoController repoController = new RepoController(); + long rid = System.currentTimeMillis(); + int tablesProcessed = 0; + boolean isPaused = false; + + while (tablesResultSet.next()) { + // Check for control signals + String signal = serverService.checkJobControlSignal(jobId); + if (signal != null) { + switch (signal) { + case "terminate": + LoggingUtils.write("info", THREAD_NAME, "Terminate signal received"); + ApplicationState.getInstance().requestImmediateTermination(); + throw new RuntimeException("Job terminated by user request"); + case "stop": + LoggingUtils.write("info", THREAD_NAME, "Stop signal received - completing gracefully"); + ApplicationState.getInstance().requestGracefulShutdown(); + return; + case "pause": + LoggingUtils.write("info", THREAD_NAME, "Pause signal received"); + isPaused = true; + break; + case "resume": + LoggingUtils.write("info", THREAD_NAME, "Resume signal received"); + isPaused = false; + break; + } + } + + // Wait while paused + while (isPaused && !ApplicationState.getInstance().isGracefulShutdownRequested()) { + Thread.sleep(5000); + String resumeSignal = serverService.checkJobControlSignal(jobId); + if ("resume".equals(resumeSignal)) { + isPaused = false; + LoggingUtils.write("info", THREAD_NAME, "Resuming job"); + } else if ("terminate".equals(resumeSignal) || "stop".equals(resumeSignal)) { + throw new RuntimeException("Job stopped while paused"); + } + } + + // Check for graceful shutdown + if (ApplicationState.getInstance().isGracefulShutdownRequested()) { + LoggingUtils.write("info", THREAD_NAME, "Graceful shutdown requested - stopping"); + return; + } + + // Get table information + DataComparisonTable dct = new DataComparisonTable( + tablesResultSet.getInt("pid"), + tablesResultSet.getInt("tid"), + tablesResultSet.getString("table_alias"), + tablesResultSet.getInt("batch_nbr"), + tablesResultSet.getInt("parallel_degree"), + tablesResultSet.getBoolean("enabled") + ); + + // Mark this table as running + serverService.updateJobProgress(jobId, dct.getTid(), "running", null, null); + + LoggingUtils.write("info", THREAD_NAME, + String.format("Processing table: %s (tid=%d)", dct.getTableAlias(), dct.getTid())); + + try { + // Get table mappings + DataComparisonTableMap dctmSource = TableController.getTableMap(connRepo, dct.getTid(), "source"); + DataComparisonTableMap dctmTarget = TableController.getTableMap(connRepo, dct.getTid(), "target"); + + // Set additional properties (consistent with standalone mode) + dctmSource.setBatchNbr(dct.getBatchNbr()); + dctmSource.setPid(dct.getPid()); + dctmSource.setTableAlias(dct.getTableAlias()); + + dctmTarget.setBatchNbr(dct.getBatchNbr()); + dctmTarget.setPid(dct.getPid()); + dctmTarget.setTableAlias(dct.getTableAlias()); + + // Perform reconciliation + JSONObject result = reconcileDataWithProgress( + jobId, connRepo, connSource, connTarget, rid, isCheck, + dct, dctmSource, dctmTarget, serverService); + + // Update progress with status and cid (counts come from dc_result) + serverService.updateJobProgress(jobId, dct.getTid(), + "completed".equals(result.optString("status")) ? "completed" : "failed", + result.optString("error", null), + result.has("cid") ? result.getInt("cid") : null); + + tablesProcessed++; + + } catch (Exception e) { + LoggingUtils.write("severe", THREAD_NAME, + String.format("Error processing table %s: %s", dct.getTableAlias(), e.getMessage())); + + serverService.updateJobProgress(jobId, dct.getTid(), + "failed", e.getMessage(), null); + } + } + + LoggingUtils.write("info", THREAD_NAME, + String.format("Job %s completed: %d tables processed", jobId, tablesProcessed)); + + } finally { + if (tablesResultSet != null) { + tablesResultSet.close(); + } + } + } + + /** + * Reconcile data with progress tracking. + */ + private static JSONObject reconcileDataWithProgress(UUID jobId, Connection connRepo, + Connection connSource, Connection connTarget, long rid, boolean check, + DataComparisonTable dct, DataComparisonTableMap dctmSource, + DataComparisonTableMap dctmTarget, ServerModeService serverService) { + + long startTime = System.currentTimeMillis(); + JSONObject result = new JSONObject(); + result.put("tableName", dct.getTableAlias()); + result.put("status", "processing"); + result.put("compareStatus", "processing"); + result.put("missingSource", 0); + result.put("missingTarget", 0); + result.put("notEqual", 0); + result.put("equal", 0); + + RepoController repoController = new RepoController(); + + try { + // Start table history tracking + repoController.startTableHistory(connRepo, (int) dct.getTid(), dct.getBatchNbr()); + + // Clear previous compare results for this table (unless it's a recheck) + if (!check) { + LoggingUtils.write("info", THREAD_NAME, + String.format("Clearing previous compare data for table %s (tid=%d, batch=%d)", + dct.getTableAlias(), dct.getTid(), dct.getBatchNbr())); + repoController.deleteDataCompare(connRepo, (int) dct.getTid(), dct.getBatchNbr()); + } + + // Get column mapping + ArrayList binds = new ArrayList<>(); + binds.add(dct.getTid()); + String columnMapping = SQLExecutionHelper.simpleSelectReturnString( + connRepo, SQL_REPO_DCTABLECOLUMNMAP_FULLBYTID, binds); + + if (columnMapping == null) { + result.put("status", "failed"); + result.put("error", "No column mapping found"); + return result; + } + + // Get column metadata + JSONObject columnMap = new JSONObject(columnMapping); + ColumnMetadata ciSource = getColumnInfo(columnMap, "source", + Props.getProperty("source-type"), + dctmSource.getSchemaName(), dctmSource.getTableName(), + "database".equals(Props.getProperty("column-hash-method"))); + + ColumnMetadata ciTarget = getColumnInfo(columnMap, "target", + Props.getProperty("target-type"), + dctmTarget.getSchemaName(), dctmTarget.getTableName(), + "database".equals(Props.getProperty("column-hash-method"))); + + // Create compare ID + Integer cid = createCompareId(connRepo, dctmTarget, rid); + + // Generate compare SQL + generateCompareSQL(dctmSource, dctmTarget, ciSource, ciTarget); + + // Execute reconciliation + if (check) { + JSONObject checkResult = DataValidationThread.checkRows( + connRepo, connSource, connTarget, dct, dctmSource, dctmTarget, + ciSource, ciTarget, cid); + result.put("checkResult", checkResult); + } else { + if (ciTarget.pkList.isBlank() || ciSource.pkList.isBlank()) { + result.put("status", "skipped"); + result.put("error", "No primary key defined"); + return result; + } + + ThreadManager.executeReconciliation(dct, cid, dctmSource, dctmTarget, + ciSource, ciTarget, connRepo); + } + + // Process results + ResultProcessor.summarizeResults(connRepo, dct.getTid(), result, cid); + + long elapsedTime = (System.currentTimeMillis() - startTime) / 1000; + result.put("elapsedTime", elapsedTime); + result.put("status", "completed"); + result.put("cid", cid); + + // Complete table history (consistent with standalone mode) + RepoController.completeTableHistory(connRepo, (int) dct.getTid(), dct.getBatchNbr(), 0, result.toString()); + + } catch (Exception e) { + LoggingUtils.write("severe", THREAD_NAME, + String.format("Error reconciling table %s: %s", dct.getTableAlias(), e.getMessage())); + result.put("status", "failed"); + result.put("error", e.getMessage()); + } + + return result; + } + + /** + * Retrieve tables from repository. + */ + private static CachedRowSet getTables(int pid, Connection conn, int batchNbr, + String table, boolean check) { + String sql = buildGetTablesSQL(batchNbr, table, check); + ArrayList binds = new ArrayList<>(); + binds.add(pid); + + if (batchNbr > 0) { + binds.add(batchNbr); + } + + if (table != null && !table.isEmpty()) { + binds.add(table); + } + + LoggingUtils.write("info", THREAD_NAME, + String.format("Retrieving tables: pid=%d, batch=%d, table='%s', check=%s", + pid, batchNbr, table, check)); + LoggingUtils.write("debug", THREAD_NAME, String.format("Tables query: %s", sql)); + + return SQLExecutionHelper.simpleSelect(conn, sql, binds); + } +} diff --git a/src/main/java/com/crunchydata/service/MappingExportService.java b/src/main/java/com/crunchydata/service/MappingExportService.java new file mode 100644 index 0000000..f9a0bc2 --- /dev/null +++ b/src/main/java/com/crunchydata/service/MappingExportService.java @@ -0,0 +1,248 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.service; + +import com.crunchydata.core.database.SQLExecutionHelper; +import com.crunchydata.model.yaml.*; +import com.crunchydata.util.LoggingUtils; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.introspector.Property; +import org.yaml.snakeyaml.nodes.NodeTuple; +import org.yaml.snakeyaml.nodes.Tag; +import org.yaml.snakeyaml.representer.Representer; + +import javax.sql.rowset.CachedRowSet; +import java.io.FileWriter; +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; + +public class MappingExportService { + + private static final String THREAD_NAME = "mapping-export"; + private static final String VERSION = "1.0"; + + private static final String SQL_SELECT_TABLES = """ + SELECT t.tid, t.table_alias, t.enabled, t.batch_nbr, t.parallel_degree + FROM dc_table t + WHERE t.pid = ? + ORDER BY t.table_alias + """; + + private static final String SQL_SELECT_TABLES_FILTERED = """ + SELECT t.tid, t.table_alias, t.enabled, t.batch_nbr, t.parallel_degree + FROM dc_table t + WHERE t.pid = ? + AND t.table_alias LIKE ? + ORDER BY t.table_alias + """; + + private static final String SQL_SELECT_TABLE_MAP = """ + SELECT dest_type, schema_name, table_name, schema_preserve_case, table_preserve_case + FROM dc_table_map + WHERE tid = ? + """; + + private static final String SQL_SELECT_COLUMNS = """ + SELECT tc.column_id, tc.column_alias, tc.enabled + FROM dc_table_column tc + WHERE tc.tid = ? + ORDER BY tc.column_alias + """; + + private static final String SQL_SELECT_COLUMN_MAP = """ + SELECT column_origin, column_name, data_type, data_class, data_length, + number_precision, number_scale, column_nullable, column_primarykey, + map_expression, supported, preserve_case + FROM dc_table_column_map + WHERE tid = ? AND column_id = ? + """; + + private static final String SQL_SELECT_PROJECT = """ + SELECT project_name FROM dc_project WHERE pid = ? + """; + + public static void exportToYaml(Connection conn, Integer pid, String tableFilter, String outputFile) throws IOException, SQLException { + LoggingUtils.write("info", THREAD_NAME, String.format("Exporting mappings for project %d to %s", pid, outputFile)); + + MappingExport export = new MappingExport(); + export.setVersion(VERSION); + export.setExportDate(LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)); + export.setProjectId(pid); + export.setProjectName(getProjectName(conn, pid)); + export.setTables(getTables(conn, pid, tableFilter)); + + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + options.setPrettyFlow(true); + options.setIndent(2); + options.setIndicatorIndent(2); + options.setIndentWithIndicator(true); + + Representer representer = new Representer(options) { + @Override + protected NodeTuple representJavaBeanProperty(Object javaBean, Property property, Object propertyValue, Tag customTag) { + if (propertyValue == null) { + return null; + } + return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag); + } + }; + representer.getPropertyUtils().setSkipMissingProperties(true); + representer.addClassTag(MappingExport.class, Tag.MAP); + + Yaml yaml = new Yaml(representer, options); + + try (FileWriter writer = new FileWriter(outputFile)) { + writer.write("# pgCompare Table and Column Mapping Export\n"); + writer.write("# Generated: " + export.getExportDate() + "\n"); + writer.write("# Project: " + export.getProjectName() + " (ID: " + export.getProjectId() + ")\n"); + writer.write("#\n"); + writer.write("# This file can be edited and re-imported using:\n"); + writer.write("# java -jar pgcompare.jar import-mapping --file [--overwrite]\n"); + writer.write("#\n"); + writer.write("# NOTES:\n"); + writer.write("# - 'alias' is the unique identifier that links source and target tables/columns\n"); + writer.write("# - Set 'enabled: false' to exclude a table or column from comparison\n"); + writer.write("# - 'mapExpression' allows custom SQL expressions for column value transformation\n"); + writer.write("# - 'primaryKey: true' marks columns used for row identification\n"); + writer.write("#\n\n"); + yaml.dump(export, writer); + } + + LoggingUtils.write("info", THREAD_NAME, String.format("Exported %d table(s) to %s", export.getTables().size(), outputFile)); + } + + private static String getProjectName(Connection conn, Integer pid) { + ArrayList binds = new ArrayList<>(); + binds.add(pid); + return SQLExecutionHelper.simpleSelectReturnString(conn, SQL_SELECT_PROJECT, binds); + } + + private static List getTables(Connection conn, Integer pid, String tableFilter) throws SQLException { + List tables = new ArrayList<>(); + + ArrayList binds = new ArrayList<>(); + binds.add(pid); + + String sql = SQL_SELECT_TABLES; + if (tableFilter != null && !tableFilter.isEmpty()) { + binds.add(tableFilter.replace("*", "%")); + sql = SQL_SELECT_TABLES_FILTERED; + } + + CachedRowSet crs = SQLExecutionHelper.simpleSelect(conn, sql, binds); + + while (crs.next()) { + TableDefinition tableDef = new TableDefinition(); + Integer tid = crs.getInt("tid"); + tableDef.setAlias(crs.getString("table_alias")); + tableDef.setEnabled(crs.getBoolean("enabled")); + tableDef.setBatchNumber(crs.getInt("batch_nbr")); + tableDef.setParallelDegree(crs.getInt("parallel_degree")); + + loadTableLocations(conn, tid, tableDef); + tableDef.setColumns(getColumns(conn, tid)); + + tables.add(tableDef); + } + crs.close(); + + return tables; + } + + private static void loadTableLocations(Connection conn, Integer tid, TableDefinition tableDef) throws SQLException { + ArrayList binds = new ArrayList<>(); + binds.add(tid); + + CachedRowSet crs = SQLExecutionHelper.simpleSelect(conn, SQL_SELECT_TABLE_MAP, binds); + + while (crs.next()) { + TableLocation loc = new TableLocation(); + loc.setSchema(crs.getString("schema_name")); + loc.setTable(crs.getString("table_name")); + loc.setSchemaPreserveCase(crs.getBoolean("schema_preserve_case")); + loc.setTablePreserveCase(crs.getBoolean("table_preserve_case")); + + String destType = crs.getString("dest_type"); + if ("source".equals(destType)) { + tableDef.setSource(loc); + } else { + tableDef.setTarget(loc); + } + } + crs.close(); + } + + private static List getColumns(Connection conn, Integer tid) throws SQLException { + List columns = new ArrayList<>(); + + ArrayList binds = new ArrayList<>(); + binds.add(tid); + + CachedRowSet crs = SQLExecutionHelper.simpleSelect(conn, SQL_SELECT_COLUMNS, binds); + + while (crs.next()) { + ColumnDefinition colDef = new ColumnDefinition(); + Integer columnId = crs.getInt("column_id"); + colDef.setAlias(crs.getString("column_alias")); + colDef.setEnabled(crs.getBoolean("enabled")); + + loadColumnMappings(conn, tid, columnId, colDef); + columns.add(colDef); + } + crs.close(); + + return columns; + } + + private static void loadColumnMappings(Connection conn, Integer tid, Integer columnId, ColumnDefinition colDef) throws SQLException { + ArrayList binds = new ArrayList<>(); + binds.add(tid); + binds.add(columnId); + + CachedRowSet crs = SQLExecutionHelper.simpleSelect(conn, SQL_SELECT_COLUMN_MAP, binds); + + while (crs.next()) { + ColumnMapping mapping = new ColumnMapping(); + mapping.setColumnName(crs.getString("column_name")); + mapping.setDataType(crs.getString("data_type")); + mapping.setDataClass(crs.getString("data_class")); + mapping.setDataLength(crs.getObject("data_length") != null ? crs.getInt("data_length") : null); + mapping.setNumberPrecision(crs.getObject("number_precision") != null ? crs.getInt("number_precision") : null); + mapping.setNumberScale(crs.getObject("number_scale") != null ? crs.getInt("number_scale") : null); + mapping.setNullable(crs.getBoolean("column_nullable")); + mapping.setPrimaryKey(crs.getBoolean("column_primarykey")); + mapping.setMapExpression(crs.getString("map_expression")); + mapping.setSupported(crs.getBoolean("supported")); + mapping.setPreserveCase(crs.getBoolean("preserve_case")); + + String origin = crs.getString("column_origin"); + if ("source".equals(origin)) { + colDef.setSource(mapping); + } else { + colDef.setTarget(mapping); + } + } + crs.close(); + } +} diff --git a/src/main/java/com/crunchydata/service/MappingImportService.java b/src/main/java/com/crunchydata/service/MappingImportService.java new file mode 100644 index 0000000..f2e58f2 --- /dev/null +++ b/src/main/java/com/crunchydata/service/MappingImportService.java @@ -0,0 +1,232 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.service; + +import com.crunchydata.core.database.SQLExecutionHelper; +import com.crunchydata.model.yaml.*; +import com.crunchydata.util.LoggingUtils; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.TypeDescription; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; + +import java.io.FileInputStream; +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +public class MappingImportService { + + private static final String THREAD_NAME = "mapping-import"; + + private static final String SQL_CHECK_TABLE_EXISTS = """ + SELECT tid FROM dc_table WHERE pid = ? AND table_alias = ? + """; + + private static final String SQL_DELETE_TABLE = """ + DELETE FROM dc_table WHERE tid = ? + """; + + private static final String SQL_INSERT_TABLE = """ + INSERT INTO dc_table (pid, table_alias, enabled, batch_nbr, parallel_degree) + VALUES (?, ?, ?, ?, ?) + RETURNING tid + """; + + private static final String SQL_INSERT_TABLE_MAP = """ + INSERT INTO dc_table_map (tid, dest_type, schema_name, table_name, + schema_preserve_case, table_preserve_case) + VALUES (?, ?, ?, ?, ?, ?) + """; + + private static final String SQL_INSERT_COLUMN = """ + INSERT INTO dc_table_column (tid, column_alias, enabled) + VALUES (?, ?, ?) + RETURNING column_id + """; + + private static final String SQL_INSERT_COLUMN_MAP = """ + INSERT INTO dc_table_column_map (tid, column_id, column_origin, column_name, + data_type, data_class, data_length, number_precision, + number_scale, column_nullable, column_primarykey, + map_expression, supported, preserve_case) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """; + + public static ImportResult importFromYaml(Connection conn, Integer pid, String inputFile, + boolean overwrite, String tableFilter) throws IOException, SQLException { + LoggingUtils.write("info", THREAD_NAME, String.format("Importing mappings from %s to project %d (overwrite=%s)", + inputFile, pid, overwrite)); + + LoaderOptions loaderOptions = new LoaderOptions(); + loaderOptions.setTagInspector(tag -> tag.getClassName().startsWith("com.crunchydata.model.yaml.")); + Constructor constructor = new Constructor(MappingExport.class, loaderOptions); + constructor.addTypeDescription(new TypeDescription(MappingExport.class, "tag:yaml.org,2002:com.crunchydata.model.yaml.MappingExport")); + Yaml yaml = new Yaml(constructor); + + MappingExport mappingExport; + try (FileInputStream fis = new FileInputStream(inputFile)) { + mappingExport = yaml.load(fis); + } + + if (mappingExport == null || mappingExport.getTables() == null) { + throw new IllegalArgumentException("Invalid or empty YAML file: " + inputFile); + } + + int tablesAdded = 0; + int tablesUpdated = 0; + int tablesSkipped = 0; + int columnsProcessed = 0; + + for (TableDefinition tableDef : mappingExport.getTables()) { + if (tableFilter != null && !tableFilter.isEmpty()) { + String pattern = tableFilter.replace("*", ".*"); + if (!tableDef.getAlias().matches(pattern)) { + tablesSkipped++; + continue; + } + } + + Integer existingTid = getExistingTableId(conn, pid, tableDef.getAlias()); + + if (existingTid != null) { + if (overwrite) { + deleteTable(conn, existingTid); + Integer newTid = insertTable(conn, pid, tableDef); + insertTableMaps(conn, newTid, tableDef); + columnsProcessed += insertColumns(conn, newTid, tableDef.getColumns()); + tablesUpdated++; + LoggingUtils.write("info", THREAD_NAME, String.format("Updated table: %s", tableDef.getAlias())); + } else { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Skipping existing table '%s' (use --overwrite to replace)", tableDef.getAlias())); + tablesSkipped++; + } + } else { + Integer newTid = insertTable(conn, pid, tableDef); + insertTableMaps(conn, newTid, tableDef); + columnsProcessed += insertColumns(conn, newTid, tableDef.getColumns()); + tablesAdded++; + LoggingUtils.write("info", THREAD_NAME, String.format("Added table: %s", tableDef.getAlias())); + } + } + + conn.commit(); + + ImportResult result = new ImportResult(tablesAdded, tablesUpdated, tablesSkipped, columnsProcessed); + LoggingUtils.write("info", THREAD_NAME, String.format("Import complete: %d added, %d updated, %d skipped, %d columns", + tablesAdded, tablesUpdated, tablesSkipped, columnsProcessed)); + + return result; + } + + private static Integer getExistingTableId(Connection conn, Integer pid, String tableAlias) { + ArrayList binds = new ArrayList<>(); + binds.add(pid); + binds.add(tableAlias.toLowerCase()); + return SQLExecutionHelper.simpleSelectReturnInteger(conn, SQL_CHECK_TABLE_EXISTS, binds); + } + + private static void deleteTable(Connection conn, Integer tid) { + ArrayList binds = new ArrayList<>(); + binds.add(tid); + SQLExecutionHelper.simpleUpdate(conn, SQL_DELETE_TABLE, binds, true); + } + + private static Integer insertTable(Connection conn, Integer pid, TableDefinition tableDef) { + ArrayList binds = new ArrayList<>(); + binds.add(pid); + binds.add(tableDef.getAlias().toLowerCase()); + binds.add(tableDef.getEnabled() != null ? tableDef.getEnabled() : true); + binds.add(tableDef.getBatchNumber() != null ? tableDef.getBatchNumber() : 1); + binds.add(tableDef.getParallelDegree() != null ? tableDef.getParallelDegree() : 1); + + return SQLExecutionHelper.simpleUpdateReturningInteger(conn, SQL_INSERT_TABLE, binds); + } + + private static void insertTableMaps(Connection conn, Integer tid, TableDefinition tableDef) { + if (tableDef.getSource() != null) { + insertTableMap(conn, tid, "source", tableDef.getSource()); + } + if (tableDef.getTarget() != null) { + insertTableMap(conn, tid, "target", tableDef.getTarget()); + } + } + + private static void insertTableMap(Connection conn, Integer tid, String destType, TableLocation loc) { + ArrayList binds = new ArrayList<>(); + binds.add(tid); + binds.add(destType); + binds.add(loc.getSchema()); + binds.add(loc.getTable()); + binds.add(loc.getSchemaPreserveCase() != null ? loc.getSchemaPreserveCase() : false); + binds.add(loc.getTablePreserveCase() != null ? loc.getTablePreserveCase() : false); + + SQLExecutionHelper.simpleUpdate(conn, SQL_INSERT_TABLE_MAP, binds, true); + } + + private static int insertColumns(Connection conn, Integer tid, List columns) { + if (columns == null) return 0; + + int count = 0; + for (ColumnDefinition colDef : columns) { + Integer columnId = insertColumn(conn, tid, colDef); + if (colDef.getSource() != null) { + insertColumnMap(conn, tid, columnId, "source", colDef.getSource()); + } + if (colDef.getTarget() != null) { + insertColumnMap(conn, tid, columnId, "target", colDef.getTarget()); + } + count++; + } + return count; + } + + private static Integer insertColumn(Connection conn, Integer tid, ColumnDefinition colDef) { + ArrayList binds = new ArrayList<>(); + binds.add(tid); + binds.add(colDef.getAlias().toLowerCase()); + binds.add(colDef.getEnabled() != null ? colDef.getEnabled() : true); + + return SQLExecutionHelper.simpleUpdateReturningInteger(conn, SQL_INSERT_COLUMN, binds); + } + + private static void insertColumnMap(Connection conn, Integer tid, Integer columnId, + String origin, ColumnMapping mapping) { + ArrayList binds = new ArrayList<>(); + binds.add(tid); + binds.add(columnId); + binds.add(origin); + binds.add(mapping.getColumnName()); + binds.add(mapping.getDataType()); + binds.add(mapping.getDataClass() != null ? mapping.getDataClass() : "string"); + binds.add(mapping.getDataLength()); + binds.add(mapping.getNumberPrecision()); + binds.add(mapping.getNumberScale()); + binds.add(mapping.getNullable() != null ? mapping.getNullable() : true); + binds.add(mapping.getPrimaryKey() != null ? mapping.getPrimaryKey() : false); + binds.add(mapping.getMapExpression()); + binds.add(mapping.getSupported() != null ? mapping.getSupported() : true); + binds.add(mapping.getPreserveCase() != null ? mapping.getPreserveCase() : false); + + SQLExecutionHelper.simpleUpdate(conn, SQL_INSERT_COLUMN_MAP, binds, true); + } + + public record ImportResult(int tablesAdded, int tablesUpdated, int tablesSkipped, int columnsProcessed) {} +} diff --git a/src/main/java/com/crunchydata/service/RepositoryInitializationService.java b/src/main/java/com/crunchydata/service/RepositoryInitializationService.java index 42983fe..cd2392f 100644 --- a/src/main/java/com/crunchydata/service/RepositoryInitializationService.java +++ b/src/main/java/com/crunchydata/service/RepositoryInitializationService.java @@ -27,6 +27,7 @@ import java.util.Properties; import static com.crunchydata.config.sql.RepoSQLConstants.*; +import static com.crunchydata.config.sql.ServerModeSQLConstants.*; /** * Utility class for creating repository schema, tables, indexes, and constraints. @@ -139,7 +140,12 @@ private static void createTables(Connection conn) throws SQLException { REPO_DDL_DC_TABLE_COLUMN_MAP, REPO_DDL_DC_TABLE_HISTORY, REPO_DDL_DC_TABLE_MAP, - REPO_DDL_DC_TARGET + REPO_DDL_DC_TARGET, + REPO_DDL_DC_SERVER, + REPO_DDL_DC_JOB, + REPO_DDL_DC_JOB_CONTROL, + REPO_DDL_DC_JOB_PROGRESS, + REPO_DDL_DC_JOB_LOG ); LoggingUtils.write("info", THREAD_NAME, "Creating repository tables"); @@ -160,7 +166,15 @@ private static void createIndexesAndConstraints(Connection conn) throws SQLExcep REPO_DDL_DC_TABLE_COLUMN_IDX1, REPO_DDL_DC_TABLE_COLUMN_FK, REPO_DDL_DC_TABLE_MAP_FK, - REPO_DDL_DC_TABLE_COLUMN_MAP_FK + REPO_DDL_DC_TABLE_COLUMN_MAP_FK, + REPO_DDL_DC_SERVER_IDX1, + REPO_DDL_DC_JOB_IDX1, + REPO_DDL_DC_JOB_IDX2, + REPO_DDL_DC_JOB_FK1, + REPO_DDL_DC_JOB_CONTROL_FK1, + REPO_DDL_DC_JOB_PROGRESS_FK1, + REPO_DDL_DC_JOB_LOG_IDX1, + REPO_DDL_DC_JOB_LOG_FK1 ); LoggingUtils.write("info", THREAD_NAME, "Creating indexes and constraints"); diff --git a/src/main/java/com/crunchydata/service/SQLFixGenerationService.java b/src/main/java/com/crunchydata/service/SQLFixGenerationService.java index 3b8731b..845d131 100644 --- a/src/main/java/com/crunchydata/service/SQLFixGenerationService.java +++ b/src/main/java/com/crunchydata/service/SQLFixGenerationService.java @@ -24,9 +24,17 @@ import org.json.JSONObject; import javax.sql.rowset.CachedRowSet; +import java.math.BigDecimal; import java.sql.Connection; import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Objects; @@ -87,7 +95,7 @@ public static String generateFixSQL(Connection sourceConn, Connection targetConn } else if (isNotEqual(rowResult)) { // Row exists on both but columns don't match -> UPDATE target return generateUpdateSQL(sourceConn, targetConn, dctmSource, dctmTarget, - binds, pk, rowResult); + binds, pk, rowResult, columnMapping); } } catch (Exception e) { @@ -298,6 +306,7 @@ private static String generateInsertSQL(Connection sourceConn, DataComparisonTab /** * Generates an UPDATE statement for the target database. + * Uses column mapping to correctly map source column names to target column names. * * @param sourceConn Source database connection * @param targetConn Target database connection @@ -306,11 +315,13 @@ private static String generateInsertSQL(Connection sourceConn, DataComparisonTab * @param binds Bind parameters for the WHERE clause * @param pk Primary key JSONObject * @param rowResult Row result from reCheck containing differences + * @param columnMapping Column mapping JSON containing source-to-target column mappings * @return UPDATE SQL statement */ private static String generateUpdateSQL(Connection sourceConn, Connection targetConn, DataComparisonTableMap dctmSource, DataComparisonTableMap dctmTarget, - ArrayList binds, JSONObject pk, JSONObject rowResult) { + ArrayList binds, JSONObject pk, JSONObject rowResult, + JSONObject columnMapping) { try { String targetQuoteChar = getQuoteChar(Props.getProperty("target-type")); @@ -326,18 +337,59 @@ private static String generateUpdateSQL(Connection sourceConn, Connection target sourceRow.next(); - // Build SET clause with all columns from source - List setItems = new ArrayList<>(); + // Parse column mapping to get source and target column information + JSONArray columns = columnMapping.getJSONArray("columns"); - int columnCount = sourceRow.getMetaData().getColumnCount(); + // Build SET clause using column mapping for proper name translation + List setItems = new ArrayList<>(); - // Start from column 3 (skip pk_hash and pk columns from compare SQL) - for (int i = 3; i <= columnCount; i++) { - String columnName = sourceRow.getMetaData().getColumnName(i); - String quotedColumnName = ShouldQuoteString(false, columnName, targetQuoteChar); + for (int i = 0; i < columns.length(); i++) { + JSONObject columnDef = columns.getJSONObject(i); + + // Skip disabled columns + if (!columnDef.getBoolean("enabled")) { + continue; + } + + // Get source and target column information + JSONObject sourceCol = columnDef.getJSONObject("source"); + JSONObject targetCol = columnDef.getJSONObject("target"); + + // Skip primary key columns - don't update PKs + if (sourceCol.getBoolean("primaryKey")) { + continue; + } + + String sourceColumnName = sourceCol.getString("columnName"); + String targetColumnName = targetCol.getString("columnName"); + boolean targetPreserveCase = targetCol.getBoolean("preserveCase"); + + // Quote target column name based on target's preserveCase setting + String quotedTargetColumnName = ShouldQuoteString(targetPreserveCase, + targetColumnName, + targetQuoteChar); - Object value = sourceRow.getObject(i); - setItems.add(quotedColumnName + " = " + formatValue(value)); + // Get value from source row using source column name + try { + Object value = sourceRow.getObject(sourceColumnName); + setItems.add(quotedTargetColumnName + " = " + formatValue(value)); + } catch (SQLException e) { + // Try with different case if the column name doesn't match exactly + try { + Object value = sourceRow.getObject(sourceColumnName.toLowerCase()); + setItems.add(quotedTargetColumnName + " = " + formatValue(value)); + } catch (SQLException e2) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Could not find column '%s' in source result set for UPDATE pk %s", + sourceColumnName, pk)); + } + } + } + + if (setItems.isEmpty()) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("No columns to update for pk: %s", pk.toString())); + return null; } // Build UPDATE statement @@ -370,6 +422,7 @@ private static String generateUpdateSQL(Connection sourceConn, Connection target /** * Builds a WHERE clause from a primary key JSONObject. + * Handles NULL values correctly using IS NULL syntax. * * @param pk Primary key JSONObject * @param quoteChar Quote character for identifiers @@ -381,48 +434,238 @@ private static String buildWhereClause(JSONObject pk, String quoteChar) { Iterator keys = pk.keys(); while (keys.hasNext()) { String key = keys.next(); - // Remove any existing quotes from the key String cleanKey = key.replace("`", "").replace("\"", ""); String quotedKey = ShouldQuoteString(false, cleanKey, quoteChar); - Object value = pk.get(key); - whereItems.add(quotedKey + " = " + formatValue(value)); + Object value = pk.opt(key); + if (value == null || value == JSONObject.NULL) { + whereItems.add(quotedKey + " IS NULL"); + } else { + whereItems.add(quotedKey + " = " + formatValue(value)); + } } return String.join(" AND ", whereItems); } /** - * Formats a value for SQL statement (adds quotes for strings, handles nulls, etc.). + * Formats a value for SQL statement based on target database type. + * Handles various data types with proper escaping and database-specific syntax. * * @param value Value to format - * @return Formatted value string + * @return Formatted value string suitable for target database */ private static String formatValue(Object value) { - switch (value) { - case null -> { - return "NULL"; - } - case String s -> { - // Escape single quotes by doubling them - String stringValue = s; - stringValue = stringValue.replace("'", "''"); - return "'" + stringValue + "'"; - } - case Number number -> { - return value.toString(); + return formatValueForTarget(value, Props.getProperty("target-type")); + } + + /** + * Formats a value for SQL statement with explicit target database type. + * + * @param value Value to format + * @param targetType Target database type (postgres, oracle, mysql, etc.) + * @return Formatted value string suitable for target database + */ + private static String formatValueForTarget(Object value, String targetType) { + if (value == null) { + return "NULL"; + } + + return switch (value) { + case String s -> formatString(s); + case Boolean b -> formatBoolean(b, targetType); + case Integer i -> i.toString(); + case Long l -> l.toString(); + case Double d -> { + if (d.isNaN()) yield "NULL"; + if (d.isInfinite()) yield "NULL"; + yield d.toString(); } - case Boolean b -> { - return value.toString(); + case Float f -> { + if (f.isNaN()) yield "NULL"; + if (f.isInfinite()) yield "NULL"; + yield f.toString(); } + case BigDecimal bd -> bd.toPlainString(); + case Number n -> n.toString(); + case Timestamp ts -> formatTimestamp(ts, targetType); + case java.sql.Date d -> formatDate(d, targetType); + case java.sql.Time t -> formatTime(t, targetType); + case Date d -> formatJavaDate(d, targetType); + case LocalDateTime ldt -> formatLocalDateTime(ldt, targetType); + case LocalDate ld -> formatLocalDate(ld, targetType); + case LocalTime lt -> formatLocalTime(lt, targetType); + case OffsetDateTime odt -> formatOffsetDateTime(odt, targetType); + case byte[] bytes -> formatBinary(bytes, targetType); default -> { + String stringValue = value.toString(); + if (stringValue == null || stringValue.isEmpty()) { + yield "NULL"; + } + yield formatString(stringValue); } + }; + } + + /** + * Formats a string value with proper escaping. + */ + private static String formatString(String value) { + if (value == null) { + return "NULL"; } - - // For other types, convert to string and quote - String stringValue = value.toString(); - stringValue = stringValue.replace("'", "''"); - return "'" + stringValue + "'"; + String escaped = value.replace("'", "''"); + escaped = escaped.replace("\\", "\\\\"); + return "'" + escaped + "'"; + } + + /** + * Formats a boolean value for the target database. + */ + private static String formatBoolean(Boolean value, String targetType) { + return switch (targetType) { + case "postgres" -> value ? "TRUE" : "FALSE"; + case "oracle", "db2" -> value ? "1" : "0"; + case "mysql", "mariadb" -> value ? "1" : "0"; + case "mssql" -> value ? "1" : "0"; + case "snowflake" -> value ? "TRUE" : "FALSE"; + default -> value.toString(); + }; + } + + /** + * Formats a Timestamp for the target database. + */ + private static String formatTimestamp(Timestamp ts, String targetType) { + String formatted = ts.toString(); + if (formatted.endsWith(".0")) { + formatted = formatted.substring(0, formatted.length() - 2); + } + return switch (targetType) { + case "oracle" -> String.format("TO_TIMESTAMP('%s', 'YYYY-MM-DD HH24:MI:SS.FF')", formatted); + case "mssql" -> String.format("CAST('%s' AS DATETIME2)", formatted); + case "mysql", "mariadb" -> String.format("'%s'", formatted); + case "snowflake" -> String.format("TO_TIMESTAMP('%s')", formatted); + default -> String.format("'%s'::timestamp", formatted); + }; + } + + /** + * Formats a SQL Date for the target database. + */ + private static String formatDate(java.sql.Date d, String targetType) { + String formatted = d.toString(); + return switch (targetType) { + case "oracle" -> String.format("TO_DATE('%s', 'YYYY-MM-DD')", formatted); + case "mssql" -> String.format("CAST('%s' AS DATE)", formatted); + case "snowflake" -> String.format("TO_DATE('%s')", formatted); + default -> String.format("'%s'::date", formatted); + }; + } + + /** + * Formats a SQL Time for the target database. + */ + private static String formatTime(java.sql.Time t, String targetType) { + String formatted = t.toString(); + return switch (targetType) { + case "oracle" -> String.format("TO_TIMESTAMP('%s', 'HH24:MI:SS')", formatted); + case "mssql" -> String.format("CAST('%s' AS TIME)", formatted); + case "snowflake" -> String.format("TO_TIME('%s')", formatted); + default -> String.format("'%s'::time", formatted); + }; + } + + /** + * Formats a Java Date for the target database. + */ + private static String formatJavaDate(Date d, String targetType) { + java.text.SimpleDateFormat sdf = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String formatted = sdf.format(d); + return switch (targetType) { + case "oracle" -> String.format("TO_TIMESTAMP('%s', 'YYYY-MM-DD HH24:MI:SS')", formatted); + case "mssql" -> String.format("CAST('%s' AS DATETIME2)", formatted); + case "snowflake" -> String.format("TO_TIMESTAMP('%s')", formatted); + default -> String.format("'%s'::timestamp", formatted); + }; + } + + /** + * Formats a LocalDateTime for the target database. + */ + private static String formatLocalDateTime(LocalDateTime ldt, String targetType) { + String formatted = ldt.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); + return switch (targetType) { + case "oracle" -> String.format("TO_TIMESTAMP('%s', 'YYYY-MM-DD HH24:MI:SS')", formatted); + case "mssql" -> String.format("CAST('%s' AS DATETIME2)", formatted); + case "snowflake" -> String.format("TO_TIMESTAMP('%s')", formatted); + default -> String.format("'%s'::timestamp", formatted); + }; + } + + /** + * Formats a LocalDate for the target database. + */ + private static String formatLocalDate(LocalDate ld, String targetType) { + String formatted = ld.format(DateTimeFormatter.ISO_LOCAL_DATE); + return switch (targetType) { + case "oracle" -> String.format("TO_DATE('%s', 'YYYY-MM-DD')", formatted); + case "mssql" -> String.format("CAST('%s' AS DATE)", formatted); + case "snowflake" -> String.format("TO_DATE('%s')", formatted); + default -> String.format("'%s'::date", formatted); + }; + } + + /** + * Formats a LocalTime for the target database. + */ + private static String formatLocalTime(LocalTime lt, String targetType) { + String formatted = lt.format(DateTimeFormatter.ISO_LOCAL_TIME); + return switch (targetType) { + case "oracle" -> String.format("TO_TIMESTAMP('%s', 'HH24:MI:SS')", formatted); + case "mssql" -> String.format("CAST('%s' AS TIME)", formatted); + case "snowflake" -> String.format("TO_TIME('%s')", formatted); + default -> String.format("'%s'::time", formatted); + }; + } + + /** + * Formats an OffsetDateTime for the target database. + */ + private static String formatOffsetDateTime(OffsetDateTime odt, String targetType) { + String formatted = odt.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); + return switch (targetType) { + case "oracle" -> String.format("TO_TIMESTAMP_TZ('%s', 'YYYY-MM-DD HH24:MI:SS TZH:TZM')", + odt.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss xxx"))); + case "mssql" -> String.format("CAST('%s' AS DATETIMEOFFSET)", + odt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + case "snowflake" -> String.format("TO_TIMESTAMP_TZ('%s')", + odt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + case "postgres" -> String.format("'%s'::timestamptz", + odt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + default -> String.format("'%s'", formatted); + }; + } + + /** + * Formats binary data for the target database. + */ + private static String formatBinary(byte[] bytes, String targetType) { + if (bytes == null || bytes.length == 0) { + return "NULL"; + } + StringBuilder hex = new StringBuilder(); + for (byte b : bytes) { + hex.append(String.format("%02x", b)); + } + return switch (targetType) { + case "postgres" -> String.format("'\\x%s'::bytea", hex); + case "oracle" -> String.format("HEXTORAW('%s')", hex); + case "mssql" -> String.format("0x%s", hex); + case "mysql", "mariadb" -> String.format("X'%s'", hex); + case "snowflake" -> String.format("TO_BINARY('%s', 'HEX')", hex); + default -> String.format("'%s'", hex); + }; } } diff --git a/src/main/java/com/crunchydata/service/ServerModeService.java b/src/main/java/com/crunchydata/service/ServerModeService.java new file mode 100644 index 0000000..f2bf19c --- /dev/null +++ b/src/main/java/com/crunchydata/service/ServerModeService.java @@ -0,0 +1,881 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.service; + +import com.crunchydata.config.ApplicationState; +import com.crunchydata.config.Settings; +import com.crunchydata.core.database.SQLExecutionHelper; +import com.crunchydata.util.LoggingUtils; + +import java.net.InetAddress; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; + +import static com.crunchydata.config.sql.ServerModeSQLConstants.*; +import static com.crunchydata.service.DatabaseConnectionService.getConnection; +import static com.crunchydata.service.DatabaseConnectionService.isConnectionValid; + +/** + * Service for running pgCompare in server mode. + * In server mode, pgCompare registers as a worker that polls a work queue + * for jobs and executes them. Multiple servers can run concurrently. + * + * @author Brian Pace + */ +public class ServerModeService { + + private static final String THREAD_NAME = "server-mode"; + private static final int HEARTBEAT_INTERVAL_MS = 30000; // 30 seconds + private static final int POLL_INTERVAL_MS = 5000; // 5 seconds + private static final int STALE_SERVER_CHECK_INTERVAL_MS = 60000; // 1 minute + private static final int MAX_RECONNECT_ATTEMPTS = 5; + private static final int RECONNECT_DELAY_MS = 5000; // 5 seconds between retries + private static final String CONN_TYPE_POSTGRES = "postgres"; + private static final String CONN_TYPE_REPO = "repo"; + + private Connection connRepo; + private final String serverName; + private UUID serverId; + private final AtomicBoolean running = new AtomicBoolean(true); + private UUID currentJobId; + + private Thread heartbeatThread; + private Thread staleServerCleanupThread; + + public ServerModeService(Connection connRepo, String serverName) { + this.connRepo = connRepo; + this.serverName = serverName; + } + + /** + * Check if the repository connection is valid and attempt to reconnect if not. + * @return true if connection is valid or successfully reconnected, false if all retries failed + */ + private boolean ensureRepoConnection() { + if (isConnectionValid(connRepo)) { + return true; + } + + LoggingUtils.write("warning", THREAD_NAME, + "Repository connection lost. Attempting to reconnect..."); + + for (int attempt = 1; attempt <= MAX_RECONNECT_ATTEMPTS; attempt++) { + try { + // Close old connection if possible + if (connRepo != null) { + try { connRepo.close(); } catch (Exception e) { /* ignore */ } + } + + // Attempt to reconnect + Connection newConn = getConnection(CONN_TYPE_POSTGRES, CONN_TYPE_REPO); + if (newConn != null && isConnectionValid(newConn)) { + connRepo = newConn; + LoggingUtils.write("info", THREAD_NAME, + String.format("Successfully reconnected to repository on attempt %d", attempt)); + return true; + } + } catch (Exception e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Reconnection attempt %d/%d failed: %s", + attempt, MAX_RECONNECT_ATTEMPTS, e.getMessage())); + } + + if (attempt < MAX_RECONNECT_ATTEMPTS) { + try { + Thread.sleep(RECONNECT_DELAY_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + } + + LoggingUtils.write("severe", THREAD_NAME, + String.format("Failed to reconnect to repository after %d attempts. Server will exit.", + MAX_RECONNECT_ATTEMPTS)); + return false; + } + + /** + * Start the server in daemon mode. + */ + public void start() { + try { + // Register this server + registerServer(); + + // Start heartbeat thread + startHeartbeatThread(); + + // Start stale server cleanup thread + startStaleServerCleanupThread(); + + LoggingUtils.write("info", THREAD_NAME, + String.format("Server '%s' started with ID: %s", serverName, serverId)); + + // Main work loop + while (running.get() && !ApplicationState.getInstance().isGracefulShutdownRequested()) { + try { + // Ensure repository connection is valid + if (!ensureRepoConnection()) { + LoggingUtils.write("severe", THREAD_NAME, + "Cannot maintain repository connection. Server shutting down."); + break; + } + + // Check for and process next job + processNextJob(); + + // Wait before polling again + Thread.sleep(POLL_INTERVAL_MS); + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } catch (Exception e) { + // Check if this is a connection-related error + if (isConnectionError(e)) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Connection error detected: %s", e.getMessage())); + // Will attempt reconnection on next loop iteration + } else { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Error processing job: %s", e.getMessage())); + } + } + } + + } catch (Exception e) { + LoggingUtils.write("severe", THREAD_NAME, + String.format("Server startup failed: %s", e.getMessage())); + } finally { + shutdown(); + } + } + + /** + * Register this server in the dc_server table. + * First removes any existing record with the same server name. + */ + private void registerServer() throws SQLException { + String hostname = getHostname(); + long pid = ProcessHandle.current().pid(); + String config = String.format("{\"version\":\"%s\"}", Settings.VERSION); + + // First, delete any existing record with the same server name + ArrayList deleteBinds = new ArrayList<>(); + deleteBinds.add(serverName); + ResultSet rsDelete = SQLExecutionHelper.simpleUpdateReturning(connRepo, SQL_SERVER_DELETE_BY_NAME, deleteBinds); + if (rsDelete != null) { + while (rsDelete.next()) { + String oldStatus = rsDelete.getString("status"); + LoggingUtils.write("info", THREAD_NAME, + String.format("Removed previous server record for '%s' (was %s)", serverName, oldStatus)); + } + rsDelete.close(); + } + + // Now register the new server + ArrayList binds = new ArrayList<>(); + binds.add(serverName); + binds.add(hostname); + binds.add(pid); + binds.add(config); + + ResultSet rs = SQLExecutionHelper.simpleUpdateReturning(connRepo, SQL_SERVER_REGISTER, binds); + if (rs != null && rs.next()) { + serverId = UUID.fromString(rs.getString("server_id")); + rs.close(); + } else { + throw new SQLException("Failed to register server"); + } + + LoggingUtils.write("info", THREAD_NAME, + String.format("Registered server: %s (%s) with PID %d", serverName, hostname, pid)); + } + + /** + * Start the heartbeat thread to keep the server registration alive. + */ + private void startHeartbeatThread() { + heartbeatThread = new Thread(() -> { + while (running.get()) { + try { + if (isConnectionValid(connRepo)) { + updateHeartbeat(); + } + Thread.sleep(HEARTBEAT_INTERVAL_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } catch (Exception e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Heartbeat failed: %s", e.getMessage())); + } + } + }, "server-heartbeat"); + heartbeatThread.setDaemon(true); + heartbeatThread.start(); + } + + /** + * Start the stale server cleanup thread. + */ + private void startStaleServerCleanupThread() { + staleServerCleanupThread = new Thread(() -> { + while (running.get()) { + try { + if (isConnectionValid(connRepo)) { + markStaleServers(); + } + Thread.sleep(STALE_SERVER_CHECK_INTERVAL_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } catch (Exception e) { + LoggingUtils.write("debug", THREAD_NAME, + String.format("Stale server cleanup failed: %s", e.getMessage())); + } + } + }, "stale-server-cleanup"); + staleServerCleanupThread.setDaemon(true); + staleServerCleanupThread.start(); + } + + /** + * Update the heartbeat timestamp. + */ + private void updateHeartbeat() throws SQLException { + String status = currentJobId != null ? "busy" : "idle"; + + ArrayList binds = new ArrayList<>(); + binds.add(status); + binds.add(serverId.toString()); + + SQLExecutionHelper.simpleUpdate(connRepo, SQL_SERVER_HEARTBEAT, binds, false); + } + + /** + * Mark stale servers as offline, delete very stale servers, and fail orphaned jobs. + */ + private void markStaleServers() throws SQLException { + ArrayList binds = new ArrayList<>(); + + // First, get the names of servers that will be marked stale + ResultSet rsStale = SQLExecutionHelper.simpleSelect(connRepo, SQL_SERVER_SELECT_STALE_TO_MARK, binds); + List staleServers = new ArrayList<>(); + if (rsStale != null) { + while (rsStale.next()) { + staleServers.add(rsStale.getString("server_name") + " (" + rsStale.getString("server_host") + ")"); + } + rsStale.close(); + } + + // Mark them as offline + int updated = SQLExecutionHelper.simpleUpdate(connRepo, SQL_SERVER_MARK_STALE, binds, false); + if (updated > 0) { + for (String serverName : staleServers) { + LoggingUtils.write("info", THREAD_NAME, + String.format("Marked server as offline (no heartbeat for 2+ minutes): %s", serverName)); + } + } + + // Handle orphaned jobs - jobs running on servers that are offline/terminated/missing + markOrphanedJobsAsFailed(); + + // Get servers that will be deleted + ResultSet rsDelete = SQLExecutionHelper.simpleSelect(connRepo, SQL_SERVER_SELECT_STALE_TO_DELETE, binds); + List deleteServers = new ArrayList<>(); + if (rsDelete != null) { + while (rsDelete.next()) { + deleteServers.add(rsDelete.getString("server_name") + " (" + rsDelete.getString("server_host") + ")"); + } + rsDelete.close(); + } + + // Delete them + int deleted = SQLExecutionHelper.simpleUpdate(connRepo, SQL_SERVER_DELETE_STALE, binds, true); + if (deleted > 0) { + for (String serverName : deleteServers) { + LoggingUtils.write("info", THREAD_NAME, + String.format("Deleted stale server (no heartbeat for 5+ minutes): %s", serverName)); + } + } + } + + /** + * Mark orphaned jobs as failed. An orphaned job is one that is marked as 'running' + * but its assigned server is offline, terminated, or hasn't sent a heartbeat recently. + */ + private void markOrphanedJobsAsFailed() { + try { + ArrayList binds = new ArrayList<>(); + ResultSet rs = SQLExecutionHelper.simpleSelect(connRepo, SQL_JOB_SELECT_ORPHANED, binds); + + if (rs != null) { + while (rs.next()) { + UUID jobId = UUID.fromString(rs.getString("job_id")); + String jobType = rs.getString("job_type"); + String serverName = rs.getString("server_name"); + String serverStatus = rs.getString("server_status"); + + // Mark job as failed + ArrayList updateBinds = new ArrayList<>(); + updateBinds.add(jobId.toString()); + int updated = SQLExecutionHelper.simpleUpdate(connRepo, SQL_JOB_MARK_ORPHANED_FAILED, updateBinds, true); + + if (updated > 0) { + String reason = serverName == null ? "server no longer exists" : + String.format("server '%s' is %s", serverName, serverStatus != null ? serverStatus : "unresponsive"); + LoggingUtils.write("warning", THREAD_NAME, + String.format("Marked orphaned job %s (%s) as failed: %s", jobId, jobType, reason)); + } + } + rs.close(); + } + } catch (Exception e) { + LoggingUtils.write("debug", THREAD_NAME, + String.format("Failed to check for orphaned jobs: %s", e.getMessage())); + } + } + + /** + * Process the next available job from the queue. + */ + private void processNextJob() { + try { + // Try to claim a job + ArrayList binds = new ArrayList<>(); + binds.add(serverId.toString()); + binds.add(serverId.toString()); + + ResultSet rs = SQLExecutionHelper.simpleSelect(connRepo, SQL_JOB_CLAIM_NEXT, binds); + + if (rs != null && rs.next()) { + currentJobId = UUID.fromString(rs.getString("job_id")); + int pid = rs.getInt("pid"); + String jobType = rs.getString("job_type"); + int batchNbr = rs.getInt("batch_nbr"); + String tableFilter = rs.getString("table_filter"); + String jobConfig = rs.getString("job_config"); + rs.close(); + + LoggingUtils.write("info", THREAD_NAME, + String.format("Claimed job %s: type=%s, pid=%d, batch=%d", + currentJobId, jobType, pid, batchNbr)); + + // Execute the job + executeJob(currentJobId, pid, jobType, batchNbr, tableFilter, jobConfig); + + } else if (rs != null) { + rs.close(); + } + } catch (Exception e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Error claiming job: %s", e.getMessage())); + + // Mark job as failed if we had claimed it + if (currentJobId != null) { + try { + updateJobStatus(currentJobId, "failed", null, e.getMessage()); + } catch (SQLException ex) { + LoggingUtils.write("severe", THREAD_NAME, + String.format("Failed to mark job as failed: %s", ex.getMessage())); + } + } + } finally { + currentJobId = null; + } + } + + /** + * Execute a job. + */ + private void executeJob(UUID jobId, int pid, String jobType, int batchNbr, + String tableFilter, String jobConfig) { + try { + // Load project configuration FIRST (before setting job context) + Settings.setProjectConfig(connRepo, pid); + + // Set job context for logging AFTER project config is loaded + // so job-logging-enabled from project config is respected + LoggingUtils.setJobContext(jobId, connRepo); + + // Set batch number + Settings.Props.setProperty("batch", String.valueOf(batchNbr)); + Settings.Props.setProperty("pid", String.valueOf(pid)); + + // Set table filter - always set it to clear any previous value + Settings.Props.setProperty("table", tableFilter != null ? tableFilter : ""); + + // Clear isCheck flag from any previous job + Settings.Props.setProperty("isCheck", "false"); + + // Apply fix setting from job_config if present + if (jobConfig != null && !jobConfig.isEmpty()) { + try { + org.json.JSONObject config = new org.json.JSONObject(jobConfig); + if (config.has("fix")) { + Settings.Props.setProperty("fix", String.valueOf(config.getBoolean("fix"))); + } + } catch (Exception e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Failed to parse job_config: %s", e.getMessage())); + } + } + + LoggingUtils.write("info", THREAD_NAME, + String.format("Starting job %s: type=%s, pid=%d, batch=%d", jobId, jobType, pid, batchNbr)); + + // Log configuration parameters for full job visibility + logConfigurationParameters(); + + // Execute based on job type + switch (jobType) { + case "compare": + executeCompareJob(jobId, pid, batchNbr); + break; + case "check": + Settings.Props.setProperty("isCheck", "true"); + executeCompareJob(jobId, pid, batchNbr); + break; + case "discover": + executeDiscoverJob(jobId, pid, tableFilter); + break; + case "test-connection": + executeTestConnectionJob(jobId, pid); + break; + default: + throw new IllegalArgumentException("Unknown job type: " + jobType); + } + + // Mark job as completed (test-connection handles its own status) + if (!"test-connection".equals(jobType)) { + // Use 'error' status if SEVERE errors occurred, otherwise 'completed' + String status = LoggingUtils.hasJobSevereError() ? "error" : "completed"; + updateJobStatus(jobId, status, buildResultSummary(jobId), null); + } + + LoggingUtils.write("info", THREAD_NAME, + String.format("Job %s completed successfully", jobId)); + + } catch (Exception e) { + LoggingUtils.write("severe", THREAD_NAME, + String.format("Job %s failed: %s", jobId, e.getMessage())); + + try { + updateJobStatus(jobId, "failed", null, e.getMessage()); + } catch (SQLException ex) { + LoggingUtils.write("severe", THREAD_NAME, + String.format("Failed to update job status: %s", ex.getMessage())); + } + } finally { + // Clear job context when job completes + LoggingUtils.clearJobContext(); + } + } + + /** + * Execute a compare/check job with progress tracking. + */ + private void executeCompareJob(UUID jobId, int pid, int batchNbr) throws Exception { + // Get source and target connections + Connection connSource = DatabaseConnectionService.getConnection( + Settings.Props.getProperty("source-type"), "source"); + Connection connTarget = DatabaseConnectionService.getConnection( + Settings.Props.getProperty("target-type"), "target"); + + if (connSource == null || connTarget == null) { + throw new RuntimeException("Failed to connect to source or target database"); + } + + try { + // Create a job-aware compare controller that reports progress + JobAwareCompareController.performCompare( + jobId, connRepo, connSource, connTarget, pid, batchNbr, this); + } finally { + try { connSource.close(); } catch (Exception e) { } + try { connTarget.close(); } catch (Exception e) { } + } + } + + /** + * Execute a discover job. + */ + private void executeDiscoverJob(UUID jobId, int pid, String tableFilter) throws Exception { + Connection connSource = DatabaseConnectionService.getConnection( + Settings.Props.getProperty("source-type"), "source"); + Connection connTarget = DatabaseConnectionService.getConnection( + Settings.Props.getProperty("target-type"), "target"); + + if (connSource == null || connTarget == null) { + throw new RuntimeException("Failed to connect to source or target database"); + } + + try { + // Discover tables + com.crunchydata.controller.DiscoverController.performTableDiscovery( + Settings.Props, pid, tableFilter != null ? tableFilter : "", + connRepo, connSource, connTarget); + + // Discover columns + com.crunchydata.controller.DiscoverController.performColumnDiscovery( + Settings.Props, pid, tableFilter != null ? tableFilter : "", + connRepo, connSource, connTarget); + } finally { + try { connSource.close(); } catch (Exception e) { } + try { connTarget.close(); } catch (Exception e) { } + } + } + + /** + * Execute a test-connection job. + */ + private void executeTestConnectionJob(UUID jobId, int pid) { + LoggingUtils.write("info", THREAD_NAME, + String.format("Testing connections for project %d", pid)); + + Map results = + ConnectionTestService.testAllConnections(); + + // Build JSON result + StringBuilder json = new StringBuilder(); + json.append("{"); + + int i = 0; + for (Map.Entry entry : results.entrySet()) { + if (i > 0) json.append(","); + json.append("\"").append(entry.getKey()).append("\":"); + json.append(resultToJson(entry.getValue())); + i++; + } + + json.append("}"); + + boolean allSuccess = results.values().stream().allMatch(r -> r.success); + String status = allSuccess ? "completed" : "completed"; + + try { + updateJobStatus(jobId, status, json.toString(), null); + } catch (SQLException e) { + LoggingUtils.write("severe", THREAD_NAME, + String.format("Failed to update test-connection job status: %s", e.getMessage())); + } + } + + private String resultToJson(ConnectionTestService.ConnectionTestResult result) { + StringBuilder json = new StringBuilder(); + json.append("{"); + json.append("\"success\":").append(result.success).append(","); + json.append("\"connectionType\":\"").append(escape(result.connectionType)).append("\","); + json.append("\"databaseType\":\"").append(escape(result.databaseType)).append("\","); + json.append("\"host\":\"").append(escape(result.host)).append("\","); + json.append("\"port\":").append(result.port).append(","); + json.append("\"database\":\"").append(escape(result.database)).append("\","); + json.append("\"schema\":\"").append(escape(result.schema)).append("\","); + json.append("\"user\":\"").append(escape(result.user)).append("\","); + json.append("\"databaseProductName\":\"").append(escape(result.databaseProductName)).append("\","); + json.append("\"databaseProductVersion\":\"").append(escape(result.databaseProductVersion)).append("\","); + json.append("\"errorMessage\":\"").append(escape(result.errorMessage)).append("\","); + json.append("\"errorDetail\":\"").append(escape(result.errorDetail)).append("\","); + json.append("\"responseTimeMs\":").append(result.responseTimeMs); + json.append("}"); + return json.toString(); + } + + private String escape(String str) { + if (str == null) return ""; + return str.replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t"); + } + + /** + * Update job status in the work queue. + */ + private void updateJobStatus(UUID jobId, String status, String resultSummary, + String errorMessage) throws SQLException { + ArrayList binds = new ArrayList<>(); + binds.add(status); + binds.add(status); + binds.add(resultSummary); + binds.add(errorMessage); + binds.add(jobId.toString()); + + SQLExecutionHelper.simpleUpdate(connRepo, SQL_JOB_UPDATE_STATUS, binds, true); + } + + /** + * Update job progress for a specific table (status and cid only, counts come from dc_result). + */ + public void updateJobProgress(UUID jobId, long tid, String status, String errorMessage, Integer cid) { + try { + ArrayList binds = new ArrayList<>(); + binds.add(status); + binds.add(status); + binds.add(errorMessage); + binds.add(cid); + binds.add(jobId.toString()); + binds.add(tid); + + SQLExecutionHelper.simpleUpdate(connRepo, SQL_JOBPROGRESS_UPDATE, binds, true); + } catch (Exception e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Failed to update job progress: %s", e.getMessage())); + } + } + + /** + * Initialize job progress records for all tables. + */ + public void initializeJobProgress(UUID jobId, long tid, String tableName) { + try { + ArrayList binds = new ArrayList<>(); + binds.add(jobId.toString()); + binds.add(tid); + binds.add(tableName); + + SQLExecutionHelper.simpleUpdate(connRepo, SQL_JOBPROGRESS_INSERT, binds, true); + } catch (Exception e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Failed to initialize job progress: %s", e.getMessage())); + } + } + + /** + * Check for control signals (pause, stop, terminate). + */ + public String checkJobControlSignal(UUID jobId) { + try { + ArrayList binds = new ArrayList<>(); + binds.add(jobId.toString()); + + ResultSet rs = SQLExecutionHelper.simpleSelect(connRepo, SQL_JOBCONTROL_CHECK_PENDING, binds); + + if (rs != null && rs.next()) { + int controlId = rs.getInt("control_id"); + String signal = rs.getString("signal"); + rs.close(); + + // Mark as processed + ArrayList markBinds = new ArrayList<>(); + markBinds.add(controlId); + SQLExecutionHelper.simpleUpdate(connRepo, SQL_JOBCONTROL_MARK_PROCESSED, markBinds, false); + + return signal; + } + + if (rs != null) { + rs.close(); + } + } catch (Exception e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Failed to check job control signal: %s", e.getMessage())); + } + + return null; + } + + /** + * Build a result summary JSON string. + */ + private String buildResultSummary(UUID jobId) { + try { + ArrayList binds = new ArrayList<>(); + binds.add(jobId.toString()); + + ResultSet rs = SQLExecutionHelper.simpleSelect(connRepo, SQL_JOBPROGRESS_SUMMARY, binds); + + if (rs != null && rs.next()) { + String summary = String.format( + "{\"totalTables\":%d,\"completedTables\":%d,\"failedTables\":%d," + + "\"totalSource\":%d,\"totalEqual\":%d,\"totalNotEqual\":%d,\"totalMissing\":%d}", + rs.getInt("total_tables"), + rs.getInt("completed_tables"), + rs.getInt("failed_tables"), + rs.getLong("total_source"), + rs.getLong("total_equal"), + rs.getLong("total_not_equal"), + rs.getLong("total_missing") + ); + rs.close(); + return summary; + } + + if (rs != null) { + rs.close(); + } + } catch (Exception e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Failed to build result summary: %s", e.getMessage())); + } + + return null; + } + + /** + * Shutdown the server. + */ + public void shutdown() { + running.set(false); + + // Stop heartbeat thread + if (heartbeatThread != null) { + heartbeatThread.interrupt(); + } + + // Stop stale server cleanup thread + if (staleServerCleanupThread != null) { + staleServerCleanupThread.interrupt(); + } + + // Delete server entry from dc_server table + if (serverId != null) { + try { + ArrayList binds = new ArrayList<>(); + binds.add(serverId.toString()); + SQLExecutionHelper.simpleUpdate(connRepo, SQL_SERVER_DELETE, binds, true); + + LoggingUtils.write("info", THREAD_NAME, + String.format("Server '%s' removed from registry", serverName)); + } catch (Exception e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Failed to remove server from registry: %s", e.getMessage())); + } + } + } + + /** + * Stop the server gracefully. + */ + public void stop() { + running.set(false); + } + + /** + * Get the server ID. + */ + public UUID getServerId() { + return serverId; + } + + /** + * Get the current job ID. + */ + public UUID getCurrentJobId() { + return currentJobId; + } + + /** + * Check if an exception is related to a database connection error. + */ + private boolean isConnectionError(Exception e) { + String message = e.getMessage(); + if (message == null) { + return false; + } + message = message.toLowerCase(); + return message.contains("connection") + || message.contains("closed") + || message.contains("socket") + || message.contains("timeout") + || message.contains("network") + || message.contains("i/o error") + || message.contains("communication"); + } + + /** + * Get the hostname. + */ + private String getHostname() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (Exception e) { + return "unknown"; + } + } + + /** + * Log configuration parameters (excluding passwords) for job visibility. + * Mirrors the standalone behavior from ApplicationContext.logConfigurationParameters(). + */ + private void logConfigurationParameters() { + LoggingUtils.write("info", THREAD_NAME, "Parameters: "); + Settings.Props.entrySet().stream() + .filter(e -> !e.getKey().toString().contains("password")) + .sorted((e1, e2) -> e1.getKey().toString().compareTo(e2.getKey().toString())) + .forEach(e -> LoggingUtils.write("info", THREAD_NAME, String.format(" %s", e))); + } + + /** + * Static method to submit a job to the work queue. + */ + public static UUID submitJob(Connection connRepo, int pid, String jobType, + int priority, int batchNbr, String tableFilter, + String targetServerId, String scheduledAt, + String createdBy, String jobConfig) throws SQLException { + ArrayList binds = new ArrayList<>(); + binds.add(pid); + binds.add(jobType); + binds.add(priority); + binds.add(batchNbr); + binds.add(tableFilter); + binds.add(targetServerId); + binds.add(scheduledAt); + binds.add(createdBy); + binds.add(jobConfig); + + ResultSet rs = SQLExecutionHelper.simpleSelect(connRepo, SQL_JOB_INSERT, binds); + + if (rs != null && rs.next()) { + UUID jobId = UUID.fromString(rs.getString("job_id")); + rs.close(); + return jobId; + } + + throw new SQLException("Failed to submit job"); + } + + /** + * Static method to send a control signal to a running job. + */ + public static int sendControlSignal(Connection connRepo, UUID jobId, + String signal, String requestedBy) throws SQLException { + ArrayList binds = new ArrayList<>(); + binds.add(jobId.toString()); + binds.add(signal); + binds.add(requestedBy); + + ResultSet rs = SQLExecutionHelper.simpleSelect(connRepo, SQL_JOBCONTROL_INSERT, binds); + + if (rs != null && rs.next()) { + int controlId = rs.getInt("control_id"); + rs.close(); + return controlId; + } + + throw new SQLException("Failed to send control signal"); + } +} diff --git a/src/main/java/com/crunchydata/service/SignalHandlerService.java b/src/main/java/com/crunchydata/service/SignalHandlerService.java new file mode 100644 index 0000000..b8486b0 --- /dev/null +++ b/src/main/java/com/crunchydata/service/SignalHandlerService.java @@ -0,0 +1,142 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.crunchydata.service; + +import com.crunchydata.config.ApplicationState; +import com.crunchydata.config.Settings; +import com.crunchydata.util.LoggingUtils; +import sun.misc.Signal; + +/** + * Service that registers and handles OS signals for shutdown and configuration reload. + * + * Supported signals: + * - SIGINT (2): Graceful shutdown - allows current table comparison to complete (Ctrl+C) + * - SIGTERM (15): Immediate termination - cancels all queries and exits + * - SIGHUP (1): Reload configuration from properties file + * + * @author Brian Pace + */ +public class SignalHandlerService { + + private static final String THREAD_NAME = "signal-handler"; + private static boolean initialized = false; + + private SignalHandlerService() { + } + + /** + * Register all signal handlers. Should be called once during application startup. + */ + public static synchronized void initialize() { + if (initialized) { + LoggingUtils.write("warning", THREAD_NAME, "Signal handlers already initialized"); + return; + } + + try { + registerGracefulShutdownHandler("INT"); + registerImmediateTerminationHandler("TERM"); + registerReloadHandler("HUP"); + + initialized = true; + LoggingUtils.write("info", THREAD_NAME, + "Signal handlers registered: SIGINT (graceful), SIGTERM (immediate), SIGHUP (reload)"); + } catch (IllegalArgumentException e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Could not register signal handlers (may not be supported on this platform): %s", e.getMessage())); + } + } + + /** + * Register a graceful shutdown signal handler (SIGINT/Ctrl+C). + * Allows current table comparison to complete before exit. + * + * @param signalName The signal name (e.g., "INT") + */ + private static void registerGracefulShutdownHandler(String signalName) { + try { + Signal.handle(new Signal(signalName), signal -> { + LoggingUtils.write("info", THREAD_NAME, + String.format("Received SIG%s - initiating graceful shutdown (current table will complete)", signalName)); + ApplicationState.getInstance().requestGracefulShutdown(); + }); + } catch (IllegalArgumentException e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Could not register SIG%s handler: %s", signalName, e.getMessage())); + } + } + + /** + * Register an immediate termination signal handler (SIGTERM). + * Cancels all queries and exits immediately. + * + * @param signalName The signal name (e.g., "TERM") + */ + private static void registerImmediateTerminationHandler(String signalName) { + try { + Signal.handle(new Signal(signalName), signal -> { + LoggingUtils.write("info", THREAD_NAME, + String.format("Received SIG%s - cancelling all queries and terminating immediately", signalName)); + ApplicationState.getInstance().requestImmediateTermination(); + }); + } catch (IllegalArgumentException e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Could not register SIG%s handler: %s", signalName, e.getMessage())); + } + } + + /** + * Register a reload signal handler for SIGHUP. + * + * @param signalName The signal name (should be "HUP") + */ + private static void registerReloadHandler(String signalName) { + try { + Signal.handle(new Signal(signalName), signal -> { + LoggingUtils.write("info", THREAD_NAME, + String.format("Received SIG%s - requesting configuration reload", signalName)); + ApplicationState.getInstance().requestReload(); + + try { + Settings.reloadProperties(); + LoggingUtils.write("info", THREAD_NAME, "Configuration reloaded successfully"); + } catch (Exception e) { + LoggingUtils.write("severe", THREAD_NAME, + String.format("Failed to reload configuration: %s", e.getMessage())); + } + }); + } catch (IllegalArgumentException e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Could not register SIG%s handler: %s", signalName, e.getMessage())); + } + } + + /** + * Check if signal handlers are supported on the current platform. + * + * @return true if signal handlers can be registered + */ + public static boolean isSignalHandlingSupported() { + try { + new Signal("TERM"); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } +} diff --git a/src/main/java/com/crunchydata/service/StandaloneJobService.java b/src/main/java/com/crunchydata/service/StandaloneJobService.java new file mode 100644 index 0000000..58dd657 --- /dev/null +++ b/src/main/java/com/crunchydata/service/StandaloneJobService.java @@ -0,0 +1,298 @@ +/* + * Copyright 2012-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.crunchydata.service; + +import com.crunchydata.core.database.SQLExecutionHelper; +import com.crunchydata.util.LoggingUtils; + +import javax.sql.rowset.CachedRowSet; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.UUID; + +import static com.crunchydata.config.Settings.Props; + +/** + * Service for tracking standalone (non-server mode) jobs in the dc_job table. + * This enables unified job history viewing from the UI regardless of how + * pgcompare was invoked. + * + * @author Brian Pace + */ +public class StandaloneJobService { + + private static final String THREAD_NAME = "standalone-job"; + + private static final String SQL_JOB_CREATE = """ + INSERT INTO dc_job (pid, job_type, status, batch_nbr, table_filter, started_at, source, rid) + VALUES (?, ?, 'running', ?, ?, current_timestamp, 'standalone', ?) + RETURNING job_id + """; + + private static final String SQL_JOB_UPDATE_STATUS = """ + UPDATE dc_job + SET status = ?, completed_at = CASE WHEN ? IN ('completed', 'error', 'failed', 'cancelled') THEN current_timestamp ELSE completed_at END, + result_summary = COALESCE(?::jsonb, result_summary), + error_message = COALESCE(?, error_message) + WHERE job_id = ?::uuid + """; + + private static final String SQL_JOBPROGRESS_INSERT = """ + INSERT INTO dc_job_progress (job_id, tid, table_name, status) + VALUES (?::uuid, ?, ?, 'pending') + ON CONFLICT (job_id, tid) DO NOTHING + """; + + private static final String SQL_JOBPROGRESS_UPDATE = """ + UPDATE dc_job_progress + SET status = ?, + started_at = CASE WHEN ? = 'running' THEN COALESCE(started_at, current_timestamp) ELSE started_at END, + completed_at = CASE WHEN ? IN ('completed', 'failed', 'skipped') THEN current_timestamp ELSE completed_at END, + error_message = COALESCE(?, error_message), + cid = COALESCE(?, cid) + WHERE job_id = ?::uuid AND tid = ? + """; + + private final Connection connRepo; + private UUID currentJobId; + + public StandaloneJobService(Connection connRepo) { + this.connRepo = connRepo; + } + + /** + * Creates a new standalone job record and sets up logging context. + * + * @param pid Project ID + * @param jobType Type of job (compare, check, discover) + * @param batchNbr Batch number + * @param tableFilter Table filter if any + * @param rid Run ID (typically System.currentTimeMillis()) + * @return The created job ID + */ + public UUID startJob(int pid, String jobType, int batchNbr, String tableFilter, long rid) { + try { + ArrayList binds = new ArrayList<>(); + binds.add(pid); + binds.add(jobType); + binds.add(batchNbr); + binds.add(tableFilter != null && !tableFilter.isEmpty() ? tableFilter : null); + binds.add(rid); + + CachedRowSet rs = SQLExecutionHelper.simpleUpdateReturning(connRepo, SQL_JOB_CREATE, binds); + if (rs != null && rs.next()) { + currentJobId = UUID.fromString(rs.getString(1)); + rs.close(); + + // Set up logging context + LoggingUtils.setJobContext(currentJobId, connRepo); + + LoggingUtils.write("info", THREAD_NAME, + String.format("Started standalone job %s: type=%s, pid=%d, batch=%d", + currentJobId, jobType, pid, batchNbr)); + + return currentJobId; + } + } catch (Exception e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Failed to create standalone job record: %s", e.getMessage())); + } + return null; + } + + /** + * Marks the current job as completed. + * Uses 'error' status if SEVERE errors occurred during the job. + * + * @param resultSummary Optional JSON result summary + */ + public void completeJob(String resultSummary) { + String status = LoggingUtils.hasJobSevereError() ? "error" : "completed"; + updateJobStatus(status, resultSummary, null); + LoggingUtils.clearJobContext(); + } + + /** + * Marks the current job as failed. + * + * @param errorMessage Error message describing the failure + */ + public void failJob(String errorMessage) { + updateJobStatus("failed", null, errorMessage); + LoggingUtils.clearJobContext(); + } + + /** + * Updates the job status. + */ + private void updateJobStatus(String status, String resultSummary, String errorMessage) { + if (currentJobId == null) { + return; + } + + try { + ArrayList binds = new ArrayList<>(); + binds.add(status); + binds.add(status); + binds.add(resultSummary); + binds.add(errorMessage); + binds.add(currentJobId.toString()); + + SQLExecutionHelper.simpleUpdate(connRepo, SQL_JOB_UPDATE_STATUS, binds, true); + + LoggingUtils.write("info", THREAD_NAME, + String.format("Job %s completed with status: %s", currentJobId, status)); + } catch (Exception e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Failed to update job status: %s", e.getMessage())); + } + } + + /** + * Gets the current job ID. + */ + public UUID getCurrentJobId() { + return currentJobId; + } + + /** + * Initialize job progress records for a table. + * This should be called before processing each table. + * + * @param tid Table ID + * @param tableName Table name + */ + public void initializeTableProgress(long tid, String tableName) { + if (currentJobId == null) { + return; + } + + try { + ArrayList binds = new ArrayList<>(); + binds.add(currentJobId.toString()); + binds.add(tid); + binds.add(tableName); + + SQLExecutionHelper.simpleUpdate(connRepo, SQL_JOBPROGRESS_INSERT, binds, true); + LoggingUtils.write("debug", THREAD_NAME, + String.format("Initialized progress for table %s (tid=%d)", tableName, tid)); + } catch (Exception e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Failed to initialize table progress: %s", e.getMessage())); + } + } + + /** + * Update progress for a specific table. + * + * @param tid Table ID + * @param status Status (pending, running, completed, failed, skipped) + * @param errorMessage Error message if failed + * @param cid Compare ID if available + */ + public void updateTableProgress(long tid, String status, String errorMessage, Integer cid) { + if (currentJobId == null) { + return; + } + + try { + ArrayList binds = new ArrayList<>(); + binds.add(status); + binds.add(status); + binds.add(status); + binds.add(errorMessage); + binds.add(cid); + binds.add(currentJobId.toString()); + binds.add(tid); + + SQLExecutionHelper.simpleUpdate(connRepo, SQL_JOBPROGRESS_UPDATE, binds, true); + LoggingUtils.write("debug", THREAD_NAME, + String.format("Updated progress for tid=%d: status=%s", tid, status)); + } catch (Exception e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Failed to update table progress: %s", e.getMessage())); + } + } + + /** + * Checks if standalone job tracking is enabled. + * Job tracking requires the dc_job table to exist. + */ + public static boolean isJobTrackingAvailable(Connection conn) { + try { + ArrayList binds = new ArrayList<>(); + CachedRowSet rs = SQLExecutionHelper.simpleSelect(conn, + "SELECT 1 FROM information_schema.tables WHERE table_name = 'dc_job' LIMIT 1", binds); + boolean exists = rs != null && rs.next(); + if (rs != null) rs.close(); + return exists; + } catch (Exception e) { + return false; + } + } + + /** + * Checks if a project exists in the repository. + * @param conn Repository connection + * @param pid Project ID to check + * @return true if project exists, false otherwise + */ + public static boolean projectExists(Connection conn, int pid) { + try { + ArrayList binds = new ArrayList<>(); + binds.add(pid); + CachedRowSet rs = SQLExecutionHelper.simpleSelect(conn, + "SELECT 1 FROM dc_project WHERE pid = ? LIMIT 1", binds); + boolean exists = rs != null && rs.next(); + if (rs != null) rs.close(); + return exists; + } catch (Exception e) { + return false; + } + } + + /** + * Ensures a project exists, creating it if necessary. + * Uses OVERRIDING SYSTEM VALUE to set a specific pid for auto-generated identity column. + * @param conn Repository connection + * @param pid Project ID to ensure exists + * @return true if project exists or was created, false on error + */ + public static boolean ensureProjectExists(Connection conn, int pid) { + if (projectExists(conn, pid)) { + return true; + } + + try { + ArrayList binds = new ArrayList<>(); + binds.add(pid); + binds.add("Project " + pid); + + SQLExecutionHelper.simpleUpdate(conn, + "INSERT INTO dc_project (pid, project_name) OVERRIDING SYSTEM VALUE VALUES (?, ?)", + binds, true); + + LoggingUtils.write("info", THREAD_NAME, + String.format("Created project %d for standalone job tracking", pid)); + return true; + } catch (Exception e) { + LoggingUtils.write("warning", THREAD_NAME, + String.format("Failed to create project %d: %s", pid, e.getMessage())); + return false; + } + } +} diff --git a/src/main/java/com/crunchydata/util/LoggingUtils.java b/src/main/java/com/crunchydata/util/LoggingUtils.java index 73f6d34..5d1494c 100644 --- a/src/main/java/com/crunchydata/util/LoggingUtils.java +++ b/src/main/java/com/crunchydata/util/LoggingUtils.java @@ -18,6 +18,9 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.util.UUID; import java.util.logging.*; import static com.crunchydata.config.Settings.Props; @@ -25,8 +28,7 @@ /** * Utility class for logging operations. * Provides methods to initialize logging configurations and write log messages at various severity levels. - * - *

This class is not instantiable.

+ * Supports writing logs to a database table for job tracking when a job context is set. * * @author Brian Pace */ @@ -38,16 +40,82 @@ public final class LoggingUtils { private static final String DEFAULT_LOG_FORMAT = "[%1$tF %1$tT] [%4$-7s] %5$s %n"; private static final String MODULE_FORMAT = "[%-24s] %s"; + private static final String SQL_JOBLOG_INSERT = """ + INSERT INTO dc_job_log (job_id, log_level, thread_name, message, context) + VALUES (?::uuid, ?, ?, ?, ?::jsonb) + """; + + private static final ThreadLocal jobContext = new ThreadLocal<>(); + private static final ThreadLocal writingToJobLog = ThreadLocal.withInitial(() -> false); + static { - // Set default format for log messages System.setProperty(LOG_FORMAT_PROPERTY, DEFAULT_LOG_FORMAT); } - // Private constructor to prevent instantiation private LoggingUtils() { throw new UnsupportedOperationException("Utility class"); } + /** + * Context for job logging - holds job ID and database connection. + */ + public static class JobLogContext { + private final UUID jobId; + private final Connection connection; + private final boolean enabled; + private volatile boolean hasSevereError = false; + + public JobLogContext(UUID jobId, Connection connection, boolean enabled) { + this.jobId = jobId; + this.connection = connection; + this.enabled = enabled; + } + + public UUID getJobId() { return jobId; } + public Connection getConnection() { return connection; } + public boolean isEnabled() { return enabled; } + public boolean hasSevereError() { return hasSevereError; } + public void setSevereError() { this.hasSevereError = true; } + } + + /** + * Sets the job context for the current thread. Log messages will be written + * to the dc_job_log table when a context is set and job logging is enabled. + * + * @param jobId Job ID to associate with log entries + * @param connection Database connection for writing logs + */ + public static void setJobContext(UUID jobId, Connection connection) { + boolean enabled = Boolean.parseBoolean(Props.getProperty("job-logging-enabled", "false")); + jobContext.set(new JobLogContext(jobId, connection, enabled)); + } + + /** + * Clears the job context for the current thread. + */ + public static void clearJobContext() { + jobContext.remove(); + } + + /** + * Gets the current job context for the current thread. + * + * @return The current JobLogContext or null if not set + */ + public static JobLogContext getJobContext() { + return jobContext.get(); + } + + /** + * Checks if the current job has encountered any SEVERE errors. + * + * @return true if SEVERE errors occurred during the job, false otherwise + */ + public static boolean hasJobSevereError() { + JobLogContext ctx = jobContext.get(); + return ctx != null && ctx.hasSevereError(); + } + /** * Initializes the logging configuration based on provided properties. */ @@ -78,7 +146,6 @@ private static void setupFileHandler(Level level) { if (!STDOUT.equalsIgnoreCase(destination)) { try { - // Ensure parent directory exists Files.createDirectories(Paths.get(destination).getParent()); FileHandler fileHandler = new FileHandler(destination, true); @@ -100,7 +167,7 @@ private static Level mapLogLevel(String setting) { case "ALL" -> Level.ALL; case "OFF" -> Level.OFF; case "INFO" -> Level.INFO; - default -> Level.INFO; // fallback + default -> Level.INFO; }; } @@ -112,8 +179,76 @@ private static Level mapLogLevel(String setting) { * @param message the message to log */ public static void write(String severity, String module, String message) { + write(severity, module, message, null); + } + + /** + * Logs a message with the specified severity and optional JSON context. + * + * @param severity the severity level (e.g., INFO, WARNING, ERROR) + * @param module the source module name + * @param message the message to log + * @param jsonContext optional JSON context for structured data (can be null) + */ + public static void write(String severity, String module, String message, String jsonContext) { Level level = mapLogLevel(severity); String formattedMessage = String.format(MODULE_FORMAT, module, message); LOGGER.log(level, formattedMessage); + + // Track SEVERE errors for job status + if (level == Level.SEVERE) { + JobLogContext ctx = jobContext.get(); + if (ctx != null) { + ctx.setSevereError(); + } + } + + // Only write to job log if message meets configured log level threshold + Level configuredLevel = mapLogLevel(Props.getProperty("log-level", "INFO")); + if (level.intValue() >= configuredLevel.intValue()) { + writeToJobLog(severity, module, message, jsonContext); + } + } + + /** + * Writes a log entry to the dc_job_log table if job context is set and enabled. + * Uses direct JDBC to avoid recursion through SQLExecutionHelper which logs. + * Commits immediately to ensure logs are visible in real-time. + */ + private static void writeToJobLog(String severity, String module, String message, String jsonContext) { + // Prevent re-entrancy - if we're already writing to job log, don't recurse + if (Boolean.TRUE.equals(writingToJobLog.get())) { + return; + } + + JobLogContext ctx = jobContext.get(); + if (ctx == null || !ctx.isEnabled() || ctx.getConnection() == null) { + return; + } + + writingToJobLog.set(true); + try { + Connection conn = ctx.getConnection(); + // Use direct JDBC instead of SQLExecutionHelper to avoid recursion + // SQLExecutionHelper.simpleUpdate calls LoggingUtils.write which would cause infinite loop + try (PreparedStatement pstmt = conn.prepareStatement(SQL_JOBLOG_INSERT)) { + pstmt.setString(1, ctx.getJobId().toString()); + pstmt.setString(2, severity.toUpperCase()); + pstmt.setString(3, module); + pstmt.setString(4, message); + pstmt.setString(5, jsonContext); + pstmt.executeUpdate(); + // Always commit immediately to make logs visible in real-time + if (!conn.getAutoCommit()) { + conn.commit(); + } + } + } catch (Exception e) { + // Don't let logging failures break the application + // Log to console only to avoid any possibility of recursion + LOGGER.log(Level.WARNING, String.format("[%-24s] Failed to write to job log: %s", "logging", e.getMessage())); + } finally { + writingToJobLog.set(false); + } } } diff --git a/ui/README.md b/ui/README.md index 1842dee..c9e8bdc 100644 --- a/ui/README.md +++ b/ui/README.md @@ -8,10 +8,19 @@ A Next.js-based web application for viewing and editing pgCompare configuration - **Project Management**: View and edit project configurations in a user-friendly table format - **Results Visualization**: Charts and graphs showing comparison results and trends - **Table Configuration**: Edit table settings, mappings, and column configurations -- **Navigation Tree**: Intuitive sidebar navigation for projects and tables +- **Navigation Tree**: Intuitive sidebar navigation for projects and tables with search/filter - **Dark/Light Mode**: Toggle between dark and light themes, with preference saved locally - **Responsive Design**: Modern, responsive UI built with Tailwind CSS +### Server Mode Features (v0.6.0) + +- **Dashboard Overview**: Real-time server status monitoring with heartbeat tracking +- **Job Scheduling**: Schedule compare, check, or discover jobs from the UI +- **Job Management**: View running, pending, and completed jobs +- **Job Control**: Pause, resume, stop, or terminate running jobs +- **Progress Tracking**: Real-time per-table progress monitoring +- **Job Logs**: View job execution logs directly in the UI + ## Prerequisites - Node.js 18+ and npm diff --git a/ui/app/api/health/route.ts b/ui/app/api/health/route.ts new file mode 100644 index 0000000..9a2ce00 --- /dev/null +++ b/ui/app/api/health/route.ts @@ -0,0 +1,20 @@ +import { NextResponse } from 'next/server'; +import { getPrisma } from '@/lib/db'; + +export async function GET() { + try { + const prisma = getPrisma(); + await prisma.$queryRaw`SELECT 1`; + return NextResponse.json({ + status: 'ok', + database: 'connected', + timestamp: new Date().toISOString() + }); + } catch (error) { + return NextResponse.json({ + status: 'error', + database: 'disconnected', + timestamp: new Date().toISOString() + }, { status: 503 }); + } +} diff --git a/ui/app/api/jobs/[id]/control/route.ts b/ui/app/api/jobs/[id]/control/route.ts new file mode 100644 index 0000000..4be8135 --- /dev/null +++ b/ui/app/api/jobs/[id]/control/route.ts @@ -0,0 +1,110 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getPrisma, getSchema } from '@/lib/db'; + +function serializeBigInt(obj: unknown): unknown { + if (obj === null || obj === undefined) return obj; + if (typeof obj === 'bigint') return Number(obj); + if (obj instanceof Date) return obj.toISOString(); + // Handle Prisma Decimal type (has toNumber method) + if (typeof obj === 'object' && obj !== null && 'toNumber' in obj && typeof (obj as any).toNumber === 'function') { + return (obj as any).toNumber(); + } + if (Array.isArray(obj)) return obj.map(serializeBigInt); + if (typeof obj === 'object') { + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = serializeBigInt(value); + } + return result; + } + return obj; +} + +export async function POST( + request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + const { id: jobId } = await params; + + try { + const prisma = getPrisma(); + const schema = getSchema(); + const body = await request.json(); + const { signal, requested_by = 'ui' } = body; + + const validSignals = ['pause', 'resume', 'stop', 'terminate']; + if (!signal || !validSignals.includes(signal)) { + return NextResponse.json( + { error: `Invalid signal. Must be one of: ${validSignals.join(', ')}` }, + { status: 400 } + ); + } + + const job = await prisma.$queryRawUnsafe(` + SELECT status FROM ${schema}.dc_job WHERE job_id = '${jobId}'::uuid + `); + + if (!job || (Array.isArray(job) && job.length === 0)) { + return NextResponse.json({ error: 'Job not found' }, { status: 404 }); + } + + const jobStatus = Array.isArray(job) ? job[0].status : (job as any).status; + + if (jobStatus !== 'running' && jobStatus !== 'paused') { + return NextResponse.json( + { error: `Cannot send signal to job with status: ${jobStatus}` }, + { status: 400 } + ); + } + + const result = await prisma.$queryRawUnsafe(` + INSERT INTO ${schema}.dc_job_control (job_id, signal, requested_by) + VALUES ('${jobId}'::uuid, '${signal}', '${requested_by}') + RETURNING control_id, requested_at + `); + + if (signal === 'pause') { + await prisma.$executeRawUnsafe(` + UPDATE ${schema}.dc_job SET status = 'paused' WHERE job_id = '${jobId}'::uuid + `); + } else if (signal === 'resume') { + await prisma.$executeRawUnsafe(` + UPDATE ${schema}.dc_job SET status = 'running' WHERE job_id = '${jobId}'::uuid + `); + } + + return NextResponse.json(serializeBigInt({ + success: true, + signal, + control: Array.isArray(result) ? result[0] : result + })); + } catch (error) { + console.error('Failed to send control signal:', error); + return NextResponse.json({ error: 'Failed to send control signal' }, { status: 500 }); + } +} + +export async function GET( + request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + const { id: jobId } = await params; + + try { + const prisma = getPrisma(); + const schema = getSchema(); + + const signals = await prisma.$queryRawUnsafe(` + SELECT control_id, signal, requested_at, processed_at, requested_by + FROM ${schema}.dc_job_control + WHERE job_id = '${jobId}'::uuid + ORDER BY requested_at DESC + LIMIT 20 + `); + + return NextResponse.json(serializeBigInt(signals)); + } catch (error) { + console.error('Failed to fetch control signals:', error); + return NextResponse.json({ error: 'Failed to fetch control signals' }, { status: 500 }); + } +} diff --git a/ui/app/api/jobs/[id]/logs/route.ts b/ui/app/api/jobs/[id]/logs/route.ts new file mode 100644 index 0000000..0f88872 --- /dev/null +++ b/ui/app/api/jobs/[id]/logs/route.ts @@ -0,0 +1,110 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getPrisma, getSchema } from '@/lib/db'; + +interface LogEntry { + log_id: number; + job_id: string; + log_ts: Date; + log_level: string; + thread_name: string | null; + message: string; + context: unknown; +} + +function serializeBigInt(obj: unknown): unknown { + if (obj === null || obj === undefined) return obj; + if (typeof obj === 'bigint') return Number(obj); + if (obj instanceof Date) return obj.toISOString(); + // Handle Prisma Decimal type (has toNumber method) + if (typeof obj === 'object' && obj !== null && 'toNumber' in obj && typeof (obj as any).toNumber === 'function') { + return (obj as any).toNumber(); + } + if (Array.isArray(obj)) return obj.map(serializeBigInt); + if (typeof obj === 'object') { + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = serializeBigInt(value); + } + return result; + } + return obj; +} + +export async function GET( + request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + const { id: jobId } = await params; + const { searchParams } = new URL(request.url); + + const page = parseInt(searchParams.get('page') || '1'); + const pageSize = Math.min(parseInt(searchParams.get('pageSize') || '100'), 500); + const level = searchParams.get('level'); + const sinceLogId = searchParams.get('sinceLogId'); + const offset = (page - 1) * pageSize; + + try { + const prisma = getPrisma(); + const schema = getSchema(); + + // Build WHERE clause + const conditions: string[] = [`job_id = '${jobId}'::uuid`]; + + if (level && level !== 'all') { + conditions.push(`log_level = '${level.toUpperCase()}'`); + } + + // Support incremental fetching for streaming + if (sinceLogId) { + conditions.push(`log_id > ${parseInt(sinceLogId)}`); + } + + const whereClause = conditions.join(' AND '); + + // Get total count (always count all logs, not just since sinceLogId) + const countWhereClause = level && level !== 'all' + ? `job_id = '${jobId}'::uuid AND log_level = '${level.toUpperCase()}'` + : `job_id = '${jobId}'::uuid`; + + const countResult = await prisma.$queryRawUnsafe(` + SELECT COUNT(*) as count FROM ${schema}.dc_job_log WHERE ${countWhereClause} + `) as { count: bigint }[]; + + const totalCount = Number(countResult[0]?.count || 0); + + // Get logs - when using sinceLogId, don't paginate, just get all new logs + let query: string; + if (sinceLogId) { + query = ` + SELECT log_id, job_id, log_ts, log_level, thread_name, message, context + FROM ${schema}.dc_job_log + WHERE ${whereClause} + ORDER BY log_ts, log_id + LIMIT 100 + `; + } else { + query = ` + SELECT log_id, job_id, log_ts, log_level, thread_name, message, context + FROM ${schema}.dc_job_log + WHERE ${whereClause} + ORDER BY log_ts, log_id + LIMIT ${pageSize} OFFSET ${offset} + `; + } + + const logs = await prisma.$queryRawUnsafe(query) as LogEntry[]; + + return NextResponse.json({ + logs: serializeBigInt(logs), + pagination: { + page, + pageSize, + totalCount, + totalPages: Math.ceil(totalCount / pageSize) + } + }); + } catch (error) { + console.error('Failed to fetch job logs:', error); + return NextResponse.json({ error: 'Failed to fetch job logs' }, { status: 500 }); + } +} diff --git a/ui/app/api/jobs/[id]/progress/route.ts b/ui/app/api/jobs/[id]/progress/route.ts new file mode 100644 index 0000000..552b92e --- /dev/null +++ b/ui/app/api/jobs/[id]/progress/route.ts @@ -0,0 +1,98 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getPrisma, getSchema } from '@/lib/db'; + +function serializeBigInt(obj: unknown): unknown { + if (obj === null || obj === undefined) return obj; + if (typeof obj === 'bigint') return Number(obj); + if (obj instanceof Date) return obj.toISOString(); + // Handle Prisma Decimal type (has toNumber method) + if (typeof obj === 'object' && obj !== null && 'toNumber' in obj && typeof (obj as any).toNumber === 'function') { + return (obj as any).toNumber(); + } + if (Array.isArray(obj)) return obj.map(serializeBigInt); + if (typeof obj === 'object') { + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = serializeBigInt(value); + } + return result; + } + return obj; +} + +export async function GET( + request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + const { id: jobId } = await params; + + try { + const prisma = getPrisma(); + const schema = getSchema(); + + // Get job progress with status and cid, then join to dc_result for counts + const progress = await prisma.$queryRawUnsafe(` + SELECT + jp.job_id, + jp.tid, + jp.table_name, + jp.status, + jp.started_at, + jp.completed_at, + jp.error_message, + jp.cid, + COALESCE(r.source_cnt, 0) as source_cnt, + COALESCE(r.target_cnt, 0) as target_cnt, + COALESCE(r.equal_cnt, 0) as equal_cnt, + COALESCE(r.not_equal_cnt, 0) as not_equal_cnt, + COALESCE(r.missing_source_cnt, 0) as missing_source_cnt, + COALESCE(r.missing_target_cnt, 0) as missing_target_cnt, + CASE + WHEN jp.started_at IS NOT NULL AND jp.completed_at IS NULL + THEN EXTRACT(EPOCH FROM (current_timestamp - jp.started_at)) + WHEN jp.started_at IS NOT NULL AND jp.completed_at IS NOT NULL + THEN EXTRACT(EPOCH FROM (jp.completed_at - jp.started_at)) + ELSE NULL + END as duration_seconds + FROM ${schema}.dc_job_progress jp + LEFT JOIN ${schema}.dc_result r ON jp.cid = r.cid + WHERE jp.job_id = '${jobId}'::uuid + ORDER BY + CASE jp.status + WHEN 'running' THEN 1 + WHEN 'pending' THEN 2 + WHEN 'completed' THEN 3 + WHEN 'failed' THEN 4 + WHEN 'skipped' THEN 5 + END, + jp.table_name + `); + + // Get summary by aggregating from dc_result via cid + const summary = await prisma.$queryRawUnsafe(` + SELECT + COUNT(*) as total_tables, + COUNT(*) FILTER (WHERE jp.status = 'completed') as completed_tables, + COUNT(*) FILTER (WHERE jp.status = 'running') as running_tables, + COUNT(*) FILTER (WHERE jp.status = 'pending') as pending_tables, + COUNT(*) FILTER (WHERE jp.status = 'failed') as failed_tables, + SUM(COALESCE(r.source_cnt, 0)) as total_source, + SUM(COALESCE(r.target_cnt, 0)) as total_target, + SUM(COALESCE(r.equal_cnt, 0)) as total_equal, + SUM(COALESCE(r.not_equal_cnt, 0)) as total_not_equal, + SUM(COALESCE(r.missing_source_cnt, 0)) as total_missing_source, + SUM(COALESCE(r.missing_target_cnt, 0)) as total_missing_target + FROM ${schema}.dc_job_progress jp + LEFT JOIN ${schema}.dc_result r ON jp.cid = r.cid + WHERE jp.job_id = '${jobId}'::uuid + `); + + return NextResponse.json(serializeBigInt({ + tables: progress, + summary: Array.isArray(summary) ? summary[0] : summary + })); + } catch (error) { + console.error('Failed to fetch job progress:', error); + return NextResponse.json({ error: 'Failed to fetch job progress' }, { status: 500 }); + } +} diff --git a/ui/app/api/jobs/[id]/route.ts b/ui/app/api/jobs/[id]/route.ts new file mode 100644 index 0000000..96ec1c4 --- /dev/null +++ b/ui/app/api/jobs/[id]/route.ts @@ -0,0 +1,136 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getPrisma, getSchema } from '@/lib/db'; + +function serializeBigInt(obj: unknown): unknown { + if (obj === null || obj === undefined) return obj; + if (typeof obj === 'bigint') return Number(obj); + if (obj instanceof Date) return obj.toISOString(); + // Handle Prisma Decimal type (has toNumber method) + if (typeof obj === 'object' && obj !== null && 'toNumber' in obj && typeof (obj as any).toNumber === 'function') { + return (obj as any).toNumber(); + } + if (Array.isArray(obj)) return obj.map(serializeBigInt); + if (typeof obj === 'object') { + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = serializeBigInt(value); + } + return result; + } + return obj; +} + +export async function GET( + request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + const { id: jobId } = await params; + + try { + const prisma = getPrisma(); + const schema = getSchema(); + + const job = await prisma.$queryRawUnsafe(` + SELECT + w.job_id, + w.pid, + p.project_name, + w.job_type, + w.status, + w.priority, + w.batch_nbr, + w.table_filter, + w.target_server_id, + w.assigned_server_id, + s.server_name as assigned_server_name, + w.created_at, + w.scheduled_at, + w.started_at, + w.completed_at, + w.created_by, + w.job_config, + w.result_summary, + w.error_message, + w.source, + CASE + WHEN w.started_at IS NOT NULL AND w.completed_at IS NOT NULL + THEN EXTRACT(EPOCH FROM (w.completed_at - w.started_at)) + WHEN w.started_at IS NOT NULL AND w.completed_at IS NULL + THEN EXTRACT(EPOCH FROM (current_timestamp - w.started_at)) + ELSE NULL + END as duration_seconds + FROM ${schema}.dc_job w + LEFT JOIN ${schema}.dc_project p ON w.pid = p.pid + LEFT JOIN ${schema}.dc_server s ON w.assigned_server_id = s.server_id + WHERE w.job_id = '${jobId}'::uuid + `); + + if (!job || (Array.isArray(job) && job.length === 0)) { + return NextResponse.json({ error: 'Job not found' }, { status: 404 }); + } + + return NextResponse.json(serializeBigInt(Array.isArray(job) ? job[0] : job)); + } catch (error) { + console.error('Failed to fetch job:', error); + return NextResponse.json({ error: 'Failed to fetch job' }, { status: 500 }); + } +} + +export async function DELETE( + request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + const { id: jobId } = await params; + + try { + const prisma = getPrisma(); + const schema = getSchema(); + + // Get cid values from job progress to clean up related data + const progressRecords = await prisma.$queryRawUnsafe(` + SELECT cid, tid FROM ${schema}.dc_job_progress + WHERE job_id = '${jobId}'::uuid AND cid IS NOT NULL + `) as { cid: number; tid: number }[]; + + if (progressRecords && progressRecords.length > 0) { + const cids = progressRecords.map(r => r.cid).filter(c => c != null); + const tids = [...new Set(progressRecords.map(r => r.tid))]; + + if (cids.length > 0) { + // Delete dc_result records for this job's compares + await prisma.$executeRawUnsafe(` + DELETE FROM ${schema}.dc_result WHERE cid = ANY(ARRAY[${cids.join(',')}]) + `); + } + + if (tids.length > 0) { + // Delete dc_source records for these tables + await prisma.$executeRawUnsafe(` + DELETE FROM ${schema}.dc_source WHERE tid = ANY(ARRAY[${tids.join(',')}]) + `); + + // Delete dc_target records for these tables + await prisma.$executeRawUnsafe(` + DELETE FROM ${schema}.dc_target WHERE tid = ANY(ARRAY[${tids.join(',')}]) + `); + + // Delete dc_table_history records for these tables + await prisma.$executeRawUnsafe(` + DELETE FROM ${schema}.dc_table_history WHERE tid = ANY(ARRAY[${tids.join(',')}]) + `); + } + } + + // Delete the job (cascades to dc_job_progress and dc_job_control via FK) + await prisma.$executeRawUnsafe(` + DELETE FROM ${schema}.dc_job + WHERE job_id = '${jobId}'::uuid + AND status IN ('pending', 'running', 'completed', 'error', 'failed', 'cancelled') + `); + + return NextResponse.json({ success: true }); + } catch (error) { + console.error('Failed to delete job:', error); + return NextResponse.json({ error: 'Failed to delete job' }, { status: 500 }); + } +} diff --git a/ui/app/api/jobs/route.ts b/ui/app/api/jobs/route.ts new file mode 100644 index 0000000..8811595 --- /dev/null +++ b/ui/app/api/jobs/route.ts @@ -0,0 +1,138 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getPrisma, getSchema } from '@/lib/db'; + +function serializeBigInt(obj: unknown): unknown { + if (obj === null || obj === undefined) return obj; + if (typeof obj === 'bigint') return Number(obj); + if (obj instanceof Date) return obj.toISOString(); + // Handle Prisma Decimal type (has toNumber method) + if (typeof obj === 'object' && obj !== null && 'toNumber' in obj && typeof (obj as any).toNumber === 'function') { + return (obj as any).toNumber(); + } + if (Array.isArray(obj)) return obj.map(serializeBigInt); + if (typeof obj === 'object') { + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = serializeBigInt(value); + } + return result; + } + return obj; +} + +export async function GET(request: NextRequest) { + const searchParams = request.nextUrl.searchParams; + const pid = searchParams.get('pid'); + const status = searchParams.get('status'); + const limit = parseInt(searchParams.get('limit') || '50'); + + try { + const prisma = getPrisma(); + const schema = getSchema(); + + let query = ` + SELECT + w.job_id, + w.pid, + p.project_name, + w.job_type, + w.status, + w.priority, + w.batch_nbr, + w.table_filter, + w.target_server_id, + w.assigned_server_id, + s.server_name as assigned_server_name, + w.created_at, + w.scheduled_at, + w.started_at, + w.completed_at, + w.created_by, + w.job_config, + w.result_summary, + w.error_message, + w.source, + CASE + WHEN w.started_at IS NOT NULL AND w.completed_at IS NULL + THEN EXTRACT(EPOCH FROM (current_timestamp - w.started_at)) + WHEN w.started_at IS NOT NULL AND w.completed_at IS NOT NULL + THEN EXTRACT(EPOCH FROM (w.completed_at - w.started_at)) + ELSE NULL + END as duration_seconds + FROM ${schema}.dc_job w + LEFT JOIN ${schema}.dc_project p ON w.pid = p.pid + LEFT JOIN ${schema}.dc_server s ON w.assigned_server_id = s.server_id + `; + + const conditions: string[] = []; + + if (pid) { + conditions.push(`w.pid = ${parseInt(pid)}`); + } + + if (status) { + const statuses = status.split(',').map(s => `'${s}'`).join(','); + conditions.push(`w.status IN (${statuses})`); + } + + if (conditions.length > 0) { + query += ` WHERE ${conditions.join(' AND ')}`; + } + + query += ` ORDER BY w.created_at DESC LIMIT ${limit}`; + + const jobs = await prisma.$queryRawUnsafe(query); + + return NextResponse.json(serializeBigInt(jobs)); + } catch (error) { + console.error('Failed to fetch jobs:', error); + return NextResponse.json({ error: 'Failed to fetch jobs' }, { status: 500 }); + } +} + +export async function POST(request: NextRequest) { + try { + const prisma = getPrisma(); + const schema = getSchema(); + const body = await request.json(); + const { + pid, + job_type = 'compare', + priority = 5, + batch_nbr = 0, + table_filter = null, + target_server_id = null, + scheduled_at = null, + created_by = 'ui', + job_config = null + } = body; + + if (!pid) { + return NextResponse.json({ error: 'Project ID (pid) is required' }, { status: 400 }); + } + + const result = await prisma.$queryRawUnsafe(` + INSERT INTO ${schema}.dc_job ( + pid, job_type, priority, batch_nbr, table_filter, + target_server_id, scheduled_at, created_by, job_config + ) + VALUES ( + ${pid}::int, + '${job_type}', + ${priority}::int, + ${batch_nbr}::int, + ${table_filter ? `'${table_filter}'` : 'NULL'}, + ${target_server_id ? `'${target_server_id}'::uuid` : 'NULL'}, + ${scheduled_at ? `'${scheduled_at}'::timestamptz` : 'NULL'}, + '${created_by}', + ${job_config ? `'${JSON.stringify(job_config)}'::jsonb` : 'NULL'} + ) + RETURNING job_id, status, created_at + `); + + return NextResponse.json(serializeBigInt(result)); + } catch (error) { + console.error('Failed to create job:', error); + return NextResponse.json({ error: 'Failed to create job' }, { status: 500 }); + } +} diff --git a/ui/app/api/projects/[id]/tables/route.ts b/ui/app/api/projects/[id]/tables/route.ts index b01eda6..07ae5c5 100644 --- a/ui/app/api/projects/[id]/tables/route.ts +++ b/ui/app/api/projects/[id]/tables/route.ts @@ -1,5 +1,5 @@ import { NextRequest, NextResponse } from 'next/server'; -import { getPrisma } from '@/lib/db'; +import { getPrisma, getSchema } from '@/lib/db'; export async function GET( request: NextRequest, @@ -32,3 +32,74 @@ export async function GET( } } +export async function POST( + request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + try { + const { id: projectId } = await params; + const prisma = getPrisma(); + const schema = getSchema(); + const body = await request.json(); + + const { + table_alias, + enabled = true, + batch_nbr = 0, + parallel_degree = 4, + source_schema, + source_table, + target_schema, + target_table, + source_schema_preserve_case = false, + source_table_preserve_case = false, + target_schema_preserve_case = false, + target_table_preserve_case = false, + } = body; + + if (!table_alias) { + return NextResponse.json({ error: 'table_alias is required' }, { status: 400 }); + } + + // Create the table entry and get the generated tid + const tableResult = await prisma.$queryRawUnsafe(` + INSERT INTO ${schema}.dc_table (pid, table_alias, enabled, batch_nbr, parallel_degree) + VALUES ($1, $2, $3, $4, $5) + RETURNING tid + `, BigInt(projectId), table_alias, enabled, batch_nbr, parallel_degree) as { tid: bigint }[]; + + if (!tableResult || tableResult.length === 0) { + throw new Error('Failed to create table'); + } + + const tid = tableResult[0].tid; + + // Create source table map if provided + if (source_schema && source_table) { + await prisma.$executeRawUnsafe(` + INSERT INTO ${schema}.dc_table_map (tid, dest_type, schema_name, table_name, schema_preserve_case, table_preserve_case) + VALUES ($1, 'source', $2, $3, $4, $5) + `, tid, source_schema, source_table, source_schema_preserve_case, source_table_preserve_case); + } + + // Create target table map if provided + if (target_schema && target_table) { + await prisma.$executeRawUnsafe(` + INSERT INTO ${schema}.dc_table_map (tid, dest_type, schema_name, table_name, schema_preserve_case, table_preserve_case) + VALUES ($1, 'target', $2, $3, $4, $5) + `, tid, target_schema, target_table, target_schema_preserve_case, target_table_preserve_case); + } + + return NextResponse.json({ + tid: Number(tid), + pid: Number(projectId), + table_alias, + enabled, + batch_nbr, + parallel_degree, + }); + } catch (error: any) { + console.error('Error creating table:', error); + return NextResponse.json({ error: error.message }, { status: 500 }); + } +} diff --git a/ui/app/api/projects/[id]/test-connection/route.ts b/ui/app/api/projects/[id]/test-connection/route.ts new file mode 100644 index 0000000..b7b5a07 --- /dev/null +++ b/ui/app/api/projects/[id]/test-connection/route.ts @@ -0,0 +1,205 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getPrisma, getSchema } from '@/lib/db'; + +interface ConnectionTestResult { + success: boolean; + connectionType: string; + databaseType: string; + host: string; + port: number; + database: string; + schema: string; + user: string; + databaseProductName?: string; + databaseProductVersion?: string; + errorMessage?: string; + errorDetail?: string; + responseTimeMs: number; +} + +function serializeBigInt(obj: unknown): unknown { + if (obj === null || obj === undefined) return obj; + if (typeof obj === 'bigint') return Number(obj); + if (obj instanceof Date) return obj.toISOString(); + // Handle Prisma Decimal type (has toNumber method) + if (typeof obj === 'object' && obj !== null && 'toNumber' in obj && typeof (obj as any).toNumber === 'function') { + return (obj as any).toNumber(); + } + if (Array.isArray(obj)) return obj.map(serializeBigInt); + if (typeof obj === 'object') { + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = serializeBigInt(value); + } + return result; + } + return obj; +} + +export async function POST( + request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + try { + const { id } = await params; + const pid = parseInt(id); + + const prisma = getPrisma(); + const schema = getSchema(); + + // Check for available servers + const servers = await prisma.$queryRawUnsafe(` + SELECT server_id, server_name + FROM ${schema}.dc_server + WHERE status IN ('idle', 'active', 'busy') + AND last_heartbeat > current_timestamp - interval '2 minutes' + LIMIT 1 + `) as any[]; + + if (!servers || servers.length === 0) { + return NextResponse.json({ + error: 'No available pgCompare servers found. Please start a server with: pgcompare server', + success: false, + }, { status: 503 }); + } + + // Submit the job + const jobResult = await prisma.$queryRawUnsafe(` + INSERT INTO ${schema}.dc_job (pid, job_type, priority, batch_nbr, created_by) + VALUES ($1, 'test-connection', 10, 0, 'ui') + RETURNING job_id + `, pid) as any[]; + + if (!jobResult || jobResult.length === 0) { + return NextResponse.json({ + error: 'Failed to submit test-connection job', + success: false, + }, { status: 500 }); + } + + const jobId = jobResult[0].job_id; + const pollIntervalMs = 500; + + // Poll until job completes, fails, or no servers available + while (true) { + // Check job status + const jobStatus = await prisma.$queryRawUnsafe(` + SELECT status, result_summary, error_message, assigned_server_id + FROM ${schema}.dc_job + WHERE job_id = $1::uuid + `, jobId) as any[]; + + if (jobStatus && jobStatus.length > 0) { + const job = jobStatus[0]; + + // Job completed successfully + if (job.status === 'completed') { + let serverName = 'unknown'; + if (job.assigned_server_id) { + const serverInfo = await prisma.$queryRawUnsafe(` + SELECT server_name FROM ${schema}.dc_server WHERE server_id = $1::uuid + `, job.assigned_server_id) as any[]; + if (serverInfo && serverInfo.length > 0) { + serverName = serverInfo[0].server_name; + } + } + + const results = job.result_summary || {}; + const allSuccess = Object.values(results).every((r: any) => r?.success); + + return NextResponse.json(serializeBigInt({ + success: allSuccess, + results, + serverUsed: { name: serverName }, + jobId, + })); + } + + // Job failed + if (job.status === 'failed') { + return NextResponse.json({ + error: job.error_message || 'Job failed', + success: false, + jobId, + }, { status: 500 }); + } + + // Job cancelled + if (job.status === 'cancelled') { + return NextResponse.json({ + error: 'Job was cancelled', + success: false, + jobId, + }, { status: 500 }); + } + } + + // Check if servers are still available (only if job is still pending) + const jobPending = jobStatus?.[0]?.status === 'pending'; + if (jobPending) { + const activeServers = await prisma.$queryRawUnsafe(` + SELECT COUNT(*) as cnt + FROM ${schema}.dc_server + WHERE status IN ('idle', 'active', 'busy') + AND last_heartbeat > current_timestamp - interval '2 minutes' + `) as any[]; + + if (!activeServers || activeServers.length === 0 || Number(activeServers[0].cnt) === 0) { + // Cancel the pending job since no servers are available + await prisma.$queryRawUnsafe(` + UPDATE ${schema}.dc_job + SET status = 'cancelled', error_message = 'No servers available to process job' + WHERE job_id = $1::uuid AND status = 'pending' + `, jobId); + + return NextResponse.json({ + error: 'No pgCompare servers available to process the job. All servers have gone offline.', + success: false, + jobId, + }, { status: 503 }); + } + } + + // If job is running, check if the assigned server is still alive + if (jobStatus?.[0]?.status === 'running' && jobStatus?.[0]?.assigned_server_id) { + const assignedServer = await prisma.$queryRawUnsafe(` + SELECT server_id, status, last_heartbeat + FROM ${schema}.dc_server + WHERE server_id = $1::uuid + `, jobStatus[0].assigned_server_id) as any[]; + + if (assignedServer && assignedServer.length > 0) { + const server = assignedServer[0]; + const lastHeartbeat = new Date(server.last_heartbeat); + const now = new Date(); + const secondsSinceHeartbeat = (now.getTime() - lastHeartbeat.getTime()) / 1000; + + // If server hasn't sent heartbeat in 2+ minutes, it's probably dead + if (secondsSinceHeartbeat > 120 || server.status === 'terminated' || server.status === 'offline') { + // Mark job as failed + await prisma.$queryRawUnsafe(` + UPDATE ${schema}.dc_job + SET status = 'failed', error_message = 'Assigned server went offline during job execution' + WHERE job_id = $1::uuid AND status = 'running' + `, jobId); + + return NextResponse.json({ + error: 'The server processing this job went offline.', + success: false, + jobId, + }, { status: 503 }); + } + } + } + + await new Promise(resolve => setTimeout(resolve, pollIntervalMs)); + } + + } catch (error: any) { + console.error('Error testing connections:', error); + return NextResponse.json({ + error: error.message, + success: false, + }, { status: 500 }); + } +} diff --git a/ui/app/api/projects/route.ts b/ui/app/api/projects/route.ts index 77ee8b9..6af89ae 100644 --- a/ui/app/api/projects/route.ts +++ b/ui/app/api/projects/route.ts @@ -31,7 +31,7 @@ export async function GET() { export async function POST(request: NextRequest) { try { const body = await request.json(); - const { project_name } = body; + const { project_name, project_config } = body; if (!project_name) { return NextResponse.json({ error: 'Project name is required' }, { status: 400 }); @@ -41,6 +41,7 @@ export async function POST(request: NextRequest) { const newProject = await prisma.dc_project.create({ data: { project_name, + ...(project_config && { project_config }), }, select: { pid: true, diff --git a/ui/app/api/results/[id]/fix-sql/route.ts b/ui/app/api/results/[id]/fix-sql/route.ts new file mode 100644 index 0000000..1872b54 --- /dev/null +++ b/ui/app/api/results/[id]/fix-sql/route.ts @@ -0,0 +1,58 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getPrisma, getSchema } from '@/lib/db'; + +export async function GET( + request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + try { + const { id } = await params; + const cid = parseInt(id); + + if (isNaN(cid)) { + return NextResponse.json({ error: 'Invalid result ID' }, { status: 400 }); + } + + const prisma = getPrisma(); + const schema = getSchema(); + + const result = await prisma.$queryRawUnsafe(` + SELECT tid FROM ${schema}.dc_result WHERE cid = $1 + `, cid) as any[]; + + if (result.length === 0) { + return NextResponse.json([]); + } + + const tid = Number(result[0].tid); + + const sourceFixSql = await prisma.$queryRawUnsafe(` + SELECT tid, pk, pk_hash, compare_result, fix_sql, 'insert' as fix_type + FROM ${schema}.dc_source + WHERE tid = $1 AND fix_sql IS NOT NULL + `, tid) as any[]; + + const targetFixSql = await prisma.$queryRawUnsafe(` + SELECT tid, pk, pk_hash, compare_result, fix_sql, 'delete' as fix_type + FROM ${schema}.dc_target + WHERE tid = $1 AND fix_sql IS NOT NULL + `, tid) as any[]; + + const allFixSql = [...sourceFixSql, ...targetFixSql]; + + const converted = allFixSql.map((stmt: any) => ({ + tid: stmt.tid ? Number(stmt.tid) : null, + cid: cid, + pk: stmt.pk, + pk_hash: stmt.pk_hash, + compare_result: stmt.compare_result, + fix_type: stmt.fix_type, + fix_sql: stmt.fix_sql, + })); + + return NextResponse.json(converted); + } catch (error: any) { + console.error('Error fetching fix SQL:', error); + return NextResponse.json({ error: error.message }, { status: 500 }); + } +} diff --git a/ui/app/api/results/[id]/source/route.ts b/ui/app/api/results/[id]/source/route.ts index 395e30a..80849eb 100644 --- a/ui/app/api/results/[id]/source/route.ts +++ b/ui/app/api/results/[id]/source/route.ts @@ -32,7 +32,8 @@ export async function GET( compare_result, thread_nbr, table_name, - batch_nbr + batch_nbr, + fix_sql FROM dc_source WHERE tid = ${result.tid} ORDER BY pk_hash diff --git a/ui/app/api/results/[id]/target/route.ts b/ui/app/api/results/[id]/target/route.ts index 8aea028..d08efb5 100644 --- a/ui/app/api/results/[id]/target/route.ts +++ b/ui/app/api/results/[id]/target/route.ts @@ -32,7 +32,8 @@ export async function GET( compare_result, thread_nbr, table_name, - batch_nbr + batch_nbr, + fix_sql FROM dc_target WHERE tid = ${result.tid} ORDER BY pk_hash diff --git a/ui/app/api/servers/route.ts b/ui/app/api/servers/route.ts new file mode 100644 index 0000000..34d7fb8 --- /dev/null +++ b/ui/app/api/servers/route.ts @@ -0,0 +1,38 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getPrisma, getSchema } from '@/lib/db'; + +export async function GET(request: NextRequest) { + try { + const prisma = getPrisma(); + const schema = getSchema(); + + const servers = await prisma.$queryRawUnsafe(` + SELECT + server_id, + server_name, + server_host, + server_pid::int as server_pid, + status, + registered_at, + last_heartbeat, + current_job_id, + server_config, + EXTRACT(EPOCH FROM (current_timestamp - last_heartbeat))::float as seconds_since_heartbeat + FROM ${schema}.dc_server + WHERE status != 'terminated' + ORDER BY + CASE status + WHEN 'busy' THEN 1 + WHEN 'idle' THEN 2 + WHEN 'active' THEN 3 + WHEN 'offline' THEN 4 + END, + last_heartbeat DESC + `); + + return NextResponse.json(servers); + } catch (error) { + console.error('Failed to fetch servers:', error); + return NextResponse.json({ error: 'Failed to fetch servers', details: String(error) }, { status: 500 }); + } +} diff --git a/ui/app/api/tables/[id]/fix-sql/route.ts b/ui/app/api/tables/[id]/fix-sql/route.ts new file mode 100644 index 0000000..3e8f04a --- /dev/null +++ b/ui/app/api/tables/[id]/fix-sql/route.ts @@ -0,0 +1,76 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getPrisma, getSchema } from '@/lib/db'; + +export async function GET( + request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + try { + const { id } = await params; + const tid = parseInt(id); + + if (isNaN(tid)) { + return NextResponse.json({ error: 'Invalid table ID' }, { status: 400 }); + } + + const prisma = getPrisma(); + const schema = getSchema(); + + const sourceFixSql = await prisma.$queryRawUnsafe(` + SELECT tid, pk, pk_hash, compare_result, fix_sql, 'insert' as fix_type + FROM ${schema}.dc_source + WHERE tid = $1 AND fix_sql IS NOT NULL + `, tid) as any[]; + + const targetFixSql = await prisma.$queryRawUnsafe(` + SELECT tid, pk, pk_hash, compare_result, fix_sql, 'delete' as fix_type + FROM ${schema}.dc_target + WHERE tid = $1 AND fix_sql IS NOT NULL + `, tid) as any[]; + + const allFixSql = [...sourceFixSql, ...targetFixSql]; + + const converted = allFixSql.map((stmt: any) => ({ + tid: stmt.tid ? Number(stmt.tid) : null, + pk: stmt.pk, + pk_hash: stmt.pk_hash, + compare_result: stmt.compare_result, + fix_type: stmt.fix_type, + fix_sql: stmt.fix_sql, + })); + + return NextResponse.json(converted); + } catch (error: any) { + console.error('Error fetching fix SQL:', error); + return NextResponse.json({ error: error.message }, { status: 500 }); + } +} + +export async function DELETE( + request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + try { + const { id } = await params; + const tid = parseInt(id); + + if (isNaN(tid)) { + return NextResponse.json({ error: 'Invalid table ID' }, { status: 400 }); + } + + const prisma = getPrisma(); + const schema = getSchema(); + + await prisma.$executeRawUnsafe(` + UPDATE ${schema}.dc_source SET fix_sql = NULL WHERE tid = $1 + `, tid); + await prisma.$executeRawUnsafe(` + UPDATE ${schema}.dc_target SET fix_sql = NULL WHERE tid = $1 + `, tid); + + return NextResponse.json({ success: true }); + } catch (error: any) { + console.error('Error clearing fix SQL:', error); + return NextResponse.json({ error: error.message }, { status: 500 }); + } +} diff --git a/ui/app/api/tables/[id]/out-of-sync/route.ts b/ui/app/api/tables/[id]/out-of-sync/route.ts new file mode 100644 index 0000000..b25907b --- /dev/null +++ b/ui/app/api/tables/[id]/out-of-sync/route.ts @@ -0,0 +1,134 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getPrisma, getSchema } from '@/lib/db'; + +function serializeBigInt(obj: unknown): unknown { + if (obj === null || obj === undefined) return obj; + if (typeof obj === 'bigint') return Number(obj); + if (obj instanceof Date) return obj.toISOString(); + if (typeof obj === 'object' && obj !== null && 'toNumber' in obj && typeof (obj as any).toNumber === 'function') { + return (obj as any).toNumber(); + } + if (Array.isArray(obj)) return obj.map(serializeBigInt); + if (typeof obj === 'object') { + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = serializeBigInt(value); + } + return result; + } + return obj; +} + +export async function GET( + request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + try { + const { id: tid } = await params; + const { searchParams } = new URL(request.url); + const cid = searchParams.get('cid'); + + const prisma = getPrisma(); + const schema = getSchema(); + + let sourceRows: any[] = []; + let targetRows: any[] = []; + let tableName = `Table ${tid}`; + const tidNum = parseInt(tid); + + if (cid) { + const cidNum = parseInt(cid); + + const resultInfo = await prisma.$queryRawUnsafe(` + SELECT table_name, tid FROM ${schema}.dc_result WHERE cid = $1 + `, cidNum) as any[]; + + if (resultInfo && resultInfo.length > 0 && Number(resultInfo[0].tid) === tidNum) { + tableName = resultInfo[0].table_name || tableName; + + sourceRows = await prisma.$queryRawUnsafe(` + SELECT + pk, + pk_hash, + column_hash, + compare_result, + thread_nbr, + table_name, + batch_nbr, + fix_sql + FROM ${schema}.dc_source + WHERE tid = $1 + AND compare_result IN ('n', 'm') + ORDER BY pk_hash + LIMIT 1000 + `, tidNum) as any[]; + + targetRows = await prisma.$queryRawUnsafe(` + SELECT + pk, + pk_hash, + column_hash, + compare_result, + thread_nbr, + table_name, + batch_nbr, + fix_sql + FROM ${schema}.dc_target + WHERE tid = $1 + AND compare_result IN ('n', 'm') + ORDER BY pk_hash + LIMIT 1000 + `, tidNum) as any[]; + } + } else { + sourceRows = await prisma.$queryRawUnsafe(` + SELECT + pk, + pk_hash, + column_hash, + compare_result, + thread_nbr, + table_name, + batch_nbr, + fix_sql + FROM ${schema}.dc_source + WHERE tid = $1 + AND compare_result IN ('n', 'm') + ORDER BY pk_hash + LIMIT 1000 + `, tidNum) as any[]; + + targetRows = await prisma.$queryRawUnsafe(` + SELECT + pk, + pk_hash, + column_hash, + compare_result, + thread_nbr, + table_name, + batch_nbr, + fix_sql + FROM ${schema}.dc_target + WHERE tid = $1 + AND compare_result IN ('n', 'm') + ORDER BY pk_hash + LIMIT 1000 + `, tidNum) as any[]; + + const tableInfo = await prisma.$queryRawUnsafe(` + SELECT table_alias FROM ${schema}.dc_table WHERE tid = $1 + `, tidNum) as any[]; + + tableName = tableInfo?.[0]?.table_alias || tableName; + } + + return NextResponse.json(serializeBigInt({ + source: sourceRows, + target: targetRows, + tableName, + })); + } catch (error: any) { + console.error('Error fetching out-of-sync data:', error); + return NextResponse.json({ error: error.message }, { status: 500 }); + } +} diff --git a/ui/app/dashboard/page.tsx b/ui/app/dashboard/page.tsx index ff08704..845362c 100644 --- a/ui/app/dashboard/page.tsx +++ b/ui/app/dashboard/page.tsx @@ -5,15 +5,32 @@ import { useRouter } from 'next/navigation'; import NavigationTree from '@/components/NavigationTree'; import ProjectView from '@/components/ProjectView'; import TableView from '@/components/TableView'; +import Dashboard from '@/components/Dashboard'; +import JobsView from '@/components/JobsView'; +import JobDetailView from '@/components/JobDetailView'; import ThemeToggle from '@/components/ThemeToggle'; -import { LogOut } from 'lucide-react'; +import ScheduleJobModal from '@/components/ScheduleJobModal'; +import JobProgressPanel from '@/components/JobProgressPanel'; +import { LogOut, LayoutDashboard, Play, Plus } from 'lucide-react'; + +type ViewMode = 'dashboard' | 'projects' | 'jobs' | 'job-detail'; export default function DashboardPage() { const router = useRouter(); + const [viewMode, setViewMode] = useState('dashboard'); const [selectedProjectId, setSelectedProjectId] = useState(); const [selectedTableId, setSelectedTableId] = useState(); + const [showScheduleModal, setShowScheduleModal] = useState(false); + const [selectedJobId, setSelectedJobId] = useState(null); + const [jobsFilter, setJobsFilter] = useState<'running' | 'pending' | 'all'>('all'); + const [navRefreshTrigger, setNavRefreshTrigger] = useState(0); + + const handleProjectUpdated = () => { + setNavRefreshTrigger(prev => prev + 1); + }; const handleProjectSelect = (projectId: number) => { + setViewMode('projects'); setSelectedProjectId(projectId); setSelectedTableId(undefined); }; @@ -28,18 +45,70 @@ export default function DashboardPage() { router.push('/'); } catch (error) { console.error('Logout error:', error); - // Still redirect even if there's an error router.push('/'); } }; + const handleJobSelect = (jobId: string) => { + setSelectedJobId(jobId); + setViewMode('job-detail'); + }; + + const handleFilterJobs = (filter: 'running' | 'pending' | 'all') => { + setJobsFilter(filter); + setViewMode('jobs'); + }; + + const handleBackFromJobs = () => { + setViewMode('dashboard'); + }; + + const handleBackFromJobDetail = () => { + setViewMode('jobs'); + }; + return (
{/* Header */}
-

pgCompare

+
+

pgCompare

+ + {/* View Mode Toggle */} +
+ + +
+
+
+
+ + {/* Modals */} + setShowScheduleModal(false)} + preselectedProject={selectedProjectId} + />
); } - diff --git a/ui/app/jobs/[id]/page.tsx b/ui/app/jobs/[id]/page.tsx new file mode 100644 index 0000000..ff2f506 --- /dev/null +++ b/ui/app/jobs/[id]/page.tsx @@ -0,0 +1,40 @@ +'use client'; + +import { useRouter } from 'next/navigation'; +import { use } from 'react'; +import JobDetailView from '@/components/JobDetailView'; +import ThemeToggle from '@/components/ThemeToggle'; +import { LayoutDashboard } from 'lucide-react'; + +export default function JobPage({ params }: { params: Promise<{ id: string }> }) { + const router = useRouter(); + const { id } = use(params); + + const handleBack = () => { + router.push('/dashboard'); + }; + + return ( +
+
+
+
+ + / + Job Details +
+ +
+
+
+ +
+
+ ); +} diff --git a/ui/app/layout.tsx b/ui/app/layout.tsx index 0885418..c13bf0a 100644 --- a/ui/app/layout.tsx +++ b/ui/app/layout.tsx @@ -2,6 +2,7 @@ import type { Metadata } from "next"; import { Geist, Geist_Mono } from "next/font/google"; import "./globals.css"; import { ThemeProvider } from "@/components/ThemeProvider"; +import { Toaster } from "@/components/Toaster"; const geistSans = Geist({ variable: "--font-geist-sans", @@ -30,6 +31,7 @@ export default function RootLayout({ > {children} + diff --git a/ui/components/AddTableModal.tsx b/ui/components/AddTableModal.tsx new file mode 100644 index 0000000..4cace29 --- /dev/null +++ b/ui/components/AddTableModal.tsx @@ -0,0 +1,250 @@ +'use client'; + +import { useState } from 'react'; +import { X } from 'lucide-react'; + +interface AddTableModalProps { + projectId: number; + onClose: () => void; + onTableCreated: (table: any) => void; +} + +export default function AddTableModal({ projectId, onClose, onTableCreated }: AddTableModalProps) { + const [tableAlias, setTableAlias] = useState(''); + const [batchNbr, setBatchNbr] = useState(0); + const [parallelDegree, setParallelDegree] = useState(4); + const [sourceSchema, setSourceSchema] = useState(''); + const [sourceTable, setSourceTable] = useState(''); + const [targetSchema, setTargetSchema] = useState(''); + const [targetTable, setTargetTable] = useState(''); + const [sourceSchemaPreserveCase, setSourceSchemaPreserveCase] = useState(false); + const [sourceTablePreserveCase, setSourceTablePreserveCase] = useState(false); + const [targetSchemaPreserveCase, setTargetSchemaPreserveCase] = useState(false); + const [targetTablePreserveCase, setTargetTablePreserveCase] = useState(false); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(''); + + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + setError(''); + + if (!tableAlias.trim()) { + setError('Table alias is required'); + return; + } + + setSaving(true); + try { + const response = await fetch(`/api/projects/${projectId}/tables`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + table_alias: tableAlias.trim(), + batch_nbr: batchNbr, + parallel_degree: parallelDegree, + source_schema: sourceSchema.trim() || undefined, + source_table: sourceTable.trim() || undefined, + target_schema: targetSchema.trim() || undefined, + target_table: targetTable.trim() || undefined, + source_schema_preserve_case: sourceSchemaPreserveCase, + source_table_preserve_case: sourceTablePreserveCase, + target_schema_preserve_case: targetSchemaPreserveCase, + target_table_preserve_case: targetTablePreserveCase, + }), + }); + + if (!response.ok) { + const data = await response.json(); + throw new Error(data.error || 'Failed to create table'); + } + + const newTable = await response.json(); + onTableCreated(newTable); + } catch (err: any) { + setError(err.message); + } finally { + setSaving(false); + } + }; + + return ( +
+
+
+

Add New Table

+ +
+ +
+ {error && ( +
+ {error} +
+ )} + +
+ + setTableAlias(e.target.value)} + placeholder="e.g., customers" + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white focus:ring-2 focus:ring-blue-500" + autoFocus + /> +

+ A unique identifier for this table mapping +

+
+ +
+
+ + setBatchNbr(parseInt(e.target.value) || 0)} + min={0} + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white focus:ring-2 focus:ring-blue-500" + /> +
+
+ + setParallelDegree(parseInt(e.target.value) || 4)} + min={1} + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white focus:ring-2 focus:ring-blue-500" + /> +
+
+ +
+

Source Table

+
+
+ + setSourceSchema(e.target.value)} + placeholder="e.g., public" + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white focus:ring-2 focus:ring-blue-500" + /> + +
+
+ + setSourceTable(e.target.value)} + placeholder="e.g., customers" + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white focus:ring-2 focus:ring-blue-500" + /> + +
+
+
+ +
+

Target Table

+
+
+ + setTargetSchema(e.target.value)} + placeholder="e.g., public" + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white focus:ring-2 focus:ring-blue-500" + /> + +
+
+ + setTargetTable(e.target.value)} + placeholder="e.g., customers" + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white focus:ring-2 focus:ring-blue-500" + /> + +
+
+
+ +
+ + +
+
+
+
+ ); +} diff --git a/ui/components/Breadcrumb.tsx b/ui/components/Breadcrumb.tsx new file mode 100644 index 0000000..626e557 --- /dev/null +++ b/ui/components/Breadcrumb.tsx @@ -0,0 +1,43 @@ +'use client'; + +import { ChevronRight, Home } from 'lucide-react'; + +export interface BreadcrumbItem { + label: string; + href?: string; + onClick?: () => void; + icon?: React.ReactNode; +} + +interface BreadcrumbProps { + items: BreadcrumbItem[]; +} + +export default function Breadcrumb({ items }: BreadcrumbProps) { + if (items.length === 0) return null; + + return ( + + ); +} diff --git a/ui/components/BulkOperations.tsx b/ui/components/BulkOperations.tsx new file mode 100644 index 0000000..fb1a8bb --- /dev/null +++ b/ui/components/BulkOperations.tsx @@ -0,0 +1,119 @@ +'use client'; + +import { useState } from 'react'; +import { CheckSquare, Square, ToggleLeft, ToggleRight, Trash2 } from 'lucide-react'; +import { toast } from '@/components/Toaster'; + +interface BulkOperationsProps { + selectedIds: number[]; + totalCount: number; + onSelectAll: () => void; + onClearSelection: () => void; + onBulkEnable: () => void; + onBulkDisable: () => void; + onBulkDelete?: () => void; + entityName?: string; +} + +export default function BulkOperations({ + selectedIds, + totalCount, + onSelectAll, + onClearSelection, + onBulkEnable, + onBulkDisable, + onBulkDelete, + entityName = 'items', +}: BulkOperationsProps) { + const [confirming, setConfirming] = useState<'enable' | 'disable' | 'delete' | null>(null); + + const handleAction = async (action: 'enable' | 'disable' | 'delete') => { + if (confirming === action) { + switch (action) { + case 'enable': + onBulkEnable(); + break; + case 'disable': + onBulkDisable(); + break; + case 'delete': + onBulkDelete?.(); + break; + } + setConfirming(null); + } else { + setConfirming(action); + setTimeout(() => setConfirming(null), 3000); + } + }; + + if (selectedIds.length === 0) { + return null; + } + + return ( +
+
+ + {selectedIds.length} of {totalCount} {entityName} selected + + +
+ + | + +
+
+ +
+ + + + + {onBulkDelete && ( + + )} +
+
+ ); +} diff --git a/ui/components/CompareDetailsModal.tsx b/ui/components/CompareDetailsModal.tsx index 8aa7c90..7ec7504 100644 --- a/ui/components/CompareDetailsModal.tsx +++ b/ui/components/CompareDetailsModal.tsx @@ -1,8 +1,9 @@ 'use client'; import { useEffect, useState } from 'react'; -import { X, AlertTriangle } from 'lucide-react'; +import { X, AlertTriangle, Code, ChevronDown, ChevronRight, Copy, Check, FlaskConical } from 'lucide-react'; import { Result } from '@/lib/types'; +import { toast } from 'sonner'; interface CompareDetailsModalProps { result: Result; @@ -16,13 +17,16 @@ interface SourceTargetRow { column_hash: string | null; compare_result: string | null; thread_nbr: number | null; + fix_sql: string | null; } export default function CompareDetailsModal({ result, isOpen, onClose }: CompareDetailsModalProps) { const [sourceRows, setSourceRows] = useState([]); const [targetRows, setTargetRows] = useState([]); const [loading, setLoading] = useState(true); - const [activeTab, setActiveTab] = useState<'not_equal' | 'missing_source' | 'missing_target'>('not_equal'); + const [activeTab, setActiveTab] = useState<'not_equal' | 'missing_source' | 'missing_target' | 'fix_sql'>('not_equal'); + const [expandedRows, setExpandedRows] = useState>(new Set()); + const [copiedSql, setCopiedSql] = useState(null); useEffect(() => { if (isOpen) { @@ -33,26 +37,14 @@ export default function CompareDetailsModal({ result, isOpen, onClose }: Compare const loadCompareDetails = async () => { setLoading(true); try { - console.log('Loading compare details for cid:', result.cid); - const [sourceResponse, targetResponse] = await Promise.all([ fetch(`/api/results/${result.cid}/source`), fetch(`/api/results/${result.cid}/target`) ]); - if (!sourceResponse.ok) { - console.error('Source response error:', await sourceResponse.text()); - } - if (!targetResponse.ok) { - console.error('Target response error:', await targetResponse.text()); - } - const sourceData = await sourceResponse.json(); const targetData = await targetResponse.json(); - console.log('Source data:', sourceData); - console.log('Target data:', targetData); - setSourceRows(Array.isArray(sourceData) ? sourceData : []); setTargetRows(Array.isArray(targetData) ? targetData : []); } catch (error) { @@ -62,15 +54,55 @@ export default function CompareDetailsModal({ result, isOpen, onClose }: Compare } }; + const toggleRow = (pkHash: string) => { + const newExpanded = new Set(expandedRows); + if (newExpanded.has(pkHash)) { + newExpanded.delete(pkHash); + } else { + newExpanded.add(pkHash); + } + setExpandedRows(newExpanded); + }; + + const copyToClipboard = async (sql: string, pkHash: string) => { + try { + await navigator.clipboard.writeText(sql); + setCopiedSql(pkHash); + toast.success('SQL copied to clipboard'); + setTimeout(() => setCopiedSql(null), 2000); + } catch (error) { + toast.error('Failed to copy to clipboard'); + } + }; + + const copyAllFixSql = async () => { + const allSql = [...sourceRows, ...targetRows] + .filter(row => row.fix_sql) + .map(row => row.fix_sql) + .join(';\n\n'); + + if (allSql) { + try { + await navigator.clipboard.writeText(allSql + ';'); + toast.success('All fix SQL copied to clipboard'); + } catch (error) { + toast.error('Failed to copy to clipboard'); + } + } + }; + if (!isOpen) return null; // Filter rows based on compare_result - const notEqualSource = sourceRows.filter(row => row.compare_result === 'n'); // Different + const notEqualSource = sourceRows.filter(row => row.compare_result === 'n'); const notEqualTarget = targetRows.filter(row => row.compare_result === 'n'); - const missingInTarget = sourceRows.filter(row => row.compare_result === 'm'); // Source only - const missingInSource = targetRows.filter(row => row.compare_result === 'm'); // Target only + const missingInTarget = sourceRows.filter(row => row.compare_result === 'm'); + const missingInSource = targetRows.filter(row => row.compare_result === 'm'); + + const allFixSql = [...sourceRows, ...targetRows].filter(row => row.fix_sql); + const hasFixSql = allFixSql.length > 0; - const renderTable = (rows: SourceTargetRow[], title: string) => ( + const renderTable = (rows: SourceTargetRow[], title: string, showFixSql: boolean = false) => (

{title}

{rows.length === 0 ? ( @@ -80,6 +112,10 @@ export default function CompareDetailsModal({ result, isOpen, onClose }: Compare + {showFixSql && ( + + )} @@ -92,25 +128,82 @@ export default function CompareDetailsModal({ result, isOpen, onClose }: Compare + {showFixSql && ( + + )} - {rows.map((row, index) => ( - - - - - - - ))} + {rows.map((row, index) => { + const pkHash = row.pk_hash || `row-${index}`; + const isExpanded = expandedRows.has(pkHash); + return ( + <> + + {showFixSql && ( + + )} + + + + + {showFixSql && ( + + )} + + {showFixSql && row.fix_sql && isExpanded && ( + + + + )} + + ); + })}
+ Primary Key Thread + Fix SQL +
- {typeof row.pk === 'object' ? JSON.stringify(row.pk) : row.pk} - - {row.pk_hash || 'N/A'} - - {row.column_hash || 'N/A'} - - {row.thread_nbr || '-'} -
+ {row.fix_sql && ( + + )} + + {typeof row.pk === 'object' ? JSON.stringify(row.pk) : row.pk} + + {row.pk_hash || 'N/A'} + + {row.column_hash || 'N/A'} + + {row.thread_nbr || '-'} + + {row.fix_sql ? ( + + + Available + + ) : ( + - + )} +
+
+ 
+                              {row.fix_sql}
+
+ +
+
@@ -118,6 +211,66 @@ export default function CompareDetailsModal({ result, isOpen, onClose }: Compare
); + const renderFixSqlTab = () => ( +
+
+
+ +
+

Experimental Feature

+

These SQL statements are designed to be executed on the target database to make it match the source. Review carefully before executing.

+
+
+
+
+

+ Generated Fix SQL ({allFixSql.length} statements) +

+ {allFixSql.length > 0 && ( + + )} +
+ + {allFixSql.length === 0 ? ( +

+ No fix SQL available. Run a check job with "Generate Fix SQL" enabled. +

+ ) : ( +
+ {allFixSql.map((row, index) => ( +
+
+
+ PK: {typeof row.pk === 'object' ? JSON.stringify(row.pk) : row.pk} +
+ +
+ 
+                {row.fix_sql}
+
+
+ ))} +
+ )} +
+ ); + return (
@@ -142,7 +295,7 @@ export default function CompareDetailsModal({ result, isOpen, onClose }: Compare {/* Summary Stats */}
-
+

Not Equal Rows

@@ -161,6 +314,12 @@ export default function CompareDetailsModal({ result, isOpen, onClose }: Compare {result.missing_target_cnt?.toLocaleString() || 0}

+
+

Fix SQL Available

+

+ {allFixSql.length} +

+
@@ -197,6 +356,19 @@ export default function CompareDetailsModal({ result, isOpen, onClose }: Compare > Missing in Source ({missingInSource.length}) + {hasFixSql && ( + + )}
@@ -208,8 +380,8 @@ export default function CompareDetailsModal({ result, isOpen, onClose }: Compare <> {activeTab === 'not_equal' && (
- {renderTable(notEqualSource, 'Source Rows (Different)')} - {renderTable(notEqualTarget, 'Target Rows (Different)')} + {renderTable(notEqualSource, 'Source Rows (Different)', hasFixSql)} + {renderTable(notEqualTarget, 'Target Rows (Different)', hasFixSql)} {notEqualSource.length === 0 && notEqualTarget.length === 0 && (

No differing rows found @@ -220,15 +392,17 @@ export default function CompareDetailsModal({ result, isOpen, onClose }: Compare {activeTab === 'missing_target' && ( <> - {renderTable(missingInTarget, 'Rows in Source but Missing in Target')} + {renderTable(missingInTarget, 'Rows in Source but Missing in Target', hasFixSql)} )} {activeTab === 'missing_source' && ( <> - {renderTable(missingInSource, 'Rows in Target but Missing in Source')} + {renderTable(missingInSource, 'Rows in Target but Missing in Source', hasFixSql)} )} + + {activeTab === 'fix_sql' && renderFixSqlTab()} )}

@@ -236,4 +410,3 @@ export default function CompareDetailsModal({ result, isOpen, onClose }: Compare
); } - diff --git a/ui/components/ConfigEditor.tsx b/ui/components/ConfigEditor.tsx new file mode 100644 index 0000000..72c6256 --- /dev/null +++ b/ui/components/ConfigEditor.tsx @@ -0,0 +1,559 @@ +'use client'; + +import { useState, useEffect } from 'react'; +import { Save, Plus, Trash2, Download, Upload, ChevronDown, ChevronRight, Info, Plug, CheckCircle, XCircle, Loader2 } from 'lucide-react'; +import { CONFIG_PROPERTIES, CATEGORY_LABELS, PropertyDefinition, getPropertyDefinition, isDefaultValue, getFriendlyLabel } from '@/lib/configProperties'; + +interface ConnectionTestResult { + success: boolean; + connectionType: string; + databaseType: string; + host: string; + port: number; + database: string; + schema: string; + user: string; + databaseProductName?: string; + databaseProductVersion?: string; + errorMessage?: string; + errorDetail?: string; + responseTimeMs: number; +} + +interface ConfigEditorProps { + configData: Array<{ key: string; value: string }>; + onConfigChange: (configData: Array<{ key: string; value: string }>) => void; + onSave: () => void; + onImport: (file: File) => void; + saving: boolean; + projectId?: number; +} + +export default function ConfigEditor({ + configData, + onConfigChange, + onSave, + onImport, + saving, + projectId +}: ConfigEditorProps) { + const [expandedCategories, setExpandedCategories] = useState>({ + system: true, + repository: false, + source: false, + target: false, + }); + const [showAddProperty, setShowAddProperty] = useState(false); + const [newPropertyKey, setNewPropertyKey] = useState(''); + const [testingConnections, setTestingConnections] = useState(false); + const [connectionResults, setConnectionResults] = useState>({}); + const [showTestResults, setShowTestResults] = useState(false); + const [testError, setTestError] = useState(null); + const [serverUsed, setServerUsed] = useState<{ name: string } | null>(null); + + const handleConfigChange = (key: string, value: string) => { + const existingIndex = configData.findIndex(item => item.key === key); + if (existingIndex >= 0) { + const newConfig = configData.map(item => + item.key === key ? { ...item, value } : item + ); + onConfigChange(newConfig); + } else { + onConfigChange([...configData, { key, value }]); + } + }; + + const handleRemoveProperty = (key: string) => { + onConfigChange(configData.filter(item => item.key !== key)); + }; + + const handleAddProperty = () => { + if (!newPropertyKey) return; + + if (configData.some(item => item.key === newPropertyKey)) { + alert('Property already exists'); + return; + } + + const propDef = getPropertyDefinition(newPropertyKey); + const defaultValue = propDef?.defaultValue || ''; + + onConfigChange([...configData, { key: newPropertyKey, value: defaultValue }]); + setNewPropertyKey(''); + setShowAddProperty(false); + }; + + const handleTestConnections = async () => { + if (!projectId) return; + + setTestingConnections(true); + setShowTestResults(true); + setConnectionResults({}); + setTestError(null); + setServerUsed(null); + + try { + const response = await fetch(`/api/projects/${projectId}/test-connection`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ connectionTypes: ['repository', 'source', 'target'] }), + }); + + const data = await response.json(); + + if (data.serverUsed) { + setServerUsed(data.serverUsed); + } + + if (data.error && !data.results) { + setTestError(data.error); + } else if (data.results) { + setConnectionResults(data.results); + } + } catch (error: any) { + console.error('Failed to test connections:', error); + setConnectionResults({ + error: { + success: false, + connectionType: 'error', + databaseType: '', + host: '', + port: 0, + database: '', + schema: '', + user: '', + errorMessage: error.message || 'Failed to test connections', + responseTimeMs: 0, + } + }); + } finally { + setTestingConnections(false); + } + }; + + const handleExport = () => { + const lines: string[] = ['# pgCompare Configuration Export', '# Only non-default values are included', '']; + + const categories = ['system', 'repository', 'source', 'target']; + + categories.forEach(category => { + const categoryProps = configData.filter(item => { + const propDef = getPropertyDefinition(item.key); + return propDef?.category === category || + (!propDef && category === 'system' && !item.key.startsWith('repo-') && !item.key.startsWith('source-') && !item.key.startsWith('target-')); + }); + + if (categoryProps.length > 0) { + lines.push(`# ${CATEGORY_LABELS[category] || category}`); + categoryProps.forEach(({ key, value }) => { + if (!isDefaultValue(key, value)) { + lines.push(`${key}=${value}`); + } + }); + lines.push(''); + } + }); + + const blob = new Blob([lines.join('\n')], { type: 'text/plain' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'pgcompare.properties'; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + }; + + const handleFileUpload = (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (file) { + onImport(file); + } + e.target.value = ''; + }; + + const toggleCategory = (category: string) => { + setExpandedCategories(prev => ({ + ...prev, + [category]: !prev[category] + })); + }; + + const getCategoryProperties = (category: string) => { + const existingKeys = new Set(configData.map(item => item.key)); + + const essentialKeys: Record = { + repository: ['repo-host', 'repo-port', 'repo-dbname', 'repo-schema', 'repo-user', 'repo-password', 'repo-sslmode'], + source: ['source-type', 'source-host', 'source-port', 'source-dbname', 'source-schema', 'source-user', 'source-password', 'source-sslmode', 'source-name', 'source-warehouse'], + target: ['target-type', 'target-host', 'target-port', 'target-dbname', 'target-schema', 'target-user', 'target-password', 'target-sslmode', 'target-name', 'target-warehouse'], + system: [] + }; + + const result: Array<{ key: string; value: string }> = []; + + const essential = essentialKeys[category] || []; + essential.forEach(key => { + const existing = configData.find(item => item.key === key); + if (existing) { + result.push(existing); + } else { + const propDef = getPropertyDefinition(key); + result.push({ key, value: propDef?.defaultValue || '' }); + } + }); + + configData.forEach(item => { + if (result.some(r => r.key === item.key)) return; + + const propDef = getPropertyDefinition(item.key); + if (propDef) { + if (propDef.category === category) result.push(item); + } else { + if (category === 'source' && item.key.startsWith('source-')) result.push(item); + else if (category === 'target' && item.key.startsWith('target-')) result.push(item); + else if (category === 'repository' && item.key.startsWith('repo-')) result.push(item); + else if (category === 'system' && !item.key.startsWith('source-') && !item.key.startsWith('target-') && !item.key.startsWith('repo-')) result.push(item); + } + }); + + return result; + }; + + const getAvailableProperties = () => { + const existingKeys = new Set(configData.map(item => item.key)); + return CONFIG_PROPERTIES + .filter(prop => !existingKeys.has(prop.key)) + .sort((a, b) => a.key.localeCompare(b.key)); + }; + + const renderPropertyInput = (key: string, value: string) => { + const propDef = getPropertyDefinition(key); + + if (propDef?.type === 'select' && propDef.options) { + return ( + + ); + } + + if (propDef?.type === 'boolean') { + return ( + + ); + } + + if (propDef?.type === 'password') { + return ( + handleConfigChange(key, e.target.value)} + placeholder="Enter password" + autoComplete="new-password" + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md dark:bg-gray-700 dark:text-white focus:ring-2 focus:ring-blue-500" + /> + ); + } + + return ( + handleConfigChange(key, e.target.value)} + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md dark:bg-gray-700 dark:text-white focus:ring-2 focus:ring-blue-500" + /> + ); + }; + + return ( +
+ {/* Toolbar */} +
+

Project Configuration

+
+ + + + {projectId && ( + + )} + +
+
+ + {/* Connection Test Results */} + {showTestResults && ( +
+
+
+

+ + Connection Test Results +

+ {serverUsed && ( +

+ Tested via server: {serverUsed.name} +

+ )} +
+ +
+ + {testingConnections ? ( +
+ + Testing connections via pgCompare server... +
+ ) : testError ? ( +
+
+ + Connection Test Failed +
+

{testError}

+
+ ) : Object.keys(connectionResults).length === 0 ? ( +

No results yet

+ ) : ( +
+ {Object.entries(connectionResults).map(([key, result]) => { + if (!result) return null; + return ( +
+
+ {result.success ? ( + + ) : ( + + )} + + {key} + + + ({result.databaseType}) + + + {result.responseTimeMs}ms + +
+ +
+

+ Host: {result.host}:{result.port} +

+

+ Database: {result.database} + {result.schema && / {result.schema}} +

+

+ User: {result.user} +

+ + {result.success && result.databaseProductName && ( +

+ Server: {result.databaseProductName} {result.databaseProductVersion} +

+ )} + + {!result.success && result.errorMessage && ( +
+

Error:

+

{result.errorMessage}

+ {result.errorDetail && ( +

{result.errorDetail}

+ )} +
+ )} +
+
+ ); + })} +
+ )} +
+ )} + + {/* Category Sections */} + {['system', 'repository', 'source', 'target'].map(category => { + const categoryProps = getCategoryProperties(category); + + return ( +
+ + + {expandedCategories[category] && ( +
+ {categoryProps.length === 0 ? ( +

No properties configured

+ ) : ( + categoryProps.map(({ key, value }) => { + const propDef = getPropertyDefinition(key); + const isDefault = isDefaultValue(key, value); + const label = getFriendlyLabel(key); + + return ( +
+
+
+ + {isDefault && ( + + default + + )} + + ({key}) + +
+ {propDef?.description && ( +

+ {propDef.description} +

+ )} + {renderPropertyInput(key, value)} +
+ +
+ ); + }) + )} +
+ )} +
+ ); + })} + + {/* Add Property */} +
+ {showAddProperty ? ( +
+ + + +
+ ) : ( + + )} +
+
+ ); +} diff --git a/ui/components/ConnectionStatus.tsx b/ui/components/ConnectionStatus.tsx new file mode 100644 index 0000000..4b7313e --- /dev/null +++ b/ui/components/ConnectionStatus.tsx @@ -0,0 +1,64 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { Wifi, WifiOff, RefreshCw } from 'lucide-react'; + +export default function ConnectionStatus() { + const [status, setStatus] = useState<'connected' | 'disconnected' | 'checking'>('checking'); + const [lastChecked, setLastChecked] = useState(null); + + const checkConnection = async () => { + setStatus('checking'); + try { + const response = await fetch('/api/health', { + method: 'GET', + cache: 'no-store' + }); + setStatus(response.ok ? 'connected' : 'disconnected'); + } catch { + setStatus('disconnected'); + } + setLastChecked(new Date()); + }; + + useEffect(() => { + checkConnection(); + const interval = setInterval(checkConnection, 30000); + return () => clearInterval(interval); + }, []); + + const statusConfig = { + connected: { + icon: Wifi, + color: 'text-green-500', + bgColor: 'bg-green-100 dark:bg-green-900/30', + label: 'Connected', + }, + disconnected: { + icon: WifiOff, + color: 'text-red-500', + bgColor: 'bg-red-100 dark:bg-red-900/30', + label: 'Disconnected', + }, + checking: { + icon: RefreshCw, + color: 'text-blue-500', + bgColor: 'bg-blue-100 dark:bg-blue-900/30', + label: 'Checking...', + }, + }; + + const config = statusConfig[status]; + const Icon = config.icon; + + return ( +
+ + {config.label} +
+ ); +} diff --git a/ui/components/Dashboard.tsx b/ui/components/Dashboard.tsx new file mode 100644 index 0000000..68ac6ff --- /dev/null +++ b/ui/components/Dashboard.tsx @@ -0,0 +1,305 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { Activity, Server, Clock, AlertTriangle, CheckCircle, PlayCircle, PauseCircle, XCircle } from 'lucide-react'; +import { Job, Server as ServerType } from '@/lib/types'; +import { formatDistanceToNow } from 'date-fns'; + +interface DashboardProps { + onJobSelect?: (jobId: string) => void; + onFilterJobs?: (filter: 'running' | 'pending' | 'all') => void; +} + +export default function Dashboard({ onJobSelect, onFilterJobs }: DashboardProps) { + const [servers, setServers] = useState([]); + const [jobs, setJobs] = useState([]); + const [loading, setLoading] = useState(true); + + useEffect(() => { + loadData(); + const interval = setInterval(loadData, 10000); + return () => clearInterval(interval); + }, []); + + const loadData = async () => { + try { + const [serversRes, jobsRes] = await Promise.all([ + fetch('/api/servers'), + fetch('/api/jobs?limit=20') + ]); + + const serversData = await serversRes.json(); + const jobsData = await jobsRes.json(); + + console.log('Servers API response:', serversRes.status, serversData); + console.log('Jobs API response:', jobsRes.status, jobsData); + + setServers(Array.isArray(serversData) ? serversData : []); + setJobs(Array.isArray(jobsData) ? jobsData : []); + } catch (error) { + console.error('Failed to load dashboard data:', error); + } finally { + setLoading(false); + } + }; + + const getStatusIcon = (status: string) => { + switch (status) { + case 'running': + return ; + case 'paused': + return ; + case 'completed': + return ; + case 'error': + return ; + case 'failed': + return ; + case 'cancelled': + return ; + default: + return ; + } + }; + + const formatDuration = (seconds: number | null | undefined): string => { + if (seconds === null || seconds === undefined || isNaN(Number(seconds))) { + return '-'; + } + return `${Math.round(Number(seconds))}s`; + }; + + const getServerStatusColor = (status: string) => { + switch (status) { + case 'busy': + return 'bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-200'; + case 'idle': + return 'bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200'; + case 'active': + return 'bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200'; + case 'offline': + return 'bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-200'; + default: + return 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-200'; + } + }; + + const runningJobs = jobs.filter(j => j.status === 'running' || j.status === 'paused'); + const pendingJobs = jobs.filter(j => j.status === 'pending' || j.status === 'scheduled'); + const activeServers = servers.filter(s => + s.status !== 'terminated' && s.status !== 'offline' && + (s.seconds_since_heartbeat === undefined || s.seconds_since_heartbeat === null || s.seconds_since_heartbeat < 300) + ); + + if (loading) { + return
Loading dashboard...
; + } + + return ( +
+ {/* Summary Stats */} +
+
+
+
+ +
+
+

Active Servers

+

{activeServers.length}

+
+
+
+ +
onFilterJobs?.('running')} + className="bg-white dark:bg-gray-800 rounded-lg shadow p-4 cursor-pointer hover:ring-2 hover:ring-blue-500 transition-all" + > +
+
+ +
+
+

Running Jobs

+

{runningJobs.length}

+
+
+
+ +
onFilterJobs?.('pending')} + className="bg-white dark:bg-gray-800 rounded-lg shadow p-4 cursor-pointer hover:ring-2 hover:ring-yellow-500 transition-all" + > +
+
+ +
+
+

Pending Jobs

+

{pendingJobs.length}

+
+
+
+ +
onFilterJobs?.('all')} + className="bg-white dark:bg-gray-800 rounded-lg shadow p-4 cursor-pointer hover:ring-2 hover:ring-purple-500 transition-all" + > +
+
+ +
+
+

Total Jobs

+

{jobs.length}

+
+
+
+
+ + {/* Servers Panel */} +
+
+

+ + Active Servers +

+
+
+ {servers.length === 0 ? ( +

+ No servers registered. Start pgCompare in server mode to register workers. +

+ ) : ( +
+ {servers.map((server) => ( +
+
+
+
+

{server.server_name}

+

+ {server.server_host} (PID: {server.server_pid}) +

+
+
+
+ + {server.status} + + {server.last_heartbeat && ( +

+ Last seen {formatDistanceToNow(new Date(server.last_heartbeat), { addSuffix: true })} +

+ )} +
+
+ ))} +
+ )} +
+
+ + {/* Running Jobs */} + {runningJobs.length > 0 && ( +
+
+

+ + Running Jobs +

+
+
+
+ {runningJobs.map((job) => ( +
onJobSelect?.(job.job_id)} + className="p-3 bg-blue-50 dark:bg-blue-900/20 rounded-lg cursor-pointer hover:bg-blue-100 dark:hover:bg-blue-900/30 transition-colors" + > +
+
+ {getStatusIcon(job.status)} + + {job.project_name || `Project ${job.pid}`} + + + ({job.job_type}) + +
+
+ {job.source === 'standalone' ? 'standalone' : (job.assigned_server_name || 'Unassigned')} +
+
+ {job.started_at && ( +

+ Started {formatDistanceToNow(new Date(job.started_at), { addSuffix: true })} +

+ )} +
+ ))} +
+
+
+ )} + + {/* Recent Jobs */} +
+
+

+ + Recent Jobs +

+
+
+ + + + + + + + + + + + + {jobs.map((job) => ( + onJobSelect?.(job.job_id)} + className="hover:bg-gray-50 dark:hover:bg-gray-700 cursor-pointer" + > + + + + + + + + ))} + +
StatusProjectTypeServerCreatedDuration
+
+ {getStatusIcon(job.status)} + {job.status} +
+ 		{job.project_name || `Project ${job.pid}`}{job.job_type} + {job.source === 'standalone' ? 'standalone' : (job.assigned_server_name || '-')} + + {formatDistanceToNow(new Date(job.created_at), { addSuffix: true })} + + {formatDuration(job.duration_seconds)} +
+
+
+
+ ); +} diff --git a/ui/components/ExportButton.tsx b/ui/components/ExportButton.tsx new file mode 100644 index 0000000..9b59b07 --- /dev/null +++ b/ui/components/ExportButton.tsx @@ -0,0 +1,107 @@ +'use client'; + +import { useState } from 'react'; +import { Download, FileText, Table } from 'lucide-react'; +import { toast } from '@/components/Toaster'; + +interface ExportButtonProps { + data: any[]; + filename: string; + headers?: { key: string; label: string }[]; +} + +export default function ExportButton({ data, filename, headers }: ExportButtonProps) { + const [isOpen, setIsOpen] = useState(false); + + const exportToCSV = () => { + if (data.length === 0) { + toast.error('No data to export'); + return; + } + + const keys = headers + ? headers.map(h => h.key) + : Object.keys(data[0]); + + const headerLabels = headers + ? headers.map(h => h.label) + : keys; + + const csvContent = [ + headerLabels.join(','), + ...data.map(row => + keys.map(key => { + const value = row[key]; + if (value === null || value === undefined) return ''; + const stringValue = String(value); + if (stringValue.includes(',') || stringValue.includes('"') || stringValue.includes('\n')) { + return `"${stringValue.replace(/"/g, '""')}"`; + } + return stringValue; + }).join(',') + ) + ].join('\n'); + + downloadFile(csvContent, `${filename}.csv`, 'text/csv'); + toast.success('CSV exported successfully'); + setIsOpen(false); + }; + + const exportToJSON = () => { + if (data.length === 0) { + toast.error('No data to export'); + return; + } + + const jsonContent = JSON.stringify(data, null, 2); + downloadFile(jsonContent, `${filename}.json`, 'application/json'); + toast.success('JSON exported successfully'); + setIsOpen(false); + }; + + const downloadFile = (content: string, filename: string, mimeType: string) => { + const blob = new Blob([content], { type: mimeType }); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = filename; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + URL.revokeObjectURL(url); + }; + + return ( +
+ + + {isOpen && ( + <> +
setIsOpen(false)} /> +
+
+ )} + + {/* Job Logs */} + + + {selectedTable && ( + setSelectedTable(null)} + /> + )} +
+ ); +} diff --git a/ui/components/JobLogsViewer.tsx b/ui/components/JobLogsViewer.tsx new file mode 100644 index 0000000..f7bc702 --- /dev/null +++ b/ui/components/JobLogsViewer.tsx @@ -0,0 +1,326 @@ +'use client'; + +import { useEffect, useState, useRef, useCallback } from 'react'; +import { FileText, RefreshCw, ChevronDown, ChevronUp, Play, Pause } from 'lucide-react'; + +interface LogEntry { + log_id: number; + job_id: string; + log_ts: string; + log_level: string; + thread_name: string | null; + message: string; + context: unknown; +} + +interface JobLogsViewerProps { + jobId: string; + isExpanded?: boolean; + jobStatus?: string; +} + +export default function JobLogsViewer({ jobId, isExpanded: initialExpanded = false, jobStatus }: JobLogsViewerProps) { + const [logs, setLogs] = useState([]); + const [loading, setLoading] = useState(false); + const [expanded, setExpanded] = useState(initialExpanded); + const [levelFilter, setLevelFilter] = useState('all'); + const [totalCount, setTotalCount] = useState(0); + const [autoScroll, setAutoScroll] = useState(true); + const [autoRefresh, setAutoRefresh] = useState(true); + const [lastLogId, setLastLogId] = useState(null); + const logsContainerRef = useRef(null); + const logsEndRef = useRef(null); + const isScrollingProgrammatically = useRef(false); + const userHasScrolled = useRef(false); + + const isJobActive = jobStatus === 'running' || jobStatus === 'paused'; + + const loadLogs = useCallback(async (sinceLogId?: number | null) => { + if (!expanded) return; + + setLoading(true); + try { + const params = new URLSearchParams({ + pageSize: '500', + ...(levelFilter !== 'all' && { level: levelFilter }), + ...(sinceLogId && { sinceLogId: sinceLogId.toString() }) + }); + + const res = await fetch(`/api/jobs/${jobId}/logs?${params}`); + if (res.ok) { + const data = await res.json(); + + if (sinceLogId && data.logs.length > 0) { + setLogs(prev => [...prev, ...data.logs]); + } else if (!sinceLogId) { + setLogs(data.logs); + } + + setTotalCount(data.pagination.totalCount); + + if (data.logs.length > 0) { + const maxLogId = Math.max(...data.logs.map((l: LogEntry) => l.log_id)); + setLastLogId(maxLogId); + } + } + } catch (error) { + console.error('Failed to load logs:', error); + } finally { + setLoading(false); + } + }, [jobId, expanded, levelFilter]); + + // Initial load and filter change + useEffect(() => { + if (expanded) { + setLastLogId(null); + userHasScrolled.current = false; + setAutoScroll(true); + loadLogs(); + } + }, [expanded, levelFilter]); + + // Auto-refresh for streaming logs + useEffect(() => { + if (!expanded || !autoRefresh) return; + + const interval = isJobActive ? 2000 : 5000; + + const timer = setInterval(() => { + if (lastLogId) { + loadLogs(lastLogId); + } else { + loadLogs(); + } + }, interval); + + return () => clearInterval(timer); + }, [expanded, autoRefresh, isJobActive, lastLogId, loadLogs]); + + // Auto-scroll to bottom when new logs arrive (only if user hasn't scrolled away) + useEffect(() => { + if (autoScroll && logsEndRef.current && logsContainerRef.current && logs.length > 0) { + isScrollingProgrammatically.current = true; + + // Use instant scroll for auto-scroll to avoid timing issues + logsContainerRef.current.scrollTop = logsContainerRef.current.scrollHeight; + + // Reset the flag after a short delay + setTimeout(() => { + isScrollingProgrammatically.current = false; + }, 100); + } + }, [logs, autoScroll]); + + // Detect manual scroll to disable auto-scroll + const handleScroll = useCallback(() => { + // Ignore programmatic scrolls + if (isScrollingProgrammatically.current) return; + if (!logsContainerRef.current) return; + + const { scrollTop, scrollHeight, clientHeight } = logsContainerRef.current; + const distanceFromBottom = scrollHeight - scrollTop - clientHeight; + const isNearBottom = distanceFromBottom < 30; + + // User scrolled away from bottom - disable auto-scroll + if (!isNearBottom) { + userHasScrolled.current = true; + setAutoScroll(false); + } + }, []); + + const scrollToBottom = useCallback(() => { + if (logsContainerRef.current) { + isScrollingProgrammatically.current = true; + logsContainerRef.current.scrollTop = logsContainerRef.current.scrollHeight; + userHasScrolled.current = false; + setAutoScroll(true); + + setTimeout(() => { + isScrollingProgrammatically.current = false; + }, 100); + } + }, []); + + const getLogLevelStyle = (level: string) => { + switch (level.toUpperCase()) { + case 'ERROR': + case 'SEVERE': + return 'text-red-400'; + case 'WARNING': + case 'WARN': + return 'text-amber-400'; + case 'INFO': + return 'text-emerald-400'; + case 'DEBUG': + case 'FINE': + return 'text-sky-400'; + case 'TRACE': + case 'FINEST': + return 'text-slate-400'; + default: + return 'text-slate-300'; + } + }; + + const getRowBackground = (level: string) => { + switch (level.toUpperCase()) { + case 'ERROR': + case 'SEVERE': + return 'bg-red-950/30'; + case 'WARNING': + case 'WARN': + return 'bg-amber-950/20'; + default: + return ''; + } + }; + + const formatTimestamp = (ts: string) => { + const date = new Date(ts); + return date.toLocaleTimeString('en-US', { + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + fractionalSecondDigits: 3 + }); + }; + + if (!expanded) { + return ( +
+ +
+ ); + } + + return ( +
+ {/* Header */} +
+
+ + + Job Logs + ({totalCount} entries) + {isJobActive && autoRefresh && ( + + + Live + + )} +
+ +
+ {/* Level Filter */} + + + {/* Auto-refresh toggle */} + + + {/* Scroll to bottom - show when auto-scroll is off */} + {!autoScroll && ( + + )} + + {/* Manual Refresh */} + +
+
+ + {/* Logs content */} +
+ {logs.length === 0 ? ( +
+ {loading ? 'Loading logs...' : 'No logs available'} +
+ ) : ( +
+ {logs.map((log) => ( +
+ + {formatTimestamp(log.log_ts)} + + + {log.log_level.toUpperCase().padEnd(7)} + + {log.thread_name && ( + + [{log.thread_name}] + + )} + {log.message} +
+ ))} +
+
+ )} +
+
+ ); +} diff --git a/ui/components/JobProgressPanel.tsx b/ui/components/JobProgressPanel.tsx new file mode 100644 index 0000000..9ed954d --- /dev/null +++ b/ui/components/JobProgressPanel.tsx @@ -0,0 +1,270 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { X, Pause, Play, StopCircle, AlertTriangle, CheckCircle, Clock, RefreshCw } from 'lucide-react'; +import { Job, JobProgress, JobProgressSummary } from '@/lib/types'; +import { formatDistanceToNow } from 'date-fns'; +import { toast } from '@/components/Toaster'; + +interface JobProgressPanelProps { + jobId: string; + onClose: () => void; +} + +export default function JobProgressPanel({ jobId, onClose }: JobProgressPanelProps) { + const [job, setJob] = useState(null); + const [progress, setProgress] = useState([]); + const [summary, setSummary] = useState(null); + const [loading, setLoading] = useState(true); + + useEffect(() => { + loadData(); + const interval = setInterval(loadData, 3000); + return () => clearInterval(interval); + }, [jobId]); + + const loadData = async () => { + try { + const [jobRes, progressRes] = await Promise.all([ + fetch(`/api/jobs/${jobId}`), + fetch(`/api/jobs/${jobId}/progress`) + ]); + + const jobData = await jobRes.json(); + const progressData = await progressRes.json(); + + setJob(jobData); + setProgress(progressData.tables || []); + setSummary(progressData.summary || null); + } catch (error) { + console.error('Failed to load job data:', error); + } finally { + setLoading(false); + } + }; + + const sendSignal = async (signal: string) => { + try { + const response = await fetch(`/api/jobs/${jobId}/control`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ signal }), + }); + + if (!response.ok) { + throw new Error('Failed to send signal'); + } + + toast.success(`${signal} signal sent`); + loadData(); + } catch (error) { + console.error('Failed to send signal:', error); + toast.error('Failed to send control signal'); + } + }; + + const getStatusColor = (status: string) => { + switch (status) { + case 'running': + return 'bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-200'; + case 'completed': + case 'in-sync': + return 'bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200'; + case 'out-of-sync': + return 'bg-orange-100 text-orange-800 dark:bg-orange-900 dark:text-orange-200'; + case 'failed': + return 'bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-200'; + case 'pending': + return 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-200'; + case 'skipped': + return 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900 dark:text-yellow-200'; + default: + return 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-200'; + } + }; + + const getDisplayStatus = (table: JobProgress) => { + if (table.status === 'completed') { + const hasOutOfSync = (table.not_equal_cnt || 0) + (table.missing_source_cnt || 0) + (table.missing_target_cnt || 0) > 0; + return hasOutOfSync ? 'out-of-sync' : 'in-sync'; + } + return table.status; + }; + + if (loading) { + return ( +
+
+
+ +
+
+ ); + } + + if (!job) { + return null; + } + + const progressPercent = summary + ? Math.round((summary.completed_tables / summary.total_tables) * 100) + : 0; + + return ( +
+
+
+ {/* Header */} +
+
+

+ Job Progress - {job.project_name || `Project ${job.pid}`} +

+

+ {job.job_type} • Started {job.started_at ? formatDistanceToNow(new Date(job.started_at), { addSuffix: true }) : 'pending'} +

+
+
+ {(job.status === 'running' || job.status === 'paused') && ( + <> + {job.status === 'running' ? ( + + ) : ( + + )} + + + )} + +
+
+ + {/* Progress Bar */} + {summary && ( +
+
+ + {summary.completed_tables} of {summary.total_tables} tables + + + {progressPercent}% + +
+
+
+
+ + {/* Stats */} +
+
+

+ {(summary.total_equal || 0).toLocaleString()} +

+

Equal

+
+
+

+ {(summary.total_not_equal || 0).toLocaleString()} +

+

Not Equal

+
+
+

+ {((summary.total_missing_source || 0) + (summary.total_missing_target || 0)).toLocaleString()} +

+

Missing

+
+
+

+ {summary.failed_tables} +

+

Failed

+
+
+
+ )} + + {/* Table Progress */} +
+ + + + + + + + + + + + + {progress.map((table) => { + const displayStatus = getDisplayStatus(table); + return ( + + + + + + + + + ); + })} + +
TableStatusEqualNot EqualMissingDuration
{table.table_name} + + {table.status === 'running' && } + {displayStatus} + + + {(table.equal_cnt || 0).toLocaleString()} + + + {(table.not_equal_cnt || 0).toLocaleString()} + + + + {((table.missing_source_cnt || 0) + (table.missing_target_cnt || 0)).toLocaleString()} + + + {table.duration_seconds ? `${Math.round(table.duration_seconds)}s` : '-'} +
+
+ + {/* Error Message */} + {job.error_message && ( +
+
+ + Error: {job.error_message} +
+
+ )} +
+
+ ); +} diff --git a/ui/components/JobsView.tsx b/ui/components/JobsView.tsx new file mode 100644 index 0000000..b50f1e6 --- /dev/null +++ b/ui/components/JobsView.tsx @@ -0,0 +1,378 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { ArrowLeft, PlayCircle, PauseCircle, StopCircle, XCircle, CheckCircle, Clock, RefreshCw, Trash2, AlertTriangle } from 'lucide-react'; +import { Job } from '@/lib/types'; +import { formatDistanceToNow, format } from 'date-fns'; +import { toast } from 'sonner'; + +interface JobsViewProps { + initialFilter?: 'running' | 'pending' | 'all'; + onBack: () => void; + onJobSelect: (jobId: string) => void; +} + +export default function JobsView({ initialFilter = 'all', onBack, onJobSelect }: JobsViewProps) { + const [jobs, setJobs] = useState([]); + const [loading, setLoading] = useState(true); + const [filter, setFilter] = useState(initialFilter); + const [selectedJobs, setSelectedJobs] = useState>(new Set()); + + useEffect(() => { + loadJobs(); + const interval = setInterval(loadJobs, 5000); + return () => clearInterval(interval); + }, [filter]); + + useEffect(() => { + setSelectedJobs(new Set()); + }, [filter]); + + const loadJobs = async () => { + try { + let url = '/api/jobs?limit=100'; + if (filter === 'running') { + url += '&status=running,paused'; + } else if (filter === 'pending') { + url += '&status=pending,scheduled'; + } + + const res = await fetch(url); + const data = await res.json(); + setJobs(Array.isArray(data) ? data : []); + } catch (error) { + console.error('Failed to load jobs:', error); + } finally { + setLoading(false); + } + }; + + const sendControl = async (jobId: string, signal: string) => { + try { + const res = await fetch(`/api/jobs/${jobId}/control`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ signal }) + }); + + if (res.ok) { + toast.success(`Job ${signal} signal sent`); + loadJobs(); + } else { + toast.error(`Failed to send ${signal} signal`); + } + } catch (error) { + toast.error(`Failed to send ${signal} signal`); + } + }; + + const deleteJob = async (jobId: string) => { + try { + const res = await fetch(`/api/jobs/${jobId}`, { method: 'DELETE' }); + if (res.ok) { + return true; + } + return false; + } catch (error) { + return false; + } + }; + + const handleDeleteSingle = async (jobId: string, isRunning: boolean = false) => { + const message = isRunning + ? 'This job is still running. Are you sure you want to delete it? This cannot be undone.' + : 'Are you sure you want to delete this job?'; + + if (!confirm(message)) return; + + const success = await deleteJob(jobId); + if (success) { + toast.success('Job deleted'); + loadJobs(); + } else { + toast.error('Failed to delete job'); + } + }; + + const handleDeleteSelected = async () => { + const deletableJobs = Array.from(selectedJobs).filter(jobId => { + const job = jobs.find(j => j.job_id === jobId); + if (!job) return false; + // Allow deletion of completed/error/failed/cancelled jobs, or running standalone jobs + return ['completed', 'error', 'failed', 'cancelled'].includes(job.status) || + (job.status === 'running' && job.source === 'standalone'); + }); + + if (deletableJobs.length === 0) { + toast.error('No deletable jobs selected'); + return; + } + + const runningCount = deletableJobs.filter(jobId => { + const job = jobs.find(j => j.job_id === jobId); + return job?.status === 'running'; + }).length; + + const message = runningCount > 0 + ? `Are you sure you want to delete ${deletableJobs.length} job(s)? ${runningCount} job(s) are still running.` + : `Are you sure you want to delete ${deletableJobs.length} job(s)?`; + + if (!confirm(message)) return; + + let successCount = 0; + let failCount = 0; + + for (const jobId of deletableJobs) { + const success = await deleteJob(jobId); + if (success) { + successCount++; + } else { + failCount++; + } + } + + if (successCount > 0) { + toast.success(`Deleted ${successCount} job(s)`); + } + if (failCount > 0) { + toast.error(`Failed to delete ${failCount} job(s)`); + } + + setSelectedJobs(new Set()); + loadJobs(); + }; + + const toggleSelectAll = () => { + if (selectedJobs.size === jobs.length) { + setSelectedJobs(new Set()); + } else { + setSelectedJobs(new Set(jobs.map(j => j.job_id))); + } + }; + + const toggleSelectJob = (jobId: string) => { + const newSelected = new Set(selectedJobs); + if (newSelected.has(jobId)) { + newSelected.delete(jobId); + } else { + newSelected.add(jobId); + } + setSelectedJobs(newSelected); + }; + + const getStatusIcon = (status: string) => { + switch (status) { + case 'running': + return ; + case 'paused': + return ; + case 'completed': + return ; + case 'error': + return ; + case 'failed': + return ; + case 'cancelled': + return ; + default: + return ; + } + }; + + const getStatusBadge = (status: string) => { + const colors: Record = { + running: 'bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-200', + paused: 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900 dark:text-yellow-200', + completed: 'bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200', + error: 'bg-orange-100 text-orange-800 dark:bg-orange-900 dark:text-orange-200', + failed: 'bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-200', + cancelled: 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-200', + pending: 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-200', + scheduled: 'bg-purple-100 text-purple-800 dark:bg-purple-900 dark:text-purple-200' + }; + return colors[status] || colors.pending; + }; + + const formatDuration = (seconds: number | null | undefined): string => { + if (seconds === null || seconds === undefined || isNaN(Number(seconds))) { + return '-'; + } + return `${Math.round(Number(seconds))}s`; + }; + + const deletableSelectedCount = Array.from(selectedJobs).filter(jobId => { + const job = jobs.find(j => j.job_id === jobId); + return job && ['completed', 'error', 'failed', 'cancelled'].includes(job.status); + }).length; + + if (loading) { + return
Loading jobs...
; + } + + return ( +
+ {/* Header */} +
+
+ +

+ {filter === 'running' ? 'Running Jobs' : filter === 'pending' ? 'Pending Jobs' : 'All Jobs'} +

+
+ +
+ {selectedJobs.size > 0 && deletableSelectedCount > 0 && ( + + )} + + +
+
+ + {/* Jobs List */} +
+ {jobs.length === 0 ? ( +
+ No jobs found +
+ ) : ( + + + + + + + + + + + + + + + {jobs.map((job) => ( + + + + + + + + + + + ))} + +
+ 0} + onChange={toggleSelectAll} + className="h-4 w-4 rounded border-gray-300 dark:border-gray-600" + /> + StatusProjectTypeServerCreatedDurationActions
+ toggleSelectJob(job.job_id)} + className="h-4 w-4 rounded border-gray-300 dark:border-gray-600" + /> + +
+ {getStatusIcon(job.status)} + + {job.status} + +
+ 		onJobSelect(job.job_id)} + > + {job.project_name || `Project ${job.pid}`} + {job.job_type} + {job.source === 'standalone' ? 'standalone' : (job.assigned_server_name || '-')} + + {formatDistanceToNow(new Date(job.created_at), { addSuffix: true })} + + {formatDuration(job.duration_seconds)} + +
+ {job.status === 'running' && job.source !== 'standalone' && ( + <> + + + + )} + {job.status === 'running' && job.source === 'standalone' && ( + + )} + {job.status === 'paused' && ( + + )} + {['completed', 'error', 'failed', 'cancelled'].includes(job.status) && ( + + )} +
+
+ )} +
+
+ ); +} diff --git a/ui/components/NavigationTree.tsx b/ui/components/NavigationTree.tsx index 4e3f0fa..53c0dd8 100644 --- a/ui/components/NavigationTree.tsx +++ b/ui/components/NavigationTree.tsx @@ -1,14 +1,17 @@ 'use client'; -import { useEffect, useState } from 'react'; -import { ChevronRight, ChevronDown, Folder, Table, X, Plus } from 'lucide-react'; +import { useEffect, useState, useMemo } from 'react'; +import { ChevronRight, ChevronDown, Folder, Table, X, Plus, Search, Upload } from 'lucide-react'; import { Project, Table as TableType, Result } from '@/lib/types'; +import { toast } from '@/components/Toaster'; +import AddTableModal from './AddTableModal'; interface NavigationTreeProps { onProjectSelect: (projectId: number) => void; onTableSelect: (tableId: number) => void; selectedProjectId?: number; selectedTableId?: number; + refreshTrigger?: number; } export default function NavigationTree({ @@ -16,6 +19,7 @@ export default function NavigationTree({ onTableSelect, selectedProjectId, selectedTableId, + refreshTrigger, }: NavigationTreeProps) { const [projects, setProjects] = useState([]); const [expandedProjects, setExpandedProjects] = useState>(new Set()); @@ -24,10 +28,29 @@ export default function NavigationTree({ const [loading, setLoading] = useState(true); const [creatingProject, setCreatingProject] = useState(false); const [newProjectName, setNewProjectName] = useState(''); + const [searchQuery, setSearchQuery] = useState(''); + const [importingProject, setImportingProject] = useState(false); + const [importProjectName, setImportProjectName] = useState(''); + const [importFile, setImportFile] = useState(null); + const [addTableProjectId, setAddTableProjectId] = useState(null); + + const filteredProjects = useMemo(() => { + if (!searchQuery.trim()) return projects; + + const query = searchQuery.toLowerCase(); + return projects.filter(project => { + const projectMatches = project.project_name.toLowerCase().includes(query); + const tables = projectTables.get(project.pid) || []; + const hasMatchingTable = tables.some(table => + table.table_alias.toLowerCase().includes(query) + ); + return projectMatches || hasMatchingTable; + }); + }, [projects, projectTables, searchQuery]); useEffect(() => { loadProjects(); - }, []); + }, [refreshTrigger]); const loadProjects = async () => { try { @@ -100,6 +123,17 @@ export default function NavigationTree({ } }; + const handleTableCreated = (projectId: number, newTable: TableType) => { + const currentTables = projectTables.get(projectId) || []; + const updatedTables = [...currentTables, newTable].sort((a, b) => + a.table_alias.localeCompare(b.table_alias) + ); + setProjectTables(new Map(projectTables).set(projectId, updatedTables)); + setAddTableProjectId(null); + toast.success(`Table "${newTable.table_alias}" created`); + onTableSelect(newTable.tid); + }; + const handleCreateProject = async () => { if (!newProjectName.trim()) { alert('Please enter a project name'); @@ -124,9 +158,69 @@ export default function NavigationTree({ // Select the newly created project onProjectSelect(newProject.pid); + toast.success(`Project "${newProjectName.trim()}" created`); } catch (error) { console.error('Failed to create project:', error); - alert('Failed to create project'); + toast.error('Failed to create project'); + } + }; + + const handleImportProject = async () => { + if (!importProjectName.trim()) { + toast.error('Please enter a project name'); + return; + } + if (!importFile) { + toast.error('Please select a properties file'); + return; + } + + try { + const text = await importFile.text(); + const lines = text.split('\n'); + const configObject: Record = {}; + + lines.forEach(line => { + const trimmedLine = line.trim(); + if (!trimmedLine || trimmedLine.startsWith('#') || trimmedLine.startsWith('!')) { + return; + } + const separatorIndex = trimmedLine.indexOf('='); + if (separatorIndex > 0) { + const key = trimmedLine.substring(0, separatorIndex).trim(); + const value = trimmedLine.substring(separatorIndex + 1).trim(); + try { + configObject[key] = JSON.parse(value); + } catch { + configObject[key] = value; + } + } + }); + + const response = await fetch('/api/projects', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + project_name: importProjectName.trim(), + project_config: configObject + }), + }); + + if (!response.ok) { + throw new Error('Failed to create project'); + } + + const newProject = await response.json(); + setProjects([...projects, newProject]); + setImportProjectName(''); + setImportFile(null); + setImportingProject(false); + + onProjectSelect(newProject.pid); + toast.success(`Project "${importProjectName.trim()}" imported with ${Object.keys(configObject).length} properties`); + } catch (error) { + console.error('Failed to import project:', error); + toast.error('Failed to import project'); } }; @@ -154,16 +248,45 @@ export default function NavigationTree({ return (
- {/* Add New Project Button */} - {!creatingProject ? ( - - ) : ( + {/* Search Input */} +
+ + setSearchQuery(e.target.value)} + className="w-full pl-8 pr-8 py-2 text-sm border border-gray-300 dark:border-gray-600 rounded bg-white dark:bg-gray-700 text-gray-900 dark:text-white placeholder-gray-400 focus:outline-none focus:ring-1 focus:ring-blue-500" + /> + {searchQuery && ( + + )} +
+ + {/* Add New Project / Import Project Buttons */} + {!creatingProject && !importingProject ? ( +
+ + +
+ ) : creatingProject ? (
+ ) : ( +
+ setImportProjectName(e.target.value)} + placeholder="Project name..." + autoFocus + className="w-full px-2 py-1 text-sm border border-gray-300 dark:border-gray-600 rounded mb-2 dark:bg-gray-700 dark:text-white" + /> + setImportFile(e.target.files?.[0] || null)} + className="w-full px-2 py-1 text-sm border border-gray-300 dark:border-gray-600 rounded mb-2 dark:bg-gray-700 dark:text-white" + /> +
+ + +
+
)} {/* Projects List */} - {projects.map((project) => { + {filteredProjects.length === 0 && searchQuery && ( +

+ No results found for "{searchQuery}" +

+ )} + {filteredProjects.map((project) => { const isExpanded = expandedProjects.has(project.pid); - const tables = projectTables.get(project.pid) || []; + const allTables = projectTables.get(project.pid) || []; + const tables = searchQuery + ? allTables.filter(t => t.table_alias.toLowerCase().includes(searchQuery.toLowerCase())) + : allTables; const isSelected = selectedProjectId === project.pid; return ( @@ -256,11 +423,26 @@ export default function NavigationTree({
); })} +
)}
); })} + + {addTableProjectId !== null && ( + setAddTableProjectId(null)} + onTableCreated={(table) => handleTableCreated(addTableProjectId, table)} + /> + )}
); } diff --git a/ui/components/OutOfSyncModal.tsx b/ui/components/OutOfSyncModal.tsx new file mode 100644 index 0000000..ece43f8 --- /dev/null +++ b/ui/components/OutOfSyncModal.tsx @@ -0,0 +1,404 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { X, AlertTriangle, Code, ChevronDown, ChevronRight, Copy, Check, FlaskConical } from 'lucide-react'; +import { toast } from 'sonner'; + +interface OutOfSyncModalProps { + tid: number; + tableName: string; + cid?: number | null; + isOpen: boolean; + onClose: () => void; +} + +interface SourceTargetRow { + pk: any; + pk_hash: string | null; + column_hash: string | null; + compare_result: string | null; + thread_nbr: number | null; + fix_sql: string | null; +} + +export default function OutOfSyncModal({ tid, tableName, cid, isOpen, onClose }: OutOfSyncModalProps) { + const [sourceRows, setSourceRows] = useState([]); + const [targetRows, setTargetRows] = useState([]); + const [loading, setLoading] = useState(true); + const [activeTab, setActiveTab] = useState<'not_equal' | 'missing_source' | 'missing_target' | 'fix_sql'>('not_equal'); + const [expandedRows, setExpandedRows] = useState>(new Set()); + const [copiedSql, setCopiedSql] = useState(null); + + useEffect(() => { + if (isOpen) { + loadData(); + } + }, [isOpen, tid, cid]); + + const loadData = async () => { + setLoading(true); + try { + let url = `/api/tables/${tid}/out-of-sync`; + if (cid) { + url += `?cid=${cid}`; + } + const response = await fetch(url); + const data = await response.json(); + + setSourceRows(Array.isArray(data.source) ? data.source : []); + setTargetRows(Array.isArray(data.target) ? data.target : []); + } catch (error) { + console.error('Failed to load out-of-sync data:', error); + } finally { + setLoading(false); + } + }; + + const toggleRow = (pkHash: string) => { + const newExpanded = new Set(expandedRows); + if (newExpanded.has(pkHash)) { + newExpanded.delete(pkHash); + } else { + newExpanded.add(pkHash); + } + setExpandedRows(newExpanded); + }; + + const copyToClipboard = async (sql: string, pkHash: string) => { + try { + await navigator.clipboard.writeText(sql); + setCopiedSql(pkHash); + toast.success('SQL copied to clipboard'); + setTimeout(() => setCopiedSql(null), 2000); + } catch (error) { + toast.error('Failed to copy to clipboard'); + } + }; + + const copyAllFixSql = async () => { + const allSql = [...sourceRows, ...targetRows] + .filter(row => row.fix_sql) + .map(row => row.fix_sql) + .join(';\n\n'); + + if (allSql) { + try { + await navigator.clipboard.writeText(allSql + ';'); + toast.success('All fix SQL copied to clipboard'); + } catch (error) { + toast.error('Failed to copy to clipboard'); + } + } + }; + + if (!isOpen) return null; + + const notEqualSource = sourceRows.filter(row => row.compare_result === 'n'); + const notEqualTarget = targetRows.filter(row => row.compare_result === 'n'); + const missingInTarget = sourceRows.filter(row => row.compare_result === 'm'); + const missingInSource = targetRows.filter(row => row.compare_result === 'm'); + + const allFixSql = [...sourceRows, ...targetRows].filter(row => row.fix_sql); + const hasFixSql = allFixSql.length > 0; + + const renderTable = (rows: SourceTargetRow[], title: string, showFixSql: boolean = false) => ( +
+

{title}

+ {rows.length === 0 ? ( +

No rows found

+ ) : ( +
+ + + + {showFixSql && ( + + )} + + + + + {showFixSql && ( + + )} + + + + {rows.map((row, index) => { + const pkHash = row.pk_hash || `row-${index}`; + const isExpanded = expandedRows.has(pkHash); + return ( + <> + + {showFixSql && ( + + )} + + + + + {showFixSql && ( + + )} + + {showFixSql && row.fix_sql && isExpanded && ( + + + + )} + + ); + })} + +
+ + Primary Key + + PK Hash + + Column Hash + + Thread + + Fix SQL +
+ {row.fix_sql && ( + + )} + + {typeof row.pk === 'object' ? JSON.stringify(row.pk) : row.pk} + + {row.pk_hash || 'N/A'} + + {row.column_hash || 'N/A'} + + {row.thread_nbr || '-'} + + {row.fix_sql ? ( + + + Available + + ) : ( + - + )} +
+
+ 
+                              {row.fix_sql}
+
+ +
+
+
+ )} +
+ ); + + const renderFixSqlTab = () => ( +
+
+
+ +
+

Experimental Feature

+

These SQL statements are designed to be executed on the target database to make it match the source. Review carefully before executing.

+
+
+
+
+

+ Generated Fix SQL ({allFixSql.length} statements) +

+ {allFixSql.length > 0 && ( + + )} +
+ + {allFixSql.length === 0 ? ( +

+ No fix SQL available. Run a check job with "Generate Fix SQL" enabled. +

+ ) : ( +
+ {allFixSql.map((row, index) => ( +
+
+
+ PK: {typeof row.pk === 'object' ? JSON.stringify(row.pk) : row.pk} +
+ +
+ 
+                {row.fix_sql}
+
+
+ ))} +
+ )} +
+ ); + + return ( +
+
+
+
+

+ + Out of Sync Details - {tableName} +

+
+ +
+ +
+
+
+

Not Equal Rows

+

+ {notEqualSource.length} +

+
+
+

Missing in Target

+

+ {missingInTarget.length} +

+
+
+

Missing in Source

+

+ {missingInSource.length} +

+
+
+

Fix SQL Available

+

+ {allFixSql.length} +

+
+
+
+ +
+
+ + + + {hasFixSql && ( + + )} +
+
+ +
+ {loading ? ( +
Loading out-of-sync details...
+ ) : ( + <> + {activeTab === 'not_equal' && ( +
+ {renderTable(notEqualSource, 'Source Rows (Different)', hasFixSql)} + {renderTable(notEqualTarget, 'Target Rows (Different)', hasFixSql)} + {notEqualSource.length === 0 && notEqualTarget.length === 0 && ( +

+ No differing rows found +

+ )} +
+ )} + + {activeTab === 'missing_target' && ( + <> + {renderTable(missingInTarget, 'Rows in Source but Missing in Target', hasFixSql)} + + )} + + {activeTab === 'missing_source' && ( + <> + {renderTable(missingInSource, 'Rows in Target but Missing in Source', hasFixSql)} + + )} + + {activeTab === 'fix_sql' && renderFixSqlTab()} + + )} +
+
+
+ ); +} diff --git a/ui/components/Pagination.tsx b/ui/components/Pagination.tsx new file mode 100644 index 0000000..43f5838 --- /dev/null +++ b/ui/components/Pagination.tsx @@ -0,0 +1,191 @@ +'use client'; + +import { useState } from 'react'; +import { ChevronLeft, ChevronRight, ChevronsLeft, ChevronsRight } from 'lucide-react'; + +interface PaginationProps { + currentPage: number; + totalPages: number; + totalItems: number; + pageSize: number; + onPageChange: (page: number) => void; + onPageSizeChange?: (size: number) => void; + pageSizeOptions?: number[]; +} + +export default function Pagination({ + currentPage, + totalPages, + totalItems, + pageSize, + onPageChange, + onPageSizeChange, + pageSizeOptions = [10, 25, 50, 100], +}: PaginationProps) { + const startItem = totalItems === 0 ? 0 : (currentPage - 1) * pageSize + 1; + const endItem = Math.min(currentPage * pageSize, totalItems); + + const getPageNumbers = () => { + const pages: (number | string)[] = []; + const maxVisible = 5; + + if (totalPages <= maxVisible + 2) { + for (let i = 1; i <= totalPages; i++) { + pages.push(i); + } + } else { + pages.push(1); + + if (currentPage > 3) { + pages.push('...'); + } + + const start = Math.max(2, currentPage - 1); + const end = Math.min(totalPages - 1, currentPage + 1); + + for (let i = start; i <= end; i++) { + if (!pages.includes(i)) { + pages.push(i); + } + } + + if (currentPage < totalPages - 2) { + pages.push('...'); + } + + if (!pages.includes(totalPages)) { + pages.push(totalPages); + } + } + + return pages; + }; + + if (totalItems === 0) { + return null; + } + + return ( +
+ {/* Info and Page Size */} +
+ + Showing {startItem} to {endItem} of {totalItems.toLocaleString()} results + + {onPageSizeChange && ( +
+ Show + + per page +
+ )} +
+ + {/* Page Navigation */} +
+ {/* First Page */} + + + {/* Previous Page */} + + + {/* Page Numbers */} +
+ {getPageNumbers().map((page, index) => ( + page === '...' ? ( + + ... + + ) : ( + + ) + ))} +
+ + {/* Next Page */} + + + {/* Last Page */} + +
+
+ ); +} + +export function usePagination(items: T[], initialPageSize = 25) { + const [currentPage, setCurrentPage] = useState(1); + const [pageSize, setPageSize] = useState(initialPageSize); + + const totalPages = Math.ceil(items.length / pageSize); + const paginatedItems = items.slice( + (currentPage - 1) * pageSize, + currentPage * pageSize + ); + + const handlePageChange = (page: number) => { + setCurrentPage(Math.max(1, Math.min(page, totalPages))); + }; + + const handlePageSizeChange = (size: number) => { + setPageSize(size); + setCurrentPage(1); + }; + + return { + currentPage, + pageSize, + totalPages, + totalItems: items.length, + paginatedItems, + handlePageChange, + handlePageSizeChange, + }; +} diff --git a/ui/components/ProjectView.tsx b/ui/components/ProjectView.tsx index 6e6a2c7..b712094 100644 --- a/ui/components/ProjectView.tsx +++ b/ui/components/ProjectView.tsx @@ -7,12 +7,15 @@ import { LineChart, Line, BarChart, Bar, PieChart, Pie, Cell, XAxis, YAxis, Cart import { formatDistanceToNow } from 'date-fns'; import CompareDetailsModal from './CompareDetailsModal'; import ProjectCurrentRunPanel from './ProjectCurrentRunPanel'; +import ConfigEditor from './ConfigEditor'; +import { toast } from '@/components/Toaster'; interface ProjectViewProps { projectId: number; + onProjectUpdated?: () => void; } -export default function ProjectView({ projectId }: ProjectViewProps) { +export default function ProjectView({ projectId, onProjectUpdated }: ProjectViewProps) { const [project, setProject] = useState(null); const [results, setResults] = useState([]); const [configData, setConfigData] = useState>([]); @@ -22,7 +25,6 @@ export default function ProjectView({ projectId }: ProjectViewProps) { const [projectName, setProjectName] = useState(''); const [selectedResult, setSelectedResult] = useState(null); const [showModal, setShowModal] = useState(false); - const fileInputRef = useRef(null); useEffect(() => { loadProject(); @@ -60,20 +62,6 @@ export default function ProjectView({ projectId }: ProjectViewProps) { } }; - const handleConfigChange = (index: number, field: 'key' | 'value', value: string) => { - const newConfig = [...configData]; - newConfig[index][field] = value; - setConfigData(newConfig); - }; - - const addConfigRow = () => { - setConfigData([...configData, { key: '', value: '' }]); - }; - - const removeConfigRow = (index: number) => { - setConfigData(configData.filter((_, i) => i !== index)); - }; - const handleSave = async () => { setSaving(true); try { @@ -89,7 +77,7 @@ export default function ProjectView({ projectId }: ProjectViewProps) { } }); - await fetch(`/api/projects/${projectId}`, { + const response = await fetch(`/api/projects/${projectId}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ @@ -98,16 +86,23 @@ export default function ProjectView({ projectId }: ProjectViewProps) { }), }); + if (!response.ok) { + throw new Error('Failed to save'); + } + // Update local state if (project) { setProject({ ...project, project_name: projectName }); } setEditingName(false); - alert('Configuration saved successfully'); + // Notify parent to refresh navigation tree + onProjectUpdated?.(); + + toast.success('Configuration saved successfully'); } catch (error) { console.error('Failed to save configuration:', error); - alert('Failed to save configuration'); + toast.error('Failed to save configuration'); } finally { setSaving(false); } @@ -124,27 +119,18 @@ export default function ProjectView({ projectId }: ProjectViewProps) { } }; - const handleImportProperties = () => { - fileInputRef.current?.click(); - }; - - const handleFileChange = async (e: React.ChangeEvent) => { - const file = e.target.files?.[0]; - if (!file) return; - + const handleImportFile = async (file: File) => { try { const text = await file.text(); const lines = text.split('\n'); const newConfigData: Array<{ key: string; value: string }> = []; lines.forEach(line => { - // Skip empty lines and comments const trimmedLine = line.trim(); if (!trimmedLine || trimmedLine.startsWith('#') || trimmedLine.startsWith('!')) { return; } - // Parse key=value const separatorIndex = trimmedLine.indexOf('='); if (separatorIndex > 0) { const key = trimmedLine.substring(0, separatorIndex).trim(); @@ -153,7 +139,6 @@ export default function ProjectView({ projectId }: ProjectViewProps) { } }); - // Merge with existing config data const mergedConfig = [...configData]; newConfigData.forEach(({ key, value }) => { const existingIndex = mergedConfig.findIndex(item => item.key === key); @@ -165,15 +150,10 @@ export default function ProjectView({ projectId }: ProjectViewProps) { }); setConfigData(mergedConfig); - alert(`Imported ${newConfigData.length} properties. Click Save to persist changes.`); + toast.success(`Imported ${newConfigData.length} properties. Click Save to persist changes.`); } catch (error) { console.error('Failed to import properties file:', error); - alert('Failed to import properties file'); - } - - // Reset file input - if (fileInputRef.current) { - fileInputRef.current.value = ''; + toast.error('Failed to import properties file'); } }; @@ -253,82 +233,14 @@ export default function ProjectView({ projectId }: ProjectViewProps) { {/* Configuration Editor */}
-
-

Configuration

-
- - - -
-
- -
- - - - - - - - - - {configData.map((item, index) => ( - - - - - - ))} - -
KeyValue
- handleConfigChange(index, 'key', e.target.value)} - className="w-full px-2 py-1 border border-gray-300 dark:border-gray-600 rounded dark:bg-gray-700 dark:text-white" - /> - - handleConfigChange(index, 'value', e.target.value)} - className="w-full px-2 py-1 border border-gray-300 dark:border-gray-600 rounded dark:bg-gray-700 dark:text-white" - /> - - -
-
- - +
{/* Current/Last Run Panel */} diff --git a/ui/components/ScheduleJobModal.tsx b/ui/components/ScheduleJobModal.tsx new file mode 100644 index 0000000..6f573d5 --- /dev/null +++ b/ui/components/ScheduleJobModal.tsx @@ -0,0 +1,315 @@ +'use client'; + +import { useState, useEffect } from 'react'; +import { useRouter } from 'next/navigation'; +import { X, Play, Calendar, Server as ServerIcon } from 'lucide-react'; +import { Project, Server } from '@/lib/types'; +import { toast } from '@/components/Toaster'; + +interface ScheduleJobModalProps { + isOpen: boolean; + onClose: () => void; + onSuccess?: () => void; + onJobCreated?: (jobId: string) => void; + preselectedProject?: number; + navigateToJob?: boolean; +} + +export default function ScheduleJobModal({ isOpen, onClose, onSuccess, onJobCreated, preselectedProject, navigateToJob = true }: ScheduleJobModalProps) { + const router = useRouter(); + const [projects, setProjects] = useState([]); + const [servers, setServers] = useState([]); + const [loading, setLoading] = useState(false); + const [submitting, setSubmitting] = useState(false); + + const [formData, setFormData] = useState({ + pid: preselectedProject || 0, + job_type: 'compare' as 'compare' | 'check' | 'discover', + priority: 5, + batch_nbr: 0, + table_filter: '', + target_server_id: '' as string, + schedule_now: true, + scheduled_at: '', + fix: false, + }); + + useEffect(() => { + if (isOpen) { + loadData(); + } + }, [isOpen]); + + useEffect(() => { + if (preselectedProject) { + setFormData(prev => ({ ...prev, pid: preselectedProject })); + } + }, [preselectedProject]); + + const loadData = async () => { + setLoading(true); + try { + const [projectsRes, serversRes] = await Promise.all([ + fetch('/api/projects'), + fetch('/api/servers') + ]); + + const projectsData = await projectsRes.json(); + const serversData = await serversRes.json(); + + setProjects(Array.isArray(projectsData) ? projectsData : []); + setServers(Array.isArray(serversData) ? serversData.filter((s: Server) => s.status !== 'terminated' && s.status !== 'offline') : []); + } catch (error) { + console.error('Failed to load data:', error); + toast.error('Failed to load projects and servers'); + } finally { + setLoading(false); + } + }; + + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + + if (!formData.pid) { + toast.error('Please select a project'); + return; + } + + setSubmitting(true); + try { + const jobConfig = formData.fix ? { fix: true } : null; + const response = await fetch('/api/jobs', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + pid: formData.pid, + job_type: formData.job_type, + priority: formData.priority, + batch_nbr: formData.batch_nbr, + table_filter: formData.table_filter || null, + target_server_id: formData.target_server_id || null, + scheduled_at: formData.schedule_now ? null : formData.scheduled_at || null, + created_by: 'ui', + job_config: jobConfig, + }), + }); + + if (!response.ok) { + throw new Error('Failed to schedule job'); + } + + const result = await response.json(); + const jobId = Array.isArray(result) && result.length > 0 ? result[0].job_id : result.job_id; + + toast.success('Job scheduled successfully'); + onSuccess?.(); + onJobCreated?.(jobId); + onClose(); + + if (navigateToJob && jobId) { + router.push(`/jobs/${jobId}`); + } + } catch (error) { + console.error('Failed to schedule job:', error); + toast.error('Failed to schedule job'); + } finally { + setSubmitting(false); + } + }; + + if (!isOpen) return null; + + return ( +
+
+
+
+

+ + Schedule Job +

+ +
+ +
+ {/* Project Selection */} +
+ + +
+ + {/* Job Type */} +
+ + +
+ + {/* Fix Option - only shown for Check mode */} + {formData.job_type === 'check' && ( +
+ setFormData({ ...formData, fix: e.target.checked })} + className="mt-0.5 h-4 w-4 rounded border-gray-300 dark:border-gray-600 text-amber-600 focus:ring-amber-500" + /> +
+ +

+ Generate INSERT, UPDATE, and DELETE statements to synchronize target with source (experimental) +

+
+
+ )} + + {/* Priority */} +
+ + setFormData({ ...formData, priority: parseInt(e.target.value) || 5 })} + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white focus:ring-blue-500 focus:border-blue-500" + /> +

Higher priority jobs are executed first

+
+ + {/* Batch Number */} +
+ + setFormData({ ...formData, batch_nbr: parseInt(e.target.value) || 0 })} + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white focus:ring-blue-500 focus:border-blue-500" + /> +

0 = all batches

+
+ + {/* Table Filter */} +
+ + setFormData({ ...formData, table_filter: e.target.value })} + placeholder="e.g., customers" + className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white focus:ring-blue-500 focus:border-blue-500" + /> +
+ + {/* Target Server */} +
+ + +
+ + {/* Schedule */} +
+ +
+ + +
+ {!formData.schedule_now && ( + setFormData({ ...formData, scheduled_at: e.target.value })} + className="mt-2 w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-700 text-gray-900 dark:text-white focus:ring-blue-500 focus:border-blue-500" + /> + )} +
+ + {/* Submit Button */} +
+ + +
+
+
+
+ ); +} diff --git a/ui/components/Skeleton.tsx b/ui/components/Skeleton.tsx new file mode 100644 index 0000000..7f515b2 --- /dev/null +++ b/ui/components/Skeleton.tsx @@ -0,0 +1,90 @@ +'use client'; + +interface SkeletonProps { + className?: string; +} + +export function Skeleton({ className = '' }: SkeletonProps) { + return ( +
+ ); +} + +export function TableSkeleton({ rows = 5, columns = 4 }: { rows?: number; columns?: number }) { + return ( +
+
+ {Array.from({ length: columns }).map((_, i) => ( + + ))} +
+ {Array.from({ length: rows }).map((_, rowIdx) => ( +
+ {Array.from({ length: columns }).map((_, colIdx) => ( + + ))} +
+ ))} +
+ ); +} + +export function CardSkeleton() { + return ( +
+
+ + +
+ + +
+ + +
+
+ ); +} + +export function DashboardSkeleton() { + return ( +
+
+ {Array.from({ length: 4 }).map((_, i) => ( +
+
+ +
+ + +
+
+
+ ))} +
+ + +
+ ); +} + +export function NavigationTreeSkeleton() { + return ( +
+ + {Array.from({ length: 5 }).map((_, i) => ( +
+ + {i < 2 && ( +
+ + +
+ )} +
+ ))} +
+ ); +} diff --git a/ui/components/TableView.tsx b/ui/components/TableView.tsx index 040da1a..4f277ff 100644 --- a/ui/components/TableView.tsx +++ b/ui/components/TableView.tsx @@ -6,6 +6,7 @@ import { Table } from '@/lib/types'; import TableMapPanel from './TableMapPanel'; import TableColumnPanel from './TableColumnPanel'; import TableResultsPanel from './TableResultsPanel'; +import { toast } from '@/components/Toaster'; interface TableViewProps { tableId: number; @@ -53,10 +54,10 @@ export default function TableView({ tableId }: TableViewProps) { }), }); - alert('Table settings saved successfully'); + toast.success('Table settings saved successfully'); } catch (error) { console.error('Failed to save table:', error); - alert('Failed to save table settings'); + toast.error('Failed to save table settings'); } finally { setSaving(false); } diff --git a/ui/components/Toaster.tsx b/ui/components/Toaster.tsx new file mode 100644 index 0000000..4daf46f --- /dev/null +++ b/ui/components/Toaster.tsx @@ -0,0 +1,20 @@ +'use client'; + +import { Toaster as SonnerToaster, toast } from 'sonner'; + +export function Toaster() { + return ( + + ); +} + +export { toast }; diff --git a/ui/hooks/useKeyboardShortcuts.ts b/ui/hooks/useKeyboardShortcuts.ts new file mode 100644 index 0000000..87f3890 --- /dev/null +++ b/ui/hooks/useKeyboardShortcuts.ts @@ -0,0 +1,73 @@ +'use client'; + +import { useEffect, useCallback } from 'react'; + +type KeyHandler = () => void; + +interface ShortcutConfig { + key: string; + ctrlKey?: boolean; + metaKey?: boolean; + shiftKey?: boolean; + altKey?: boolean; + handler: KeyHandler; + description?: string; +} + +export function useKeyboardShortcuts(shortcuts: ShortcutConfig[]) { + const handleKeyDown = useCallback((event: KeyboardEvent) => { + if ( + event.target instanceof HTMLInputElement || + event.target instanceof HTMLTextAreaElement || + event.target instanceof HTMLSelectElement + ) { + return; + } + + for (const shortcut of shortcuts) { + const keyMatches = event.key.toLowerCase() === shortcut.key.toLowerCase(); + const ctrlMatches = shortcut.ctrlKey ? event.ctrlKey : !event.ctrlKey; + const metaMatches = shortcut.metaKey ? event.metaKey : !event.metaKey; + const shiftMatches = shortcut.shiftKey ? event.shiftKey : !event.shiftKey; + const altMatches = shortcut.altKey ? event.altKey : !event.altKey; + + const cmdOrCtrl = shortcut.ctrlKey || shortcut.metaKey; + const cmdOrCtrlPressed = event.ctrlKey || event.metaKey; + + if (cmdOrCtrl) { + if (keyMatches && cmdOrCtrlPressed && shiftMatches && altMatches) { + event.preventDefault(); + shortcut.handler(); + return; + } + } else if (keyMatches && ctrlMatches && metaMatches && shiftMatches && altMatches) { + event.preventDefault(); + shortcut.handler(); + return; + } + } + }, [shortcuts]); + + useEffect(() => { + window.addEventListener('keydown', handleKeyDown); + return () => window.removeEventListener('keydown', handleKeyDown); + }, [handleKeyDown]); +} + +export function getShortcutLabel(config: Omit): string { + const parts: string[] = []; + + if (config.ctrlKey || config.metaKey) { + parts.push(navigator.platform.includes('Mac') ? '⌘' : 'Ctrl'); + } + if (config.shiftKey) { + parts.push('Shift'); + } + if (config.altKey) { + parts.push(navigator.platform.includes('Mac') ? '⌥' : 'Alt'); + } + + parts.push(config.key.toUpperCase()); + + return parts.join('+'); +} diff --git a/ui/lib/configProperties.ts b/ui/lib/configProperties.ts new file mode 100644 index 0000000..899acb2 --- /dev/null +++ b/ui/lib/configProperties.ts @@ -0,0 +1,85 @@ +export interface PropertyDefinition { + key: string; + label: string; + description: string; + defaultValue: string; + type: 'string' | 'number' | 'boolean' | 'select' | 'password'; + options?: string[]; + category: 'system' | 'repository' | 'source' | 'target'; +} + +export const CONFIG_PROPERTIES: PropertyDefinition[] = [ + // System Settings (sorted alphabetically by key) + { key: 'batch-commit-size', label: 'Batch Commit Size', description: 'Number of rows to commit per batch', defaultValue: '2000', type: 'number', category: 'system' }, + { key: 'batch-fetch-size', label: 'Batch Fetch Size', description: 'Number of rows to fetch per batch', defaultValue: '2000', type: 'number', category: 'system' }, + { key: 'batch-progress-report-size', label: 'Progress Report Size', description: 'Rows between progress reports', defaultValue: '1000000', type: 'number', category: 'system' }, + { key: 'column-hash-method', label: 'Hash Method', description: 'Method for hashing columns', defaultValue: 'database', type: 'select', options: ['database', 'hybrid', 'raw'], category: 'system' }, + { key: 'database-sort', label: 'Database Sort', description: 'Let database sort results', defaultValue: 'true', type: 'boolean', category: 'system' }, + { key: 'float-cast', label: 'Float Cast', description: 'Method for casting float/double to string', defaultValue: 'notation', type: 'select', options: ['notation', 'standard'], category: 'system' }, + { key: 'float-scale', label: 'Float Scale', description: 'Decimal places for float comparison', defaultValue: '3', type: 'number', category: 'system' }, + { key: 'job-logging-enabled', label: 'Job Logging', description: 'Log job output to dc_job_log table', defaultValue: 'false', type: 'boolean', category: 'system' }, + { key: 'loader-threads', label: 'Loader Threads', description: 'Number of loader threads (0=disabled)', defaultValue: '0', type: 'number', category: 'system' }, + { key: 'log-destination', label: 'Log Destination', description: 'Where to send logs', defaultValue: 'stdout', type: 'string', category: 'system' }, + { key: 'log-level', label: 'Log Level', description: 'Logging level', defaultValue: 'INFO', type: 'select', options: ['DEBUG', 'INFO', 'WARNING', 'ERROR'], category: 'system' }, + { key: 'message-queue-size', label: 'Message Queue Size', description: 'Size of message queue', defaultValue: '100', type: 'number', category: 'system' }, + { key: 'number-cast', label: 'Number Cast', description: 'Number casting method', defaultValue: 'notation', type: 'select', options: ['notation', 'standard'], category: 'system' }, + { key: 'observer-throttle', label: 'Observer Throttle', description: 'Enable observer throttling', defaultValue: 'true', type: 'boolean', category: 'system' }, + { key: 'observer-throttle-size', label: 'Throttle Size', description: 'Observer throttle threshold', defaultValue: '2000000', type: 'number', category: 'system' }, + { key: 'observer-vacuum', label: 'Observer Vacuum', description: 'Vacuum after observer operations', defaultValue: 'true', type: 'boolean', category: 'system' }, + { key: 'stage-table-parallel', label: 'Stage Parallel', description: 'Parallel workers for staging', defaultValue: '0', type: 'number', category: 'system' }, + { key: 'standard-number-format', label: 'Number Format', description: 'Format for standard number casting', defaultValue: '0000000000000000000000.0000000000000000000000', type: 'string', category: 'system' }, + + // Repository Settings (sorted alphabetically by key) + { key: 'repo-dbname', label: 'Database', description: 'Repository database name', defaultValue: 'pgcompare', type: 'string', category: 'repository' }, + { key: 'repo-host', label: 'Host', description: 'Repository host address', defaultValue: 'localhost', type: 'string', category: 'repository' }, + { key: 'repo-password', label: 'Password', description: 'Repository password', defaultValue: '', type: 'password', category: 'repository' }, + { key: 'repo-port', label: 'Port', description: 'Repository port number', defaultValue: '5432', type: 'number', category: 'repository' }, + { key: 'repo-schema', label: 'Schema', description: 'Repository schema name', defaultValue: 'pgcompare', type: 'string', category: 'repository' }, + { key: 'repo-sslmode', label: 'SSL Mode', description: 'Repository SSL mode', defaultValue: 'disable', type: 'select', options: ['disable', 'prefer', 'require', 'verify-ca', 'verify-full'], category: 'repository' }, + { key: 'repo-user', label: 'User', description: 'Repository username', defaultValue: 'pgcompare', type: 'string', category: 'repository' }, + + // Source Database Settings (sorted alphabetically by key) + { key: 'source-dbname', label: 'Database', description: 'Database name', defaultValue: 'postgres', type: 'string', category: 'source' }, + { key: 'source-host', label: 'Host', description: 'Host address', defaultValue: 'localhost', type: 'string', category: 'source' }, + { key: 'source-name', label: 'Service Name', description: 'Oracle service/SID name', defaultValue: '', type: 'string', category: 'source' }, + { key: 'source-password', label: 'Password', description: 'Password', defaultValue: '', type: 'password', category: 'source' }, + { key: 'source-port', label: 'Port', description: 'Port number', defaultValue: '5432', type: 'number', category: 'source' }, + { key: 'source-schema', label: 'Schema', description: 'Schema name', defaultValue: '', type: 'string', category: 'source' }, + { key: 'source-sslmode', label: 'SSL Mode', description: 'SSL mode', defaultValue: 'disable', type: 'select', options: ['disable', 'prefer', 'require', 'verify-ca', 'verify-full'], category: 'source' }, + { key: 'source-type', label: 'Type', description: 'Database type', defaultValue: 'postgres', type: 'select', options: ['postgres', 'oracle', 'db2', 'mariadb', 'mysql', 'mssql', 'snowflake'], category: 'source' }, + { key: 'source-user', label: 'User', description: 'Username', defaultValue: 'postgres', type: 'string', category: 'source' }, + { key: 'source-warehouse', label: 'Warehouse', description: 'Snowflake warehouse', defaultValue: 'compute_wh', type: 'string', category: 'source' }, + + // Target Database Settings (sorted alphabetically by key) + { key: 'target-dbname', label: 'Database', description: 'Database name', defaultValue: 'postgres', type: 'string', category: 'target' }, + { key: 'target-host', label: 'Host', description: 'Host address', defaultValue: 'localhost', type: 'string', category: 'target' }, + { key: 'target-name', label: 'Service Name', description: 'Oracle service/SID name', defaultValue: '', type: 'string', category: 'target' }, + { key: 'target-password', label: 'Password', description: 'Password', defaultValue: '', type: 'password', category: 'target' }, + { key: 'target-port', label: 'Port', description: 'Port number', defaultValue: '5432', type: 'number', category: 'target' }, + { key: 'target-schema', label: 'Schema', description: 'Schema name', defaultValue: '', type: 'string', category: 'target' }, + { key: 'target-sslmode', label: 'SSL Mode', description: 'SSL mode', defaultValue: 'disable', type: 'select', options: ['disable', 'prefer', 'require', 'verify-ca', 'verify-full'], category: 'target' }, + { key: 'target-type', label: 'Type', description: 'Database type', defaultValue: 'postgres', type: 'select', options: ['postgres', 'oracle', 'db2', 'mariadb', 'mysql', 'mssql', 'snowflake'], category: 'target' }, + { key: 'target-user', label: 'User', description: 'Username', defaultValue: 'postgres', type: 'string', category: 'target' }, + { key: 'target-warehouse', label: 'Warehouse', description: 'Snowflake warehouse', defaultValue: 'compute_wh', type: 'string', category: 'target' }, +]; + +export const CATEGORY_LABELS: Record = { + system: 'System Settings', + repository: 'Repository Database', + source: 'Source Database', + target: 'Target Database', +}; + +export function getPropertyDefinition(key: string): PropertyDefinition | undefined { + return CONFIG_PROPERTIES.find(p => p.key === key); +} + +export function isDefaultValue(key: string, value: string): boolean { + const prop = getPropertyDefinition(key); + return prop ? prop.defaultValue === value : false; +} + +export function getFriendlyLabel(key: string): string { + const prop = getPropertyDefinition(key); + return prop?.label || key; +} diff --git a/ui/lib/db.ts b/ui/lib/db.ts index 02fc2e8..d61cb22 100644 --- a/ui/lib/db.ts +++ b/ui/lib/db.ts @@ -64,6 +64,21 @@ export function getPrisma(): PrismaClient { return initializePrisma(creds); } + // Fallback to DATABASE_URL environment variable + if (process.env.DATABASE_URL && process.env.DATABASE_URL !== 'postgresql://dummy:dummy@localhost:5432/dummy') { + console.log('Initializing Prisma from DATABASE_URL'); + const url = new URL(process.env.DATABASE_URL); + const schema = url.searchParams.get('schema') || 'pgcompare'; + currentSchema = schema; + global.dbSchema = schema; + + prisma = new PrismaClient({ + log: process.env.NODE_ENV === 'development' ? ['error', 'warn'] : ['error'], + }); + global.prisma = prisma; + return prisma; + } + throw new Error('Database not initialized. Please login again.'); } diff --git a/ui/lib/types.ts b/ui/lib/types.ts index d15ed25..51e7f28 100644 --- a/ui/lib/types.ts +++ b/ui/lib/types.ts @@ -76,3 +76,79 @@ export interface Result { compare_end?: Date; } +export interface Server { + server_id: string; + server_name: string; + server_host: string; + server_pid: number; + status: 'active' | 'idle' | 'busy' | 'offline' | 'terminated'; + registered_at: Date; + last_heartbeat: Date; + current_job_id?: string; + server_config?: Record; + seconds_since_heartbeat?: number; +} + +export interface Job { + job_id: string; + pid: number; + project_name?: string; + job_type: 'compare' | 'check' | 'discover'; + status: 'pending' | 'scheduled' | 'running' | 'paused' | 'completed' | 'failed' | 'cancelled'; + priority: number; + batch_nbr: number; + table_filter?: string; + target_server_id?: string; + assigned_server_id?: string; + assigned_server_name?: string; + created_at: Date; + scheduled_at?: Date; + started_at?: Date; + completed_at?: Date; + created_by?: string; + job_config?: Record; + result_summary?: { + totalTables?: number; + completedTables?: number; + failedTables?: number; + totalSource?: number; + totalEqual?: number; + totalNotEqual?: number; + totalMissing?: number; + }; + error_message?: string; + duration_seconds?: number; + source?: 'server' | 'standalone' | 'api'; +} + +export interface JobProgress { + job_id: string; + tid: number; + table_name: string; + status: 'pending' | 'running' | 'completed' | 'failed' | 'skipped'; + started_at?: Date; + completed_at?: Date; + source_cnt?: number; + target_cnt?: number; + equal_cnt?: number; + not_equal_cnt?: number; + missing_source_cnt?: number; + missing_target_cnt?: number; + error_message?: string; + duration_seconds?: number; +} + +export interface JobProgressSummary { + total_tables: number; + completed_tables: number; + running_tables: number; + pending_tables: number; + failed_tables: number; + total_source: number; + total_target: number; + total_equal: number; + total_not_equal: number; + total_missing_source: number; + total_missing_target: number; +} + diff --git a/ui/next.config.ts b/ui/next.config.ts index e9ffa30..68a6c64 100644 --- a/ui/next.config.ts +++ b/ui/next.config.ts @@ -1,7 +1,7 @@ import type { NextConfig } from "next"; const nextConfig: NextConfig = { - /* config options here */ + output: "standalone", }; export default nextConfig; diff --git a/ui/package-lock.json b/ui/package-lock.json index 90eeea4..4b8b339 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -14,7 +14,8 @@ "next": "16.0.1", "react": "19.2.0", "react-dom": "19.2.0", - "recharts": "^3.3.0" + "recharts": "^3.3.0", + "sonner": "^1.7.0" }, "devDependencies": { "@tailwindcss/postcss": "^4", @@ -72,6 +73,7 @@ "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", @@ -1740,6 +1742,7 @@ "integrity": "sha512-6mDvHUFSjyT2B2yeNx2nUgMxh9LtOWvkhIU3uePn2I2oyNymUAX1NIsdgviM4CH+JSrp2D2hsMvJOkxY+0wNRA==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.0.2" } @@ -1806,6 +1809,7 @@ "integrity": "sha512-BnOroVl1SgrPLywqxyqdJ4l3S2MsKVLDVxZvjI1Eoe8ev2r3kGDo+PcMihNmDE+6/KjkTubSJnmqGZZjQSBq/g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.46.2", "@typescript-eslint/types": "8.46.2", @@ -2336,6 +2340,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2677,6 +2682,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.19", "caniuse-lite": "^1.0.30001751", @@ -3530,6 +3536,7 @@ "integrity": "sha512-t5aPOpmtJcZcz5UJyY2GbvpDlsK5E8JqRqoKtfiKE3cNh437KIqfJr3A3AKf5k64NPx6d0G3dno6XDY05PqPtw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -3715,6 +3722,7 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -5994,6 +6002,7 @@ "devOptional": true, "hasInstallScript": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "@prisma/config": "6.18.0", "@prisma/engines": "6.18.0" @@ -6089,6 +6098,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.0.tgz", "integrity": "sha512-tmbWg6W31tQLeB5cdIBOicJDJRR2KzXsV7uSK9iNfLWQ5bIZfxuPEHp7M8wiHyHnn0DD1i7w3Zmin0FtkrwoCQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -6098,6 +6108,7 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.0.tgz", "integrity": "sha512-UlbRu4cAiGaIewkPyiRGJk0imDN2T3JjieT6spoL2UeSf5od4n5LB/mQ4ejmxhCFT1tYe8IvaFulzynWovsEFQ==", "license": "MIT", + "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -6109,13 +6120,15 @@ "version": "16.13.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/react-redux": { "version": "9.2.0", "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.2.0.tgz", "integrity": "sha512-ROY9fvHhwOD9ySfrF0wmvu//bKCQ6AeZZq1nJNtbDC+kk5DuSuNX/n6YWYF/SYy7bSba4D4FSz8DJeKY/S/r+g==", "license": "MIT", + "peer": true, "dependencies": { "@types/use-sync-external-store": "^0.0.6", "use-sync-external-store": "^1.4.0" @@ -6179,7 +6192,8 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/redux-thunk": { "version": "3.1.0", @@ -6591,6 +6605,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/sonner": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/sonner/-/sonner-1.7.4.tgz", + "integrity": "sha512-DIS8z4PfJRbIyfVFDVnK9rO3eYDtse4Omcm6bt0oEr5/jtLgysmjuBl1frJ9E/EQZrFmKx2A8m/s5s9CRXIzhw==", + "license": "MIT", + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0 || ^19.0.0-rc", + "react-dom": "^18.0.0 || ^19.0.0 || ^19.0.0-rc" + } + }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -6881,6 +6905,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -7043,6 +7068,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -7349,6 +7375,7 @@ "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==", "dev": true, "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/ui/package.json b/ui/package.json index 3ef93d8..5636c7b 100644 --- a/ui/package.json +++ b/ui/package.json @@ -15,7 +15,8 @@ "next": "16.0.1", "react": "19.2.0", "react-dom": "19.2.0", - "recharts": "^3.3.0" + "recharts": "^3.3.0", + "sonner": "^1.7.0" }, "devDependencies": { "@tailwindcss/postcss": "^4", diff --git a/ui/prisma/schema.prisma b/ui/prisma/schema.prisma index 47fc6f3..01e7ed2 100644 --- a/ui/prisma/schema.prisma +++ b/ui/prisma/schema.prisma @@ -1,5 +1,6 @@ generator client { provider = "prisma-client-js" + binaryTargets = ["native", "linux-arm64-openssl-1.1.x", "linux-arm64-openssl-3.0.x", "linux-musl-arm64-openssl-3.0.x", "debian-openssl-1.1.x", "debian-openssl-3.0.x"] } datasource db { @@ -13,6 +14,7 @@ model dc_project { project_config Json? dc_table dc_table[] dc_result dc_result[] + dc_job dc_job[] } model dc_result { @@ -115,3 +117,84 @@ model dc_target { @@ignore @@map("dc_target") } + +model dc_table_history { + tid BigInt + batch_nbr Int + start_dt DateTime + end_dt DateTime? + action_result Json? + row_count BigInt? + + @@ignore + @@map("dc_table_history") +} + +model dc_server { + server_id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + server_name String + server_host String + server_pid BigInt + status String @default("active") @db.VarChar(20) + registered_at DateTime @default(now()) + last_heartbeat DateTime @default(now()) + current_job_id String? @db.Uuid + server_config Json? + + @@index([status, last_heartbeat], name: "dc_server_idx1") +} + +model dc_job { + job_id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + pid BigInt + job_type String @default("compare") @db.VarChar(20) + status String @default("pending") @db.VarChar(20) + priority Int @default(5) + batch_nbr Int @default(0) + table_filter String? + target_server_id String? @db.Uuid + assigned_server_id String? @db.Uuid + created_at DateTime @default(now()) + scheduled_at DateTime? + started_at DateTime? + completed_at DateTime? + created_by String? + job_config Json? + result_summary Json? + error_message String? + dc_project dc_project @relation(fields: [pid], references: [pid], onDelete: Cascade) + dc_job_control dc_job_control[] + dc_job_progress dc_job_progress[] + + @@index([status, priority(sort: Desc), created_at], name: "dc_job_idx1") + @@index([pid, status], name: "dc_job_idx2") +} + +model dc_job_control { + control_id Int @id @default(autoincrement()) + job_id String @db.Uuid + signal String @db.VarChar(20) + requested_at DateTime @default(now()) + processed_at DateTime? + requested_by String? + dc_job dc_job @relation(fields: [job_id], references: [job_id], onDelete: Cascade) +} + +model dc_job_progress { + job_id String @db.Uuid + tid BigInt + table_name String + status String @default("pending") @db.VarChar(20) + started_at DateTime? + completed_at DateTime? + source_cnt BigInt? @default(0) + target_cnt BigInt? @default(0) + equal_cnt BigInt? @default(0) + not_equal_cnt BigInt? @default(0) + missing_source_cnt BigInt? @default(0) + missing_target_cnt BigInt? @default(0) + error_message String? + dc_job dc_job @relation(fields: [job_id], references: [job_id], onDelete: Cascade) + + @@id([job_id, tid]) +}