diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7a47e49b..b1f84446 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,16 @@ -name: Go Build & Test +name: Build and Test + +permissions: + contents: write + packages: write + pages: write + on: + push: + branches: + - master pull_request: + paths-ignore: ['docs/**', 'mkdocs.yml'] workflow_dispatch: concurrency: @@ -92,8 +102,6 @@ jobs: - name: Test run: go test -v -p 1 -parallel 1 -failfast ./... - - test-postgresql-ubuntu: if: true # false to skip job during debug name: Test and Build on Ubuntu @@ -145,3 +153,48 @@ jobs: with: version: latest args: release --snapshot --skip=publish --clean + + build-docs: + if: true # false to skip job during debug + needs: [test-postgresql-ubuntu, test-postgresql-windows, test-postgresql-macos] + name: Build Docs + runs-on: ubuntu-latest + steps: + + - name: Check out code + uses: actions/checkout@v5 + + - name: Configure Git + run: | + git config user.name "${GITHUB_ACTOR}" + git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" + + - name: Set up Golang + uses: actions/setup-go@v6 + with: + go-version: '1.25' + + - name: Set up gopages + run: go install github.com/johnstarich/go/gopages@v0.1.29 + + - name: Build Developer Docs + run: gopages -out "docs/godoc" -base "/pg_timetable/devel/godoc" -internal + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.13 + cache: 'pip' + cache-dependency-path: '**/requirements-doc.txt' + + - name: Install dependencies + run: pip install -r docs/requirements-doc.txt + + - name: Check if we should push to gh-pages + # if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + run: | + git fetch origin gh-pages --depth=1 + echo "push_opt=--push" >> $GITHUB_ENV + + - name: Build mkdocs + run: mike deploy ${{ env.push_opt }} devel \ No newline at end of file diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..ef83de7e --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,57 @@ +name: Deploy MkDocs Documentation + +on: + push: + branches: [ master ] + paths: + - 'docs/**' + - 'mkdocs.yml' + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + pip install mkdocs-material + pip install mkdocstrings[python] + + - name: Setup Pages + uses: actions/configure-pages@v3 + + - name: Build with MkDocs + run: mkdocs build --config-file mkdocs.yml + + - name: Upload artifact + uses: actions/upload-pages-artifact@v2 + with: + path: ./site + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 \ No newline at end of file diff --git a/README.md b/README.md index 4acf7de3..ea342e4c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ +[![Documentation](https://img.shields.io/badge/Documentation-pg__timetable-brightgreen)](https://cybertec-postgresql.github.io/pg_timetable/) [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT) -![Build & Test](https://github.com/cybertec-postgresql/pg_timetable/workflows/Go%20Build%20&%20Test/badge.svg) +![Build & Test](https://github.com/cybertec-postgresql/pg_timetable/workflows/Build%20and%20Test/badge.svg) [![Coverage Status](https://coveralls.io/repos/github/cybertec-postgresql/pg_timetable/badge.svg?branch=master&service=github)](https://coveralls.io/github/cybertec-postgresql/pg_timetable?branch=master) -[![Documentation Status](https://readthedocs.org/projects/pg-timetable/badge/?version=master)](https://pg-timetable.readthedocs.io/en/master/?badge=master) [![Release](https://img.shields.io/github/v/release/cybertec-postgresql/pg_timetable?include_prereleases)](https://github.com/cybertec-postgresql/pg_timetable/releases) [![Github All Releases](https://img.shields.io/github/downloads/cybertec-postgresql/pg_timetable/total?style=flat-square)](https://github.com/cybertec-postgresql/pg_timetable/releases) [![Docker Pulls](https://img.shields.io/docker/pulls/cybertecpostgresql/pg_timetable)](https://hub.docker.com/r/cybertecpostgresql/pg_timetable) diff --git a/docs/README.rst b/docs/README.rst deleted file mode 100644 index a3195b93..00000000 --- a/docs/README.rst +++ /dev/null @@ -1,120 +0,0 @@ -Introduction -================================================ - -**pg_timetable** is an advanced job scheduler for PostgreSQL, offering many advantages over traditional schedulers such as **cron** and others. -It is completely database driven and provides a couple of advanced concepts. - -Main features --------------- - -- Tasks can be arranged in chains -- A chain can consist of built-int commands, SQL and executables -- Parameters can be passed to chains -- Missed tasks (possibly due to downtime) can be retried automatically -- Support for configurable repetitions -- Built-in tasks such as sending emails, etc. -- Fully database driven configuration -- Full support for database driven logging -- Cron-style scheduling at the PostgreSQL server time zone -- Optional concurrency protection -- Task and chain can have execution timeout settings - -Quick Start ------------- - -1. Download pg_timetable executable -2. Make sure your PostgreSQL server is up and running and has a role with ``CREATE`` privilege - for a target database, e.g. - - .. code-block:: SQL - - my_database=> CREATE ROLE scheduler PASSWORD 'somestrong'; - my_database=> GRANT CREATE ON DATABASE my_database TO scheduler; - -3. Create a new job, e.g. run ``VACUUM`` each night at 00:30 Postgres server time zone - - .. code-block:: SQL - - my_database=> SELECT timetable.add_job('frequent-vacuum', '30 * * * *', 'VACUUM'); - add_job - --------- - 3 - (1 row) - -4. Run the **pg_timetable** - - .. code-block:: - - # pg_timetable postgresql://scheduler:somestrong@localhost/my_database --clientname=vacuumer - -5. PROFIT! - -Command line options ------------------------- -.. code-block:: - - # ./pg_timetable - - Application Options: - -c, --clientname= Unique name for application instance [$PGTT_CLIENTNAME] - --config= YAML configuration file - --no-program-tasks Disable executing of PROGRAM tasks [$PGTT_NOPROGRAMTASKS] - -v, --version Output detailed version information [$PGTT_VERSION] - - Connection: - -h, --host= PostgreSQL host (default: localhost) [$PGTT_PGHOST] - -p, --port= PostgreSQL port (default: 5432) [$PGTT_PGPORT] - -d, --dbname= PostgreSQL database name (default: timetable) [$PGTT_PGDATABASE] - -u, --user= PostgreSQL user (default: scheduler) [$PGTT_PGUSER] - --password= PostgreSQL user password [$PGTT_PGPASSWORD] - --sslmode=[disable|require] Connection SSL mode (default: disable) [$PGTT_PGSSLMODE] - --pgurl= PostgreSQL connection URL [$PGTT_URL] - --timeout= PostgreSQL connection timeout (default: 90) [$PGTT_TIMEOUT] - - Logging: - --log-level=[debug|info|error] Verbosity level for stdout and log file (default: info) - --log-database-level=[debug|info|error|none] Verbosity level for database storing (default: info) - --log-file= File name to store logs - --log-file-format=[json|text] Format of file logs (default: json) - --log-file-rotate Rotate log files - --log-file-size= Maximum size in MB of the log file before it gets rotated (default: 100) - --log-file-age= Number of days to retain old log files, 0 means forever (default: 0) - --log-file-number= Maximum number of old log files to retain, 0 to retain all (default: 0) - - Start: - -f, --file= SQL script file to execute during startup - --init Initialize database schema to the latest version and exit. Can be used - with --upgrade - --upgrade Upgrade database to the latest version - --debug Run in debug mode. Only asynchronous chains will be executed - - Resource: - --cron-workers= Number of parallel workers for scheduled chains (default: 16) - --interval-workers= Number of parallel workers for interval chains (default: 16) - --chain-timeout= Abort any chain that takes more than the specified number of - milliseconds - --task-timeout= Abort any task within a chain that takes more than the specified number - of milliseconds - - REST: - --rest-port= REST API port (default: 0) [$PGTT_RESTPORT] - - -Contributing ------------- -If you want to contribute to **pg_timetable** and help make it better, feel free to open an -`issue `_ or even consider submitting a -`pull request `_. You also can give a -`star `_ to **pg_timetable** project, -and to tell the world about it. - -Support ------------- -For professional support, please contact `Cybertec `_. - - -Authors ---------- -Implementation: `Pavlo Golub `_ - -Initial idea and draft design: `Hans-Jürgen Schönig `_ diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 00000000..e1a55f73 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,26 @@ +# REST API + +**pg_timetable** has a rich REST API, which can be used by external tools in order to perform start/stop/reinitialize/restarts/reloads, +by any kind of tools to perform HTTP health checks, and of course, could also be used for monitoring. + +Below you will find the list of **pg_timetable** REST API endpoints. + +## Health check endpoints + +### `GET /liveness` +Always returns HTTP status code `200`, indicating that **pg_timetable** is running. + +### `GET /readiness` +Returns HTTP status code `200` when the **pg_timetable** is running, and the scheduler is in the main loop processing chains. +If the scheduler connects to the database, creates the database schema, or upgrades it, it will return the HTTP status code `503`. + +## Chain management endpoints + +### `GET /startchain?id=` +Returns HTTP status code `200` if the chain with the given id can be added to the worker queue. It doesn't, however, mean the chain execution starts immediately. It is up to the worker to perform load and other checks before starting the chain. +In the case of an error, the HTTP status code `400` followed by an error message returned. + +### `GET /stopchain?id=` +Returns HTTP status code `200` if the chain with the given id is working at the moment and can be stopped. If the chain is running the +cancel signal would be sent immediately. +In the case of an error, the HTTP status code `400` followed by an error message returned. \ No newline at end of file diff --git a/docs/api.rst b/docs/api.rst deleted file mode 100644 index 0b3faae8..00000000 --- a/docs/api.rst +++ /dev/null @@ -1,29 +0,0 @@ -REST API -================================================ - -**pg_timetable** has a rich REST API, which can be used by external tools in order to perform start/stop/reinitialize/restarts/reloads, -by any kind of tools to perform HTTP health checks, and of course, could also be used for monitoring. - -Below you will find the list of **pg_timetable** REST API endpoints. - -Health check endpoints ------------------------------------------------- - -``GET /liveness`` - Always returns HTTP status code ``200``, indicating that **pg_timetable** is running. - -``GET /readiness`` - Returns HTTP status code ``200`` when the **pg_timetable** is running, and the scheduler is in the main loop processing chains. - If the scheduler connects to the database, creates the database schema, or upgrades it, it will return the HTTP status code ``503``. - -Chain management endpoints ------------------------------------------------- - -``GET /startchain?id=`` - Returns HTTP status code ``200`` if the chain with the given id can be added to the worker queue. It doesn't, however, mean the chain execution starts immediately. It is up to the worker to perform load and other checks before starting the chain. - In the case of an error, the HTTP status code ``400`` followed by an error message returned. - -``GET /stopchain?id=`` - Returns HTTP status code ``200`` if the chain with the given id is working at the moment and can be stopped. If the chain is running the - cancel signal would be sent immediately. - In the case of an error, the HTTP status code ``400`` followed by an error message returned. \ No newline at end of file diff --git a/docs/background.md b/docs/background.md new file mode 100644 index 00000000..37156f47 --- /dev/null +++ b/docs/background.md @@ -0,0 +1,28 @@ +# Project background + +The pg_timetable project got started back in 2019 for internal scheduling needs at Cybertec. + +For more background on the project motivations and design goals see the original series of blogposts announcing the project +and the following feature updates. + +Cybertec also provides commercial 9-to-5 and 24/7 support for pg_timetable. + +* [Project announcement](https://www.cybertec-postgresql.com/en/pg_timetable-advanced-postgresql-job-scheduling/) + +* [v2 released](https://www.cybertec-postgresql.com/en/pg_timetable-advanced-postgresql-cron-like-scheduler-released/) + +* [Start-up improvements](https://www.cybertec-postgresql.com/en/pg_timetable-start-up-improvements/) + +* [v3 released](https://www.cybertec-postgresql.com/en/pg_timetable-v3-is-out/) + +* [Exclusive jobs explained](https://www.cybertec-postgresql.com/en/postgresql-exclusive-cron-jobs-using-pg_timetable-scheduler/) + +* [Asynchronous chain execution](https://www.cybertec-postgresql.com/en/pg_timetable-asynchronous-chain-execution/) + +* [v4 released](https://www.cybertec-postgresql.com/en/pg_timetable_v4-is-out/) + +* [PostgreSQL schedulers: comparison table](https://www.cybertec-postgresql.com/en/postgresql-schedulers-comparison-table/) + +## Project feedback + +For feature requests or troubleshooting assistance please open an issue on project's [Github page](https://github.com/cybertec-postgresql/pg_timetable). \ No newline at end of file diff --git a/docs/background.rst b/docs/background.rst deleted file mode 100644 index cf4a1cfa..00000000 --- a/docs/background.rst +++ /dev/null @@ -1,30 +0,0 @@ -Project background -================== - -The pg_timetable project got started back in 2019 for internal scheduling needs at Cybertec. - -For more background on the project motivations and design goals see the original series of blogposts announcing the project -and the following feature updates. - -Cybertec also provides commercial 9-to-5 and 24/7 support for pg_timetable. - -* `Project announcement `_ - -* `v2 released `_ - -* `Start-up improvements `_ - -* `v3 released `_ - -* `Exclusive jobs explained `_ - -* `Asynchronous chain execution `_ - -* `v4 released `_ - -* `PostgreSQL schedulers: comparison table `_ - -Project feedback ----------------- - -For feature requests or troubleshooting assistance please open an issue on project's `Github page `_. \ No newline at end of file diff --git a/docs/basic_jobs.md b/docs/basic_jobs.md new file mode 100644 index 00000000..1264fce9 --- /dev/null +++ b/docs/basic_jobs.md @@ -0,0 +1,86 @@ +# Getting started + +A variety of examples can be found in the [samples](samples.md). If you want to migrate from a different scheduler, you can use scripts from [migration](migration.md) chapter. + +## Add simple job + +In a real world usually it's enough to use simple jobs. Under this term we understand: + +* job is a chain with only one **task** (step) in it; +* it doesn't use complicated logic, but rather simple **command**; +* it doesn't require complex transaction handling, since one task is implicitely executed as a single transaction. + +For such a group of chains we've introduced a special function `timetable.add_job()`. + +### Function: `timetable.add_job()` + +Creates a simple one-task chain + +**Returns:** `BIGINT` + +#### Parameters + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `job_name` | `text` | The unique name of the **chain** and **command** | Required | +| `job_schedule` | `timetable.cron` | Time schedule in сron syntax at Postgres server time zone | Required | +| `job_command` | `text` | The SQL which will be executed | Required | +| `job_parameters` | `jsonb` | Arguments for the chain **command** | `NULL` | +| `job_kind` | `timetable.command_kind` | Kind of the command: *SQL*, *PROGRAM* or *BUILTIN* | `SQL` | +| `job_client_name` | `text` | Specifies which client should execute the chain. Set this to `NULL` to allow any client | `NULL` | +| `job_max_instances` | `integer` | The amount of instances that this chain may have running at the same time | `NULL` | +| `job_live` | `boolean` | Control if the chain may be executed once it reaches its schedule | `TRUE` | +| `job_self_destruct` | `boolean` | Self destruct the chain after execution | `FALSE` | +| `job_ignore_errors` | `boolean` | Ignore error during execution | `TRUE` | +| `job_exclusive` | `boolean` | Execute the chain in the exclusive mode | `FALSE` | + +**Returns:** the ID of the created chain + +## Examples + +1. Run `public.my_func()` at 00:05 every day in August Postgres server time zone: + + ```sql + SELECT timetable.add_job('execute-func', '5 0 * 8 *', 'SELECT public.my_func()'); + ``` + +2. Run `VACUUM` at minute 23 past every 2nd hour from 0 through 20 every day Postgres server time zone: + + ```sql + SELECT timetable.add_job('run-vacuum', '23 0-20/2 * * *', 'VACUUM'); + ``` + +3. Refresh materialized view every 2 hours: + + ```sql + SELECT timetable.add_job('refresh-matview', '@every 2 hours', 'REFRESH MATERIALIZED VIEW public.mat_view'); + ``` + +4. Clear log table after **pg_timetable** restart: + + ```sql + SELECT timetable.add_job('clear-log', '@reboot', 'TRUNCATE timetable.log'); + ``` + +5. Reindex at midnight Postgres server time zone on Sundays with [reindexdb](https://www.postgresql.org/docs/current/app-reindexdb.html) utility: + + - using default database under default user (no command line arguments) + + ```sql + SELECT timetable.add_job('reindex', '0 0 * * 7', 'reindexdb', job_kind := 'PROGRAM'); + ``` + + - specifying target database and tables, and be verbose + + ```sql + SELECT timetable.add_job('reindex', '0 0 * * 7', 'reindexdb', + '["--table=foo", "--dbname=postgres", "--verbose"]'::jsonb, 'PROGRAM'); + ``` + + - passing password using environment variable through `bash` shell + + ```sql + SELECT timetable.add_job('reindex', '0 0 * * 7', 'bash', + '["-c", "PGPASSWORD=5m3R7K4754p4m reindexdb -U postgres -h 192.168.0.221 -v"]'::jsonb, + 'PROGRAM'); + ``` \ No newline at end of file diff --git a/docs/basic_jobs.rst b/docs/basic_jobs.rst deleted file mode 100644 index 2ed5978b..00000000 --- a/docs/basic_jobs.rst +++ /dev/null @@ -1,106 +0,0 @@ -Getting started -================================================================ - -A variety of examples can be found in the :doc:`samples`. If you want to migrate from a different scheduler, -you can use scripts from :doc:`migration` chapter. - -Add simple job -~~~~~~~~~~~~~~ - -In a real world usually it's enough to use simple jobs. Under this term we understand: - -* job is a chain with only one **task** (step) in it; -* it doesn't use complicated logic, but rather simple **command**; -* it doesn't require complex transaction handling, since one task is implicitely executed as a single transaction. - -For such a group of chains we've introduced a special function ``timetable.add_job()``. - -.. function:: timetable.add_job(job_name, job_schedule, job_command, ...) RETURNS BIGINT - - Creates a simple one-task chain - - :param job_name: The unique name of the **chain** and **command**. - :type job_name: text - - :param job_schedule: Time schedule in сron syntax at Postgres server time zone - :type job_schedule: timetable.cron - - :param job_command: The SQL which will be executed. - :type job_command: text - - :param job_parameters: Arguments for the chain **command**. Default: ``NULL``. - :type job_parameters: jsonb - - :param job_kind: Kind of the command: *SQL*, *PROGRAM* or *BUILTIN*. Default: ``SQL``. - :type job_kind: timetable.command_kind - - :param job_client_name: Specifies which client should execute the chain. Set this to `NULL` to allow any client. Default: ``NULL``. - :type job_client_name: text - - :param job_max_instances: The amount of instances that this chain may have running at the same time. Default: ``NULL``. - :type job_max_instances: integer - - :param job_live: Control if the chain may be executed once it reaches its schedule. Default: ``TRUE``. - :type job_live: boolean - - :param job_self_destruct: Self destruct the chain after execution. Default: ``FALSE``. - :type job_self_destruct: boolean - - :param job_ignore_errors: Ignore error during execution. Default: ``TRUE``. - :type job_ignore_errors: boolean - - :param job_exclusive: Execute the chain in the exclusive mode. Default: ``FALSE``. - :type job_exclusive: boolean - - :returns: the ID of the created chain - :rtype: integer - -Examples -~~~~~~~~~ - -#. Run ``public.my_func()`` at 00:05 every day in August Postgres server time zone: - - .. code-block:: SQL - - SELECT timetable.add_job('execute-func', '5 0 * 8 *', 'SELECT public.my_func()'); - -#. Run `VACUUM` at minute 23 past every 2nd hour from 0 through 20 every day Postgres server time zone: - - .. code-block:: SQL - - SELECT timetable.add_job('run-vacuum', '23 0-20/2 * * *', 'VACUUM'); - -#. Refresh materialized view every 2 hours: - - .. code-block:: SQL - - SELECT timetable.add_job('refresh-matview', '@every 2 hours', 'REFRESH MATERIALIZED VIEW public.mat_view'); - -#. Clear log table after **pg_timetable** restart: - - .. code-block:: SQL - - SELECT timetable.add_job('clear-log', '@reboot', 'TRUNCATE timetable.log'); - -#. Reindex at midnight Postgres server time zone on Sundays with `reindexdb `_ utility: - - - using default database under default user (no command line arguments) - - .. code-block:: SQL - - SELECT timetable.add_job('reindex', '0 0 * * 7', 'reindexdb', job_kind := 'PROGRAM'); - - - specifying target database and tables, and be verbose - - .. code-block:: SQL - - SELECT timetable.add_job('reindex', '0 0 * * 7', 'reindexdb', - '["--table=foo", "--dbname=postgres", "--verbose"]'::jsonb, 'PROGRAM'); - - - passing password using environment variable through ``bash`` shell - - .. code-block:: SQL - - SELECT timetable.add_job('reindex', '0 0 * * 7', 'bash', - '["-c", "PGPASSWORD=5m3R7K4754p4m reindexdb -U postgres -h 192.168.0.221 -v"]'::jsonb, - 'PROGRAM'); \ No newline at end of file diff --git a/docs/components.md b/docs/components.md new file mode 100644 index 00000000..b7e7eada --- /dev/null +++ b/docs/components.md @@ -0,0 +1,205 @@ +# Components + +The scheduling in **pg_timetable** encompasses three different abstraction levels to facilitate the reuse with other parameters or additional schedules. + +**Command:** The base level, **command**, defines *what* to do. + +**Task:** The second level, **task**, represents a chain element (step) to run one of the commands. With **tasks** we define order of commands, arguments passed (if any), and how errors are handled. + +**Chain:** The third level represents a connected tasks forming a chain of tasks. **Chain** defines *if*, *when*, and *how often* a job should be executed. + +## Command + +Currently, there are three different kinds of commands: + +### `SQL` +SQL snippet. Starting a cleanup, refreshing a materialized view or processing data. + +### `PROGRAM` +External Command. Anything that can be called as an external binary, including shells, e.g. `bash`, `pwsh`, etc. The external command will be called using golang's [exec.CommandContext](https://pkg.go.dev/os/exec#CommandContext). + +### `BUILTIN` +Internal Command. A prebuilt functionality included in **pg_timetable**. These include: + +* *NoOp* +* *Sleep* +* *Log* +* *SendMail* +* *Download* +* *CopyFromFile* +* *CopyToFile* +* *Shutdown* + +## Task + +The next building block is a **task**, which simply represents a step in a list of chain commands. An example of tasks combined in a chain would be: + +1. Download files from a server +2. Import files +3. Run aggregations +4. Build report +5. Remove the files from disk + +!!! note + + All tasks of the chain in **pg_timetable** are executed within one transaction. However, please, pay attention there is no opportunity to rollback `PROGRAM` and `BUILTIN` tasks. + +### Table timetable.task + +| Field | Type | Description | +|-------|------|-------------| +| `chain_id` | `bigint` | Link to the chain, if `NULL` task considered to be disabled | +| `task_order` | `DOUBLE PRECISION` | Indicates the order of task within a chain | +| `kind` | `timetable.command_kind` | The type of the command. Can be *SQL* (default), *PROGRAM* or *BUILTIN* | +| `command` | `text` | Contains either a SQL command, a path to application or name of the *BUILTIN* command which will be executed | +| `run_as` | `text` | The role as which the task should be executed as | +| `database_connection` | `text` | The connection string for the external database that should be used | +| `ignore_error` | `boolean` | Specify if the next task should proceed after encountering an error (default: `false`) | +| `autonomous` | `boolean` | Specify if the task should be executed out of the chain transaction. Useful for `VACUUM`, `CREATE DATABASE`, `CALL` etc. | +| `timeout` | `integer` | Abort any task within a chain that takes more than the specified number of milliseconds | + +!!! warning + + If the **task** has been configured with `ignore_error` set to `true` (the default value is `false`), the worker process will report a success on execution *even if the task within the chain fails*. + +As mentioned above, **commands** are simple skeletons (e.g. *send email*, *vacuum*, etc.). +In most cases, they have to be brought to live by passing input parameters to the execution. + +### Table timetable.parameter + +| Field | Type | Description | +|-------|------|-------------| +| `task_id` | `bigint` | The ID of the task | +| `order_id` | `integer` | The order of the parameter. Several parameters are processed one by one according to the order | +| `value` | `jsonb` | A JSON value containing the parameters | + +### Parameter value format + +Depending on the **command** kind argument can be represented by different *JSON* values. + +#### `SQL` +Schema: `array` + +Example: +```sql +'[ "one", 2, 3.14, false ]'::jsonb +``` + +#### `PROGRAM` +Schema: `array of strings` + +Example: +```sql +'["-x", "Latin-ASCII", "-o", "orte_ansi.txt", "orte.txt"]'::jsonb +``` + +#### `BUILTIN: Sleep` +Schema: `integer` + +Example: +```sql +'5' :: jsonb +``` + +#### `BUILTIN: Log` +Schema: `any` + +Examples: +```sql +'"WARNING"'::jsonb +'{"Status": "WARNING"}'::jsonb +``` + +#### `BUILTIN: SendMail` +Schema: `object` + +Example: +```sql +'{ + "username": "user@example.com", + "password": "password", + "serverhost": "smtp.example.com", + "serverport": 587, + "senderaddr": "user@example.com", + "ccaddr": ["recipient_cc@example.com"], + "bccaddr": ["recipient_bcc@example.com"], + "toaddr": ["recipient@example.com"], + "subject": "pg_timetable - No Reply", + "attachment": ["/temp/attachments/Report.pdf","config.yaml"], + "attachmentdata": [{"name": "File.txt", "base64data": "RmlsZSBDb250ZW50"}], + "msgbody": "

Hello User,

check some attachments!

", + "contenttype": "text/html; charset=UTF-8" +}'::jsonb +``` + +#### `BUILTIN: Download` +Schema: `object` + +Example: +```sql +'{ + "workersnum": 2, + "fileurls": ["http://example.com/foo.gz", "https://example.com/bar.csv"], + "destpath": "." +}'::jsonb +``` + +#### `BUILTIN: CopyFromFile` +Schema: `object` + +Example: +```sql +'{ + "sql": "COPY location FROM STDIN", + "filename": "download/orte_ansi.txt" +}'::jsonb +``` + +#### `BUILTIN: CopyToFile` +Schema: `object` + +Example: +```sql +'{ + "sql": "COPY location TO STDOUT", + "filename": "download/location.txt" +}'::jsonb +``` + +#### `BUILTIN: Shutdown` +*value ignored* + +#### `BUILTIN: NoOp` +*value ignored* + +## Chain + +Once tasks have been arranged, they have to be scheduled as a **chain**. For this, **pg_timetable** builds upon the enhanced **cron**-string, all the while adding multiple configuration options. + +### Table timetable.chain + +| Field | Type | Description | +|-------|------|-------------| +| `chain_name` | `text` | The unique name of the chain | +| `run_at` | `timetable.cron` | Standard *cron*-style value at Postgres server time zone or `@after`, `@every`, `@reboot` clause | +| `max_instances` | `integer` | The amount of instances that this chain may have running at the same time | +| `timeout` | `integer` | Abort any chain that takes more than the specified number of milliseconds | +| `live` | `boolean` | Control if the chain may be executed once it reaches its schedule | +| `self_destruct` | `boolean` | Self destruct the chain after successful execution. Failed chains will be executed according to the schedule one more time | +| `exclusive_execution` | `boolean` | Specifies whether the chain should be executed exclusively while all other chains are paused | +| `client_name` | `text` | Specifies which client should execute the chain. Set this to `NULL` to allow any client | +| `timeout` | `integer` | Abort a chain that takes more than the specified number of milliseconds | +| `on_error` | — | Holds SQL to execute if an error occurs. If task produced an error is marked with `ignore_error` then nothing is done | + +!!! note + + All chains in **pg_timetable** are scheduled at the PostgreSQL server time zone. + You can change the [timezone](https://www.postgresql.org/docs/current/datatype-datetime.html#DATATYPE-TIMEZONES) + for the **current session** when adding new chains, e.g. + + ```sql + SET TIME ZONE 'UTC'; + + -- Run VACUUM at 00:05 every day in August UTC + SELECT timetable.add_job('execute-func', '5 0 * 8 *', 'VACUUM'); + ``` \ No newline at end of file diff --git a/docs/components.rst b/docs/components.rst deleted file mode 100644 index c4677688..00000000 --- a/docs/components.rst +++ /dev/null @@ -1,215 +0,0 @@ -Components -================================================ - -The scheduling in **pg_timetable** encompasses three different abstraction levels to facilitate the reuse with other parameters or additional schedules. - -:Command: The base level, **command**, defines *what* to do. -:Task: The second level, **task**, represents a chain element (step) to run one of the commands. With **tasks** we define order of commands, arguments passed (if any), and how errors are handled. -:Chain: The third level represents a connected tasks forming a chain of tasks. **Chain** defines *if*, *when*, and *how often* a job should be executed. - -Command ------------------------------------------------- - -Currently, there are three different kinds of commands: - -``SQL`` - SQL snippet. Starting a cleanup, refreshing a materialized view or processing data. - -``PROGRAM`` - External Command. Anything that can be called as an external binary, including shells, e.g. ``bash``, ``pwsh``, etc. The external command will be called using golang's `exec.CommandContext `_. - -``BUILTIN`` - Internal Command. A prebuilt functionality included in **pg_timetable**. These include: - - * *NoOp*, - * *Sleep*, - * *Log*, - * *SendMail*, - * *Download*, - * *CopyFromFile*, - * *CopyToFile*, - * *Shutdown*. - -Task ------------------------------------------------- - -The next building block is a **task**, which simply represents a step in a list of chain commands. An example of tasks combined in a chain would be: - -#. Download files from a server -#. Import files -#. Run aggregations -#. Build report -#. Remove the files from disk - -.. note:: - - All tasks of the chain in **pg_timetable** are executed within one transaction. However, please, pay attention there is no opportunity to rollback ``PROGRAM`` and ``BUILTIN`` tasks. - -Table timetable.task -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - ``chain_id bigint`` - Link to the chain, if ``NULL`` task considered to be disabled - ``task_order DOUBLE PRECISION`` - Indicates the order of task within a chain. - ``kind timetable.command_kind`` - The type of the command. Can be *SQL* (default), *PROGRAM* or *BUILTIN*. - ``command text`` - Contains either a SQL command, a path to application or name of the *BUILTIN* command which will be executed. - ``run_as text`` - The role as which the task should be executed as. - ``database_connection text`` - The connection string for the external database that should be used. - ``ignore_error boolean`` - Specify if the next task should proceed after encountering an error (default: ``false``). - ``autonomous boolean`` - Specify if the task should be executed out of the chain transaction. Useful for ``VACUUM``, ``CREATE DATABASE``, ``CALL`` etc. - ``timeout integer`` - Abort any task within a chain that takes more than the specified number of milliseconds. - - - -.. warning:: If the **task** has been configured with ``ignore_error`` set to ``true`` (the default value is ``false``), the worker process will report a success on execution *even if the task within the chain fails*. - -As mentioned above, **commands** are simple skeletons (e.g. *send email*, *vacuum*, etc.). -In most cases, they have to be brought to live by passing input parameters to the execution. - -Table timetable.parameter -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - ``task_id bigint`` - The ID of the task. - ``order_id integer`` - The order of the parameter. Several parameters are processed one by one according to the order. - ``value jsonb`` - A JSON value containing the parameters. - -Parameter value format -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Depending on the **command** kind argument can be represented by different *JSON* values. - -Kind - Schema - Example - -``SQL`` - ``array`` - .. code-block:: SQL - - '[ "one", 2, 3.14, false ]'::jsonb - -``PROGRAM`` - ``array of strings`` - .. code-block:: SQL - - '["-x", "Latin-ASCII", "-o", "orte_ansi.txt", "orte.txt"]'::jsonb - -``BUILTIN: Sleep`` - ``integer`` - .. code-block:: SQL - - '5' :: jsonb - - -``BUILTIN: Log`` - ``any`` - .. code-block:: SQL - - '"WARNING"'::jsonb - '{"Status": "WARNING"}'::jsonb - -``BUILTIN: SendMail`` - ``object`` - .. code-block:: SQL - - '{ - "username": "user@example.com", - "password": "password", - "serverhost": "smtp.example.com", - "serverport": 587, - "senderaddr": "user@example.com", - "ccaddr": ["recipient_cc@example.com"], - "bccaddr": ["recipient_bcc@example.com"], - "toaddr": ["recipient@example.com"], - "subject": "pg_timetable - No Reply", - "attachment": ["/temp/attachments/Report.pdf","config.yaml"], - "attachmentdata": [{"name": "File.txt", "base64data": "RmlsZSBDb250ZW50"}], - "msgbody": "

Hello User,

check some attachments!

", - "contenttype": "text/html; charset=UTF-8" - }'::jsonb - -``BUILTIN: Download`` - ``object`` - .. code-block:: SQL - - '{ - "workersnum": 2, - "fileurls": ["http://example.com/foo.gz", "https://example.com/bar.csv"], - "destpath": "." - }'::jsonb - -``BUILTIN: CopyFromFile`` - ``object`` - .. code-block:: SQL - - '{ - "sql": "COPY location FROM STDIN", - "filename": "download/orte_ansi.txt" - }'::jsonb - -``BUILTIN: CopyToFile`` - ``object`` - .. code-block:: SQL - - '{ - "sql": "COPY location TO STDOUT", - "filename": "download/location.txt" - }'::jsonb - -``BUILTIN: Shutdown`` - *value ignored* - -``BUILTIN: NoOp`` - *value ignored* - -Chain ------------------------------------------------- - -Once tasks have been arranged, they have to be scheduled as a **chain**. For this, **pg_timetable** builds upon the enhanced **cron**-string, all the while adding multiple configuration options. - -Table timetable.chain -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - ``chain_name text`` - The unique name of the chain. - ``run_at timetable.cron`` - Standard *cron*-style value at Postgres server time zone or ``@after``, ``@every``, ``@reboot`` clause. - ``max_instances integer`` - The amount of instances that this chain may have running at the same time. - ``timeout integer`` - Abort any chain that takes more than the specified number of milliseconds. - ``live boolean`` - Control if the chain may be executed once it reaches its schedule. - ``self_destruct boolean`` - Self destruct the chain after successful execution. Failed chains will be executed according to the schedule one more time. - ``exclusive_execution boolean`` - Specifies whether the chain should be executed exclusively while all other chains are paused. - ``client_name text`` - Specifies which client should execute the chain. Set this to `NULL` to allow any client. - ``timeout integer`` - Abort a chain that takes more than the specified number of milliseconds. - ``on_error`` - Holds SQL to execute if an error occurs. If task produced an error is marked with ``ignore_error`` then nothing is done. - -.. note:: - - All chains in **pg_timetable** are scheduled at the PostgreSQL server time zone. - You can change the `timezone `_ - for the **current session** when adding new chains, e.g. - - .. code-block:: SQL - - SET TIME ZONE 'UTC'; - - -- Run VACUUM at 00:05 every day in August UTC - SELECT timetable.add_job('execute-func', '5 0 * 8 *', 'VACUUM'); diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index be6f1cb0..00000000 --- a/docs/conf.py +++ /dev/null @@ -1,29 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -from __future__ import division, print_function, unicode_literals - -from datetime import datetime - -extensions = [] -templates_path = ['templates', '_templates', '.templates'] -source_suffix = ['.rst', '.md'] -project = u'pg_timetable' -copyright = str(datetime.now().year) - -# -- Options for EPUB output -epub_show_urls = "footnote" - -exclude_patterns = ['_build'] -pygments_style = 'sphinx' -htmlhelp_basename = 'pg-timetable' -html_theme = "sphinx_rtd_theme" -file_insertion_enabled = False -latex_documents = [ - ('index', 'pg-timetable.tex', u'pg_timetable Documentation', - u'', 'manual'), -] - diff --git a/docs/database_schema.md b/docs/database_schema.md new file mode 100644 index 00000000..7a0ee95f --- /dev/null +++ b/docs/database_schema.md @@ -0,0 +1,27 @@ +# Database Schema + +**pg_timetable** is a database driven application. During the first start the necessary schema is created if absent. + +## Main tables and objects + +```sql +--8<-- "internal/pgengine/sql/ddl.sql" +``` + +## Jobs related functions + +```sql +--8<-- "internal/pgengine/sql/job_functions.sql" +``` + +## Сron related functions + +```sql +--8<-- "internal/pgengine/sql/cron_functions.sql" +``` + +## ER-Diagram + +![Database Schema](timetable_schema.png) + +*ER-Diagram showing the database structure* \ No newline at end of file diff --git a/docs/database_schema.rst b/docs/database_schema.rst deleted file mode 100644 index 870a629a..00000000 --- a/docs/database_schema.rst +++ /dev/null @@ -1,34 +0,0 @@ -Database Schema -======================================== - -**pg_timetable** is a database driven application. During the first start the necessary schema is created if absent. - -Main tables and objects -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. literalinclude:: ../internal/pgengine/sql/ddl.sql - :linenos: - :language: SQL - -Jobs related functions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. literalinclude:: ../internal/pgengine/sql/job_functions.sql - :linenos: - :language: SQL - -Сron related functions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. literalinclude:: ../internal/pgengine/sql/cron_functions.sql - :linenos: - :language: SQL - -ER-Diagram -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. figure:: timetable_schema.png - :align: center - :alt: Database Schema - - ER-Diagram \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..4d2e06cf --- /dev/null +++ b/docs/index.md @@ -0,0 +1,106 @@ +# Introduction + +**pg_timetable** is an advanced job scheduler for PostgreSQL, offering many advantages over traditional schedulers such as **cron** and others. +It is completely database driven and provides a couple of advanced concepts. + +## Main features + +- Tasks can be arranged in chains +- A chain can consist of built-int commands, SQL and executables +- Parameters can be passed to chains +- Missed tasks (possibly due to downtime) can be retried automatically +- Support for configurable repetitions +- Built-in tasks such as sending emails, etc. +- Fully database driven configuration +- Full support for database driven logging +- Cron-style scheduling at the PostgreSQL server time zone +- Optional concurrency protection +- Task and chain can have execution timeout settings + +## Quick Start + +1. Download pg_timetable executable +2. Make sure your PostgreSQL server is up and running and has a role with `CREATE` privilege + for a target database, e.g. + + ```sql + my_database=> CREATE ROLE scheduler PASSWORD 'somestrong'; + my_database=> GRANT CREATE ON DATABASE my_database TO scheduler; + ``` + +3. Create a new job, e.g. run `VACUUM` each night at 00:30 Postgres server time zone + + ```sql + my_database=> SELECT timetable.add_job('frequent-vacuum', '30 * * * *', 'VACUUM'); + add_job + --------- + 3 + (1 row) + ``` + +4. Run the **pg_timetable** + + ```bash + # pg_timetable postgresql://scheduler:somestrong@localhost/my_database --clientname=vacuumer + ``` + +5. PROFIT! + +## Command line options + +```bash +# ./pg_timetable + +Application Options: + -c, --clientname= Unique name for application instance [$PGTT_CLIENTNAME] + --config= YAML configuration file + --no-program-tasks Disable executing of PROGRAM tasks [$PGTT_NOPROGRAMTASKS] + -v, --version Output detailed version information [$PGTT_VERSION] + --connstr PostgreSQL connection string [$PGTT_CONNSTR] + +Logging: + --log-level=[debug|info|error] Verbosity level for stdout and log file (default: info) + --log-database-level=[debug|info|error|none] Verbosity level for database storing (default: info) + --log-file= File name to store logs + --log-file-format=[json|text] Format of file logs (default: json) + --log-file-rotate Rotate log files + --log-file-size= Maximum size in MB of the log file before it gets rotated (default: 100) + --log-file-age= Number of days to retain old log files, 0 means forever (default: 0) + --log-file-number= Maximum number of old log files to retain, 0 to retain all (default: 0) + +Start: + -f, --file= SQL script file to execute during startup + --init Initialize database schema to the latest version and exit. Can be used + with --upgrade + --upgrade Upgrade database to the latest version + --debug Run in debug mode. Only asynchronous chains will be executed + +Resource: + --cron-workers= Number of parallel workers for scheduled chains (default: 16) + --interval-workers= Number of parallel workers for interval chains (default: 16) + --chain-timeout= Abort any chain that takes more than the specified number of + milliseconds + --task-timeout= Abort any task within a chain that takes more than the specified number + of milliseconds + +REST: + --rest-port= REST API port (default: 0) [$PGTT_RESTPORT] +``` + +## Contributing + +If you want to contribute to **pg_timetable** and help make it better, feel free to open an +[issue](https://github.com/cybertec-postgresql/pg_timetable/issues) or even consider submitting a +[pull request](https://github.com/cybertec-postgresql/pg_timetable/pulls). You also can give a +[star](https://github.com/cybertec-postgresql/pg_timetable/stargazers) to **pg_timetable** project, +and to tell the world about it. + +## Support + +For professional support, please contact [Cybertec](https://www.cybertec-postgresql.com/). + +## Authors + +**Implementation:** [Pavlo Golub](https://github.com/pashagolub) + +**Initial idea and draft design:** [Hans-Jürgen Schönig](https://github.com/postgresql007) \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index e1a4daa2..00000000 --- a/docs/index.rst +++ /dev/null @@ -1,24 +0,0 @@ -Welcome to pg_timetable's documentation! -======================================== - -.. toctree:: - :numbered: - :maxdepth: 2 - :caption: Contents: - - README - background - installation - components - basic_jobs - samples - migration - api - database_schema - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 00000000..83d510f8 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,84 @@ +# Installation + +**pg_timetable** is compatible with the latest supported [PostgreSQL versions](https://www.postgresql.org/support/versioning/): 11, 12, 13, 14 (stable); 15 (dev). + +!!! note "Older PostgreSQL versions (9.5, 9.6, and 10)" + + If you want to use **pg_timetable** with older versions (9.5, 9.6 and 10), please execute this SQL command before running pg_timetable: + + ```sql + CREATE OR REPLACE FUNCTION starts_with(text, text) + RETURNS bool AS + $$ + SELECT + CASE WHEN length($2) > length($1) THEN + FALSE + ELSE + left($1, length($2)) = $2 + END + $$ + LANGUAGE SQL + IMMUTABLE STRICT PARALLEL SAFE + COST 5; + ``` + +## Official release packages + +You may find binary package for your platform on the official [Releases](https://github.com/cybertec-postgresql/pg_timetable/releases) page. Right now **Windows**, **Linux** and **macOS** packages are available. + +## Docker + +The official docker image can be found here: https://hub.docker.com/r/cybertecpostgresql/pg_timetable + +!!! note + + The `latest` tag is up to date with the `master` branch thanks to [this github action](https://github.com/cybertec-postgresql/pg_timetable/blob/master/.github/workflows/docker.yml). In production you probably want to use the latest [stable tag](https://hub.docker.com/r/cybertecpostgresql/pg_timetable/tags). + +Run **pg_timetable** in Docker: + +```bash +docker run --rm \ +cybertecpostgresql/pg_timetable:latest \ +-h 10.0.0.3 -p 54321 -c worker001 +``` + +Run **pg_timetable** in Docker with Environment variables: + +```bash +docker run --rm \ +-e PGTT_PGHOST=10.0.0.3 \ +-e PGTT_PGPORT=54321 \ +cybertecpostgresql/pg_timetable:latest \ +-c worker001 +``` + +## Build from sources + +1. Download and install [Go](https://golang.org/doc/install) on your system. +2. Clone **pg_timetable** repo: + + ```bash + $ git clone https://github.com/cybertec-postgresql/pg_timetable.git + $ cd pg_timetable + ``` + +3. Run **pg_timetable**: + + ```bash + $ go run main.go --clientname=worker001 postgresql://scheduler:strongpwd@localhost:5432/dbname + ``` + +4. Alternatively, build a binary and run it: + + ```bash + $ go build + $ ./pg_timetable --clientname=worker001 postgresql://scheduler:strongpwd@localhost:5432/dbname + ``` + +5. (Optional) Run tests in all sub-folders of the project: + + ```bash + $ psql --command="CREATE USER scheduler PASSWORD 'somestrong'" + $ createdb --owner=scheduler timetable + $ go test -failfast -timeout=300s -count=1 -p 1 ./... + ``` \ No newline at end of file diff --git a/docs/installation.rst b/docs/installation.rst deleted file mode 100644 index b2b975a3..00000000 --- a/docs/installation.rst +++ /dev/null @@ -1,90 +0,0 @@ -Installation -================================================ - -**pg_timetable** is compatible with the latest supported `PostgreSQL versions `_: 11, 12, 13, 14 (stable); 15 (dev). - -.. note:: - - .. raw:: html - -
- If you want to use pg_timetable with older versions (9.5, 9.6 and 10)... - please, execute this SQL command before running pg_timetable: - - .. code-block:: SQL - - CREATE OR REPLACE FUNCTION starts_with(text, text) - RETURNS bool AS - $$ - SELECT - CASE WHEN length($2) > length($1) THEN - FALSE - ELSE - left($1, length($2)) = $2 - END - $$ - LANGUAGE SQL - IMMUTABLE STRICT PARALLEL SAFE - COST 5; - - .. raw:: html - -
- - -Official release packages ------------------------------------------------- - -You may find binary package for your platform on the official `Releases `_ page. Right now **Windows**, **Linux** and **macOS** packages are available. - -Docker ------------------------------------------------- - -The official docker image can be found here: https://hub.docker.com/r/cybertecpostgresql/pg_timetable - -.. note:: - - The ``latest`` tag is up to date with the `master` branch thanks to `this github action `_. In production you probably want to use the latest `stable tag `_. - -Run **pg_timetable** in Docker: - -.. code-block:: console - - docker run --rm \ - cybertecpostgresql/pg_timetable:latest \ - -h 10.0.0.3 -p 54321 -c worker001 - -Run **pg_timetable** in Docker with Environment variables: - -.. code-block:: console - - docker run --rm \ - -e PGTT_PGHOST=10.0.0.3 \ - -e PGTT_PGPORT=54321 \ - cybertecpostgresql/pg_timetable:latest \ - -c worker001 - -Build from sources ------------------------------------------------- - -1. Download and install `Go `_ on your system. -#. Clone **pg_timetable** repo:: - - $ git clone https://github.com/cybertec-postgresql/pg_timetable.git - $ cd pg_timetable - -#. Run **pg_timetable**:: - - $ go run main.go --dbname=dbname --clientname=worker001 --user=scheduler --password=strongpwd - -#. Alternatively, build a binary and run it:: - - $ go build - $ ./pg_timetable --dbname=dbname --clientname=worker001 --user=scheduler --password=strongpwd - -#. (Optional) Run tests in all sub-folders of the project:: - - $ psql --command="CREATE USER scheduler PASSWORD 'somestrong'" - $ createdb --owner=scheduler timetable - $ go test -failfast -timeout=300s -count=1 -p 1 ./... - diff --git a/docs/license.md b/docs/license.md new file mode 100644 index 00000000..0e8c6fca --- /dev/null +++ b/docs/license.md @@ -0,0 +1,3 @@ +# License + +--8<-- "LICENSE" diff --git a/docs/migration.rst b/docs/migration.md similarity index 63% rename from docs/migration.rst rename to docs/migration.md index 6d9f0f5a..21612376 100644 --- a/docs/migration.rst +++ b/docs/migration.md @@ -1,13 +1,12 @@ -Migration from others schedulers -================================================ +# Migration from others schedulers + +## Migrate jobs from pg_cron to pg_timetable -Migrate jobs from pg_cron to pg_timetable -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you want to quickly export jobs scheduled from *pg_cron* to *pg_timetable*, you can use this SQL snippet: -.. literalinclude:: ../extras/pg_cron_to_pg_timetable_simple.sql - :linenos: - :language: SQL +```sql +--8<-- "extras/pg_cron_to_pg_timetable_simple.sql" +``` The *timetable.add_job()*, however, has some limitations. First of all, the function will mark the task created as **autonomous**, specifying scheduler should execute the task out of the chain transaction. It's not an error, @@ -17,16 +16,16 @@ Secondly, database connection parameters are lost for source *pg_cron* jobs, mak every information available precisely as possible, use this SQL snippet under the role they were scheduled in *pg_cron*: -.. literalinclude:: ../extras/pg_cron_to_pg_timetable.sql - :linenos: - :language: SQL +```sql +--8<-- "extras/pg_cron_to_pg_timetable.sql" +``` + +## Migrate jobs from pgAgent to pg_timetable -Migrate jobs from pgAgent to pg_timetable -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To migrate jobs from **pgAgent**, please use this script. **pgAgent** doesn't have concept of *PROGRAM* task, thus to emulate *BATCH* steps, **pg_timetable** will execute them inside the shell. You may change the shell by editing *cte_shell* CTE clause. -.. literalinclude:: ../extras/pgagent_to_pg_timetable.sql - :linenos: - :language: SQL \ No newline at end of file +```sql +--8<-- "extras/pgagent_to_pg_timetable.sql" +``` \ No newline at end of file diff --git a/docs/requirements-doc.txt b/docs/requirements-doc.txt new file mode 100644 index 00000000..dd1b8cb4 --- /dev/null +++ b/docs/requirements-doc.txt @@ -0,0 +1,2 @@ +mike +mkdocs-material \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index b617ee0a..00000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -Sphinx -sphinx_rtd_theme \ No newline at end of file diff --git a/docs/samples.rst b/docs/samples.md similarity index 58% rename from docs/samples.rst rename to docs/samples.md index b4989b1d..6accf4f9 100644 --- a/docs/samples.rst +++ b/docs/samples.md @@ -1,60 +1,59 @@ -Samples -======== +# Samples + +## Basic -Basic -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sample demonstrates how to create a basic one-step chain with parameters. It uses CTE to directly update the **timetable** schema tables. -.. literalinclude:: ../samples/Basic.sql - :linenos: - :language: SQL +```sql +--8<-- "samples/Basic.sql" +``` + +## Send email -Send email -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sample demonstrates how to create an advanced email job. It will check if there are emails to send, will send them and log the status of the command execution. You don't need to setup anything, every parameter can be specified during the chain creation. -.. literalinclude:: ../samples/Mail.sql - :linenos: - :language: SQL +```sql +--8<-- "samples/Mail.sql" +``` + +## Download, Transform and Import -Download, Transform and Import -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sample demonstrates how to create enhanced three-step chain with parameters. It uses DO statement to directly update the **timetable** schema tables. -.. literalinclude:: ../samples/Download.sql - :linenos: - :language: SQL +```sql +--8<-- "samples/Download.sql" +``` + +## Run tasks in autonomous transaction -Run tasks in autonomous transaction -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sample demonstrates how to run special tasks out of chain transaction context. This is useful for special routines and/or -non-transactional operations, e.g. *CREATE DATABASE*, *REINDEX*, *VACUUM*, *CREATE TABLESPACE*, etc. +non-transactional operations, e.g. *CREATE DATABASE*, *REINDEX*, *VACUUM*, *CREATE TABLESPACE*, etc. + +```sql +--8<-- "samples/Autonomous.sql" +``` -.. literalinclude:: ../samples/Autonomous.sql - :linenos: - :language: SQL +## Shutdown the scheduler and terminate the session -Shutdown the scheduler and terminate the session -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sample demonstrates how to shutdown the scheduler using special built-in task. This can be used to control maintenance windows, to restart the scheduler for update purposes, or to stop session before the database should be dropped. -.. literalinclude:: ../samples/Shutdown.sql - :linenos: - :language: SQL +```sql +--8<-- "samples/Shutdown.sql" +``` + +## Access previous task result code and output from the next task -Access previous task result code and output from the next task -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sample demonstrates how to check the result code and output of a previous task. If the last task failed, that is possible only if *ignore_error boolean = true* is set for that task. Otherwise, a scheduler will stop the chain. This sample shows how to calculate failed, successful, and the total number of tasks executed. Based on these values, we can calculate the success ratio. -.. literalinclude:: ../samples/ManyTasks.sql - :linenos: - :language: SQL \ No newline at end of file +```sql +--8<-- "samples/ManyTasks.sql" +``` \ No newline at end of file diff --git a/go.mod b/go.mod index fa290024..2adf33f9 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sagikazarmark/locafero v0.12.0 // indirect - github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect diff --git a/go.sum b/go.sum index 3d7ee6d4..a773a2b4 100644 --- a/go.sum +++ b/go.sum @@ -45,16 +45,12 @@ github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnH github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= -github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= -github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..c149f4c4 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,78 @@ +site_name: 🕰️ pg_timetable +site_description: Advanced job scheduler for PostgreSQL +site_url: https://cybertec-postgresql.github.io/pg_timetable/ +repo_url: https://github.com/cybertec-postgresql/pg_timetable +repo_name: cybertec-postgresql/pg_timetable +copyright: Made by and for PostgreSQL lovers! 🐘❤️ + +theme: + name: material + language: en + include_search_page: false + search_index_only: true + features: + - navigation.tabs + - navigation.sections + - navigation.expand + - navigation.top + - navigation.footer + - navigation.instant + - navigation.tracking + - search.highlight + - search.share + - content.code.copy + - content.tabs.link + palette: + - media: "(prefers-color-scheme: light)" # Light mode + scheme: default + primary: blue + accent: blue + toggle: + icon: material/lightbulb-outline + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" # Dark mode + scheme: slate + primary: light blue + accent: indigo + toggle: + icon: material/lightbulb + name: Switch to light mode + +plugins: + - search + +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - attr_list + - md_in_html + +nav: + - Intro: + - Introduction: index.md + - Background: background.md + - Concepts: + - Components: components.md + - Database Schema: database_schema.md + - Tutorials: + - Installation: installation.md + - Basic Jobs: basic_jobs.md + - Migration: migration.md + - Samples: samples.md + - Reference: + - API: api.md + - Developer: + - License: license.md + - Documentation: ../devel/godoc/index.html + +extra: + version: + provider: mike + alias: true \ No newline at end of file