From e3cbc5cf6180e01420bb3a535c246000a3f8e9af Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Fri, 14 May 2021 13:53:52 +0300 Subject: [PATCH 001/121] Introducing Apalis --- packages/apalis-sql/Cargo.toml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 packages/apalis-sql/Cargo.toml diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml new file mode 100644 index 0000000..48e59f4 --- /dev/null +++ b/packages/apalis-sql/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "apalis-sql" +version = "0.1.0" +authors = ["Geoffrey Mureithi "] +edition = "2018" + +[features] +default = [ "sqlite" ] +postgres = [ "sqlx/postgres"] +sqlite = [ "sqlx/sqlite"] +mysql = [ "sqlx/mysql"] + +[dependencies.sqlx] +version = "0.4.2" +default-features = false +features = [ "runtime-actix-rustls", "macros", "chrono", "decimal", "uuid", "json" ] + +[dependencies] +serde = { version = "1", features = ["derive"] } +apalis = { path = "../apalis"} \ No newline at end of file From c3523d3d7ef2ed9ce245e775cbdb6aab96fc5602 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Fri, 14 May 2021 14:24:07 +0300 Subject: [PATCH 002/121] Readme Changes --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 48e59f4..7fee6e9 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.1.0" +version = "0.2.0" authors = ["Geoffrey Mureithi "] edition = "2018" From c653652d9b7e6b0602c94945d3bed56feb2ae521 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Fri, 14 May 2021 14:52:22 +0300 Subject: [PATCH 003/121] Make release ready --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 7fee6e9..9188780 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -17,4 +17,4 @@ features = [ "runtime-actix-rustls", "macros", "chrono", "decimal", "uuid", "jso [dependencies] serde = { version = "1", features = ["derive"] } -apalis = { path = "../apalis"} \ No newline at end of file +apalis = { path = "../../"} \ No newline at end of file From b8f70ea4d870af4f0b47f6ca77102a4cefef4ea5 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Fri, 14 May 2021 15:51:19 +0300 Subject: [PATCH 004/121] Lower version --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 9188780..7358b63 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.2.0" +version = "0.1.0" authors = ["Geoffrey Mureithi "] edition = "2018" From 53101b50a615515f467df5e74fcb32b12f5cf388 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Fri, 14 May 2021 16:09:57 +0300 Subject: [PATCH 005/121] Renaming imports --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 7358b63..eb21e31 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -17,4 +17,4 @@ features = [ "runtime-actix-rustls", "macros", "chrono", "decimal", "uuid", "jso [dependencies] serde = { version = "1", features = ["derive"] } -apalis = { path = "../../"} \ No newline at end of file +apalis-core = { path = "../../packages/apalis-core"} \ No newline at end of file From fbcfb5d4d17a8089293b559278713277eeee3183 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Fri, 14 May 2021 17:04:21 +0300 Subject: [PATCH 006/121] Up the version --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index eb21e31..a1a2852 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.1.0" +version = "0.2.0" authors = ["Geoffrey Mureithi "] edition = "2018" From 057d4103fb03aa711f9c46082a714cb88b226789 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Fri, 14 May 2021 17:34:15 +0300 Subject: [PATCH 007/121] Setup the version --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index a1a2852..c61770c 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -17,4 +17,4 @@ features = [ "runtime-actix-rustls", "macros", "chrono", "decimal", "uuid", "jso [dependencies] serde = { version = "1", features = ["derive"] } -apalis-core = { path = "../../packages/apalis-core"} \ No newline at end of file +apalis-core = { path = "../../packages/apalis-core", version = "0.2"} \ No newline at end of file From b03c654bc23b1050368cf49218362548a02308ae Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Fri, 14 May 2021 18:26:29 +0300 Subject: [PATCH 008/121] Add license and description --- packages/apalis-sql/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index c61770c..f06e99e 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -4,6 +4,9 @@ version = "0.2.0" authors = ["Geoffrey Mureithi "] edition = "2018" +license = "MIT" +description = "SQL Storage for Apalis: simple and reliable background processing for Rust using Actix actors" + [features] default = [ "sqlite" ] postgres = [ "sqlx/postgres"] From c04c1bcce5705b1cb5729d1340f882e655bef4a3 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Sat, 30 Apr 2022 17:35:53 +0300 Subject: [PATCH 009/121] Introducing tower, major changes, refactoring --- packages/apalis-sql/Cargo.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index f06e99e..6166c1d 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Geoffrey Mureithi "] edition = "2018" license = "MIT" -description = "SQL Storage for Apalis: simple and reliable background processing for Rust using Actix actors" +description = "SQL Storage for Apalis: simple, extensible multithreaded background processing for Rust" [features] default = [ "sqlite" ] @@ -14,10 +14,12 @@ sqlite = [ "sqlx/sqlite"] mysql = [ "sqlx/mysql"] [dependencies.sqlx] -version = "0.4.2" +version = "0.5" default-features = false features = [ "runtime-actix-rustls", "macros", "chrono", "decimal", "uuid", "json" ] [dependencies] serde = { version = "1", features = ["derive"] } -apalis-core = { path = "../../packages/apalis-core", version = "0.2"} \ No newline at end of file +serde_json = "1" +apalis-core = { path = "../../packages/apalis-core", version = "0.2"} +chrono = { version = "0.4", features = ["serde"] } \ No newline at end of file From ee26994497f2360af8ea05e09e2b6ef44cd2284e Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Sat, 30 Apr 2022 17:59:37 +0300 Subject: [PATCH 010/121] Updated docs --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 6166c1d..2f6d410 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Geoffrey Mureithi "] edition = "2018" license = "MIT" -description = "SQL Storage for Apalis: simple, extensible multithreaded background processing for Rust" +description = "SQL Storage for Apalis: simple, extensible multithreaded background job processing for Rust" [features] default = [ "sqlite" ] From 80fdf9236a26c4fa409d18d2bd62dae71868670a Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Mon, 2 May 2022 15:51:05 +0300 Subject: [PATCH 011/121] Improvements on Extensions and Storages --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 2f6d410..7ed541a 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -8,7 +8,7 @@ license = "MIT" description = "SQL Storage for Apalis: simple, extensible multithreaded background job processing for Rust" [features] -default = [ "sqlite" ] +default = [ "sqlite", "postgres" ] postgres = [ "sqlx/postgres"] sqlite = [ "sqlx/sqlite"] mysql = [ "sqlx/mysql"] From 2ab53ef554977ba0722e09448296f535a50af26b Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Mon, 9 May 2022 11:08:38 +0300 Subject: [PATCH 012/121] Standardize core + redis + sqlite --- packages/apalis-sql/Cargo.toml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 7ed541a..236813f 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.2.0" +version = "0.3.0-beta.0" authors = ["Geoffrey Mureithi "] edition = "2018" @@ -21,5 +21,8 @@ features = [ "runtime-actix-rustls", "macros", "chrono", "decimal", "uuid", "jso [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.2"} -chrono = { version = "0.4", features = ["serde"] } \ No newline at end of file +apalis-core = { path = "../../packages/apalis-core", version = "0.3.0-beta.0"} +chrono = { version = "0.4", features = ["serde"] } +futures = "0.3" +async-stream = "0.3" +tokio = "1" \ No newline at end of file From 96e38de5e5788e186a567f99518d5f6ea07d961c Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Sun, 22 May 2022 10:03:49 +0300 Subject: [PATCH 013/121] Introducing prometheus, mysql, and axum --- packages/apalis-sql/Cargo.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 236813f..7dd10f2 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -8,7 +8,7 @@ license = "MIT" description = "SQL Storage for Apalis: simple, extensible multithreaded background job processing for Rust" [features] -default = [ "sqlite", "postgres" ] +default = [ "sqlite", "postgres", "mysql" ] postgres = [ "sqlx/postgres"] sqlite = [ "sqlx/sqlite"] mysql = [ "sqlx/mysql"] @@ -25,4 +25,5 @@ apalis-core = { path = "../../packages/apalis-core", version = "0.3.0-beta.0"} chrono = { version = "0.4", features = ["serde"] } futures = "0.3" async-stream = "0.3" -tokio = "1" \ No newline at end of file +tokio = "1" +futures-lite = "1.12.0" \ No newline at end of file From 223477645030e8eef9dc60758dfd1f448fa56817 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Sat, 28 May 2022 00:13:34 +0300 Subject: [PATCH 014/121] Added More Improvements esp to features --- packages/apalis-sql/Cargo.toml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 7dd10f2..1adf04e 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -8,22 +8,27 @@ license = "MIT" description = "SQL Storage for Apalis: simple, extensible multithreaded background job processing for Rust" [features] -default = [ "sqlite", "postgres", "mysql" ] -postgres = [ "sqlx/postgres"] -sqlite = [ "sqlx/sqlite"] -mysql = [ "sqlx/mysql"] +default = [ "sqlite" ] +postgres = [ "sqlx/postgres", "sqlx/json", "sqlx/chrono"] +sqlite = [ "sqlx/sqlite", "sqlx/json", "sqlx/chrono"] +mysql = [ "sqlx/mysql", "sqlx/json", "sqlx/chrono"] [dependencies.sqlx] version = "0.5" default-features = false -features = [ "runtime-actix-rustls", "macros", "chrono", "decimal", "uuid", "json" ] +features = [ "runtime-actix-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.3.0-beta.0"} +apalis-core = { path = "../../packages/apalis-core", version = "0.3.0-beta.0", default-features = false} chrono = { version = "0.4", features = ["serde"] } futures = "0.3" async-stream = "0.3" tokio = "1" -futures-lite = "1.12.0" \ No newline at end of file +futures-lite = "1.12.0" + +[package.metadata.docs.rs] +# defines the configuration attribute `docsrs` +rustdoc-args = ["--cfg", "docsrs"] +all-features = true \ No newline at end of file From 3d140d9fc94d9c98f5497ef2d86aae14ef9034c8 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Sun, 29 May 2022 14:22:03 +0300 Subject: [PATCH 015/121] Moved from Actix --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 1adf04e..457ec1d 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -16,7 +16,7 @@ mysql = [ "sqlx/mysql", "sqlx/json", "sqlx/chrono"] [dependencies.sqlx] version = "0.5" default-features = false -features = [ "runtime-actix-rustls", "uuid" ] +features = [ "runtime-tokio-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } From 126e99dc12dd017b47b74a461d26ef44b6c9a618 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Wed, 1 Jun 2022 14:04:14 +0300 Subject: [PATCH 016/121] Stabilized parts of Worker API --- packages/apalis-sql/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 457ec1d..26aeb42 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -27,6 +27,7 @@ futures = "0.3" async-stream = "0.3" tokio = "1" futures-lite = "1.12.0" +async-trait = "0.1.53" [package.metadata.docs.rs] # defines the configuration attribute `docsrs` From 08553df3d140c039de3135819f7b1a3c648b9b54 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Sat, 4 Jun 2022 14:18:06 +0300 Subject: [PATCH 017/121] Added more fixes, Migrations for sqlx and more documentation --- packages/apalis-sql/Cargo.toml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 26aeb42..6124992 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -8,20 +8,21 @@ license = "MIT" description = "SQL Storage for Apalis: simple, extensible multithreaded background job processing for Rust" [features] -default = [ "sqlite" ] +default = [ "sqlite", "migrate" ] postgres = [ "sqlx/postgres", "sqlx/json", "sqlx/chrono"] sqlite = [ "sqlx/sqlite", "sqlx/json", "sqlx/chrono"] mysql = [ "sqlx/mysql", "sqlx/json", "sqlx/chrono"] +migrate = ["sqlx/migrate"] [dependencies.sqlx] -version = "0.5" +version = "0.5.5" default-features = false features = [ "runtime-tokio-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.3.0-beta.0", default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.3.0-beta.0", features= ["storage"], default-features = false} chrono = { version = "0.4", features = ["serde"] } futures = "0.3" async-stream = "0.3" @@ -29,6 +30,10 @@ tokio = "1" futures-lite = "1.12.0" async-trait = "0.1.53" +[dev-dependencies] +tokio = { version = "1", features = ["macros"] } +email-service = { path = "../../examples/email-service"} + [package.metadata.docs.rs] # defines the configuration attribute `docsrs` rustdoc-args = ["--cfg", "docsrs"] From 1bcc70facf1d2b893e065438481459110682a8bd Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Sat, 4 Jun 2022 14:21:12 +0300 Subject: [PATCH 018/121] Features + introduce twitter example --- packages/apalis-sql/build.rs | 3 ++ .../sqlite/20220530084123_jobs_workers.sql | 36 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 packages/apalis-sql/build.rs create mode 100644 packages/apalis-sql/migrations/sqlite/20220530084123_jobs_workers.sql diff --git a/packages/apalis-sql/build.rs b/packages/apalis-sql/build.rs new file mode 100644 index 0000000..4a3cea3 --- /dev/null +++ b/packages/apalis-sql/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo:rerun-if-changed=migrations"); +} \ No newline at end of file diff --git a/packages/apalis-sql/migrations/sqlite/20220530084123_jobs_workers.sql b/packages/apalis-sql/migrations/sqlite/20220530084123_jobs_workers.sql new file mode 100644 index 0000000..5883b48 --- /dev/null +++ b/packages/apalis-sql/migrations/sqlite/20220530084123_jobs_workers.sql @@ -0,0 +1,36 @@ +CREATE TABLE IF NOT EXISTS Workers ( + id TEXT NOT NULL UNIQUE, + worker_type TEXT NOT NULL, + storage_name TEXT NOT NULL, + layers TEXT, + last_seen INTEGER NOT NULL DEFAULT (strftime('%s', 'now')) +); + +CREATE INDEX IF NOT EXISTS Idx ON Workers(id); + +CREATE INDEX IF NOT EXISTS WTIdx ON Workers(worker_type); + +CREATE INDEX IF NOT EXISTS LSIdx ON Workers(last_seen); + +CREATE TABLE IF NOT EXISTS Jobs ( + job TEXT NOT NULL, + id TEXT NOT NULL UNIQUE, + job_type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'Pending', + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 25, + run_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')), + last_error TEXT, + lock_at INTEGER, + lock_by TEXT, + done_at INTEGER, + FOREIGN KEY(lock_by) REFERENCES Workers(id) +); + +CREATE INDEX IF NOT EXISTS TIdx ON Jobs(id); + +CREATE INDEX IF NOT EXISTS SIdx ON Jobs(status); + +CREATE INDEX IF NOT EXISTS LIdx ON Jobs(lock_by); + +CREATE INDEX IF NOT EXISTS JTIdx ON Jobs(job_type); \ No newline at end of file From 8ca00d95a1d78f80993cd3ef0f7aaa2cabe2a1a1 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Sun, 5 Jun 2022 08:11:01 +0300 Subject: [PATCH 019/121] Minor fixes and added examples --- packages/apalis-sql/Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 6124992..d78db1a 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -11,11 +11,11 @@ description = "SQL Storage for Apalis: simple, extensible multithreaded backgrou default = [ "sqlite", "migrate" ] postgres = [ "sqlx/postgres", "sqlx/json", "sqlx/chrono"] sqlite = [ "sqlx/sqlite", "sqlx/json", "sqlx/chrono"] -mysql = [ "sqlx/mysql", "sqlx/json", "sqlx/chrono"] -migrate = ["sqlx/migrate"] +mysql = [ "sqlx/mysql", "sqlx/json", "sqlx/chrono", "sqlx/bigdecimal"] +migrate = ["sqlx/migrate", "sqlx/macros"] [dependencies.sqlx] -version = "0.5.5" +version = "0.5.13" default-features = false features = [ "runtime-tokio-rustls", "uuid" ] From 3900c4d1a44d5049e50151815b625d427030e025 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Sun, 5 Jun 2022 08:19:50 +0300 Subject: [PATCH 020/121] Bump up to v0.3 --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index d78db1a..70ce7b1 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.3.0-beta.0" +version = "0.3.0" authors = ["Geoffrey Mureithi "] edition = "2018" @@ -22,7 +22,7 @@ features = [ "runtime-tokio-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.3.0-beta.0", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.3.0", features= ["storage"], default-features = false} chrono = { version = "0.4", features = ["serde"] } futures = "0.3" async-stream = "0.3" From 9ff74abb5ee4dee795086b2a2928039d390504ab Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Sun, 10 Jul 2022 00:20:43 +0300 Subject: [PATCH 021/121] Improvements in Docs and bump up to v0.3.1 --- packages/apalis-sql/Cargo.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 70ce7b1..ac4c879 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,8 +1,9 @@ [package] name = "apalis-sql" -version = "0.3.0" +version = "0.3.1" authors = ["Geoffrey Mureithi "] edition = "2018" +readme = "../../README.md" license = "MIT" description = "SQL Storage for Apalis: simple, extensible multithreaded background job processing for Rust" @@ -22,7 +23,7 @@ features = [ "runtime-tokio-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.3.0", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.3.1", features= ["storage"], default-features = false} chrono = { version = "0.4", features = ["serde"] } futures = "0.3" async-stream = "0.3" From 21eed226470f7a3c58c26e86ab8d6c614816d5a9 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Sun, 10 Jul 2022 00:24:54 +0300 Subject: [PATCH 022/121] Fixes by Rustfmt --- packages/apalis-sql/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/build.rs b/packages/apalis-sql/build.rs index 4a3cea3..3a8149e 100644 --- a/packages/apalis-sql/build.rs +++ b/packages/apalis-sql/build.rs @@ -1,3 +1,3 @@ fn main() { println!("cargo:rerun-if-changed=migrations"); -} \ No newline at end of file +} From f80a38b073a2f3a3c3daf3ca02ca1915547d68a9 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Thu, 11 Aug 2022 16:30:42 +0300 Subject: [PATCH 023/121] Bump up version --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index ac4c879..cb145bd 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.3.1" +version = "0.3.2" authors = ["Geoffrey Mureithi "] edition = "2018" readme = "../../README.md" @@ -23,7 +23,7 @@ features = [ "runtime-tokio-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.3.1", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.3.2", features= ["storage"], default-features = false} chrono = { version = "0.4", features = ["serde"] } futures = "0.3" async-stream = "0.3" From be50d258029a90065086e275ac56bff4bb094b21 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Sat, 13 Aug 2022 10:04:52 +0300 Subject: [PATCH 024/121] Bumping up versions --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index cb145bd..97b22b0 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.3.2" +version = "0.3.3" authors = ["Geoffrey Mureithi "] edition = "2018" readme = "../../README.md" @@ -23,7 +23,7 @@ features = [ "runtime-tokio-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.3.2", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.3.3", features= ["storage"], default-features = false} chrono = { version = "0.4", features = ["serde"] } futures = "0.3" async-stream = "0.3" From d229e634e688786595932789ce173d03eab907ac Mon Sep 17 00:00:00 2001 From: autotaker Date: Sat, 17 Sep 2022 05:48:50 +0000 Subject: [PATCH 025/121] add once_cell to dev-dependencies --- packages/apalis-sql/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 97b22b0..4370986 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -34,6 +34,7 @@ async-trait = "0.1.53" [dev-dependencies] tokio = { version = "1", features = ["macros"] } email-service = { path = "../../examples/email-service"} +once_cell = "1.14.0" [package.metadata.docs.rs] # defines the configuration attribute `docsrs` From 1bedd7995bfa363c4972c498c72c8a2c8ff5b903 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Tue, 20 Sep 2022 11:00:56 +0300 Subject: [PATCH 026/121] Bump up to v0.3.4 --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 4370986..0f51cd1 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.3.3" +version = "0.3.4" authors = ["Geoffrey Mureithi "] edition = "2018" readme = "../../README.md" @@ -23,7 +23,7 @@ features = [ "runtime-tokio-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.3.3", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.3.4", features= ["storage"], default-features = false} chrono = { version = "0.4", features = ["serde"] } futures = "0.3" async-stream = "0.3" From 5112157b670839b9ffdc924f8b176029a0587cd7 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Fri, 23 Dec 2022 16:27:51 +0300 Subject: [PATCH 027/121] Bump up version --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 0f51cd1..bbe48d8 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.3.4" +version = "0.3.5" authors = ["Geoffrey Mureithi "] edition = "2018" readme = "../../README.md" @@ -23,7 +23,7 @@ features = [ "runtime-tokio-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.3.4", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.3.5", features= ["storage"], default-features = false} chrono = { version = "0.4", features = ["serde"] } futures = "0.3" async-stream = "0.3" From 5f117eb8a88cf62a3e1252bb2c173174918d467d Mon Sep 17 00:00:00 2001 From: Rob Ede Date: Fri, 30 Dec 2022 16:39:03 +0000 Subject: [PATCH 028/121] Remove time 0.1 from dependency tree --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index bbe48d8..73eada4 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -24,7 +24,7 @@ features = [ "runtime-tokio-rustls", "uuid" ] serde = { version = "1", features = ["derive"] } serde_json = "1" apalis-core = { path = "../../packages/apalis-core", version = "0.3.5", features= ["storage"], default-features = false} -chrono = { version = "0.4", features = ["serde"] } +chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } futures = "0.3" async-stream = "0.3" tokio = "1" @@ -39,4 +39,4 @@ once_cell = "1.14.0" [package.metadata.docs.rs] # defines the configuration attribute `docsrs` rustdoc-args = ["--cfg", "docsrs"] -all-features = true \ No newline at end of file +all-features = true From 5800fd0c54d3e3ed5128fbbf5a4e0dad2d872318 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi Date: Mon, 2 Jan 2023 20:25:33 +0300 Subject: [PATCH 029/121] Bump up version --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 73eada4..122dab8 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.3.5" +version = "0.3.6" authors = ["Geoffrey Mureithi "] edition = "2018" readme = "../../README.md" @@ -23,7 +23,7 @@ features = [ "runtime-tokio-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.3.5", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.3.6", features= ["storage"], default-features = false} chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } futures = "0.3" async-stream = "0.3" From 86cc7dc10539788971a78f414ee54662bde13bbe Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Thu, 23 Mar 2023 00:35:53 +0300 Subject: [PATCH 030/121] Updates getting ready for 0.4.0-alpha --- packages/apalis-sql/Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 122dab8..eb387dd 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "apalis-sql" -version = "0.3.6" +version = "0.4.0-alpha.0" authors = ["Geoffrey Mureithi "] -edition = "2018" +edition = "2021" readme = "../../README.md" license = "MIT" @@ -23,7 +23,7 @@ features = [ "runtime-tokio-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.3.6", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.0", features= ["storage"], default-features = false} chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } futures = "0.3" async-stream = "0.3" From bdc6790b4b430685b902447915f61e6e656cc02f Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Thu, 23 Mar 2023 22:58:57 +0300 Subject: [PATCH 031/121] Bump up version --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index eb387dd..5095ada 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.0-alpha.0" +version = "0.4.0-alpha.1" authors = ["Geoffrey Mureithi "] edition = "2021" readme = "../../README.md" @@ -23,7 +23,7 @@ features = [ "runtime-tokio-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.0", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.1", features= ["storage"], default-features = false} chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } futures = "0.3" async-stream = "0.3" From fecbfd04b8edccfe6addf29f0616a04eb85f12db Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Fri, 24 Mar 2023 09:04:03 +0300 Subject: [PATCH 032/121] Bump up version --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 5095ada..9724c79 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.0-alpha.1" +version = "0.4.0-alpha.2" authors = ["Geoffrey Mureithi "] edition = "2021" readme = "../../README.md" @@ -23,7 +23,7 @@ features = [ "runtime-tokio-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.1", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.2", features= ["storage"], default-features = false} chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } futures = "0.3" async-stream = "0.3" From 010c8b109d499a8d4c50f896ed6a3147fa843b3b Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Fri, 24 Mar 2023 09:19:25 +0300 Subject: [PATCH 033/121] Fixes in changing features --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 9724c79..28dd403 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.0-alpha.2" +version = "0.4.0-alpha.3" authors = ["Geoffrey Mureithi "] edition = "2021" readme = "../../README.md" @@ -23,7 +23,7 @@ features = [ "runtime-tokio-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.2", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.3", features= ["storage"], default-features = false} chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } futures = "0.3" async-stream = "0.3" From 3cf324fdff388c9fc4324c492b2da37af83fffa6 Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Tue, 28 Mar 2023 09:08:53 +0300 Subject: [PATCH 034/121] Bump version --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 28dd403..6915cb4 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.0-alpha.3" +version = "0.4.0-alpha.4" authors = ["Geoffrey Mureithi "] edition = "2021" readme = "../../README.md" @@ -23,7 +23,7 @@ features = [ "runtime-tokio-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.3", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.4", features= ["storage"], default-features = false} chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } futures = "0.3" async-stream = "0.3" From b1422c4db6acbe1d8206859c01c193b47804f37d Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Wed, 29 Mar 2023 13:00:24 +0300 Subject: [PATCH 035/121] Fixed concrete types, for worker and jobs --- packages/apalis-sql/Cargo.toml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 6915cb4..33c318c 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -9,16 +9,17 @@ license = "MIT" description = "SQL Storage for Apalis: simple, extensible multithreaded background job processing for Rust" [features] -default = [ "sqlite", "migrate" ] +default = [ "sqlite", "migrate", "tokio-comp" ] postgres = [ "sqlx/postgres", "sqlx/json", "sqlx/chrono"] sqlite = [ "sqlx/sqlite", "sqlx/json", "sqlx/chrono"] mysql = [ "sqlx/mysql", "sqlx/json", "sqlx/chrono", "sqlx/bigdecimal"] migrate = ["sqlx/migrate", "sqlx/macros"] +async-std-comp = ["async-std", "sqlx/runtime-async-std-rustls"] +tokio-comp = ["tokio", "sqlx/runtime-tokio-rustls"] [dependencies.sqlx] -version = "0.5.13" +version = "0.6" default-features = false -features = [ "runtime-tokio-rustls", "uuid" ] [dependencies] serde = { version = "1", features = ["derive"] } @@ -27,7 +28,8 @@ apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.4", chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } futures = "0.3" async-stream = "0.3" -tokio = "1" +tokio = { version = "1", features = ["rt", "net"], optional = true } +async-std = { version = "1.8.0", optional = true} futures-lite = "1.12.0" async-trait = "0.1.53" From ba46c09eaddffde400781c6657e4f9403bb560e6 Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Thu, 30 Mar 2023 22:05:56 +0300 Subject: [PATCH 036/121] Minor improvements, fixes on failing tests --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 33c318c..79fe611 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" readme = "../../README.md" license = "MIT" -description = "SQL Storage for Apalis: simple, extensible multithreaded background job processing for Rust" +description = "SQL Storage for apalis: simple, extensible multithreaded background job processing for Rust" [features] default = [ "sqlite", "migrate", "tokio-comp" ] From 4a4505cd674a7ab7f446714386dcab1fb2c2b128 Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Sat, 1 Apr 2023 17:55:38 +0300 Subject: [PATCH 037/121] Improved layers and added beats --- packages/apalis-sql/Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 79fe611..7a7a724 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -9,13 +9,13 @@ license = "MIT" description = "SQL Storage for apalis: simple, extensible multithreaded background job processing for Rust" [features] -default = [ "sqlite", "migrate", "tokio-comp" ] +default = [ "sqlite", "migrate" ] postgres = [ "sqlx/postgres", "sqlx/json", "sqlx/chrono"] sqlite = [ "sqlx/sqlite", "sqlx/json", "sqlx/chrono"] mysql = [ "sqlx/mysql", "sqlx/json", "sqlx/chrono", "sqlx/bigdecimal"] migrate = ["sqlx/migrate", "sqlx/macros"] -async-std-comp = ["async-std", "sqlx/runtime-async-std-rustls"] -tokio-comp = ["tokio", "sqlx/runtime-tokio-rustls"] +async-std-comp = ["async-std", "apalis-core/async-std-comp", "sqlx/runtime-async-std-rustls"] +tokio-comp = ["tokio", "apalis-core/tokio-comp", "sqlx/runtime-tokio-rustls"] [dependencies.sqlx] version = "0.6" From 0a08d0420bcb348d7f8eab56f167115d7a13e7af Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Fri, 7 Apr 2023 16:31:19 +0300 Subject: [PATCH 038/121] Added expose feature, and worker config --- packages/apalis-sql/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 7a7a724..93fdcad 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -9,13 +9,14 @@ license = "MIT" description = "SQL Storage for apalis: simple, extensible multithreaded background job processing for Rust" [features] -default = [ "sqlite", "migrate" ] +default = [ "sqlite", "migrate"] postgres = [ "sqlx/postgres", "sqlx/json", "sqlx/chrono"] sqlite = [ "sqlx/sqlite", "sqlx/json", "sqlx/chrono"] mysql = [ "sqlx/mysql", "sqlx/json", "sqlx/chrono", "sqlx/bigdecimal"] migrate = ["sqlx/migrate", "sqlx/macros"] async-std-comp = ["async-std", "apalis-core/async-std-comp", "sqlx/runtime-async-std-rustls"] tokio-comp = ["tokio", "apalis-core/tokio-comp", "sqlx/runtime-tokio-rustls"] +expose = ["apalis-core/expose"] [dependencies.sqlx] version = "0.6" From e09a33551956a6c4514844911b0b7bb28c486d5a Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Sun, 9 Apr 2023 02:20:49 +0300 Subject: [PATCH 039/121] Bump up version --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 93fdcad..1eb3527 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.0-alpha.4" +version = "0.4.0-alpha.5" authors = ["Geoffrey Mureithi "] edition = "2021" readme = "../../README.md" @@ -25,7 +25,7 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.4", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.5", features= ["storage"], default-features = false} chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } futures = "0.3" async-stream = "0.3" From 77070c259dadd0eb64e9c301be85420e0c12ce4e Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Sun, 9 Apr 2023 09:57:54 +0300 Subject: [PATCH 040/121] Minor updates on cargo.toml --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 1eb3527..fb7e701 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -9,7 +9,7 @@ license = "MIT" description = "SQL Storage for apalis: simple, extensible multithreaded background job processing for Rust" [features] -default = [ "sqlite", "migrate"] +default = [ "sqlite", "migrate", "tokio-comp"] postgres = [ "sqlx/postgres", "sqlx/json", "sqlx/chrono"] sqlite = [ "sqlx/sqlite", "sqlx/json", "sqlx/chrono"] mysql = [ "sqlx/mysql", "sqlx/json", "sqlx/chrono", "sqlx/bigdecimal"] From 1ce06a01a71ac5db766ac848bff322345d03b043 Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Sun, 9 Apr 2023 19:44:38 +0300 Subject: [PATCH 041/121] Bump up version --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index fb7e701..985b3d3 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.0-alpha.5" +version = "0.4.0-alpha.6" authors = ["Geoffrey Mureithi "] edition = "2021" readme = "../../README.md" @@ -25,7 +25,7 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.5", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.6", features= ["storage"], default-features = false} chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } futures = "0.3" async-stream = "0.3" From 08b8c81df1cf580dca9e7c505fcb67bf7b2eec13 Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Wed, 12 Apr 2023 16:16:41 +0300 Subject: [PATCH 042/121] Bump version --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 985b3d3..562325e 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.0-alpha.6" +version = "0.4.0-alpha.7" authors = ["Geoffrey Mureithi "] edition = "2021" readme = "../../README.md" @@ -25,7 +25,7 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.6", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.7", features= ["storage"], default-features = false} chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } futures = "0.3" async-stream = "0.3" From ba7cece92e41ee79561540f96f4484fb13b35b60 Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Wed, 12 Apr 2023 17:25:49 +0300 Subject: [PATCH 043/121] Fix Ack and bump up version --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 562325e..ac341e9 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.0-alpha.7" +version = "0.4.0-alpha.8" authors = ["Geoffrey Mureithi "] edition = "2021" readme = "../../README.md" @@ -25,7 +25,7 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.7", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.8", features= ["storage"], default-features = false} chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } futures = "0.3" async-stream = "0.3" From c6643541b9690b5559da36ec983cb04a139a23f9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 13 Apr 2023 05:18:25 +0000 Subject: [PATCH 044/121] Update Rust crate async-trait to 0.1.68 --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 985b3d3..bdaf076 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -32,7 +32,7 @@ async-stream = "0.3" tokio = { version = "1", features = ["rt", "net"], optional = true } async-std = { version = "1.8.0", optional = true} futures-lite = "1.12.0" -async-trait = "0.1.53" +async-trait = "0.1.68" [dev-dependencies] tokio = { version = "1", features = ["macros"] } From c2c7b24bcc9e7eae581d73689ed3367ff2a41d0e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 13 Apr 2023 07:47:55 +0000 Subject: [PATCH 045/121] Update Rust crate async-std to 1.12.0 --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 985b3d3..b9ce6b3 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -30,7 +30,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock", "serd futures = "0.3" async-stream = "0.3" tokio = { version = "1", features = ["rt", "net"], optional = true } -async-std = { version = "1.8.0", optional = true} +async-std = { version = "1.12.0", optional = true} futures-lite = "1.12.0" async-trait = "0.1.53" From 36da067b92e8b11ed7727db376e93bec6cb17edd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 13 Apr 2023 11:23:56 +0000 Subject: [PATCH 046/121] Update Rust crate futures-lite to 1.13.0 --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 985b3d3..7360745 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -31,7 +31,7 @@ futures = "0.3" async-stream = "0.3" tokio = { version = "1", features = ["rt", "net"], optional = true } async-std = { version = "1.8.0", optional = true} -futures-lite = "1.12.0" +futures-lite = "1.13.0" async-trait = "0.1.53" [dev-dependencies] From f3c9183eaf9a7eed5eadf3975c3965849dd64370 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 13 Apr 2023 16:26:08 +0000 Subject: [PATCH 047/121] Update Rust crate once_cell to 1.17.1 --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 985b3d3..28d1a18 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -37,7 +37,7 @@ async-trait = "0.1.53" [dev-dependencies] tokio = { version = "1", features = ["macros"] } email-service = { path = "../../examples/email-service"} -once_cell = "1.14.0" +once_cell = "1.17.1" [package.metadata.docs.rs] # defines the configuration attribute `docsrs` From 5f2491f2c89d08023224ce5ee25659b91aae6adf Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Tue, 2 May 2023 09:01:54 +0300 Subject: [PATCH 048/121] Bump up version --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index ac341e9..77e2e5f 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.0-alpha.8" +version = "0.4.0-alpha.9" authors = ["Geoffrey Mureithi "] edition = "2021" readme = "../../README.md" @@ -25,7 +25,7 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.8", features= ["storage"], default-features = false} +apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.9", features= ["storage"], default-features = false} chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } futures = "0.3" async-stream = "0.3" From aeeeb246c50cac10a5ae4c20b92117e405418ae3 Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Tue, 16 May 2023 05:29:46 +0300 Subject: [PATCH 049/121] Added improvements and simplified --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 156043c..f81fa1e 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -9,7 +9,7 @@ license = "MIT" description = "SQL Storage for apalis: simple, extensible multithreaded background job processing for Rust" [features] -default = [ "sqlite", "migrate", "tokio-comp"] +default = [ "sqlite", "migrate"] postgres = [ "sqlx/postgres", "sqlx/json", "sqlx/chrono"] sqlite = [ "sqlx/sqlite", "sqlx/json", "sqlx/chrono"] mysql = [ "sqlx/mysql", "sqlx/json", "sqlx/chrono", "sqlx/bigdecimal"] From 428fccd979b9555224dbcbac21579bf98ca7696d Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Tue, 16 May 2023 08:46:37 +0300 Subject: [PATCH 050/121] Prep release version 0.4.0-alpha.9 --- packages/apalis-sql/Cargo.toml | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index f81fa1e..f3b164b 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,20 +1,24 @@ [package] name = "apalis-sql" version = "0.4.0-alpha.9" -authors = ["Geoffrey Mureithi "] +authors = ["Njuguna Mureithi "] edition = "2021" readme = "../../README.md" license = "MIT" -description = "SQL Storage for apalis: simple, extensible multithreaded background job processing for Rust" +description = "SQL Storage for apalis. Use sqlite, postgres and mysql for background job processing" [features] -default = [ "sqlite", "migrate"] -postgres = [ "sqlx/postgres", "sqlx/json", "sqlx/chrono"] -sqlite = [ "sqlx/sqlite", "sqlx/json", "sqlx/chrono"] -mysql = [ "sqlx/mysql", "sqlx/json", "sqlx/chrono", "sqlx/bigdecimal"] +default = ["sqlite", "migrate"] +postgres = ["sqlx/postgres", "sqlx/json", "sqlx/chrono"] +sqlite = ["sqlx/sqlite", "sqlx/json", "sqlx/chrono"] +mysql = ["sqlx/mysql", "sqlx/json", "sqlx/chrono", "sqlx/bigdecimal"] migrate = ["sqlx/migrate", "sqlx/macros"] -async-std-comp = ["async-std", "apalis-core/async-std-comp", "sqlx/runtime-async-std-rustls"] +async-std-comp = [ + "async-std", + "apalis-core/async-std-comp", + "sqlx/runtime-async-std-rustls", +] tokio-comp = ["tokio", "apalis-core/tokio-comp", "sqlx/runtime-tokio-rustls"] expose = ["apalis-core/expose"] @@ -25,21 +29,26 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.9", features= ["storage"], default-features = false} -chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } +apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.9", features = [ + "storage", +], default-features = false } +chrono = { version = "0.4", default-features = false, features = [ + "clock", + "serde", +] } futures = "0.3" async-stream = "0.3" tokio = { version = "1", features = ["rt", "net"], optional = true } futures-lite = "1.13.0" -async-std = { version = "1.12.0", optional = true} +async-std = { version = "1.12.0", optional = true } async-trait = "0.1.68" [dev-dependencies] tokio = { version = "1", features = ["macros"] } -email-service = { path = "../../examples/email-service"} +email-service = { path = "../../examples/email-service" } once_cell = "1.17.1" [package.metadata.docs.rs] # defines the configuration attribute `docsrs` rustdoc-args = ["--cfg", "docsrs"] -all-features = true +features = ["postgres", "mysql", "sqlite", "expose", "migrate", "tokio-comp"] From 5ef6bf6c409754082f15d3c7d75a43c4ff8d665b Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Wed, 24 May 2023 00:25:59 +0300 Subject: [PATCH 051/121] Bump up version --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index f3b164b..4fa38bb 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.0-alpha.9" +version = "0.4.0" authors = ["Njuguna Mureithi "] edition = "2021" readme = "../../README.md" @@ -29,7 +29,7 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.0-alpha.9", features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.4.0", features = [ "storage", ], default-features = false } chrono = { version = "0.4", default-features = false, features = [ From 60ab9553bd4404f34493ee97012b973789b14192 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 29 May 2023 10:48:13 +0000 Subject: [PATCH 052/121] Update Rust crate once_cell to 1.17.2 --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 4fa38bb..4004176 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -46,7 +46,7 @@ async-trait = "0.1.68" [dev-dependencies] tokio = { version = "1", features = ["macros"] } email-service = { path = "../../examples/email-service" } -once_cell = "1.17.1" +once_cell = "1.17.2" [package.metadata.docs.rs] # defines the configuration attribute `docsrs` From ae3734c85bf2d2138dbdaa8433352c36f7e50155 Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Fri, 2 Jun 2023 19:35:52 +0300 Subject: [PATCH 053/121] Bump up version --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 4004176..0473458 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.0" +version = "0.4.1" authors = ["Njuguna Mureithi "] edition = "2021" readme = "../../README.md" @@ -29,7 +29,7 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.0", features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.4.1", features = [ "storage", ], default-features = false } chrono = { version = "0.4", default-features = false, features = [ From 273ad383bdf8147f640044b135ad3b1452123de7 Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Sun, 4 Jun 2023 13:06:46 +0300 Subject: [PATCH 054/121] Added improvements to postgres storage --- packages/apalis-sql/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 0473458..7e4a61e 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -42,6 +42,7 @@ tokio = { version = "1", features = ["rt", "net"], optional = true } futures-lite = "1.13.0" async-std = { version = "1.12.0", optional = true } async-trait = "0.1.68" +debounced = "0.1.0" [dev-dependencies] tokio = { version = "1", features = ["macros"] } From cf0129072e6af23c8cdc2ed6171db584560b1013 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 4 Jun 2023 14:08:56 +0000 Subject: [PATCH 055/121] Update Rust crate once_cell to 1.18.0 --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 0473458..c68930f 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -46,7 +46,7 @@ async-trait = "0.1.68" [dev-dependencies] tokio = { version = "1", features = ["macros"] } email-service = { path = "../../examples/email-service" } -once_cell = "1.17.2" +once_cell = "1.18.0" [package.metadata.docs.rs] # defines the configuration attribute `docsrs` From f8428ee77e3386401f9c8aef646e6d8c08f10801 Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Fri, 9 Jun 2023 11:15:07 +0300 Subject: [PATCH 056/121] Bump up version to v0.4.2 --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 262dbfc..297aaf1 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.1" +version = "0.4.2" authors = ["Njuguna Mureithi "] edition = "2021" readme = "../../README.md" @@ -29,7 +29,7 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.1", features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.4.2", features = [ "storage", ], default-features = false } chrono = { version = "0.4", default-features = false, features = [ From 086b70117429e3f2f9cb9cc1cf0d0a3326df979e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 08:15:08 +0000 Subject: [PATCH 057/121] Update Rust crate async-trait to 0.1.72 --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 297aaf1..c8727db 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -41,7 +41,7 @@ async-stream = "0.3" tokio = { version = "1", features = ["rt", "net"], optional = true } futures-lite = "1.13.0" async-std = { version = "1.12.0", optional = true } -async-trait = "0.1.68" +async-trait = "0.1.72" debounced = "0.1.0" [dev-dependencies] From 2f05d6f5bbf9a9e8a4672e02e2b9641a9604bfa3 Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Sat, 22 Jul 2023 04:27:07 +0300 Subject: [PATCH 058/121] Bump up version --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index c8727db..7927cd5 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.2" +version = "0.4.3" authors = ["Njuguna Mureithi "] edition = "2021" readme = "../../README.md" @@ -29,7 +29,7 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.2", features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.4.3", features = [ "storage", ], default-features = false } chrono = { version = "0.4", default-features = false, features = [ From c229d77639d1a0b8a46b4b65c57b0538726f9ee7 Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Fri, 28 Jul 2023 17:20:22 +0300 Subject: [PATCH 059/121] update: sqlx to v0.7 --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 7927cd5..95e67a5 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -23,7 +23,7 @@ tokio-comp = ["tokio", "apalis-core/tokio-comp", "sqlx/runtime-tokio-rustls"] expose = ["apalis-core/expose"] [dependencies.sqlx] -version = "0.6" +version = "0.7.0" default-features = false [dependencies] From 5d91d5bbaea27fc3e6ed64645ea00bea7f9a230b Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Mon, 31 Jul 2023 13:41:31 +0300 Subject: [PATCH 060/121] bump: to version v0.4.4 --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 95e67a5..2d5fa8c 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.3" +version = "0.4.4" authors = ["Njuguna Mureithi "] edition = "2021" readme = "../../README.md" @@ -29,7 +29,7 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.3", features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.4.4", features = [ "storage", ], default-features = false } chrono = { version = "0.4", default-features = false, features = [ From 765e85ec3d11f3ea36595092ddf27908ac4a90a0 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 12 Aug 2023 04:37:02 +0000 Subject: [PATCH 061/121] fix(deps): update rust crate async-trait to 0.1.73 --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 2d5fa8c..e234ad3 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -41,7 +41,7 @@ async-stream = "0.3" tokio = { version = "1", features = ["rt", "net"], optional = true } futures-lite = "1.13.0" async-std = { version = "1.12.0", optional = true } -async-trait = "0.1.72" +async-trait = "0.1.73" debounced = "0.1.0" [dev-dependencies] From bad39878a13aa40c0a15842b80d3467d6727b3ae Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 27 Sep 2023 01:00:46 +0000 Subject: [PATCH 062/121] fix(deps): update rust crate sqlx to 0.7.2 --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index e234ad3..8356a76 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -23,7 +23,7 @@ tokio-comp = ["tokio", "apalis-core/tokio-comp", "sqlx/runtime-tokio-rustls"] expose = ["apalis-core/expose"] [dependencies.sqlx] -version = "0.7.0" +version = "0.7.2" default-features = false [dependencies] From 45d4dcbd30942e2a5af5b2a3ab9c76a043cb8750 Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi Date: Wed, 11 Oct 2023 08:47:30 +0300 Subject: [PATCH 063/121] bump: v0.4.5 --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 8356a76..21440c8 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.4" +version = "0.4.5" authors = ["Njuguna Mureithi "] edition = "2021" readme = "../../README.md" @@ -29,7 +29,7 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.4", features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.4.5", features = [ "storage", ], default-features = false } chrono = { version = "0.4", default-features = false, features = [ From ad48d93af11761d11a376ae93825004a3177984f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 15 Oct 2023 07:44:33 +0000 Subject: [PATCH 064/121] fix(deps): update rust crate async-trait to 0.1.74 --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 21440c8..9454245 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -41,7 +41,7 @@ async-stream = "0.3" tokio = { version = "1", features = ["rt", "net"], optional = true } futures-lite = "1.13.0" async-std = { version = "1.12.0", optional = true } -async-trait = "0.1.73" +async-trait = "0.1.74" debounced = "0.1.0" [dev-dependencies] From 229798aa17418d7471c0d9f3901cbdef79826324 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 10:56:58 +0300 Subject: [PATCH 065/121] fix(deps): update rust crate futures-lite to v2 (#192) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 9454245..656d82b 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -39,7 +39,7 @@ chrono = { version = "0.4", default-features = false, features = [ futures = "0.3" async-stream = "0.3" tokio = { version = "1", features = ["rt", "net"], optional = true } -futures-lite = "1.13.0" +futures-lite = "2.0.1" async-std = { version = "1.12.0", optional = true } async-trait = "0.1.74" debounced = "0.1.0" From bc545b409ca624f32a6d3099f82ee07a71a2cd18 Mon Sep 17 00:00:00 2001 From: Vlad Stepanov <8uk.8ak@gmail.com> Date: Thu, 9 Nov 2023 10:12:21 +0400 Subject: [PATCH 066/121] Add option to use `time` crate instead of `chrono` (#185) * Add option to use `time` crate instead of `chrono` This is useful for users who are already working with `sqlx` and `time`, bc `sqlx` doesn't allow you to use `chrono` and `time` together. * leaves the default as `chrono` but allows users to opt-in to using `time` instead * adds a `time` feature to `apalis-core`, `apalis-sql` and `apalis-redis` * enabling both `cron` and `time` features now triggers a compile error because `apalis-cron` uses `cron` and `cron` depends on `chrono` * adds `time` to the CI matrix * (subjective) use `datetime - duration` instead of `datetime.sub(duration)` * [fix] include CI changes * [fix] use correct timestamping method in `apalis-redis` * fix tests * rustfmt --- packages/apalis-sql/Cargo.toml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 656d82b..8ef5e42 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -9,10 +9,12 @@ license = "MIT" description = "SQL Storage for apalis. Use sqlite, postgres and mysql for background job processing" [features] -default = ["sqlite", "migrate"] -postgres = ["sqlx/postgres", "sqlx/json", "sqlx/chrono"] -sqlite = ["sqlx/sqlite", "sqlx/json", "sqlx/chrono"] -mysql = ["sqlx/mysql", "sqlx/json", "sqlx/chrono", "sqlx/bigdecimal"] +default = ["sqlite", "migrate", "chrono"] +chrono = ["sqlx/chrono", "dep:chrono", "apalis-core/chrono"] +time = ["sqlx/time", "dep:time", "apalis-core/time"] +postgres = ["sqlx/postgres", "sqlx/json"] +sqlite = ["sqlx/sqlite", "sqlx/json"] +mysql = ["sqlx/mysql", "sqlx/json", "sqlx/bigdecimal"] migrate = ["sqlx/migrate", "sqlx/macros"] async-std-comp = [ "async-std", @@ -31,11 +33,15 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" apalis-core = { path = "../../packages/apalis-core", version = "0.4.5", features = [ "storage", + "tokio-comp", ], default-features = false } -chrono = { version = "0.4", default-features = false, features = [ +chrono = { version = "0.4", default-features = false, optional = true, features = [ "clock", "serde", ] } +time = { version = "0.3", optional = true, features = [ + "serde", +] } futures = "0.3" async-stream = "0.3" tokio = { version = "1", features = ["rt", "net"], optional = true } From 1f3eeba29f84f9d5c4d1542e815037e45f2e2d01 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Thu, 9 Nov 2023 09:58:57 +0300 Subject: [PATCH 067/121] Bump to version 0.4.6 (#197) * docs: minor updates * fix: update to latest version of cargo-get * bump: up to version 0.4.6 --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 8ef5e42..6f1ce7a 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.5" +version = "0.4.6" authors = ["Njuguna Mureithi "] edition = "2021" readme = "../../README.md" @@ -31,7 +31,7 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.5", features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.4.6", features = [ "storage", "tokio-comp", ], default-features = false } From c851ebc35cf18c872de1f216f6ad124a2ea22afd Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Sat, 18 Nov 2023 10:38:05 +0300 Subject: [PATCH 068/121] Bump to v0.4.7 (#210) * fix(deps): update rust crate http to v1 * fix: update for http 1.x * bump: to v0.4.7 --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 6f1ce7a..6159278 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.6" +version = "0.4.7" authors = ["Njuguna Mureithi "] edition = "2021" readme = "../../README.md" @@ -31,7 +31,7 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.6", features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.4.7", features = [ "storage", "tokio-comp", ], default-features = false } From 34864f86b8af4fdef6ba569fde96054bca27589c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 23 Nov 2023 21:10:38 +0300 Subject: [PATCH 069/121] fix(deps): update rust crate sqlx to 0.7.3 (#212) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 6159278..284b6ba 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -25,7 +25,7 @@ tokio-comp = ["tokio", "apalis-core/tokio-comp", "sqlx/runtime-tokio-rustls"] expose = ["apalis-core/expose"] [dependencies.sqlx] -version = "0.7.2" +version = "0.7.3" default-features = false [dependencies] From 77333e7a4229e35434dc879a58e270aca776f8c0 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 15 Dec 2023 16:12:21 +0300 Subject: [PATCH 070/121] fix(deps): update rust crate futures-lite to 2.1.0 (#214) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 284b6ba..53fe311 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -45,7 +45,7 @@ time = { version = "0.3", optional = true, features = [ futures = "0.3" async-stream = "0.3" tokio = { version = "1", features = ["rt", "net"], optional = true } -futures-lite = "2.0.1" +futures-lite = "2.1.0" async-std = { version = "1.12.0", optional = true } async-trait = "0.1.74" debounced = "0.1.0" From 5400d96ff3031f6b5f0a692fb2d257835cde51d9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 15 Dec 2023 16:13:37 +0300 Subject: [PATCH 071/121] chore(deps): update rust crate once_cell to 1.19.0 (#217) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 53fe311..ebdf9ff 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -53,7 +53,7 @@ debounced = "0.1.0" [dev-dependencies] tokio = { version = "1", features = ["macros"] } email-service = { path = "../../examples/email-service" } -once_cell = "1.18.0" +once_cell = "1.19.0" [package.metadata.docs.rs] # defines the configuration attribute `docsrs` From 1be21d489ef2d2fc6323e629b909d9ce15193422 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 3 Jan 2024 09:51:26 +0300 Subject: [PATCH 072/121] fix(deps): update rust crate async-trait to 0.1.77 (#225) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index ebdf9ff..da8d169 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -47,7 +47,7 @@ async-stream = "0.3" tokio = { version = "1", features = ["rt", "net"], optional = true } futures-lite = "2.1.0" async-std = { version = "1.12.0", optional = true } -async-trait = "0.1.74" +async-trait = "0.1.77" debounced = "0.1.0" [dev-dependencies] From d1bbf48ecd010418077385e8699f37cf80d99ef7 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Fri, 5 Jan 2024 07:56:33 +0300 Subject: [PATCH 073/121] bump: to v0.4.8 (#229) --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index da8d169..a175bc0 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.7" +version = "0.4.8" authors = ["Njuguna Mureithi "] edition = "2021" readme = "../../README.md" @@ -31,7 +31,7 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.7", features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.4.8", features = [ "storage", "tokio-comp", ], default-features = false } From 2b1eb9dde6a52734785b19f33703db821b1d9345 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 8 Jan 2024 18:54:39 +0300 Subject: [PATCH 074/121] fix(deps): update rust crate futures-lite to 2.2.0 (#230) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index a175bc0..f05420a 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -45,7 +45,7 @@ time = { version = "0.3", optional = true, features = [ futures = "0.3" async-stream = "0.3" tokio = { version = "1", features = ["rt", "net"], optional = true } -futures-lite = "2.1.0" +futures-lite = "2.2.0" async-std = { version = "1.12.0", optional = true } async-trait = "0.1.77" debounced = "0.1.0" From 075128f48990a738ea4c6e4472810aa23c88a367 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Wed, 10 Jan 2024 05:19:54 +0300 Subject: [PATCH 075/121] bump: to v0.4.9 (#232) --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index f05420a..074f7a5 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.8" +version = "0.4.9" authors = ["Njuguna Mureithi "] edition = "2021" readme = "../../README.md" @@ -31,7 +31,7 @@ default-features = false [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.8", features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.4.9", features = [ "storage", "tokio-comp", ], default-features = false } From 4bebae3380edee9379d73f473b531bab1d0b8b72 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Sun, 25 Feb 2024 14:44:32 +0300 Subject: [PATCH 076/121] Introducing v0.5! (#234) * Introducing v0.5! * fix: api and other fixes * Fix: improvements on backends * fix: improve api * fix: remove utils and simplify crates * lint: fmt issues * lint: clippy fixes * fix: moved files * clippy: fixes * tests: updates on tests * fixes: on internal api * ci: update the test features * test: remove some assertions * docs: updated docs for tests * fix: minor clippy and fixes * bench: fixes on bench: * bench: fixes on bench: * bench: ! * bench: ! * fix: version for test * fix: add vacuuming and other apis * fix: improve benches * fix: benches and worker * fix: typo * fix: prepared statement * lint: clippy * fix: redis test * add: use ulid without jid * add: improve the Readme.md * fix: tower link * bump: to 0.5 --- packages/apalis-sql/Cargo.toml | 39 ++++++++++++---------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 074f7a5..39d51b1 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.4.9" +version = "0.5.0" authors = ["Njuguna Mureithi "] edition = "2021" readme = "../../README.md" @@ -9,53 +9,42 @@ license = "MIT" description = "SQL Storage for apalis. Use sqlite, postgres and mysql for background job processing" [features] -default = ["sqlite", "migrate", "chrono"] -chrono = ["sqlx/chrono", "dep:chrono", "apalis-core/chrono"] -time = ["sqlx/time", "dep:time", "apalis-core/time"] +default = ["sqlite", "migrate", "postgres"] postgres = ["sqlx/postgres", "sqlx/json"] sqlite = ["sqlx/sqlite", "sqlx/json"] mysql = ["sqlx/mysql", "sqlx/json", "sqlx/bigdecimal"] migrate = ["sqlx/migrate", "sqlx/macros"] -async-std-comp = [ - "async-std", - "apalis-core/async-std-comp", - "sqlx/runtime-async-std-rustls", -] -tokio-comp = ["tokio", "apalis-core/tokio-comp", "sqlx/runtime-tokio-rustls"] -expose = ["apalis-core/expose"] +async-std-comp = ["async-std", "sqlx/runtime-async-std-rustls"] +tokio-comp = ["tokio", "sqlx/runtime-tokio-rustls"] [dependencies.sqlx] version = "0.7.3" default-features = false +features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.4.9", features = [ - "storage", - "tokio-comp", -], default-features = false } -chrono = { version = "0.4", default-features = false, optional = true, features = [ - "clock", - "serde", -] } -time = { version = "0.3", optional = true, features = [ - "serde", +apalis-core = { path = "../../packages/apalis-core", version = "0.5.0", default-features = false, features = [ + "sleep", + "json", ] } +log = "0.4" futures = "0.3" async-stream = "0.3" tokio = { version = "1", features = ["rt", "net"], optional = true } futures-lite = "2.2.0" async-std = { version = "1.12.0", optional = true } -async-trait = "0.1.77" -debounced = "0.1.0" [dev-dependencies] -tokio = { version = "1", features = ["macros"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } email-service = { path = "../../examples/email-service" } +apalis = { path = "../../", default-features = false, features = [ + "tokio-comp", +] } once_cell = "1.19.0" [package.metadata.docs.rs] # defines the configuration attribute `docsrs` rustdoc-args = ["--cfg", "docsrs"] -features = ["postgres", "mysql", "sqlite", "expose", "migrate", "tokio-comp"] +all-features = true From ab96be53005b0542dea3dcff023bfa33525ba02e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 21:52:38 +0300 Subject: [PATCH 077/121] fix(deps): update rust crate sqlx to 0.7.4 (#262) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 39d51b1..6a927fa 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -18,7 +18,7 @@ async-std-comp = ["async-std", "sqlx/runtime-async-std-rustls"] tokio-comp = ["tokio", "sqlx/runtime-tokio-rustls"] [dependencies.sqlx] -version = "0.7.3" +version = "0.7.4" default-features = false features = ["chrono"] From 3805431908e29dc7f710721317927cfb016afdae Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Wed, 13 Mar 2024 22:25:29 +0300 Subject: [PATCH 078/121] bump: to version 0.5.1 (#265) --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 6a927fa..9c937f3 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.5.0" +version = "0.5.1" authors = ["Njuguna Mureithi "] edition = "2021" readme = "../../README.md" @@ -25,7 +25,7 @@ features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.5.0", default-features = false, features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.5.1", default-features = false, features = [ "sleep", "json", ] } From 3e92f692fd1e96ef50a7c9f4e7c035acec0da45a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 6 Apr 2024 12:18:27 +0300 Subject: [PATCH 079/121] fix(deps): update rust crate futures-lite to 2.3.0 (#276) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 9c937f3..f6e1934 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -33,7 +33,7 @@ log = "0.4" futures = "0.3" async-stream = "0.3" tokio = { version = "1", features = ["rt", "net"], optional = true } -futures-lite = "2.2.0" +futures-lite = "2.3.0" async-std = { version = "1.12.0", optional = true } [dev-dependencies] From 1a619479211f25590671529162d1f451e0fff9d1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 17:16:54 +0300 Subject: [PATCH 080/121] fix(deps): update rust crate futures to 0.3.30 (#307) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index f6e1934..23b09a3 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -30,7 +30,7 @@ apalis-core = { path = "../../packages/apalis-core", version = "0.5.1", default- "json", ] } log = "0.4" -futures = "0.3" +futures = "0.3.30" async-stream = "0.3" tokio = { version = "1", features = ["rt", "net"], optional = true } futures-lite = "2.3.0" From de0de46ca83b2e75905ab70d61c258cbfee289fa Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 18:10:04 +0300 Subject: [PATCH 081/121] fix(deps): update rust crate async-stream to 0.3.5 (#304) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 23b09a3..db64558 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -31,7 +31,7 @@ apalis-core = { path = "../../packages/apalis-core", version = "0.5.1", default- ] } log = "0.4" futures = "0.3.30" -async-stream = "0.3" +async-stream = "0.3.5" tokio = { version = "1", features = ["rt", "net"], optional = true } futures-lite = "2.3.0" async-std = { version = "1.12.0", optional = true } From 33edef51d68d67604b277c1ef72fce6bb9812296 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 3 May 2024 06:19:25 +0300 Subject: [PATCH 082/121] fix(deps): update rust crate log to 0.4.21 (#308) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index db64558..2ebec06 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -29,7 +29,7 @@ apalis-core = { path = "../../packages/apalis-core", version = "0.5.1", default- "sleep", "json", ] } -log = "0.4" +log = "0.4.21" futures = "0.3.30" async-stream = "0.3.5" tokio = { version = "1", features = ["rt", "net"], optional = true } From bc0e9d82d97ab50f1cb24021eaa7e71f5be65562 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Mon, 20 May 2024 12:20:36 +0300 Subject: [PATCH 083/121] bump: to version 0.5.2 (#322) --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 2ebec06..2ec572b 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.5.1" +version = "0.5.2" authors = ["Njuguna Mureithi "] edition = "2021" readme = "../../README.md" @@ -25,7 +25,7 @@ features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.5.1", default-features = false, features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.5.2", default-features = false, features = [ "sleep", "json", ] } From d2931a3fa206c20055482dabd9a465ea6aa7493b Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Mon, 27 May 2024 09:58:05 +0300 Subject: [PATCH 084/121] bump: to version 0.5.3 (#328) --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 2ec572b..f7dec23 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.5.2" +version = "0.5.3" authors = ["Njuguna Mureithi "] edition = "2021" readme = "../../README.md" @@ -25,7 +25,7 @@ features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.5.2", default-features = false, features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.5.3", default-features = false, features = [ "sleep", "json", ] } From 6f9808fede022195df9109b1391760d3ce3b74e5 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Fri, 5 Jul 2024 13:54:49 +0800 Subject: [PATCH 085/121] chore: Add repository to metadata (#345) --- packages/apalis-sql/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index f7dec23..a3f67f3 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -2,7 +2,8 @@ name = "apalis-sql" version = "0.5.3" authors = ["Njuguna Mureithi "] -edition = "2021" +edition.workspace = true +repository.workspace = true readme = "../../README.md" license = "MIT" From a12a8955c5c12e221a098d30b890db300f6aac95 Mon Sep 17 00:00:00 2001 From: Michael Brogdon Date: Mon, 2 Sep 2024 22:55:46 -0600 Subject: [PATCH 086/121] Update sqlx to 0.8.1 (#403) --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index a3f67f3..e6e18b8 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -19,7 +19,7 @@ async-std-comp = ["async-std", "sqlx/runtime-async-std-rustls"] tokio-comp = ["tokio", "sqlx/runtime-tokio-rustls"] [dependencies.sqlx] -version = "0.7.4" +version = "0.8.1" default-features = false features = ["chrono"] From 8a75ad2d004787586e19bcdab21abc8ae9c64e6d Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:09:20 +0300 Subject: [PATCH 087/121] bump: 0.5.4 (#405) --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index e6e18b8..f59e705 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.5.3" +version = "0.5.4" authors = ["Njuguna Mureithi "] edition.workspace = true repository.workspace = true @@ -26,7 +26,7 @@ features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.5.3", default-features = false, features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.5.4", default-features = false, features = [ "sleep", "json", ] } From ccc9debca1af594aa7a5104e7f925230e73bd45c Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Thu, 5 Sep 2024 14:02:28 +0300 Subject: [PATCH 088/121] bump: to 0.5.5 (#408) --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index f59e705..a018d80 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.5.4" +version = "0.5.5" authors = ["Njuguna Mureithi "] edition.workspace = true repository.workspace = true @@ -26,7 +26,7 @@ features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.5.4", default-features = false, features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.5.5", default-features = false, features = [ "sleep", "json", ] } From 2856858ed2556598c5864e4ca31878fcbda20409 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Thu, 28 Nov 2024 09:01:08 +0300 Subject: [PATCH 089/121] [bump] introduce new version: 0.6 (#459) * fix: improve external api for redis * fix: improve exports for redis * fix: expose redis codec * Feature: v0.6.0-alpha.0 version of apalis Breaking Changes: - Dropped traits Job and Message, please use namespace * fix: minor fixes on some failures * lint: cargo fmt * fix: remove Job impl * lint: cargo fmt * bench: improve polling * fix: introduce namespace and codec config (#339) * fix: introduce namespace and codec config * fix: missing apis * Version: 0.6.0-alpha.1 (#342) * api: for redis and sqlite * Version: 0.6.0-alpha.1 Changelog: - Redis storage doesnt require pool to be clone. Allows use of deadpool-redis among others. - Namespace is picked by default for `new` methods. * fix: docs and tests * lint: cargo clippy and fmt * postgres: add a listener example * bump: to v0.6.0-alpha.1 (#343) * api: for redis and sqlite * Version: 0.6.0-alpha.1 Changelog: - Redis storage doesnt require pool to be clone. Allows use of deadpool-redis among others. - Namespace is picked by default for `new` methods. * fix: docs and tests * lint: cargo clippy and fmt * postgres: add a listener example * bump: to v0.6.0-alpha.1 * fix: allow cd for prereleases (#349) * Remove `Clone` constraints and buffer the service (#348) * feat: remove the `Clone` requirements for services * test save * fix: get buffered layer working * update: remove clone & update api * fix: tests and api * lint: clippy fixes * lint: cargo fmt * bump: to 0.6.0-rc.1 (#350) * feat: add rsmq example (#353) * Fix: load layer from poller (#354) * fix: backend layers were not loaded * fix: handle clone * Fix: mq example (#355) * fix: mq ack * lint: fmt * fix: handle unwraps in storages (#356) * fix: handle unwraps in storages * fix: ensure no unwrap * fix: better apalis deps allowing tree shaking for backends (#357) * fix: better apalis deps allowing tree shaking for backends * fix: remove backend features in the root crate * standardize backend for storage and mq (#358) * fix: standardize backend for storage and mq * fix: minor fixes * feat: standardize cron as backend (#359) * fix: remove non-working restapi example (#360) * fix: expose the missing apis (#361) * bump: to new version (#362) * Make Config accessible publicly (#364) * fix: add missing exposed config * fix: add getters * fix: die if retries is zero (#365) * Feature: Add a layer that catches panics (#366) * Feature: Add a layer that catches panics This allows preventing job execution from killing workers and returns an error containing the backtrace * fix: backtrace as it may be different * add: example for catch-panic * fix: make not default * Feature: Save results for storages (#369) * Feature: Save results for storages Currently just the status is stored, this PR adds the ability to save the result * fix: result from storage * fix: kill and abort issue * Bump: to 0.6.0-rc.3 (#370) * fix: serde for sql request (#371) * fix: serde for sql request * fix: serde for attempts * lint: fmt * fix: handle attempts in storages (#373) * fix: handle attempts in storages * fix: chrono serialization * fix: tests failing because of tests * add: test utils that allow backend polling during tests (#374) * add: test utils that allow backend polling during tests * fix: introduce testwrapper and add more tests * fix: add sample for testing * fix: more fixes and actions fixes * fix: more fixes on vacuuming * tests: improve cleanup and generic testing * fix: improve testing and fix some found bugs * fix: postgres query and remove incompatible tests * fix: remove redis incompatible check * fix: minor fixes * fix: postgres json elements * bump: to 0.6.0-rc.4 (#377) * fix: handle 0 retries (#378) * fix: ack api to allow backend to handle differently (#383) * fix: ack api to allow backend to handle differently * fix: related to storage tests * fix: calculate status for postgres * fix(deps): update rust crate sqlx to 0.8.0 (#380) * chore: fix typos (#346) * chore: Add repository to metadata (#345) * fix(deps): update rust crate sqlx to 0.8.0 * fix: sqlite example --------- Co-authored-by: John Vandenberg Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: geofmureithi Co-authored-by: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> * bump: to v0.6.0-rc.5 (#385) * chore: standardize codec usage (#388) * bump: to v0.6.0-rc.5 * fix: standardize codec usage * lint: cargo fmt * Chore/more examples (#389) * add: catch-panic example * add: graceful shutdown example * add: unmonitored example * add: arguments example * fix: minor updates * fix: sql tests * fix: minor updates * fix: improve on benches (#379) * fix: improve on benches * fix: bench trigger * fix: include tokio for sqlx * fix: improve the benching approach * fix: mysql api * fix: redis api * fix: improve bench approach, remove counter * remove: setup * remove: pg * fix: pg * fix: pg * fix(deps): update rust crate sqlx to 0.8.1 [security] (#400) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> * fix: add some missing data required for dependency injection (#409) * fix: add some missing data required for dependency injection * lint: clippy and fmt * remove: benchmarks (#410) They will be moved to https://github.com/geofmureithi/apalis-benchmarks * bump: to 0.6.0-rc.6 (#412) * Update async-std to 1.13 (#413) * Feature: Introducing Request Context (#416) * wip: introduce context to request * fix: get request context working * lint: cargo fmt * fix: get tests compiling * add: push_request and shedule_request * fix: task_id for Testwrapper * fix: minor checks and fixes on postgres tests * fix: bug on postgres fetch_next * bump: to 0.6.0-rc.7 (#418) * fix: apply `FromRequest` for items in `Parts` (#425) Problem: We are missing crucial `FromRequest` impls for: - TaskId - Attempt - Namespace Also removed `Context` Solution: Implement `FromRequest` for these Types. * fix:[bug] include backend provided layer in service layers. (#426) * fix:[bug] include backend provided layer in service layers. Problem: The current worker logic is missing an implementation where the backend provided layer should be added to the service's layer. This is a critical issue that affects all v0.6.0-rc-7 users and they should update as soon as a new release is done. Solution: - Add backend layers to service's layer. - Add worker_consume tests on the storages to prevent regression on this. * chore: comment an enforcement rule not yet followed by redis * chore: bump to 0.6.0-rc.8 (#430) * fix: apply max_attempts set via SqlContext (#447) So that a custom number of attempts can be configured: let mut ctx = SqlContext::new(); ctx.set_max_attempts(2); let req = Request::new_with_ctx(job, ctx); storage.push_request(req).await.unwrap(); While the default is still to try up to 25 times: storage.push(job).await.unwrap(); * Bump redis (#442) * feat: re-export sqlx (#451) Making sqlx accessible to users of apalis without requiring them to explicitly add it as a dependency. * feat: Improve Worker management and drop Executor (#428) * feat: introducing WorkerBuilderExt which makes the work of building a new worker way easier. * improve: worker api almost there * fix: radical improvements and updates. Removed executor and got graceful shutdown working * chore: deprecate register with count and force builder order * chore: more improvements on the worker * fix: allow DI for Worker * add: get the task count by a worker * lint: fmt and clippy * fix: allow worker stopping * Chore/better api (#452) * fix: relax the api provided for sqlx req * lint: clippy and fmt * feat: add recovery of abandoned jobs to backend heartbeats (#453) * feat: add recovery of abandoned jobs to backend heartbeats * lint: fmt * fix: attempt to get tests passing * fix: attempt to get tests passing * fix: minor fix typo * fix: minor different solutions * fix: better handle attempts * handle postgres edge case * fix: better handling * feat: allow backends to emit errors (#454) * feat: allow backends to emit errors * lint: fmt * fix: pass in a reference to prevent mutation * Feat: Introduce simple ability to pipe cron jobs to any backend (#455) * Feat: Introduce simple ability to pipe cron jobs to any backend This feature allows you to quickly persist cron jobs guaranteeing they will be run and can be distributed * lint: cargo fmt * chore/dev-branch * feat: return of exposing backends to help in building apis (#457) (#458) * fmt: Cargo.toml * fix: removed features * fix: run only specific tests --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: John Vandenberg Co-authored-by: Mathias Lafeldt Co-authored-by: zakstucke <44890343+zakstucke@users.noreply.github.com> --- packages/apalis-sql/Cargo.toml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index a018d80..6b78766 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.5.5" +version = "0.6.0" authors = ["Njuguna Mureithi "] edition.workspace = true repository.workspace = true @@ -10,7 +10,7 @@ license = "MIT" description = "SQL Storage for apalis. Use sqlite, postgres and mysql for background job processing" [features] -default = ["sqlite", "migrate", "postgres"] +default = ["migrate"] postgres = ["sqlx/postgres", "sqlx/json"] sqlite = ["sqlx/sqlite", "sqlx/json"] mysql = ["sqlx/mysql", "sqlx/json", "sqlx/bigdecimal"] @@ -26,7 +26,7 @@ features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.5.5", default-features = false, features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.6.0", default-features = false, features = [ "sleep", "json", ] } @@ -35,15 +35,18 @@ futures = "0.3.30" async-stream = "0.3.5" tokio = { version = "1", features = ["rt", "net"], optional = true } futures-lite = "2.3.0" -async-std = { version = "1.12.0", optional = true } +async-std = { version = "1.13.0", optional = true } +chrono = { version = "0.4", features = ["serde"] } +thiserror = "2.0.0" + [dev-dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } email-service = { path = "../../examples/email-service" } -apalis = { path = "../../", default-features = false, features = [ - "tokio-comp", -] } +apalis = { path = "../../", default-features = false } once_cell = "1.19.0" +apalis-sql = { path = ".", features = ["tokio-comp"] } +apalis-core = { path = "../apalis-core", features = ["test-utils"] } [package.metadata.docs.rs] # defines the configuration attribute `docsrs` From 4de149a24ffab70d9fc055da8329fc0d5aa1b7b4 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Tue, 10 Dec 2024 07:21:04 +0300 Subject: [PATCH 090/121] bump: to v0.6.1 (#479) --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 6b78766..d8ae2cb 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.6.0" +version = "0.6.1" authors = ["Njuguna Mureithi "] edition.workspace = true repository.workspace = true @@ -26,7 +26,7 @@ features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.6.0", default-features = false, features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.6.1", default-features = false, features = [ "sleep", "json", ] } From a096d12ad9dc5ae06583dc800c030b1005e17dbf Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Sun, 15 Dec 2024 07:32:21 +0300 Subject: [PATCH 091/121] bump: to v0.6.2 (#482) --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index d8ae2cb..040b086 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.6.1" +version = "0.6.2" authors = ["Njuguna Mureithi "] edition.workspace = true repository.workspace = true @@ -26,7 +26,7 @@ features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.6.1", default-features = false, features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.6.2", default-features = false, features = [ "sleep", "json", ] } From 871194f81c78a10dbee55d1bc1619c0800e16bb4 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Sun, 5 Jan 2025 08:13:53 +0300 Subject: [PATCH 092/121] bump: to v0.6.3 (#490) --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 040b086..db85d8f 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.6.2" +version = "0.6.3" authors = ["Njuguna Mureithi "] edition.workspace = true repository.workspace = true @@ -26,7 +26,7 @@ features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.6.2", default-features = false, features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.6.3", default-features = false, features = [ "sleep", "json", ] } From 469169922bcfd8069ce12cb8f251cc20e1fd9180 Mon Sep 17 00:00:00 2001 From: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Wed, 15 Jan 2025 13:03:36 +0300 Subject: [PATCH 093/121] bump: to v0.6.4 (#500) --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index db85d8f..f4e3679 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.6.3" +version = "0.6.4" authors = ["Njuguna Mureithi "] edition.workspace = true repository.workspace = true @@ -26,7 +26,7 @@ features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.6.3", default-features = false, features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.6.4", default-features = false, features = [ "sleep", "json", ] } From 7e661c91012f27c4cdcf3b190831c2fd2a0aed4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABlle=20Huisman?= Date: Mon, 3 Mar 2025 12:54:29 +0100 Subject: [PATCH 094/121] feat: add `native-tls` feature flags (#525) Co-authored-by: Geoffrey Mureithi <95377562+geofmureithi@users.noreply.github.com> --- packages/apalis-sql/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index f4e3679..70dd519 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -16,7 +16,9 @@ sqlite = ["sqlx/sqlite", "sqlx/json"] mysql = ["sqlx/mysql", "sqlx/json", "sqlx/bigdecimal"] migrate = ["sqlx/migrate", "sqlx/macros"] async-std-comp = ["async-std", "sqlx/runtime-async-std-rustls"] +async-std-comp-native-tls = ["async-std", "sqlx/runtime-async-std-native-tls"] tokio-comp = ["tokio", "sqlx/runtime-tokio-rustls"] +tokio-comp-native-tls = ["tokio", "sqlx/runtime-tokio-native-tls"] [dependencies.sqlx] version = "0.8.1" From 2a26293671d3a5bca40db41d403161ecdf46c0aa Mon Sep 17 00:00:00 2001 From: Derek Leverenz Date: Mon, 17 Mar 2025 13:24:50 -0700 Subject: [PATCH 095/121] Support priority in SQL storage backends (#533) * Add support for optional job priority to postgres storage * add migration * revert postgres changes * update mysql and postgres code to use sqlcontext priority * WIP * finish (hopefully) mysql and sqlite job fetch * add priority tests * update changelog * Update CHANGELOG.md * test * cleanup and add missing priority in FromRow impls * fix mysql implementation --- .../migrations/sqlite/20250313213411_add_job_priority.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 packages/apalis-sql/migrations/sqlite/20250313213411_add_job_priority.sql diff --git a/packages/apalis-sql/migrations/sqlite/20250313213411_add_job_priority.sql b/packages/apalis-sql/migrations/sqlite/20250313213411_add_job_priority.sql new file mode 100644 index 0000000..030fda7 --- /dev/null +++ b/packages/apalis-sql/migrations/sqlite/20250313213411_add_job_priority.sql @@ -0,0 +1,2 @@ +ALTER TABLE Jobs +ADD priority INTEGER NOT NULL DEFAULT 0; From d1e53fe129f05556fc90d45e0e59a9d3e1a460d1 Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Mon, 24 Mar 2025 04:42:26 +0300 Subject: [PATCH 096/121] bump: to 0.7.0 (#536) --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 70dd519..bced8f0 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.6.4" +version = "0.7.0" authors = ["Njuguna Mureithi "] edition.workspace = true repository.workspace = true @@ -28,7 +28,7 @@ features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.6.4", default-features = false, features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.7.0", default-features = false, features = [ "sleep", "json", ] } From d0e46bdca6d5fbe82ad795c4916cdc1e01eef10b Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Thu, 24 Apr 2025 09:45:17 +0300 Subject: [PATCH 097/121] bump: to v0.7.1 (#559) * changelog: bump to v0.7.1 * bump: to v0.7.1 --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index bced8f0..59745d9 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.7.0" +version = "0.7.1" authors = ["Njuguna Mureithi "] edition.workspace = true repository.workspace = true @@ -28,7 +28,7 @@ features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.7.0", default-features = false, features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.7.1", default-features = false, features = [ "sleep", "json", ] } From 125ffe79f2bc0a76102c24b8f077f36fac501890 Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Tue, 27 May 2025 15:14:33 +0300 Subject: [PATCH 098/121] bump: apalis to v0.7.2 (#580) --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 59745d9..cfa6473 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.7.1" +version = "0.7.2" authors = ["Njuguna Mureithi "] edition.workspace = true repository.workspace = true @@ -28,7 +28,7 @@ features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.7.1", default-features = false, features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.7.2", default-features = false, features = [ "sleep", "json", ] } From b06738c1bdd85f0c7a56b5a2cd28be877f6b3f84 Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Mon, 28 Jul 2025 12:30:12 +0300 Subject: [PATCH 099/121] more: fixes on updates --- packages/apalis-sql/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 59745d9..8809dc2 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -40,7 +40,7 @@ futures-lite = "2.3.0" async-std = { version = "1.13.0", optional = true } chrono = { version = "0.4", features = ["serde"] } thiserror = "2.0.0" - +pin-project-lite = "0.2.14" [dev-dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } From b1dd7cd1e147898cd776c2f65df0f90b698b02f8 Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Sat, 9 Aug 2025 23:58:32 +0300 Subject: [PATCH 100/121] fix: more improvements --- packages/apalis-sql/Cargo.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 8809dc2..bfbf85e 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -10,7 +10,7 @@ license = "MIT" description = "SQL Storage for apalis. Use sqlite, postgres and mysql for background job processing" [features] -default = ["migrate"] +default = ["migrate", "tokio-comp"] postgres = ["sqlx/postgres", "sqlx/json"] sqlite = ["sqlx/sqlite", "sqlx/json"] mysql = ["sqlx/mysql", "sqlx/json", "sqlx/bigdecimal"] @@ -40,15 +40,16 @@ futures-lite = "2.3.0" async-std = { version = "1.13.0", optional = true } chrono = { version = "0.4", features = ["serde"] } thiserror = "2.0.0" -pin-project-lite = "0.2.14" +pin-project = "1.1.10" [dev-dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } -email-service = { path = "../../examples/email-service" } +# email-service = { path = "../../examples/email-service" } apalis = { path = "../../", default-features = false } once_cell = "1.19.0" apalis-sql = { path = ".", features = ["tokio-comp"] } apalis-core = { path = "../apalis-core", features = ["test-utils"] } +tower = {version = "0.5", features = ["full"]} [package.metadata.docs.rs] # defines the configuration attribute `docsrs` From 2e8e5547d5f270499ddd74b2eb1b90c2d2380feb Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Tue, 12 Aug 2025 08:39:45 +0300 Subject: [PATCH 101/121] introduce custom backends --- packages/apalis-sql/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index bfbf85e..d8a0ef5 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -12,6 +12,7 @@ description = "SQL Storage for apalis. Use sqlite, postgres and mysql for backgr [features] default = ["migrate", "tokio-comp"] postgres = ["sqlx/postgres", "sqlx/json"] +postgres-bytes = ["sqlx/postgres"] sqlite = ["sqlx/sqlite", "sqlx/json"] mysql = ["sqlx/mysql", "sqlx/json", "sqlx/bigdecimal"] migrate = ["sqlx/migrate", "sqlx/macros"] From 1c9bdd94ae3e457a14132b15ea27af2d15930347 Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Wed, 13 Aug 2025 00:49:22 +0300 Subject: [PATCH 102/121] feat: remove task_id constrain as ulid --- packages/apalis-sql/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index d8a0ef5..1c92f3e 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -51,6 +51,7 @@ once_cell = "1.19.0" apalis-sql = { path = ".", features = ["tokio-comp"] } apalis-core = { path = "../apalis-core", features = ["test-utils"] } tower = {version = "0.5", features = ["full"]} +nanoid = "0.4.0" [package.metadata.docs.rs] # defines the configuration attribute `docsrs` From 1261fc96bb7163fa69afc32c3536814146e1a22d Mon Sep 17 00:00:00 2001 From: Njuguna Mureithi <95377562+geofmureithi@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:39:28 +0300 Subject: [PATCH 103/121] bump: apalis to v0.7.3 (#595) --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index cfa6473..9a72d29 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.7.2" +version = "0.7.3" authors = ["Njuguna Mureithi "] edition.workspace = true repository.workspace = true @@ -28,7 +28,7 @@ features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.7.2", default-features = false, features = [ +apalis-core = { path = "../../packages/apalis-core", version = "0.7.3", default-features = false, features = [ "sleep", "json", ] } From 3115b7780a2f3baed1e0d1672b716818d6e9660f Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Mon, 18 Aug 2025 19:10:43 +0300 Subject: [PATCH 104/121] wip: generic sinks and fetchers --- packages/apalis-sql/Cargo.toml | 1 - packages/apalis-sql/src/sqlite/mod.rs | 0 packages/apalis-sql/src/sqlite/sqlite_old.rs | 892 +++++++++++++++++++ 3 files changed, 892 insertions(+), 1 deletion(-) create mode 100644 packages/apalis-sql/src/sqlite/mod.rs create mode 100644 packages/apalis-sql/src/sqlite/sqlite_old.rs diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 1c92f3e..f9c72e6 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -35,7 +35,6 @@ apalis-core = { path = "../../packages/apalis-core", version = "0.7.1", default- ] } log = "0.4.21" futures = "0.3.30" -async-stream = "0.3.5" tokio = { version = "1", features = ["rt", "net"], optional = true } futures-lite = "2.3.0" async-std = { version = "1.13.0", optional = true } diff --git a/packages/apalis-sql/src/sqlite/mod.rs b/packages/apalis-sql/src/sqlite/mod.rs new file mode 100644 index 0000000..e69de29 diff --git a/packages/apalis-sql/src/sqlite/sqlite_old.rs b/packages/apalis-sql/src/sqlite/sqlite_old.rs new file mode 100644 index 0000000..2af75b5 --- /dev/null +++ b/packages/apalis-sql/src/sqlite/sqlite_old.rs @@ -0,0 +1,892 @@ +use crate::context::SqlContext; +use crate::{calculate_status, Config, SqlError}; +use apalis_core::backend::{BackendExpose, Stat, WorkerState}; +use apalis_core::codec::json::JsonCodec; +use apalis_core::error::Error; +use apalis_core::layers::{Ack, AckLayer}; +use apalis_core::poller::controller::Controller; +use apalis_core::poller::stream::BackendStream; +use apalis_core::poller::Poller; +use apalis_core::response::Response; +use apalis_core::storage::Storage; +use apalis_core::task::namespace::Namespace; +use apalis_core::task::task_id::TaskId; +use apalis_core::task::{Metadata, RequestStream, State, Task}; +use apalis_core::worker::{Event, SimpleWorker, WorkerContext, WorkerId}; +use apalis_core::{backend::Backend, codec::Codec}; +use async_stream::try_stream; +use chrono::{DateTime, Utc}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; +use log::error; +use serde::{de::DeserializeOwned, Serialize}; +use sqlx::{Pool, Row, Sqlite}; +use std::any::type_name; +use std::convert::TryInto; +use std::fmt::Debug; +use std::sync::Arc; +use std::{fmt, io}; +use std::{marker::PhantomData, time::Duration}; + +use crate::from_row::SqlRequest; + +pub use sqlx::sqlite::SqlitePool; + +/// Represents a [Storage] that persists to Sqlite +// #[derive(Debug)] +pub struct SqliteStorage> { + pool: Pool, + job_type: PhantomData, + controller: Controller, + config: Config, + codec: PhantomData, +} + +impl fmt::Debug for SqliteStorage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SqliteStorage") + .field("pool", &self.pool) + .field("job_type", &"PhantomData") + .field("controller", &self.controller) + .field("config", &self.config) + .field("codec", &std::any::type_name::()) + .finish() + } +} + +impl Clone for SqliteStorage { + fn clone(&self) -> Self { + SqliteStorage { + pool: self.pool.clone(), + job_type: PhantomData, + controller: self.controller.clone(), + config: self.config.clone(), + codec: self.codec, + } + } +} + +impl SqliteStorage<()> { + /// Perform migrations for storage + #[cfg(feature = "migrate")] + pub async fn setup(pool: &Pool) -> Result<(), sqlx::Error> { + sqlx::query("PRAGMA journal_mode = 'WAL';") + .execute(pool) + .await?; + sqlx::query("PRAGMA temp_store = 2;").execute(pool).await?; + sqlx::query("PRAGMA synchronous = NORMAL;") + .execute(pool) + .await?; + sqlx::query("PRAGMA cache_size = 64000;") + .execute(pool) + .await?; + Self::migrations().run(pool).await?; + Ok(()) + } + + /// Get sqlite migrations without running them + #[cfg(feature = "migrate")] + pub fn migrations() -> sqlx::migrate::Migrator { + sqlx::migrate!("migrations/sqlite") + } +} + +impl SqliteStorage { + /// Create a new instance + pub fn new(pool: SqlitePool) -> Self { + Self { + pool, + job_type: PhantomData, + controller: Controller::new(), + config: Config::new(type_name::()), + codec: PhantomData, + } + } + + /// Create a new instance with a custom config + pub fn new_with_config(pool: SqlitePool, config: Config) -> Self { + Self { + pool, + job_type: PhantomData, + controller: Controller::new(), + config, + codec: PhantomData, + } + } +} +impl SqliteStorage { + /// Keeps a storage notified that the worker is still alive manually + pub async fn keep_alive_at( + &mut self, + worker: &WorkerContext, + last_seen: i64, + ) -> Result<(), sqlx::Error> { + let worker_type = self.config.namespace.clone(); + let storage_name = std::any::type_name::(); + let query = "INSERT INTO Workers (id, worker_type, storage_name, layers, last_seen) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (id) DO + UPDATE SET last_seen = EXCLUDED.last_seen"; + sqlx::query(query) + .bind(worker.id().to_string()) + .bind(worker_type) + .bind(storage_name) + .bind(worker.get_service()) + .bind(last_seen) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Expose the pool for other functionality, eg custom migrations + pub fn pool(&self) -> &Pool { + &self.pool + } + + /// Get the config used by the storage + pub fn get_config(&self) -> &Config { + &self.config + } +} + +impl SqliteStorage { + /// Expose the code used + pub fn codec(&self) -> &PhantomData { + &self.codec + } +} + +async fn fetch_next( + pool: &Pool, + worker_id: &WorkerId, + id: String, + config: &Config, +) -> Result>, sqlx::Error> { + let now: i64 = Utc::now().timestamp(); + let update_query = "UPDATE Jobs SET status = 'Running', lock_by = ?2, lock_at = ?3 WHERE id = ?1 AND job_type = ?4 AND status = 'Pending' AND lock_by IS NULL RETURNING *"; + let job: Option> = sqlx::query_as(update_query) + .bind(id.to_string()) + .bind(worker_id.to_string()) + .bind(now) + .bind(config.namespace.clone()) + .fetch_optional(pool) + .await?; + + if job.is_none() { + // Retry path. + let job: Option> = + sqlx::query_as("SELECT * FROM Jobs WHERE id = ?1 AND lock_by = ?2 AND job_type = ?3") + .bind(id.to_string()) + .bind(worker_id.to_string()) + .bind(config.namespace.clone()) + .fetch_optional(pool) + .await?; + + return Ok(job); + } + + Ok(job) +} + +impl SqliteStorage +where + T: DeserializeOwned + Send + Unpin, + C: Codec, +{ + fn stream_jobs( + &self, + worker: &WorkerContext, + interval: Duration, + buffer_size: usize, + ) -> impl Stream>, sqlx::Error>> { + let pool = self.pool.clone(); + let worker = worker.clone(); + let config = self.config.clone(); + let namespace = Namespace(self.config.namespace.clone()); + try_stream! { + loop { + apalis_core::sleep(interval).await; + if !worker.is_ready() { + continue; + } + let worker_id = worker.id(); + let job_type = &config.namespace; + let fetch_query = "SELECT id FROM Jobs + WHERE (status = 'Pending' OR (status = 'Failed' AND attempts < max_attempts)) AND run_at < ?1 AND job_type = ?2 ORDER BY priority DESC LIMIT ?3"; + let now: i64 = Utc::now().timestamp(); + let ids: Vec<(String,)> = sqlx::query_as(fetch_query) + .bind(now) + .bind(job_type) + .bind(i64::try_from(buffer_size).map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?) + .fetch_all(&pool) + .await?; + + for id in ids { + let res = fetch_next(&pool, worker_id, id.0, &config).await?; + yield match res { + None => None::>, + Some(job) => { + let (req, parts) = job.req.take(); + let args = C::decode(req) + .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; + let mut req = Task::new_with_parts(args, parts); + req.meta.namespace = Some(namespace.clone()); + Some(req) + } + } + }; + } + } + } +} + +impl Storage for SqliteStorage +where + T: Serialize + DeserializeOwned + Send + 'static + Unpin + Sync, + C: Codec + Send + 'static + Sync, + C::Error: std::error::Error + Send + Sync + 'static, +{ + type Job = T; + + type Error = sqlx::Error; + + type Context = SqlContext; + + type Compact = String; + + async fn push_request( + &mut self, + job: Task, + ) -> Result, Self::Error> { + let query = "INSERT INTO Jobs VALUES (?1, ?2, ?3, 'Pending', 0, ?4, strftime('%s','now'), NULL, NULL, NULL, NULL, ?5)"; + let (task, parts) = job.take(); + let raw = C::encode(&task) + .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; + let job_type = self.config.namespace.clone(); + sqlx::query(query) + .bind(raw) + .bind(parts.task_id.to_string()) + .bind(job_type.to_string()) + .bind(parts.context.max_attempts()) + .bind(parts.context.priority()) + .execute(&self.pool) + .await?; + Ok(parts) + } + + async fn push_raw_request( + &mut self, + job: Task, + ) -> Result, Self::Error> { + let query = "INSERT INTO Jobs VALUES (?1, ?2, ?3, 'Pending', 0, ?4, strftime('%s','now'), NULL, NULL, NULL, NULL, ?5)"; + let (task, parts) = job.take(); + let raw = C::encode(&task) + .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; + let job_type = self.config.namespace.clone(); + sqlx::query(query) + .bind(raw) + .bind(parts.task_id.to_string()) + .bind(job_type.to_string()) + .bind(parts.context.max_attempts()) + .bind(parts.context.priority()) + .execute(&self.pool) + .await?; + Ok(parts) + } + + async fn schedule_request( + &mut self, + req: Task, + on: i64, + ) -> Result, Self::Error> { + let query = + "INSERT INTO Jobs VALUES (?1, ?2, ?3, 'Pending', 0, ?4, ?5, NULL, NULL, NULL, NULL, ?6)"; + let id = &req.meta.task_id; + let job = C::encode(&req.args) + .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; + let job_type = self.config.namespace.clone(); + sqlx::query(query) + .bind(job) + .bind(id.to_string()) + .bind(job_type) + .bind(req.meta.context.max_attempts()) + .bind(on) + .bind(req.meta.context.priority()) + .execute(&self.pool) + .await?; + Ok(req.meta) + } + + async fn fetch_by_id( + &mut self, + job_id: &TaskId, + ) -> Result>, Self::Error> { + let fetch_query = "SELECT * FROM Jobs WHERE id = ?1"; + let res: Option> = sqlx::query_as(fetch_query) + .bind(job_id.to_string()) + .fetch_optional(&self.pool) + .await?; + match res { + None => Ok(None), + Some(job) => Ok(Some({ + let (req, parts) = job.req.take(); + let args = C::decode(req) + .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; + + let mut req: Task = Task::new_with_parts(args, parts); + req.meta.namespace = Some(Namespace(self.config.namespace.clone())); + req + })), + } + } + + async fn len(&mut self) -> Result { + let query = "Select COUNT(*) AS count FROM Jobs WHERE (status = 'Pending' OR (status = 'Failed' AND attempts < max_attempts))"; + let record = sqlx::query(query).fetch_one(&self.pool).await?; + record.try_get("count") + } + + async fn reschedule( + &mut self, + job: Task, + wait: Duration, + ) -> Result<(), Self::Error> { + let task_id = job.meta.task_id; + + let wait: i64 = wait + .as_secs() + .try_into() + .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; + + let query = + "UPDATE Jobs SET status = 'Failed', done_at = NULL, lock_by = NULL, lock_at = NULL, run_at = ?2 WHERE id = ?1"; + let now: i64 = Utc::now().timestamp(); + let wait_until = now + wait; + + sqlx::query(query) + .bind(task_id.to_string()) + .bind(wait_until) + .execute(self.pool()) + .await?; + Ok(()) + } + + async fn update(&mut self, job: Task) -> Result<(), Self::Error> { + let ctx = job.meta.context; + let status = ctx.status().to_string(); + let attempts = job.meta.attempt; + let done_at = *ctx.done_at(); + let lock_by = ctx.lock_by().clone(); + let lock_at = *ctx.lock_at(); + let last_error = ctx.last_error().clone(); + let priority = *ctx.priority(); + let job_id = job.meta.task_id; + let query = + "UPDATE Jobs SET status = ?1, attempts = ?2, done_at = ?3, lock_by = ?4, lock_at = ?5, last_error = ?6, priority = ?7 WHERE id = ?8"; + sqlx::query(query) + .bind(status.to_owned()) + .bind::( + attempts + .current() + .try_into() + .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?, + ) + .bind(done_at) + .bind(lock_by.map(|w| w.name().to_string())) + .bind(lock_at) + .bind(last_error) + .bind(priority) + .bind(job_id.to_string()) + .execute(self.pool()) + .await?; + Ok(()) + } + + async fn is_empty(&mut self) -> Result { + self.len().map_ok(|c| c == 0).await + } + + async fn vacuum(&mut self) -> Result { + let query = "Delete from Jobs where status='Done'"; + let record = sqlx::query(query).execute(&self.pool).await?; + Ok(record.rows_affected().try_into().unwrap_or_default()) + } +} + +impl SqliteStorage { + /// Puts the job instantly back into the queue + /// Another Worker may consume + pub async fn retry( + &mut self, + worker_id: &WorkerId, + job_id: &TaskId, + ) -> Result<(), sqlx::Error> { + let query = + "UPDATE Jobs SET status = 'Pending', done_at = NULL, lock_by = NULL WHERE id = ?1 AND lock_by = ?2"; + sqlx::query(query) + .bind(job_id.to_string()) + .bind(worker_id.to_string()) + .execute(self.pool()) + .await?; + Ok(()) + } + + /// Kill a job + pub async fn kill(&mut self, worker_id: &WorkerId, job_id: &TaskId) -> Result<(), sqlx::Error> { + let query = + "UPDATE Jobs SET status = 'Killed', done_at = strftime('%s','now') WHERE id = ?1 AND lock_by = ?2"; + sqlx::query(query) + .bind(job_id.to_string()) + .bind(worker_id.to_string()) + .execute(self.pool()) + .await?; + Ok(()) + } + + /// Add jobs that workers have disappeared to the queue + pub async fn reenqueue_orphaned( + &self, + count: i32, + dead_since: DateTime, + ) -> Result<(), sqlx::Error> { + let job_type = self.config.namespace.clone(); + let query = r#"Update Jobs + SET status = "Pending", done_at = NULL, lock_by = NULL, lock_at = NULL, attempts = attempts + 1, last_error ="Job was abandoned" + WHERE id in + (SELECT Jobs.id from Jobs INNER join Workers ON lock_by = Workers.id + WHERE status= "Running" AND workers.last_seen < ?1 + AND Workers.worker_type = ?2 ORDER BY lock_at ASC LIMIT ?3);"#; + + sqlx::query(query) + .bind(dead_since.timestamp()) + .bind(job_type) + .bind(count) + .execute(self.pool()) + .await?; + Ok(()) + } +} + +/// Errors that can occur while polling an SQLite database. +#[derive(thiserror::Error, Debug)] +pub enum SqlitePollError { + /// Error during a keep-alive heartbeat. + #[error("Encountered an error during KeepAlive heartbeat: `{0}`")] + KeepAliveError(sqlx::Error), + + /// Error during re-enqueuing orphaned tasks. + #[error("Encountered an error during ReenqueueOrphaned heartbeat: `{0}`")] + ReenqueueOrphanedError(sqlx::Error), +} + +impl Backend> for SqliteStorage +where + C: Codec + Send + 'static + Sync, + C::Error: std::error::Error + 'static + Send + Sync, + T: Serialize + DeserializeOwned + Sync + Send + Unpin + 'static, +{ + type Stream = BackendStream>>; + type Layer = AckLayer, T, SqlContext, C>; + + type Codec = JsonCodec; + + fn poll(mut self, worker: &WorkerContext) -> Poller { + let layer = AckLayer::new(self.clone()); + let config = self.config.clone(); + let controller = self.controller.clone(); + let stream = self + .stream_jobs(worker, config.poll_interval, config.buffer_size) + .map_err(|e| Error::SourceError(Arc::new(Box::new(e)))); + let stream = BackendStream::new(stream.boxed(), controller); + let requeue_storage = self.clone(); + let w = worker.clone(); + let heartbeat = async move { + // Lets reenqueue any jobs that belonged to this worker in case of a death + if let Err(e) = self + .reenqueue_orphaned((config.buffer_size * 10) as i32, Utc::now()) + .await + { + w.emit(Event::Error(Box::new( + SqlitePollError::ReenqueueOrphanedError(e), + ))); + } + loop { + let now: i64 = Utc::now().timestamp(); + if let Err(e) = self.keep_alive_at(&w, now).await { + w.emit(Event::Error(Box::new(SqlitePollError::KeepAliveError(e)))); + } + apalis_core::sleep(Duration::from_secs(30)).await; + } + } + .boxed(); + let w = worker.clone(); + let reenqueue_beat = async move { + loop { + let dead_since = Utc::now() + - chrono::Duration::from_std(config.reenqueue_orphaned_after).unwrap(); + if let Err(e) = requeue_storage + .reenqueue_orphaned( + config + .buffer_size + .try_into() + .expect("could not convert usize to i32"), + dead_since, + ) + .await + { + w.emit(Event::Error(Box::new( + SqlitePollError::ReenqueueOrphanedError(e), + ))); + } + apalis_core::sleep(config.poll_interval).await; + } + }; + Poller::new_with_layer( + stream, + async { + futures::join!(heartbeat, reenqueue_beat); + }, + layer, + ) + } +} + +impl Ack for SqliteStorage { + type Context = SqlContext; + type AckError = sqlx::Error; + async fn ack(&mut self, ctx: &Self::Context, res: &Response) -> Result<(), sqlx::Error> { + let pool = self.pool.clone(); + let query = + "UPDATE Jobs SET status = ?4, attempts = ?5, done_at = strftime('%s','now'), last_error = ?3 WHERE id = ?1 AND lock_by = ?2"; + let result = serde_json::to_string(&res.inner.as_ref().map_err(|r| r.to_string())) + .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; + sqlx::query(query) + .bind(res.task_id.to_string()) + .bind( + ctx.lock_by() + .as_ref() + .expect("Task is not locked") + .to_string(), + ) + .bind(result) + .bind(calculate_status(ctx, res).to_string()) + .bind(res.attempt.current() as u32) + .execute(&pool) + .await?; + Ok(()) + } +} + +impl BackendExpose + for SqliteStorage> +{ + type Request = Task>; + type Error = SqlError; + async fn stats(&self) -> Result { + let fetch_query = "SELECT + COUNT(1) FILTER (WHERE status = 'Pending') AS pending, + COUNT(1) FILTER (WHERE status = 'Running') AS running, + COUNT(1) FILTER (WHERE status = 'Done') AS done, + COUNT(1) FILTER (WHERE status = 'Failed') AS failed, + COUNT(1) FILTER (WHERE status = 'Killed') AS killed + FROM Jobs WHERE job_type = ?"; + + let res: (i64, i64, i64, i64, i64, i64) = sqlx::query_as(fetch_query) + .bind(self.get_config().namespace()) + .fetch_one(self.pool()) + .await?; + + Ok(Stat { + pending: res.0.try_into()?, + running: res.1.try_into()?, + dead: res.4.try_into()?, + failed: res.3.try_into()?, + success: res.2.try_into()?, + }) + } + + async fn list_jobs( + &self, + status: &State, + page: i32, + ) -> Result, Self::Error> { + let status = status.to_string(); + let fetch_query = "SELECT * FROM Jobs WHERE status = ? AND job_type = ? ORDER BY done_at DESC, run_at DESC LIMIT 10 OFFSET ?"; + let res: Vec> = sqlx::query_as(fetch_query) + .bind(status) + .bind(self.get_config().namespace()) + .bind(((page - 1) * 10).to_string()) + .fetch_all(self.pool()) + .await?; + Ok(res + .into_iter() + .map(|j| { + let (req, ctx) = j.req.take(); + let req = JsonCodec::::decode(req).unwrap(); + Task::new_with_ctx(req, ctx) + }) + .collect()) + } + + async fn list_workers(&self) -> Result>, Self::Error> { + let fetch_query = + "SELECT id, layers, last_seen FROM Workers WHERE worker_type = ? ORDER BY last_seen DESC LIMIT 20 OFFSET ?"; + let res: Vec<(String, String, i64)> = sqlx::query_as(fetch_query) + .bind(self.get_config().namespace()) + .bind(0) + .fetch_all(self.pool()) + .await?; + Ok(res + .into_iter() + .map(|w| SimpleWorker::new(WorkerId::new(w.0), WorkerState::new::(w.1))) + .collect()) + } +} + +#[cfg(test)] +mod tests { + + use crate::sql_storage_tests; + + use super::*; + use apalis_core::task::State; + use chrono::Utc; + use email_service::example_good_email; + use email_service::Email; + use futures::StreamExt; + + use apalis_core::generic_storage_test; + use apalis_core::test_utils::apalis_test_service_fn; + use apalis_core::test_utils::TestWrapper; + + generic_storage_test!(setup); + sql_storage_tests!(setup::, SqliteStorage, Email); + + /// migrate DB and return a storage instance. + async fn setup() -> SqliteStorage { + // Because connections cannot be shared across async runtime + // (different runtimes are created for each test), + // we don't share the storage and tests must be run sequentially. + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); + SqliteStorage::setup(&pool) + .await + .expect("failed to migrate DB"); + let config = Config::new("apalis::test"); + let storage = SqliteStorage::::new_with_config(pool, config); + + storage + } + + #[tokio::test] + async fn test_inmemory_sqlite_worker() { + let mut sqlite = setup().await; + sqlite + .push(Email { + subject: "Test Subject".to_string(), + to: "example@sqlite".to_string(), + text: "Some Text".to_string(), + }) + .await + .expect("Unable to push job"); + let len = sqlite.len().await.expect("Could not fetch the jobs count"); + assert_eq!(len, 1); + } + + async fn consume_one( + storage: &mut SqliteStorage, + worker: &WorkerContext, + ) -> Task { + let mut stream = storage + .stream_jobs(worker, std::time::Duration::from_secs(10), 1) + .boxed(); + stream + .next() + .await + .expect("stream is empty") + .expect("failed to poll job") + .expect("no job is pending") + } + + async fn register_worker_at( + storage: &mut SqliteStorage, + last_seen: i64, + ) -> WorkerContext { + let worker_id = WorkerId::new("test-worker"); + + let worker = SimpleWorker::new(worker_id, Default::default()); + storage + .keep_alive_at(&worker, last_seen) + .await + .expect("failed to register worker"); + worker.start(); + worker + } + + async fn register_worker(storage: &mut SqliteStorage) -> WorkerContext { + register_worker_at(storage, Utc::now().timestamp()).await + } + + async fn push_email(storage: &mut SqliteStorage, email: Email) { + storage.push(email).await.expect("failed to push a job"); + } + + async fn get_job( + storage: &mut SqliteStorage, + job_id: &TaskId, + ) -> Task { + storage + .fetch_by_id(job_id) + .await + .expect("failed to fetch job by id") + .expect("no job found by id") + } + + #[tokio::test] + async fn test_consume_last_pushed_job() { + let mut storage = setup().await; + let worker = register_worker(&mut storage).await; + + push_email(&mut storage, example_good_email()).await; + let len = storage.len().await.expect("Could not fetch the jobs count"); + assert_eq!(len, 1); + + let job = consume_one(&mut storage, &worker).await; + let ctx = job.meta.context; + assert_eq!(*ctx.status(), State::Running); + assert_eq!(*ctx.lock_by(), Some(worker.id().clone())); + assert!(ctx.lock_at().is_some()); + } + + #[tokio::test] + async fn test_acknowledge_job() { + let mut storage = setup().await; + let worker = register_worker(&mut storage).await; + + push_email(&mut storage, example_good_email()).await; + let job = consume_one(&mut storage, &worker).await; + let job_id = &job.meta.task_id; + let ctx = &job.meta.context; + let res = 1usize; + storage + .ack( + ctx, + &Response::success(res, job_id.clone(), job.meta.attempt.clone()), + ) + .await + .expect("failed to acknowledge the job"); + + let job = get_job(&mut storage, job_id).await; + let ctx = job.meta.context; + assert_eq!(*ctx.status(), State::Done); + assert!(ctx.done_at().is_some()); + } + + #[tokio::test] + async fn test_kill_job() { + let mut storage = setup().await; + + push_email(&mut storage, example_good_email()).await; + + let worker = register_worker(&mut storage).await; + + let job = consume_one(&mut storage, &worker).await; + let job_id = &job.meta.task_id; + + storage + .kill(&worker.id(), job_id) + .await + .expect("failed to kill job"); + + let job = get_job(&mut storage, job_id).await; + let ctx = job.meta.context; + assert_eq!(*ctx.status(), State::Killed); + assert!(ctx.done_at().is_some()); + } + + #[tokio::test] + async fn test_heartbeat_renqueueorphaned_pulse_last_seen_6min() { + let mut storage = setup().await; + + push_email(&mut storage, example_good_email()).await; + + let six_minutes_ago = Utc::now() - Duration::from_secs(6 * 60); + + let five_minutes_ago = Utc::now() - Duration::from_secs(5 * 60); + let worker = register_worker_at(&mut storage, six_minutes_ago.timestamp()).await; + + let job = consume_one(&mut storage, &worker).await; + let job_id = &job.meta.task_id; + storage + .reenqueue_orphaned(1, five_minutes_ago) + .await + .expect("failed to heartbeat"); + let job = get_job(&mut storage, job_id).await; + let ctx = &job.meta.context; + assert_eq!(*ctx.status(), State::Pending); + assert!(ctx.done_at().is_none()); + assert!(ctx.lock_by().is_none()); + assert!(ctx.lock_at().is_none()); + assert_eq!(*ctx.last_error(), Some("Job was abandoned".to_owned())); + assert_eq!(job.meta.attempt.current(), 1); + + let job = consume_one(&mut storage, &worker).await; + let ctx = &job.meta.context; + // Simulate worker + job.meta.attempt.increment(); + storage + .ack( + ctx, + &Response::new(Ok("success".to_owned()), job_id.clone(), job.meta.attempt), + ) + .await + .unwrap(); + //end simulate worker + + let job = get_job(&mut storage, &job_id).await; + let ctx = &job.meta.context; + assert_eq!(*ctx.status(), State::Done); + assert_eq!(*ctx.lock_by(), Some(worker.id().clone())); + assert!(ctx.lock_at().is_some()); + assert_eq!(*ctx.last_error(), Some("{\"Ok\":\"success\"}".to_owned())); + assert_eq!(job.meta.attempt.current(), 2); + } + + #[tokio::test] + async fn test_heartbeat_renqueueorphaned_pulse_last_seen_4min() { + let mut storage = setup().await; + + push_email(&mut storage, example_good_email()).await; + + let six_minutes_ago = Utc::now() - Duration::from_secs(6 * 60); + let four_minutes_ago = Utc::now() - Duration::from_secs(4 * 60); + let worker = register_worker_at(&mut storage, four_minutes_ago.timestamp()).await; + + let job = consume_one(&mut storage, &worker).await; + let job_id = job.meta.task_id; + storage + .reenqueue_orphaned(1, six_minutes_ago) + .await + .expect("failed to heartbeat"); + + let job = get_job(&mut storage, &job_id).await; + let ctx = &job.meta.context; + + // Simulate worker + job.meta.attempt.increment(); + storage + .ack( + ctx, + &Response::new(Ok("success".to_owned()), job_id.clone(), job.meta.attempt), + ) + .await + .unwrap(); + //end simulate worker + + let job = get_job(&mut storage, &job_id).await; + let ctx = &job.meta.context; + assert_eq!(*ctx.status(), State::Done); + assert_eq!(*ctx.lock_by(), Some(worker.id().clone())); + assert!(ctx.lock_at().is_some()); + assert_eq!(*ctx.last_error(), Some("{\"Ok\":\"success\"}".to_owned())); + assert_eq!(job.meta.attempt.current(), 1); + } +} From e70ae2dfe7e50207f7747b13b5a42d2792997c6e Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Tue, 19 Aug 2025 13:37:30 +0300 Subject: [PATCH 105/121] improve metadata --- packages/apalis-sql/src/sqlite/mod.rs | 39 +++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/packages/apalis-sql/src/sqlite/mod.rs b/packages/apalis-sql/src/sqlite/mod.rs index e69de29..97889a0 100644 --- a/packages/apalis-sql/src/sqlite/mod.rs +++ b/packages/apalis-sql/src/sqlite/mod.rs @@ -0,0 +1,39 @@ +#[derive(Debug)] +struct DbEvent { + op: &'static str, + db_name: String, + table_name: String, + rowid: i64, +} + +// Callback for SQLite update hook +extern "C" fn update_hook_callback( + arg: *mut c_void, + op: c_int, + db_name: *const c_char, + table_name: *const c_char, + rowid: i64, +) { + let op_str = match op { + libsqlite3_sys::SQLITE_INSERT => "INSERT", + libsqlite3_sys::SQLITE_UPDATE => "UPDATE", + libsqlite3_sys::SQLITE_DELETE => "DELETE", + _ => "UNKNOWN", + }; + + unsafe { + let db = CStr::from_ptr(db_name).to_string_lossy().to_string(); + let table = CStr::from_ptr(table_name).to_string_lossy().to_string(); + + // Recover sender from raw pointer + let tx = &mut *(arg as *mut mpsc::UnboundedSender); + + // Ignore send errors (receiver closed) + let _ = tx.send(DbEvent { + op: op_str, + db_name: db, + table_name: table, + rowid, + }); + } +} From 5ed353ecf03a7070c452029c0a5f7c1008a0bfa8 Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Fri, 22 Aug 2025 14:55:59 +0300 Subject: [PATCH 106/121] docs: improvement --- packages/apalis-sql/Cargo.toml | 3 +- packages/apalis-sql/src/sqlite/mod.rs | 58 +++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index f9c72e6..de68e26 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -10,7 +10,7 @@ license = "MIT" description = "SQL Storage for apalis. Use sqlite, postgres and mysql for background job processing" [features] -default = ["migrate", "tokio-comp"] +default = ["migrate", "tokio-comp", "sqlite"] postgres = ["sqlx/postgres", "sqlx/json"] postgres-bytes = ["sqlx/postgres"] sqlite = ["sqlx/sqlite", "sqlx/json"] @@ -41,6 +41,7 @@ async-std = { version = "1.13.0", optional = true } chrono = { version = "0.4", features = ["serde"] } thiserror = "2.0.0" pin-project = "1.1.10" +libsqlite3-sys = "0.30.1" [dev-dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } diff --git a/packages/apalis-sql/src/sqlite/mod.rs b/packages/apalis-sql/src/sqlite/mod.rs index 97889a0..1ca8783 100644 --- a/packages/apalis-sql/src/sqlite/mod.rs +++ b/packages/apalis-sql/src/sqlite/mod.rs @@ -1,3 +1,12 @@ +use futures::channel::mpsc::{self, Receiver}; +use futures::{SinkExt, StreamExt}; +use libsqlite3_sys::{sqlite3, sqlite3_update_hook}; +use sqlx::SqlitePool; +use sqlx::{sqlite::SqliteConnection, Connection, Executor}; +use std::ffi::{c_void, CStr}; +use std::os::raw::{c_char, c_int}; +use std::ptr; + #[derive(Debug)] struct DbEvent { op: &'static str, @@ -37,3 +46,52 @@ extern "C" fn update_hook_callback( }); } } + +pub struct PushListener { + pool: SqlitePool, + rx: Receiver, +} + +async fn run_it() -> Result<(), sqlx::Error> { + let (tx, mut rx) = mpsc::channel::(10); + + let pool = SqlitePool::connect(":memory:").await.unwrap(); + + let mut conn = pool.acquire().await?; + + // Create table + conn.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)") + .await?; + + // Get raw sqlite3* handle + let handle: *mut sqlite3 = conn.lock_handle().await.unwrap().as_raw_handle().as_ptr(); + + // Put sender in a Box so it has a stable memory address + let tx_box = Box::new(tx); + let tx_ptr = Box::into_raw(tx_box) as *mut c_void; + + unsafe { + sqlite3_update_hook(handle, Some(update_hook_callback), tx_ptr); + } + + // Spawn a task to process events asynchronously + tokio::spawn(async move { + while let Some(event) = rx.next().await { + println!("Got DB event: {:?}", event); + } + }); + + // These will trigger the hook + conn.execute("INSERT INTO users (name) VALUES ('Alice')") + .await?; + conn.execute("INSERT INTO users (name) VALUES ('Bob')") + .await?; + conn.execute("UPDATE users SET name = 'Charlie' WHERE id = 1") + .await?; + conn.execute("DELETE FROM users WHERE id = 2").await?; + + // Let events flush + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + Ok(()) +} From e6596226ed9ebd792b11fc08fc9235e50247b0b4 Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Fri, 12 Sep 2025 17:57:25 +0300 Subject: [PATCH 107/121] bump: version --- packages/apalis-sql/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml index 50ca185..e9df625 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/packages/apalis-sql/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "apalis-sql" -version = "0.7.3" +version = "1.0.0-alpha.1" authors = ["Njuguna Mureithi "] edition.workspace = true repository.workspace = true @@ -29,7 +29,7 @@ features = ["chrono"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.7.3", default-features = false, features = [ +apalis-core = { path = "../../packages/apalis-core", version = "1.0.0-alpha.1", default-features = false, features = [ "sleep", "json", ] } From 3e54babafc850ab986bd5eb6d43d4e55784e0504 Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Mon, 22 Sep 2025 23:08:29 +0300 Subject: [PATCH 108/121] chore: initial split --- .github/ISSUE_TEMPLATE/bug_report.yml | 89 + .github/ISSUE_TEMPLATE/feature_request.yml | 57 + .github/PULL_REQUEST_TEMPLATE.md | 32 + .github/dependabot.yml | 39 + .github/workflows/bench.yml | 80 + .github/workflows/ci.yml | 192 ++ .github/workflows/docs.yml | 36 + .github/workflows/pr-preview.yml | 117 + .github/workflows/release.yml | 132 + .github/workflows/rust-security.yml | 68 + .github/workflows/security.yml | 83 + .github/workflows/stale.yml | 38 + .gitignore | 29 + ...10e5c52e64d5289ca829e2a0135a14801b930.json | 86 + ...bc68adca35acef6ce80e58bad3b70c3fd6d97.json | 12 + ...c4db57308e5cf88f44efaf23b05c585626465.json | 12 + ...acaa0ee3ad0de2ad6453554e95be9df4e487e.json | 12 + Cargo.lock | 2822 +++++++++++++++++ packages/apalis-sql/Cargo.toml => Cargo.toml | 34 +- packages/apalis-sql/build.rs => build.rs | 0 .../20220530084123_jobs_workers.sql | 0 .../20250313213411_add_job_priority.sql | 0 queries/task/ack.sql | 10 + queries/task/fetch_next.sql | 17 + queries/task/sink.sql | 17 + queries/worker/register.sql | 7 + src/ack.rs | 79 + src/config.rs | 131 + src/context.rs | 109 + src/fetcher.rs | 208 ++ src/from_row.rs | 72 + .../src/sqlite/mod.rs => src/hook.rs | 20 +- src/lib.rs | 403 +++ src/shared.rs | 290 ++ src/sink.rs | 152 + .../src/sqlite => src}/sqlite_old.rs | 0 36 files changed, 5453 insertions(+), 32 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/bench.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/pr-preview.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/rust-security.yml create mode 100644 .github/workflows/security.yml create mode 100644 .github/workflows/stale.yml create mode 100644 .gitignore create mode 100644 .sqlx/query-0819e39ed31495e0c70a6dd549b10e5c52e64d5289ca829e2a0135a14801b930.json create mode 100644 .sqlx/query-0be6db0bdde33835113a74e4749bc68adca35acef6ce80e58bad3b70c3fd6d97.json create mode 100644 .sqlx/query-4ead10dafe0d2d024654403f289c4db57308e5cf88f44efaf23b05c585626465.json create mode 100644 .sqlx/query-cfc8346ce97a0f65e057b439787acaa0ee3ad0de2ad6453554e95be9df4e487e.json create mode 100644 Cargo.lock rename packages/apalis-sql/Cargo.toml => Cargo.toml (53%) rename packages/apalis-sql/build.rs => build.rs (100%) rename {packages/apalis-sql/migrations/sqlite => migrations}/20220530084123_jobs_workers.sql (100%) rename {packages/apalis-sql/migrations/sqlite => migrations}/20250313213411_add_job_priority.sql (100%) create mode 100644 queries/task/ack.sql create mode 100644 queries/task/fetch_next.sql create mode 100644 queries/task/sink.sql create mode 100644 queries/worker/register.sql create mode 100644 src/ack.rs create mode 100644 src/config.rs create mode 100644 src/context.rs create mode 100644 src/fetcher.rs create mode 100644 src/from_row.rs rename packages/apalis-sql/src/sqlite/mod.rs => src/hook.rs (88%) create mode 100644 src/lib.rs create mode 100644 src/shared.rs create mode 100644 src/sink.rs rename {packages/apalis-sql/src/sqlite => src}/sqlite_old.rs (100%) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..b3d06ce --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,89 @@ +name: Bug Report +description: File a bug report to help us improve +title: "[Bug]: " +labels: ["bug"] +assignees: [] + +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + + - type: textarea + id: what-happened + attributes: + label: What happened? + description: A clear and concise description of what the bug is. + placeholder: Tell us what you see! + validations: + required: true + + - type: textarea + id: expected + attributes: + label: Expected behavior + description: A clear and concise description of what you expected to happen. + validations: + required: true + + - type: textarea + id: reproduce + attributes: + label: Steps to reproduce + description: Steps to reproduce the behavior + placeholder: | + 1. Go to '...' + 2. Click on '....' + 3. Scroll down to '....' + 4. See error + validations: + required: true + + - type: textarea + id: code + attributes: + label: Minimal code example + description: If applicable, add a minimal code example that reproduces the issue + render: rust + + - type: dropdown + id: version + attributes: + label: Version + description: What version of apalis-sqlite are you running? + options: + - 1.0.0-alpha.1 + - main branch + - Other (specify in additional context) + validations: + required: true + + - type: textarea + id: environment + attributes: + label: Environment + description: | + Please provide information about your environment: + - OS: [e.g. Ubuntu 20.04, macOS 12.0, Windows 11] + - Rust version: [e.g. 1.70.0] + - Cargo version: [e.g. 1.70.0] + value: | + - OS: + - Rust version: + - Cargo version: + validations: + required: true + + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + render: shell + + - type: textarea + id: context + attributes: + label: Additional context + description: Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..412b715 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,57 @@ +name: Feature Request +description: Suggest an idea for this project +title: "[Feature]: " +labels: ["enhancement"] +assignees: [] + +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to suggest a new feature! + + - type: textarea + id: problem + attributes: + label: Is your feature request related to a problem? + description: A clear and concise description of what the problem is. + placeholder: I'm always frustrated when... + validations: + required: true + + - type: textarea + id: solution + attributes: + label: Describe the solution you'd like + description: A clear and concise description of what you want to happen. + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Describe alternatives you've considered + description: A clear and concise description of any alternative solutions or features you've considered. + + - type: textarea + id: use-case + attributes: + label: Use case + description: Describe your specific use case and how this feature would help. + validations: + required: true + + - type: textarea + id: implementation + attributes: + label: Additional context + description: Add any other context, screenshots, or implementation ideas about the feature request here. + + - type: checkboxes + id: terms + attributes: + label: Code of Conduct + description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/apalis-dev/apalis-sqlite/blob/main/CODE_OF_CONDUCT.md) + options: + - label: I agree to follow this project's Code of Conduct + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..bd6b9c0 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,32 @@ +## Description + +Brief description of what this PR does. + +## Type of Change + +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] Documentation update +- [ ] Performance improvement +- [ ] Refactoring (no functional changes) + +## Testing + +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] I have run the existing tests and they pass +- [ ] I have run `cargo fmt` and `cargo clippy` + +## Checklist + +- [ ] My code follows the code style of this project +- [ ] I have performed a self-review of my own code +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] My changes generate no new warnings +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] New and existing unit tests pass locally with my changes + +## Additional Notes + +Any additional information, context, or screenshots about the pull request here. \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..4449dce --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,39 @@ +version: 2 +updates: + # Enable version updates for Cargo + - package-ecosystem: "cargo" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "06:00" + open-pull-requests-limit: 10 + reviewers: + - "mureithinjuguna" + assignees: + - "mureithinjuguna" + commit-message: + prefix: "deps" + include: "scope" + labels: + - "dependencies" + - "rust" + + # Enable version updates for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "06:00" + open-pull-requests-limit: 5 + reviewers: + - "geofmureithi" + assignees: + - "geofmureithi" + commit-message: + prefix: "ci" + include: "scope" + labels: + - "dependencies" + - "github-actions" \ No newline at end of file diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..afdabe7 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,80 @@ +name: Benchmarks + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +env: + CARGO_TERM_COLOR: always + +jobs: + benchmark: + name: Performance Benchmarks + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }} + + - name: Run benchmarks + run: | + if find . -name "*.rs" -exec grep -l "#\[bench\]" {} \; | grep -q .; then + echo "Found benchmark tests, running cargo bench" + cargo bench + else + echo "No benchmark tests found, skipping" + fi + + - name: Install cargo-criterion (if criterion benchmarks exist) + run: | + if find . -name "*.rs" -exec grep -l "criterion::" {} \; | grep -q .; then + echo "Found criterion benchmarks, installing cargo-criterion" + cargo install cargo-criterion + cargo criterion + else + echo "No criterion benchmarks found, skipping" + fi + + msrv: + name: Minimum Supported Rust Version + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Get MSRV from Cargo.toml + id: msrv + run: | + MSRV=$(grep "rust-version" Cargo.toml | sed 's/.*rust-version.*=.*"\(.*\)".*/\1/' || echo "1.70.0") + echo "msrv=$MSRV" >> $GITHUB_OUTPUT + echo "Detected MSRV: $MSRV" + + - name: Install MSRV Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.msrv.outputs.msrv }} + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-msrv-${{ steps.msrv.outputs.msrv }}-${{ hashFiles('**/Cargo.lock') }} + + - name: Test with MSRV + run: cargo test --all-features diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..fdd040a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,192 @@ +name: CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + test: + name: Test Suite + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + rust: [stable, beta] + include: + - os: ubuntu-latest + rust: nightly + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ matrix.rust }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-${{ matrix.rust }}- + ${{ runner.os }}-cargo- + + - name: Run cargo test + run: cargo test --verbose + + - name: Run cargo test with all features + run: cargo test --all-features --verbose + + features: + name: Feature Matrix Tests + runs-on: ubuntu-latest + strategy: + matrix: + features: + - "" + - "--features migrate" + - "--features async-std-comp" + - "--features async-std-comp-native-tls" + - "--features tokio-comp" + - "--features tokio-comp-native-tls" + - "--all-features" + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-features-${{ hashFiles('**/Cargo.lock') }} + + - name: Test with features + run: cargo test ${{ matrix.features }} --verbose + + fmt: + name: Rustfmt + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + + - name: Run cargo fmt + run: cargo fmt --all -- --check + + clippy: + name: Clippy + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }} + + - name: Run cargo clippy + run: cargo clippy --all-targets --all-features + + docs: + name: Documentation + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-docs-${{ hashFiles('**/Cargo.lock') }} + + - name: Build documentation + run: cargo doc --all-features --no-deps --document-private-items + env: + RUSTDOCFLAGS: "-Dwarnings" + + security: + name: Security Audit + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install cargo-audit + run: cargo install cargo-audit + + - name: Run cargo audit + run: cargo audit + + coverage: + name: Code Coverage + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install cargo-tarpaulin + run: cargo install cargo-tarpaulin + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-coverage-${{ hashFiles('**/Cargo.lock') }} + + - name: Generate code coverage + run: cargo tarpaulin --all-features --verbose --workspace --timeout 120 --out xml + + - name: Upload to codecov.io + uses: codecov/codecov-action@v3 + with: + file: cobertura.xml + fail_ci_if_error: false diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..a6a6290 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,36 @@ +name: Documentation + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +env: + CARGO_TERM_COLOR: always + +jobs: + docs: + name: Build and Test Documentation + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-docs-${{ hashFiles('**/Cargo.lock') }} + + - name: Build documentation + run: | + cargo doc --all-features --no-deps --document-private-items + env: + RUSTDOCFLAGS: "--cfg docsrs -Dwarnings" diff --git a/.github/workflows/pr-preview.yml b/.github/workflows/pr-preview.yml new file mode 100644 index 0000000..fde1d61 --- /dev/null +++ b/.github/workflows/pr-preview.yml @@ -0,0 +1,117 @@ +name: PR Preview + +on: + pull_request: + types: [opened, synchronize, reopened] + +env: + CARGO_TERM_COLOR: always + +jobs: + quick-check: + name: Quick Check + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-quick-${{ hashFiles('**/Cargo.lock') }} + + - name: Check format + run: cargo fmt --all -- --check + + - name: Run clippy + run: cargo clippy --all-targets --all-features + + - name: Check build + run: cargo check --all-features + + - name: Run quick tests + run: cargo test --lib + + changes: + name: Detect Changes + runs-on: ubuntu-latest + outputs: + src: ${{ steps.changes.outputs.src }} + docs: ${{ steps.changes.outputs.docs }} + workflows: ${{ steps.changes.outputs.workflows }} + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - uses: dorny/paths-filter@v2 + id: changes + with: + filters: | + src: + - 'src/**' + - 'Cargo.toml' + - 'Cargo.lock' + docs: + - '**.md' + - 'docs/**' + workflows: + - '.github/workflows/**' + + test-if-needed: + name: Test if Source Changed + runs-on: ubuntu-latest + needs: changes + if: needs.changes.outputs.src == 'true' + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }} + + - name: Run full test suite + run: cargo test --all-features + + summary: + name: PR Preview Summary + runs-on: ubuntu-latest + needs: [quick-check, changes, test-if-needed] + if: always() + steps: + - name: Summary + run: | + echo "## PR Preview Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **Quick Check**: ${{ needs.quick-check.result }}" >> $GITHUB_STEP_SUMMARY + if [[ "${{ needs.changes.outputs.src }}" == "true" ]]; then + echo "- **Full Tests**: ${{ needs.test-if-needed.result }}" >> $GITHUB_STEP_SUMMARY + else + echo "- **Full Tests**: Skipped (no source changes)" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Changed Files" >> $GITHUB_STEP_SUMMARY + if [[ "${{ needs.changes.outputs.src }}" == "true" ]]; then + echo "- ✅ Source code" >> $GITHUB_STEP_SUMMARY + fi + if [[ "${{ needs.changes.outputs.docs }}" == "true" ]]; then + echo "- 📚 Documentation" >> $GITHUB_STEP_SUMMARY + fi + if [[ "${{ needs.changes.outputs.workflows }}" == "true" ]]; then + echo "- ⚙️ GitHub Actions workflows" >> $GITHUB_STEP_SUMMARY + fi \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..ea14cd6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,132 @@ +name: Release and Publish + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + version: + description: 'Version to publish (without v prefix)' + required: true + type: string + +env: + CARGO_TERM_COLOR: always + +jobs: + validate: + name: Validate Release + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }} + + - name: Extract version + id: version + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + VERSION="${{ github.event.inputs.version }}" + else + VERSION=${GITHUB_REF#refs/tags/v} + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Extracted version: $VERSION" + + - name: Verify version matches Cargo.toml + run: | + CARGO_VERSION=$(grep '^version = ' Cargo.toml | head -1 | sed 's/version = "\(.*\)"/\1/') + if [[ "$CARGO_VERSION" != "${{ steps.version.outputs.version }}" ]]; then + echo "Version mismatch: Cargo.toml has $CARGO_VERSION, but release is ${{ steps.version.outputs.version }}" + exit 1 + fi + + - name: Run tests + run: cargo test --all-features + + - name: Check format + run: cargo fmt --all -- --check + + - name: Run clippy + run: cargo clippy --all-targets --all-features + + - name: Build docs + run: cargo doc --all-features --no-deps + + publish: + name: Publish to crates.io + runs-on: ubuntu-latest + needs: validate + environment: release + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-publish-${{ hashFiles('**/Cargo.lock') }} + + - name: Login to crates.io + run: cargo login ${{ secrets.CRATES_IO_TOKEN }} + + - name: Publish to crates.io + run: cargo publish + + github-release: + name: Create GitHub Release + runs-on: ubuntu-latest + needs: [validate, publish] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Generate changelog + id: changelog + run: | + if [[ -f CHANGELOG.md ]]; then + # Extract changelog for this version + VERSION="v${{ needs.validate.outputs.version }}" + sed -n "/^## \[$VERSION\]/,/^## \[/p" CHANGELOG.md | sed '$d' > release_notes.md + if [[ -s release_notes.md ]]; then + echo "Found changelog entry for $VERSION" + else + echo "## Changes" > release_notes.md + echo "See commit history for detailed changes." >> release_notes.md + fi + else + echo "## Changes" > release_notes.md + echo "See commit history for detailed changes." >> release_notes.md + fi + + - name: Create GitHub Release + uses: softprops/action-gh-release@v1 + with: + name: Release v${{ needs.validate.outputs.version }} + body_path: release_notes.md + draft: false + prerelease: ${{ contains(needs.validate.outputs.version, 'alpha') || contains(needs.validate.outputs.version, 'beta') || contains(needs.validate.outputs.version, 'rc') }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/rust-security.yml b/.github/workflows/rust-security.yml new file mode 100644 index 0000000..e2f0bb8 --- /dev/null +++ b/.github/workflows/rust-security.yml @@ -0,0 +1,68 @@ +name: Rust Security + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + schedule: + - cron: '0 3 * * 1' # Run weekly on Monday at 3 AM UTC + +env: + CARGO_TERM_COLOR: always + +jobs: + cargo-deny: + name: Cargo Deny + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install cargo-deny + uses: taiki-e/install-action@cargo-deny + + - name: Run cargo deny + run: cargo deny check + + supply-chain: + name: Supply Chain Security + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install cargo-audit + uses: taiki-e/install-action@cargo-audit + + - name: Audit dependencies + run: cargo audit + + - name: Install cargo-outdated + uses: taiki-e/install-action@cargo-outdated + + - name: Check for outdated dependencies + run: cargo outdated --exit-code 1 + continue-on-error: true + + unused-deps: + name: Unused Dependencies + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust nightly toolchain + uses: dtolnay/rust-toolchain@nightly + + - name: Install cargo-udeps + uses: taiki-e/install-action@cargo-udeps + + - name: Check for unused dependencies + run: cargo +nightly udeps --all-targets --all-features diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..7f501f7 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,83 @@ +name: Security and Dependencies + +on: + schedule: + # Run security audit daily at 02:00 UTC + - cron: '0 2 * * *' + push: + branches: [ main ] + paths: + - '**/Cargo.toml' + - '**/Cargo.lock' + pull_request: + paths: + - '**/Cargo.toml' + - '**/Cargo.lock' + +env: + CARGO_TERM_COLOR: always + +jobs: + security-audit: + name: Security Audit + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install cargo-audit + uses: taiki-e/install-action@cargo-audit + + - name: Run cargo audit + run: cargo audit + + - name: Run cargo audit (JSON output) + run: cargo audit --json > audit-results.json + continue-on-error: true + + - name: Upload audit results + uses: actions/upload-artifact@v4 + if: always() + with: + name: security-audit-results + path: audit-results.json + + dependency-review: + name: Dependency Review + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Dependency Review + uses: actions/dependency-review-action@v3 + + license-check: + name: License Check + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install cargo-license + uses: taiki-e/install-action@cargo-license + + - name: Check licenses + run: | + echo "## License Report" > license-report.md + echo "" >> license-report.md + cargo license --json | jq -r '.[] | "- **\(.name)** (\(.version)): \(.license // "Unknown")"' >> license-report.md + cat license-report.md + + - name: Upload license report + uses: actions/upload-artifact@v4 + with: + name: license-report + path: license-report.md diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..9aff8d0 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,38 @@ +name: Mark stale issues and pull requests + +on: + schedule: + - cron: '0 0 * * *' # Run daily at midnight + workflow_dispatch: + +jobs: + stale: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v8 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: | + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs within 7 days. + Thank you for your contributions. + stale-pr-message: | + This pull request has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs within 7 days. + Thank you for your contributions. + close-issue-message: | + This issue has been automatically closed due to inactivity. + If you believe this is still relevant, please reopen it or create a new issue. + close-pr-message: | + This pull request has been automatically closed due to inactivity. + If you believe this is still relevant, please reopen it or create a new pull request. + days-before-stale: 60 + days-before-close: 7 + stale-issue-label: 'stale' + stale-pr-label: 'stale' + exempt-issue-labels: 'pinned,security,good first issue' + exempt-pr-labels: 'pinned,security' + operations-per-run: 50 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2e19174 --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +# Generated by Cargo +# will have compiled files and executables +debug +target + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb + +# Generated by cargo mutants +# Contains mutation testing data +**/mutants.out*/ + +# RustRover +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# dotenv environment variables file +.env + +# SQLite database files +test.db +test.db-shm +test.db-wal diff --git a/.sqlx/query-0819e39ed31495e0c70a6dd549b10e5c52e64d5289ca829e2a0135a14801b930.json b/.sqlx/query-0819e39ed31495e0c70a6dd549b10e5c52e64d5289ca829e2a0135a14801b930.json new file mode 100644 index 0000000..426e55e --- /dev/null +++ b/.sqlx/query-0819e39ed31495e0c70a6dd549b10e5c52e64d5289ca829e2a0135a14801b930.json @@ -0,0 +1,86 @@ +{ + "db_name": "SQLite", + "query": "UPDATE Jobs\nSET\n status = 'Running',\n lock_by = ?1,\n lock_at = CURRENT_TIMESTAMP\nWHERE\n ROWID IN (\n SELECT ROWID\n FROM Jobs\n WHERE job_type = ?2\n AND status = 'Pending'\n AND lock_by IS NULL\n ORDER BY priority DESC, id ASC\n LIMIT ?3\n )\nRETURNING *\n", + "describe": { + "columns": [ + { + "name": "job", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "job_type", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "status", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "attempts", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "max_attempts", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "run_at", + "ordinal": 6, + "type_info": "Integer" + }, + { + "name": "last_error", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "lock_at", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "lock_by", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "done_at", + "ordinal": 10, + "type_info": "Integer" + }, + { + "name": "priority", + "ordinal": 11, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 3 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + false + ] + }, + "hash": "0819e39ed31495e0c70a6dd549b10e5c52e64d5289ca829e2a0135a14801b930" +} diff --git a/.sqlx/query-0be6db0bdde33835113a74e4749bc68adca35acef6ce80e58bad3b70c3fd6d97.json b/.sqlx/query-0be6db0bdde33835113a74e4749bc68adca35acef6ce80e58bad3b70c3fd6d97.json new file mode 100644 index 0000000..e914ad4 --- /dev/null +++ b/.sqlx/query-0be6db0bdde33835113a74e4749bc68adca35acef6ce80e58bad3b70c3fd6d97.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "INSERT INTO\n Workers (id, worker_type, storage_name, layers, last_seen)\nVALUES\n ($1, $2, $3, $4, $5) ON CONFLICT (id) DO\nUPDATE\nSET\n last_seen = EXCLUDED.last_seen\n", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + }, + "hash": "0be6db0bdde33835113a74e4749bc68adca35acef6ce80e58bad3b70c3fd6d97" +} diff --git a/.sqlx/query-4ead10dafe0d2d024654403f289c4db57308e5cf88f44efaf23b05c585626465.json b/.sqlx/query-4ead10dafe0d2d024654403f289c4db57308e5cf88f44efaf23b05c585626465.json new file mode 100644 index 0000000..e2ab422 --- /dev/null +++ b/.sqlx/query-4ead10dafe0d2d024654403f289c4db57308e5cf88f44efaf23b05c585626465.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "INSERT INTO\n Jobs\nVALUES\n (\n ?1,\n ?2,\n ?3,\n 'Pending',\n 0,\n ?4,\n ?5,\n NULL,\n NULL,\n NULL,\n NULL,\n ?6\n )\n", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + }, + "hash": "4ead10dafe0d2d024654403f289c4db57308e5cf88f44efaf23b05c585626465" +} diff --git a/.sqlx/query-cfc8346ce97a0f65e057b439787acaa0ee3ad0de2ad6453554e95be9df4e487e.json b/.sqlx/query-cfc8346ce97a0f65e057b439787acaa0ee3ad0de2ad6453554e95be9df4e487e.json new file mode 100644 index 0000000..fea0ff3 --- /dev/null +++ b/.sqlx/query-cfc8346ce97a0f65e057b439787acaa0ee3ad0de2ad6453554e95be9df4e487e.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "UPDATE\n Jobs\nSET\n status = ?4,\n attempts = ?2,\n last_error = ?3,\n done_at = CURRENT_TIMESTAMP\nWHERE\n id = ?1\n AND lock_by = ?5\n", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + }, + "hash": "cfc8346ce97a0f65e057b439787acaa0ee3ad0de2ad6453554e95be9df4e487e" +} diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..cd9bd23 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2822 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "apalis-core" +version = "1.0.0-alpha.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4096618accabcf371a44e1ef9b9739cf3a2ba5878a6e64ef6ef78a1f035e8cb" +dependencies = [ + "futures-channel", + "futures-core", + "futures-sink", + "futures-timer", + "futures-util", + "pin-project", + "serde", + "serde_json", + "thiserror", + "tower-layer", + "tower-service", +] + +[[package]] +name = "apalis-sqlite" +version = "1.0.0-alpha.1" +dependencies = [ + "apalis-core", + "apalis-sqlite", + "async-std", + "chrono", + "futures", + "libsqlite3-sys", + "log", + "pin-project", + "serde", + "serde_json", + "sqlx", + "thiserror", + "tokio", + "ulid", +] + +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497c00e0fd83a72a79a39fcbd8e3e2f055d6f6c7e025f3b3d91f4f8e76527fb8" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand 2.3.0", + "futures-lite 2.6.1", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" +dependencies = [ + "async-channel 2.5.0", + "async-executor", + "async-io 2.6.0", + "async-lock 3.4.1", + "blocking", + "futures-lite 2.6.1", + "once_cell", +] + +[[package]] +name = "async-io" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" +dependencies = [ + "async-lock 2.8.0", + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-lite 1.13.0", + "log", + "parking", + "polling 2.8.0", + "rustix 0.37.28", + "slab", + "socket2 0.4.10", + "waker-fn", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite 2.6.1", + "parking", + "polling 3.11.0", + "rustix 1.1.2", + "slab", + "windows-sys 0.61.0", +] + +[[package]] +name = "async-lock" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +dependencies = [ + "event-listener 2.5.3", +] + +[[package]] +name = "async-lock" +version = "3.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd03604047cee9b6ce9de9f70c6cd540a0520c813cbd49bae61f33ab80ed1dc" +dependencies = [ + "event-listener 5.4.1", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-std" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c8e079a4ab67ae52b7403632e4618815d6db36d2a010cfe41b02c1b1578f93b" +dependencies = [ + "async-channel 1.9.0", + "async-global-executor", + "async-io 2.6.0", + "async-lock 3.4.1", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite 2.6.1", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" +dependencies = [ + "serde", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel 2.5.0", + "async-task", + "futures-io", + "futures-lite 2.6.1", + "piper", +] + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cc" +version = "1.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" + +[[package]] +name = "chrono" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +dependencies = [ + "serde", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.0", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener 5.4.1", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand 2.3.0", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.7+wasi-0.2.4", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" +dependencies = [ + "equivalent", + "hashbrown 0.16.0", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "io-uring" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" +dependencies = [ + "bitflags 2.9.4", + "cfg-if", + "libc", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "js-sys" +version = "0.3.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852f13bec5eba4ba9afbeb93fd7c13fe56147f055939ae21c43a29a0ecb2702e" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + +[[package]] +name = "libc" +version = "0.2.175" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" + +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + +[[package]] +name = "libredox" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" +dependencies = [ + "bitflags 2.9.4", + "libc", + "redox_syscall", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + +[[package]] +name = "lock_api" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +dependencies = [ + "value-bag", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", +] + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "openssl" +version = "0.10.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" +dependencies = [ + "bitflags 2.9.4", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "piper" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" +dependencies = [ + "atomic-waker", + "fastrand 2.3.0", + "futures-io", +] + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "polling" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" +dependencies = [ + "autocfg", + "bitflags 1.3.2", + "cfg-if", + "concurrent-queue", + "libc", + "log", + "pin-project-lite", + "windows-sys 0.48.0", +] + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi 0.5.2", + "pin-project-lite", + "rustix 1.1.2", + "windows-sys 0.61.0", +] + +[[package]] +name = "potential_utf" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.3", +] + +[[package]] +name = "redox_syscall" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +dependencies = [ + "bitflags 2.9.4", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rsa" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" + +[[package]] +name = "rustix" +version = "0.37.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "519165d378b97752ca44bbe15047d5d3409e875f39327546b42ac81d7e18c1b6" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags 2.9.4", + "errno", + "libc", + "linux-raw-sys 0.11.0", + "windows-sys 0.61.0", +] + +[[package]] +name = "rustls" +version = "0.23.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3c25631629d034ce7cd9940adc9d45762d46de2b0f57193c4443b92c6d4d40" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8572f3c2cb9934231157b45499fc41e1f58c589fdfb81a844ba873265e80f8eb" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.9.4", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.226" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dca6411025b24b60bfa7ec1fe1f8e710ac09782dca409ee8237ba74b51295fd" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.226" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba2ba63999edb9dac981fb34b3e5c0d111a69b0924e253ed29d83f7c99e966a4" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.226" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8db53ae22f34573731bafa1db20f04027b2d25e02d8205921b569171699cdb33" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "async-io 1.13.0", + "async-std", + "base64", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener 5.4.1", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap", + "log", + "memchr", + "native-tls", + "once_cell", + "percent-encoding", + "rustls", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", + "webpki-roots 0.26.11", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "async-std", + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64", + "bitflags 2.9.4", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.5", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64", + "bitflags 2.9.4", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.5", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror", + "tracing", + "url", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84fa4d11fadde498443cca10fd3ac23c951f0dc59e080e9f4b93d4df4e4eea53" +dependencies = [ + "fastrand 2.3.0", + "getrandom 0.3.3", + "once_cell", + "rustix 1.1.2", + "windows-sys 0.61.0", +] + +[[package]] +name = "thiserror" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.47.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +dependencies = [ + "backtrace", + "bytes", + "io-uring", + "libc", + "mio", + "pin-project-lite", + "slab", + "socket2 0.6.0", + "tokio-macros", + "windows-sys 0.59.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + +[[package]] +name = "ulid" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" +dependencies = [ + "rand 0.9.2", + "web-time", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" + +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "value-bag" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943ce29a8a743eb10d6082545d861b24f9d1b160b7d741e0f2cdf726bec909c5" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "waker-fn" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab10a69fbd0a177f5f649ad4d8d3305499c42bab9aef2f7ff592d0ec8f833819" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb702423545a6007bbc368fde243ba47ca275e549c8a28617f56f6ba53b1d1c" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0b221ff421256839509adbb55998214a70d829d3a28c69b4a6672e9d2a42f67" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc65f4f411d91494355917b605e1480033152658d71f722a90647f56a70c88a0" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc003a991398a8ee604a401e194b6b3a39677b3173d6e74495eb51b82e99a32" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "293c37f4efa430ca14db3721dfbe48d8c33308096bd44d80ebaa775ab71ba1cf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbe734895e869dc429d78c4b433f8d17d95f8d05317440b4fad5ab2d33e596dc" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.2", +] + +[[package]] +name = "webpki-roots" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fe7168f7de578d2d8a05b07fd61870d2e73b4020e9f49aa00da8471723497c" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" + +[[package]] +name = "windows-result" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7084dcc306f89883455a206237404d3eaf961e5bd7e0f312f7c91f57eb44167f" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7218c655a553b0bed4426cf54b20d7ba363ef543b52d515b3e48d7fd55318dda" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/packages/apalis-sql/Cargo.toml b/Cargo.toml similarity index 53% rename from packages/apalis-sql/Cargo.toml rename to Cargo.toml index e9df625..f18c300 100644 --- a/packages/apalis-sql/Cargo.toml +++ b/Cargo.toml @@ -1,20 +1,15 @@ [package] -name = "apalis-sql" +name = "apalis-sqlite" version = "1.0.0-alpha.1" authors = ["Njuguna Mureithi "] -edition.workspace = true -repository.workspace = true -readme = "../../README.md" - +readme = "README.md" +edition = "2024" +repository = "https://github.com/apalis-dev/apalis-sqlite" license = "MIT" -description = "SQL Storage for apalis. Use sqlite, postgres and mysql for background job processing" +description = "Background task processing for rust using apalis and sqlite" [features] -default = ["migrate", "tokio-comp", "sqlite"] -postgres = ["sqlx/postgres", "sqlx/json"] -postgres-bytes = ["sqlx/postgres"] -sqlite = ["sqlx/sqlite", "sqlx/json"] -mysql = ["sqlx/mysql", "sqlx/json", "sqlx/bigdecimal"] +default = ["migrate"] migrate = ["sqlx/migrate", "sqlx/macros"] async-std-comp = ["async-std", "sqlx/runtime-async-std-rustls"] async-std-comp-native-tls = ["async-std", "sqlx/runtime-async-std-native-tls"] @@ -22,36 +17,31 @@ tokio-comp = ["tokio", "sqlx/runtime-tokio-rustls"] tokio-comp-native-tls = ["tokio", "sqlx/runtime-tokio-native-tls"] [dependencies.sqlx] -version = "0.8.1" +version = "0.8.6" default-features = false -features = ["chrono"] +features = ["chrono", "sqlite"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "1.0.0-alpha.1", default-features = false, features = [ +apalis-core = { version = "1.0.0-alpha.2", default-features = false, features = [ "sleep", "json", ] } log = "0.4.21" futures = "0.3.30" tokio = { version = "1", features = ["rt", "net"], optional = true } -futures-lite = "2.3.0" async-std = { version = "1.13.0", optional = true } chrono = { version = "0.4", features = ["serde"] } thiserror = "2.0.0" pin-project = "1.1.10" libsqlite3-sys = "0.30.1" +ulid = { version = "1" } [dev-dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } -# email-service = { path = "../../examples/email-service" } -apalis = { path = "../../", default-features = false } -once_cell = "1.19.0" -apalis-sql = { path = ".", features = ["tokio-comp"] } -apalis-core = { path = "../apalis-core", features = ["test-utils"] } -tower = {version = "0.5", features = ["full"]} -nanoid = "0.4.0" +apalis-core = { version = "1.0.0-alpha.2", features = ["test-utils"] } +apalis-sqlite = { path = "." , features = ["migrate", "tokio-comp"] } [package.metadata.docs.rs] # defines the configuration attribute `docsrs` diff --git a/packages/apalis-sql/build.rs b/build.rs similarity index 100% rename from packages/apalis-sql/build.rs rename to build.rs diff --git a/packages/apalis-sql/migrations/sqlite/20220530084123_jobs_workers.sql b/migrations/20220530084123_jobs_workers.sql similarity index 100% rename from packages/apalis-sql/migrations/sqlite/20220530084123_jobs_workers.sql rename to migrations/20220530084123_jobs_workers.sql diff --git a/packages/apalis-sql/migrations/sqlite/20250313213411_add_job_priority.sql b/migrations/20250313213411_add_job_priority.sql similarity index 100% rename from packages/apalis-sql/migrations/sqlite/20250313213411_add_job_priority.sql rename to migrations/20250313213411_add_job_priority.sql diff --git a/queries/task/ack.sql b/queries/task/ack.sql new file mode 100644 index 0000000..337f9a0 --- /dev/null +++ b/queries/task/ack.sql @@ -0,0 +1,10 @@ +UPDATE + Jobs +SET + status = ?4, + attempts = ?2, + last_error = ?3, + done_at = CURRENT_TIMESTAMP +WHERE + id = ?1 + AND lock_by = ?5 diff --git a/queries/task/fetch_next.sql b/queries/task/fetch_next.sql new file mode 100644 index 0000000..daf8858 --- /dev/null +++ b/queries/task/fetch_next.sql @@ -0,0 +1,17 @@ +UPDATE Jobs +SET + status = 'Running', + lock_by = ?1, + lock_at = CURRENT_TIMESTAMP +WHERE + ROWID IN ( + SELECT ROWID + FROM Jobs + WHERE job_type = ?2 + AND status = 'Pending' + AND lock_by IS NULL + AND (run_at IS NULL OR run_at <= CURRENT_TIMESTAMP) + ORDER BY priority DESC, id ASC + LIMIT ?3 + ) +RETURNING * diff --git a/queries/task/sink.sql b/queries/task/sink.sql new file mode 100644 index 0000000..ee42376 --- /dev/null +++ b/queries/task/sink.sql @@ -0,0 +1,17 @@ +INSERT INTO + Jobs +VALUES + ( + ?1, + ?2, + ?3, + 'Pending', + 0, + ?4, + ?5, + NULL, + NULL, + NULL, + NULL, + ?6 + ) diff --git a/queries/worker/register.sql b/queries/worker/register.sql new file mode 100644 index 0000000..54f2da4 --- /dev/null +++ b/queries/worker/register.sql @@ -0,0 +1,7 @@ +INSERT INTO + Workers (id, worker_type, storage_name, layers, last_seen) +VALUES + ($1, $2, $3, $4, $5) ON CONFLICT (id) DO +UPDATE +SET + last_seen = EXCLUDED.last_seen diff --git a/src/ack.rs b/src/ack.rs new file mode 100644 index 0000000..8e733a1 --- /dev/null +++ b/src/ack.rs @@ -0,0 +1,79 @@ +use apalis_core::{ + error::BoxDynError, + task::{Parts, status::Status}, + worker::{self, ext::ack::Acknowledge}, +}; +use chrono::Utc; +use futures::{FutureExt, future::BoxFuture}; +use serde::Serialize; +use sqlx::SqlitePool; +use ulid::Ulid; + +use crate::context::SqliteContext; + +#[derive(Clone)] +pub struct SqliteAck { + pool: SqlitePool, +} +impl SqliteAck { + pub fn new(pool: SqlitePool) -> Self { + Self { pool } + } +} + +impl Acknowledge for SqliteAck { + type Error = sqlx::Error; + type Future = BoxFuture<'static, Result<(), Self::Error>>; + fn ack( + &mut self, + res: &Result, + parts: &Parts, + ) -> Self::Future { + let task_id = parts.task_id.clone(); + let worker_id = parts.ctx.lock_by().clone(); + + let response = serde_json::to_string(&res.as_ref().map_err(|e| e.to_string())); + let status = calculate_status(parts, res).to_string(); + let attempt = parts.attempt.current() as i32; + let pool = self.pool.clone(); + let res = response.map_err(|e| sqlx::Error::Decode(e.into())); + + async move { + let task_id = task_id + .ok_or(sqlx::Error::ColumnNotFound("TASK_ID_FOR_ACK".to_owned()))? + .to_string(); + let worker_id = + worker_id.ok_or(sqlx::Error::ColumnNotFound("WORKER_ID_LOCK_BY".to_owned()))?; + let res_ok = res?; + let res = sqlx::query_file!( + "queries/task/ack.sql", + task_id, + attempt, + res_ok, + status, + worker_id + ) + .execute(&pool) + .await?; + + if res.rows_affected() == 0 { + return Err(sqlx::Error::RowNotFound); + } + Ok(()) + } + .boxed() + } +} + +pub fn calculate_status( + parts: &Parts, + res: &Result, +) -> Status { + match &res { + Ok(_) => Status::Done, + Err(e) => match &e { + e if parts.ctx.max_attempts() as usize <= parts.attempt.current() => Status::Killed, + _ => Status::Failed, + }, + } +} diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..cf99233 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,131 @@ +use std::time::Duration; + +use apalis_core::backend::poll_strategy::{IntervalStrategy, MultiStrategy, StrategyBuilder}; +use sqlx::sqlite::SqliteConnectOptions; + +#[derive(Debug, Clone)] +pub struct Config { + keep_alive: Duration, + buffer_size: usize, + poll_strategy: MultiStrategy, + reenqueue_orphaned_after: Duration, + namespace: String, + ack: bool, +} + +impl Default for Config { + fn default() -> Self { + Self { + keep_alive: Duration::from_secs(30), + buffer_size: 10, + poll_strategy: StrategyBuilder::new() + .apply(IntervalStrategy::new(Duration::from_millis(100))) + .build(), + reenqueue_orphaned_after: Duration::from_secs(300), // 5 minutes + namespace: String::from("apalis::sqlite"), + ack: true, + } + } +} + +impl Config { + /// Create a new config with a jobs namespace + pub fn new(namespace: &str) -> Self { + Config::default().set_namespace(namespace) + } + + /// Interval between database poll queries + /// + /// Defaults to 100ms + pub fn with_poll_interval(mut self, strategy: MultiStrategy) -> Self { + self.poll_strategy = strategy; + self + } + + /// Interval between worker keep-alive database updates + /// + /// Defaults to 30s + pub fn set_keep_alive(mut self, keep_alive: Duration) -> Self { + self.keep_alive = keep_alive; + self + } + + /// Buffer size to use when querying for jobs + /// + /// Defaults to 10 + pub fn set_buffer_size(mut self, buffer_size: usize) -> Self { + self.buffer_size = buffer_size; + self + } + + /// Set the namespace to consume and push jobs to + /// + /// Defaults to "apalis::sql" + pub fn set_namespace(mut self, namespace: &str) -> Self { + self.namespace = namespace.to_string(); + self + } + + /// Gets a reference to the keep_alive duration. + pub fn keep_alive(&self) -> &Duration { + &self.keep_alive + } + + /// Gets a mutable reference to the keep_alive duration. + pub fn keep_alive_mut(&mut self) -> &mut Duration { + &mut self.keep_alive + } + + /// Gets the buffer size. + pub fn buffer_size(&self) -> usize { + self.buffer_size + } + + /// Gets a reference to the poll_strategy. + pub fn poll_strategy(&self) -> &MultiStrategy { + &self.poll_strategy + } + + /// Gets a mutable reference to the poll_strategy. + pub fn poll_strategy_mut(&mut self) -> &mut MultiStrategy { + &mut self.poll_strategy + } + + /// Gets a reference to the namespace. + pub fn namespace(&self) -> &String { + &self.namespace + } + + /// Gets a mutable reference to the namespace. + pub fn namespace_mut(&mut self) -> &mut String { + &mut self.namespace + } + + /// Gets the reenqueue_orphaned_after duration. + pub fn reenqueue_orphaned_after(&self) -> Duration { + self.reenqueue_orphaned_after + } + + /// Gets a mutable reference to the reenqueue_orphaned_after. + pub fn reenqueue_orphaned_after_mut(&mut self) -> &mut Duration { + &mut self.reenqueue_orphaned_after + } + + /// Occasionally some workers die, or abandon jobs because of panics. + /// This is the time a task takes before its back to the queue + /// + /// Defaults to 5 minutes + pub fn set_reenqueue_orphaned_after(mut self, after: Duration) -> Self { + self.reenqueue_orphaned_after = after; + self + } + + pub fn ack(&self) -> bool { + self.ack + } + + pub fn set_ack(mut self, auto_ack: bool) -> Self { + self.ack = auto_ack; + self + } +} diff --git a/src/context.rs b/src/context.rs new file mode 100644 index 0000000..f112560 --- /dev/null +++ b/src/context.rs @@ -0,0 +1,109 @@ +use std::convert::Infallible; + +use apalis_core::{ + task::{Task, status::Status}, + task_fn::FromRequest, +}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use ulid::Ulid; + +use crate::SqliteTask; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SqliteContext { + max_attempts: i32, + last_result: Option, + lock_at: Option, + lock_by: Option, + done_at: Option, + priority: i32, +} + +impl Default for SqliteContext { + fn default() -> Self { + Self::new() + } +} + +impl SqliteContext { + /// Build a new context with defaults + pub fn new() -> Self { + SqliteContext { + lock_at: None, + done_at: None, + max_attempts: 5, + last_result: None, + lock_by: None, + priority: 0, + } + } + + /// Set the number of attempts + pub fn set_max_attempts(&mut self, max_attempts: i32) { + self.max_attempts = max_attempts; + } + + /// Gets the maximum attempts for a job. Default 25 + pub fn max_attempts(&self) -> i32 { + self.max_attempts + } + + /// Get the time a job was done + pub fn done_at(&self) -> &Option { + &self.done_at + } + + /// Set the time a job was done + pub fn set_done_at(&mut self, done_at: Option) { + self.done_at = done_at; + } + + /// Get the time a job was locked + pub fn lock_at(&self) -> &Option { + &self.lock_at + } + + /// Set the lock_at value + pub fn set_lock_at(&mut self, lock_at: Option) { + self.lock_at = lock_at; + } + + /// Get the time a job was locked + pub fn lock_by(&self) -> &Option { + &self.lock_by + } + + /// Set `lock_by` + pub fn set_lock_by(&mut self, lock_by: Option) { + self.lock_by = lock_by; + } + + /// Get the time a job was locked + pub fn last_result(&self) -> &Option { + &self.last_result + } + + /// Set the last result + pub fn set_last_result(&mut self, result: Option) { + self.last_result = result; + } + + /// Set the job priority. Larger values will run sooner. Default is 0. + pub fn set_priority(&mut self, priority: i32) { + self.priority = priority + } + + /// Get the job priority + pub fn priority(&self) -> i32 { + self.priority + } +} + +impl FromRequest> for SqliteContext { + type Error = Infallible; + async fn from_request(req: &SqliteTask) -> Result { + Ok(req.parts.ctx.clone()) + } +} diff --git a/src/fetcher.rs b/src/fetcher.rs new file mode 100644 index 0000000..74be18a --- /dev/null +++ b/src/fetcher.rs @@ -0,0 +1,208 @@ +use std::{ + collections::VecDeque, + marker::PhantomData, + pin::Pin, + sync::{Arc, atomic::AtomicUsize}, + task::{Context, Poll}, + time::{Duration, Instant}, +}; + +use apalis_core::{ + backend::{ + codec::{Codec, json::JsonCodec}, + poll_strategy::{PollContext, PollStrategyExt}, + }, + task::Task, + timer::Delay, + worker::context::WorkerContext, +}; +use futures::{ + Future, FutureExt, StreamExt, + future::BoxFuture, + stream::{self, BoxStream, Stream}, +}; +use pin_project::pin_project; +use serde_json::Value; +use sqlx::{Pool, Sqlite, SqlitePool}; +use ulid::Ulid; + +use crate::{SqliteTask, config::Config, context::SqliteContext, from_row::TaskRow}; + +pub(crate) async fn fetch_next>( + pool: SqlitePool, + config: Config, + worker: WorkerContext, +) -> Result>, sqlx::Error> +where + D::Error: std::error::Error + Send + Sync + 'static, +{ + let job_type = config.namespace(); + let buffer_size = config.buffer_size() as i32; + let worker = worker.name().to_string(); + sqlx::query_file_as!( + TaskRow, + "queries/task/fetch_next.sql", + worker, + job_type, + buffer_size + ) + .fetch_all(&pool) + .await? + .into_iter() + .map(|r| r.try_into_task::()) + .collect() +} + +enum StreamState { + Ready, + Delay, + Fetch(BoxFuture<'static, Result>, sqlx::Error>>), + Buffered(VecDeque>), + Empty, +} + +#[pin_project] +pub struct SqliteFetcher> { + pool: SqlitePool, + config: Config, + wrk: WorkerContext, + _marker: PhantomData<(Compact, Decode)>, + #[pin] + state: StreamState, + + #[pin] + delay_stream: Option + Send>>>, + + prev_count: Arc, +} + +impl Clone for SqliteFetcher { + fn clone(&self) -> Self { + Self { + pool: self.pool.clone(), + config: self.config.clone(), + wrk: self.wrk.clone(), + _marker: PhantomData, + state: StreamState::Ready, + delay_stream: None, + prev_count: Arc::new(AtomicUsize::new(0)), + } + } +} + +impl SqliteFetcher { + pub fn new(pool: &Pool, config: &Config, wrk: &WorkerContext) -> Self + where + Decode: Codec + 'static, + Decode::Error: std::error::Error + Send + Sync + 'static, + { + Self { + pool: pool.clone(), + config: config.clone(), + wrk: wrk.clone(), + _marker: PhantomData, + state: StreamState::Ready, + delay_stream: None, + prev_count: Arc::new(AtomicUsize::new(0)), + } + } +} + +impl Stream for SqliteFetcher +where + Decode::Error: std::error::Error + Send + Sync + 'static, + Args: Send + 'static + Unpin, + Decode: Codec + 'static, + // Compact: Unpin + Send + 'static, +{ + type Item = Result>, sqlx::Error>; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut this = self.get_mut(); + if this.delay_stream.is_none() { + let strategy = this + .config + .poll_strategy() + .clone() + .build_stream(&PollContext::new(this.wrk.clone(), this.prev_count.clone())); + this.delay_stream = Some(Box::pin(strategy)); + } + + loop { + match this.state { + StreamState::Ready => { + let stream = fetch_next::( + this.pool.clone(), + this.config.clone(), + this.wrk.clone(), + ); + this.state = StreamState::Fetch(stream.boxed()); + } + StreamState::Delay => { + if let Some(delay_stream) = this.delay_stream.as_mut() { + match delay_stream.as_mut().poll_next(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Some(_)) => { + this.state = StreamState::Ready; + } + Poll::Ready(None) => { + this.state = StreamState::Empty; + return Poll::Ready(None); + } + } + } else { + this.state = StreamState::Empty; + return Poll::Ready(None); + } + } + + StreamState::Fetch(ref mut fut) => match fut.poll_unpin(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(item) => match item { + Ok(requests) => { + if requests.is_empty() { + this.state = StreamState::Delay; + } else { + let mut buffer = VecDeque::new(); + for request in requests { + buffer.push_back(request); + } + + this.state = StreamState::Buffered(buffer); + } + } + Err(e) => { + this.state = StreamState::Empty; + return Poll::Ready(Some(Err(e))); + } + }, + }, + + StreamState::Buffered(ref mut buffer) => { + if let Some(request) = buffer.pop_front() { + // Yield the next buffered item + if buffer.is_empty() { + // Buffer is now empty, transition to ready for next fetch + this.state = StreamState::Ready; + } + return Poll::Ready(Some(Ok(Some(request)))); + } else { + // Buffer is empty, transition to ready + this.state = StreamState::Ready; + } + } + + StreamState::Empty => return Poll::Ready(None), + } + } + } +} + +impl SqliteFetcher { + pub fn take_pending(&mut self) -> VecDeque> { + match &mut self.state { + StreamState::Buffered(tasks) => std::mem::take(tasks), + _ => VecDeque::new(), + } + } +} diff --git a/src/from_row.rs b/src/from_row.rs new file mode 100644 index 0000000..e336634 --- /dev/null +++ b/src/from_row.rs @@ -0,0 +1,72 @@ +use std::str::FromStr; + +use apalis_core::{ + backend::codec::Codec, + task::{attempt::Attempt, builder::TaskBuilder, status::Status, task_id::TaskId}, +}; +use chrono::{DateTime, Utc}; +use serde_json::{Map, Value}; + +use crate::{SqliteTask, context::SqliteContext}; + +#[derive(Debug)] +pub(crate) struct TaskRow { + pub(crate) job: String, + pub(crate) id: Option, + pub(crate) job_type: Option, + pub(crate) status: Option, + pub(crate) attempts: Option, + pub(crate) max_attempts: Option, + pub(crate) run_at: Option, + pub(crate) last_error: Option, + pub(crate) lock_at: Option, + pub(crate) lock_by: Option, + pub(crate) done_at: Option, + pub(crate) priority: Option, + // pub(crate) meta: Option>, +} + +impl TaskRow { + pub fn try_into_task, Args>( + self, + ) -> Result, sqlx::Error> + where + D::Error: std::error::Error + Send + Sync + 'static, + { + let mut ctx = SqliteContext::default(); + ctx.set_lock_by(self.lock_by); + ctx.set_max_attempts(self.max_attempts.unwrap_or(25) as i32); + ctx.set_done_at(self.done_at); + ctx.set_last_result(self.last_error); + ctx.set_priority(self.priority.unwrap_or(0) as i32); + ctx.set_lock_at(self.lock_at); + let args = D::decode(&self.job).map_err(|e| sqlx::Error::Decode(e.into()))?; + let task = TaskBuilder::new(args) + .with_ctx(ctx) + .with_attempt(Attempt::new_with_value( + self.attempts + .ok_or(sqlx::Error::ColumnNotFound("attempts".to_owned()))? + as usize, + )) + .with_status( + self.status + .ok_or(sqlx::Error::ColumnNotFound("status".to_owned())) + .and_then(|s| { + Status::from_str(&s).map_err(|e| sqlx::Error::Decode(e.into())) + })?, + ) + .with_task_id( + self.id + .ok_or(sqlx::Error::ColumnNotFound("task_id".to_owned())) + .and_then(|s| { + TaskId::from_str(&s).map_err(|e| sqlx::Error::Decode(e.into())) + })?, + ) + .run_at_timestamp( + self.run_at + .ok_or(sqlx::Error::ColumnNotFound("run_at".to_owned()))? + as u64, + ); + Ok(task.build()) + } +} diff --git a/packages/apalis-sql/src/sqlite/mod.rs b/src/hook.rs similarity index 88% rename from packages/apalis-sql/src/sqlite/mod.rs rename to src/hook.rs index 1ca8783..1d3869e 100644 --- a/packages/apalis-sql/src/sqlite/mod.rs +++ b/src/hook.rs @@ -1,22 +1,21 @@ use futures::channel::mpsc::{self, Receiver}; use futures::{SinkExt, StreamExt}; use libsqlite3_sys::{sqlite3, sqlite3_update_hook}; +use sqlx::Executor; use sqlx::SqlitePool; -use sqlx::{sqlite::SqliteConnection, Connection, Executor}; -use std::ffi::{c_void, CStr}; +use std::ffi::{CStr, c_void}; use std::os::raw::{c_char, c_int}; -use std::ptr; #[derive(Debug)] -struct DbEvent { - op: &'static str, +pub struct DbEvent { + pub op: &'static str, db_name: String, - table_name: String, + pub table_name: String, rowid: i64, } // Callback for SQLite update hook -extern "C" fn update_hook_callback( +pub(crate) extern "C" fn update_hook_callback( arg: *mut c_void, op: c_int, db_name: *const c_char, @@ -47,11 +46,12 @@ extern "C" fn update_hook_callback( } } + pub struct PushListener { - pool: SqlitePool, - rx: Receiver, + pub rx: Receiver, } +#[cfg(test)] async fn run_it() -> Result<(), sqlx::Error> { let (tx, mut rx) = mpsc::channel::(10); @@ -91,7 +91,7 @@ async fn run_it() -> Result<(), sqlx::Error> { conn.execute("DELETE FROM users WHERE id = 2").await?; // Let events flush - tokio::time::sleep(std::time::Duration::from_millis(100)).await; + // tokio::time::sleep(std::time::Duration::from_millis(100)).await; Ok(()) } diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..ff2cda0 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,403 @@ +use std::{fmt, marker::PhantomData}; + +use apalis_core::{ + backend::{ + Backend, TaskStream, + codec::{Codec, json::JsonCodec}, + }, + task::Task, + worker::{context::WorkerContext, ext::ack::AcknowledgeLayer}, +}; +use chrono::{DateTime, Utc}; +use futures::{ + StreamExt, + channel::mpsc, + future::ready, + stream::{self, BoxStream, select}, +}; +use serde_json::Value; +use sqlx::{Pool, Sqlite, SqlitePool}; +use ulid::Ulid; + +use crate::{ + ack::SqliteAck, + config::Config, + context::SqliteContext, + fetcher::{SqliteFetcher, fetch_next}, + hook::{DbEvent, PushListener, update_hook_callback}, + sink::SqliteSink, +}; + +mod ack; +mod config; +mod context; +mod fetcher; +mod from_row; +mod hook; +mod sink; +// mod shared; + +pub type SqliteTask = Task; + +type DefaultFetcher = PhantomData>>; + +#[pin_project::pin_project] +pub struct SqliteStorage, Fetcher = DefaultFetcher> { + pool: Pool, + job_type: PhantomData, + config: Config, + codec: PhantomData, + #[pin] + sink: SqliteSink, + #[pin] + fetcher: Fetcher, +} + +impl fmt::Debug for SqliteStorage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SqliteStorage") + .field("pool", &self.pool) + .field("job_type", &"PhantomData") + .field("config", &self.config) + .field("codec", &std::any::type_name::()) + .finish() + } +} + +impl Clone for SqliteStorage { + fn clone(&self) -> Self { + SqliteStorage { + sink: self.sink.clone(), + pool: self.pool.clone(), + job_type: PhantomData, + config: self.config.clone(), + codec: self.codec, + fetcher: self.fetcher.clone(), + } + } +} + +impl SqliteStorage<()> { + /// Perform migrations for storage + #[cfg(feature = "migrate")] + pub async fn setup(pool: &Pool) -> Result<(), sqlx::Error> { + sqlx::query("PRAGMA journal_mode = 'WAL';") + .execute(pool) + .await?; + sqlx::query("PRAGMA temp_store = 2;").execute(pool).await?; + sqlx::query("PRAGMA synchronous = NORMAL;") + .execute(pool) + .await?; + sqlx::query("PRAGMA cache_size = 64000;") + .execute(pool) + .await?; + Self::migrations().run(pool).await?; + Ok(()) + } + + /// Get sqlite migrations without running them + #[cfg(feature = "migrate")] + pub fn migrations() -> sqlx::migrate::Migrator { + sqlx::migrate!("./migrations") + } +} + +impl SqliteStorage { + /// Create a new SqliteStorage + pub fn new(pool: &Pool) -> Self { + let config = Config::default(); + Self { + pool: pool.clone(), + job_type: PhantomData, + sink: SqliteSink::new(pool, &config), + config, + codec: PhantomData, + fetcher: PhantomData, + } + } + + pub fn new_with_codec(pool: &Pool, config: &Config) -> SqliteStorage { + SqliteStorage { + pool: pool.clone(), + job_type: PhantomData, + config: config.clone(), + codec: PhantomData, + sink: SqliteSink::new(pool, config), + fetcher: PhantomData, + } + } + + pub fn new_with_config(pool: &Pool, config: &Config) -> Self { + Self { + pool: pool.clone(), + job_type: PhantomData, + config: config.clone(), + codec: PhantomData, + sink: SqliteSink::new(pool, config), + fetcher: PhantomData, + } + } + + pub async fn new_with_hook( + pool: &Pool, + config: &Config, + ) -> SqliteStorage, PushListener> { + use libsqlite3_sys::{sqlite3, sqlite3_update_hook}; + use std::ffi::{CStr, c_void}; + use std::os::raw::{c_char, c_int}; + + let pool = pool.clone(); + let (tx, rx) = mpsc::channel::(10); + + let listener = PushListener { rx }; + + let mut conn = pool.acquire().await.unwrap(); + // Get raw sqlite3* handle + let handle: *mut sqlite3 = conn.lock_handle().await.unwrap().as_raw_handle().as_ptr(); + + // Put sender in a Box so it has a stable memory address + let tx_box = Box::new(tx); + let tx_ptr = Box::into_raw(tx_box) as *mut c_void; + + unsafe { + sqlite3_update_hook(handle, Some(update_hook_callback), tx_ptr); + } + SqliteStorage { + pool: pool.clone(), + job_type: PhantomData, + config: config.clone(), + codec: PhantomData, + sink: SqliteSink::new(&pool, config), + fetcher: listener, + } + } +} + +impl Backend for SqliteStorage +where + Args: Send + 'static + Unpin, + Decode: Codec + 'static, + Decode::Error: std::error::Error + Send + Sync + 'static, +{ + type IdType = Ulid; + + type Context = SqliteContext; + + type Codec = Decode; + + type Error = sqlx::Error; + + type Stream = SqliteFetcher; + + type Beat = BoxStream<'static, Result<(), sqlx::Error>>; + + type Layer = AcknowledgeLayer; + + fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { + let worker_type = self.config.namespace().to_owned(); + let fut = register( + self.pool.clone(), + worker_type, + worker.clone(), + Utc::now().timestamp(), + "SqliteStorage", + ); + stream::once(fut).boxed() + } + + fn middleware(&self) -> Self::Layer { + AcknowledgeLayer::new(SqliteAck::new(self.pool.clone())) + } + + fn poll(self, worker: &WorkerContext) -> Self::Stream { + SqliteFetcher::new(&self.pool, &self.config, worker) + } +} + +impl Backend for SqliteStorage +where + Args: Send + 'static + Unpin, + Decode: Codec + Send + 'static, + Decode::Error: std::error::Error + Send + Sync + 'static, +{ + type IdType = Ulid; + + type Context = SqliteContext; + + type Codec = Decode; + + type Error = sqlx::Error; + + type Stream = TaskStream, sqlx::Error>; + + type Beat = BoxStream<'static, Result<(), sqlx::Error>>; + + type Layer = AcknowledgeLayer; + + fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { + let worker_type = self.config.namespace().to_owned(); + let fut = register( + self.pool.clone(), + worker_type, + worker.clone(), + Utc::now().timestamp(), + "SqliteStorage", + ); + stream::once(fut).boxed() + } + + fn middleware(&self) -> Self::Layer { + AcknowledgeLayer::new(SqliteAck::new(self.pool.clone())) + } + + fn poll(self, worker: &WorkerContext) -> Self::Stream { + let pool = self.pool.clone(); + let config = self.config.clone(); + let worker = worker.clone(); + let eager_fetcher: SqliteFetcher = + SqliteFetcher::new(&self.pool, &self.config, &worker); + let lazy_fetcher = self + .fetcher + .rx + .inspect(|res| { + println!("Received DB Event: {:?}", res); + }) + .filter(|a| ready(a.op == "UPDATE" && a.table_name == "Jobs")) + .then(move |_| fetch_next::(pool.clone(), config.clone(), worker.clone())) + .flat_map(|res| match res { + Ok(tasks) => stream::iter(tasks).map(Ok).boxed(), + Err(e) => stream::iter(vec![Err(e)]).boxed(), + }) + .map(|res| match res { + Ok(task) => Ok(Some(task)), + Err(e) => Err(e), + }); + + select(eager_fetcher, lazy_fetcher).boxed() + } +} + +pub(crate) async fn register( + pool: SqlitePool, + worker_type: String, + worker: WorkerContext, + last_seen: i64, + backend_type: &str, +) -> Result<(), sqlx::Error> { + let last_seen = DateTime::from_timestamp(last_seen, 0).ok_or(sqlx::Error::Io( + std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid Timestamp"), + ))?; + let svc = worker.get_service().to_owned(); + let worker = worker.name().to_owned(); + let res = sqlx::query_file!( + "queries/worker/register.sql", + worker, + worker_type, + backend_type, + svc, + last_seen + ) + .execute(&pool) + .await?; + if res.rows_affected() == 0 { + return Err(sqlx::Error::Io(std::io::Error::new( + std::io::ErrorKind::AddrInUse, + "WORKER_ALREADY_EXISTS", + ))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, str::FromStr, time::Duration}; + + use chrono::Local; + + use apalis_core::{ + backend::poll_strategy::{IntervalStrategy, StrategyBuilder}, + error::BoxDynError, + worker::{builder::WorkerBuilder, event::Event, ext::event_listener::EventListenerExt}, + }; + use futures::SinkExt; + + use super::*; + + #[tokio::test] + async fn basic_worker() { + const ITEMS: usize = 10; + let pool = SqlitePool::connect("sqlite://./test.db").await.unwrap(); + let mut backend = SqliteStorage::new(&pool); + + // SqliteStorage::setup(&backend.pool).await.unwrap(); + let mut start = 0; + + let mut items = stream::repeat_with(move || { + start += 1; + let task = Task::builder(start) + .run_after(Duration::from_secs(1)) + // .with_ctx(SqliteContext::new().with_priority(1)) + .build(); + Ok(task) + }) + .take(ITEMS); + backend.send_all(&mut items).await.unwrap(); + + println!("Starting worker at {}", Local::now()); + + async fn send_reminder(item: usize, wrk: WorkerContext) -> Result<(), BoxDynError> { + if ITEMS == item { + wrk.stop().unwrap(); + } + Ok(()) + } + + let worker = WorkerBuilder::new("rango-tango-1") + .backend(backend) + .build(send_reminder); + worker.run().await.unwrap(); + } + + #[tokio::test] + async fn hooked_worker() { + const ITEMS: usize = 20; + let pool = SqlitePool::connect("sqlite://./test.db").await.unwrap(); + let lazy_strategy = StrategyBuilder::new() + .apply(IntervalStrategy::new(Duration::from_secs(5))) + .build(); + let config = Config::default() + .with_poll_interval(lazy_strategy) + .set_buffer_size(5); + let mut backend = SqliteStorage::new_with_hook(&pool, &config).await; + + // SqliteStorage::setup(&backend.pool).await.unwrap(); + let mut start = 0; + + let mut items = stream::repeat_with(move || { + start += 1; + let task = Task::builder(start) + .run_after(Duration::from_secs(2)) + // .with_ctx(SqliteContext::new().with_priority(1)) + .build(); + Ok(task) + }) + .take(ITEMS); + backend.send_all(&mut items).await.unwrap(); + + println!("Starting worker at {}", Local::now()); + + async fn send_reminder(item: usize, wrk: WorkerContext) -> Result<(), BoxDynError> { + println!("Processed item: {}", item); + if ITEMS == item { + apalis_core::timer::sleep(Duration::from_secs(1)).await; + wrk.stop().unwrap(); + } + Ok(()) + } + + let worker = WorkerBuilder::new("rango-tango-1") + .backend(backend) + .build(send_reminder); + worker.run().await.unwrap(); + } +} diff --git a/src/shared.rs b/src/shared.rs new file mode 100644 index 0000000..0369e47 --- /dev/null +++ b/src/shared.rs @@ -0,0 +1,290 @@ +use std::{ + collections::{HashMap, HashSet}, + future::ready, + marker::PhantomData, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use crate::{ + Config, InsertEvent, SqliteTask, PostgresStorage, ack::SqliteAck, context::SqliteContext, + fetcher::SqliteFetcher, register, +}; +use crate::{from_row::TaskRow, sink::SqliteSink}; +use apalis_core::{ + backend::{ + Backend, TaskStream, + codec::{Codec, json::JsonCodec}, + shared::MakeShared, + }, + task::{Task, task_id::TaskId}, + worker::{context::WorkerContext, ext::ack::AcknowledgeLayer}, +}; +use chrono::Utc; +use futures::{ + FutureExt, SinkExt, Stream, StreamExt, TryStreamExt, + channel::mpsc::{self, Receiver, Sender}, + future::{self, BoxFuture, Shared}, + lock::Mutex, + stream::{self, BoxStream, select}, +}; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use serde_json::Value; +use sqlx::{SqlitePool, postgres::SqliteListener}; +use ulid::Ulid; + +pub struct SharedPostgresStorage> { + pool: SqlitePool, + registry: Arc>>>, + drive: Shared>, + _marker: PhantomData<(Compact, Codec)>, +} + +impl SharedPostgresStorage { + pub fn new(pool: SqlitePool) -> Self { + let registry: Arc>>> = + Arc::new(Mutex::new(HashMap::default())); + let p = pool.clone(); + let instances = registry.clone(); + Self { + pool, + drive: async move { + let mut listener = SqliteListener::connect_with(&p).await.unwrap(); + listener.listen("apalis::job::insert").await.unwrap(); + listener + .into_stream() + .filter_map(|notification| { + let instances = instances.clone(); + async move { + let Sqlite_notification = notification.ok()?; + let payload = Sqlite_notification.payload(); + let ev: InsertEvent = serde_json::from_str(payload).ok()?; + let instances = instances.lock().await; + if instances.get(&ev.job_type).is_some() { + return Some(ev); + } + None + } + }) + .for_each(|ev| { + let instances = instances.clone(); + async move { + let mut instances = instances.lock().await; + let sender = instances.get_mut(&ev.job_type).unwrap(); + sender.send(ev.id).await.unwrap(); + } + }) + .await; + } + .boxed() + .shared(), + registry, + _marker: PhantomData, + } + } +} +#[derive(Debug)] +pub enum SharedPostgresError { + NamespaceExists(String), + RegistryLocked, +} + +impl MakeShared for SharedPostgresStorage { + type Backend = PostgresStorage; + type Config = Config; + type MakeError = SharedPostgresError; + fn make_shared(&mut self) -> Result + where + Self::Config: Default, + { + Self::make_shared_with_config(self, Config::new(std::any::type_name::())) + } + fn make_shared_with_config( + &mut self, + config: Self::Config, + ) -> Result { + let (tx, rx) = mpsc::channel(config.buffer_size()); + let mut r = self + .registry + .try_lock() + .ok_or(SharedPostgresError::RegistryLocked)?; + if let Some(_) = r.insert(config.namespace().to_owned(), tx) { + return Err(SharedPostgresError::NamespaceExists( + config.namespace().to_owned(), + )); + } + let sink = SqliteSink::new(&self.pool, &config); + Ok(PostgresStorage { + _marker: PhantomData, + config, + fetcher: SharedFetcher { + poller: self.drive.clone(), + receiver: rx, + }, + pool: self.pool.clone(), + sink, + }) + } +} + +pub struct SharedFetcher { + poller: Shared>, + receiver: Receiver, +} + +impl Stream for SharedFetcher { + type Item = TaskId; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + // Keep the poller alive by polling it, but ignoring the output + let _ = this.poller.poll_unpin(cx); + + // Delegate actual items to receiver + this.receiver.poll_next_unpin(cx) + } +} + +impl Backend for PostgresStorage +where + Args: Send + 'static + Unpin, + Decode: Codec + 'static + Unpin + Send, + Decode::Error: std::error::Error + Send + Sync + 'static, +{ + type IdType = Ulid; + + type Error = sqlx::Error; + + type Stream = TaskStream, sqlx::Error>; + + type Beat = BoxStream<'static, Result<(), sqlx::Error>>; + + type Codec = Decode; + + type Context = SqliteContext; + + type Layer = AcknowledgeLayer; + + fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { + let worker_type = self.config.namespace().to_owned(); + let fut = register( + self.pool.clone(), + worker_type, + worker.clone(), + Utc::now().timestamp(), + "SharedPostgresStorage", + ); + stream::once(fut).boxed() + } + + fn middleware(&self) -> Self::Layer { + AcknowledgeLayer::new(SqliteAck::new(self.pool.clone())) + } + + fn poll(self, worker: &WorkerContext) -> Self::Stream { + let pool = self.pool.clone(); + let worker_id = worker.name().to_owned(); + let lazy_fetcher = self + .fetcher + .map(|t| t.to_string()) + .ready_chunks(self.config.buffer_size()) + .then(move |ids| { + let pool = pool.clone(); + let worker_id = worker_id.clone(); + async move { + let mut tx = pool.begin().await?; + let res: Vec<_> = sqlx::query_file_as!( + TaskRow, + "queries/task/lock_by_id.sql", + &ids, + &worker_id + ) + .fetch(&mut *tx) + .map(|r| Ok(Some(r?.try_into_task::()?))) + .collect() + .await; + tx.commit().await?; + Ok::<_, sqlx::Error>(res) + } + }) + .flat_map(|vec| match vec { + Ok(vec) => stream::iter(vec.into_iter().map(|res| match res { + Ok(t) => Ok(t), + Err(e) => Err(e), + })) + .boxed(), + Err(e) => stream::once(ready(Err(e))).boxed(), + }) + .boxed(); + + let eager_fetcher = StreamExt::boxed(SqliteFetcher::::new( + &self.pool, + &self.config, + worker, + )); + select(lazy_fetcher, eager_fetcher).boxed() + } +} + +#[cfg(test)] +mod tests { + use std::{str::FromStr, time::Duration}; + + use chrono::Local; + + use apalis_core::{ + backend::{TaskSink, memory::MemoryStorage}, + error::BoxDynError, + worker::{builder::WorkerBuilder, event::Event, ext::event_listener::EventListenerExt}, + }; + + use crate::context::SqliteContext; + + use super::*; + + #[tokio::test] + async fn basic_worker() { + let pool = SqlitePool::connect("postgres://postgres:postgres@localhost/apalis_dev") + .await + .unwrap(); + let mut store = SharedPostgresStorage::new(pool); + + let mut map_store = store.make_shared().unwrap(); + + let mut int_store = store.make_shared().unwrap(); + + let task = Task::builder(99u32) + .run_after(Duration::from_secs(2)) + .with_ctx({ + let mut ctx = SqliteContext::default(); + ctx.set_priority(1); + ctx + }) + .build(); + + map_store + .send_all(&mut stream::iter(vec![task].into_iter().map(Ok))) + .await + .unwrap(); + int_store.push(99).await.unwrap(); + + async fn send_reminder( + _: T, + task_id: TaskId, + wrk: WorkerContext, + ) -> Result<(), BoxDynError> { + tokio::time::sleep(Duration::from_secs(2)).await; + wrk.stop().unwrap(); + Ok(()) + } + + let int_worker = WorkerBuilder::new("rango-tango-2") + .backend(int_store) + .build(send_reminder); + let map_worker = WorkerBuilder::new("rango-tango-1") + .backend(map_store) + .build(send_reminder); + let res = tokio::try_join!(int_worker.run(), map_worker.run()).unwrap(); + } +} diff --git a/src/sink.rs b/src/sink.rs new file mode 100644 index 0000000..e813950 --- /dev/null +++ b/src/sink.rs @@ -0,0 +1,152 @@ +use std::{ + pin::{self, Pin}, + task::{Context, Poll}, +}; + +use apalis_core::{ + backend::codec::{Codec, json::JsonCodec}, + error::BoxDynError, +}; +use chrono::DateTime; +use futures::Sink; +use serde_json::Value; +use sqlx::SqlitePool; +use ulid::Ulid; + +use crate::{SqliteStorage, SqliteTask, config::Config}; + +#[pin_project::pin_project] +pub struct SqliteSink> { + pool: SqlitePool, + config: Config, + buffer: Vec>, + #[pin] + flush_future: Option> + Send>>>, + _marker: std::marker::PhantomData<(Args, Codec)>, +} + +impl Clone for SqliteSink { + fn clone(&self) -> Self { + Self { + pool: self.pool.clone(), + config: self.config.clone(), + buffer: Vec::new(), + flush_future: None, + _marker: std::marker::PhantomData, + } + } +} + +async fn push_tasks( + pool: SqlitePool, + cfg: Config, + buffer: Vec>, +) -> Result<(), sqlx::Error> { + let mut tx = pool.begin().await?; + let job_type = cfg.namespace(); + + for task in buffer { + let id = task + .parts + .task_id + .map(|id| id.to_string()) + .unwrap_or(Ulid::new().to_string()); + let run_at = task.parts.run_at as i64; + let max_attempts = task.parts.ctx.max_attempts() as i32; + let priority = task.parts.ctx.priority() as i32; + let args = task.args; + sqlx::query_file!( + "queries/task/sink.sql", + args, + id, + job_type, + max_attempts, + run_at, + priority, + ) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + + Ok(()) +} + +impl SqliteSink { + pub fn new(pool: &SqlitePool, config: &Config) -> Self { + Self { + pool: pool.clone(), + config: config.clone(), + buffer: Vec::new(), + _marker: std::marker::PhantomData, + flush_future: None, + } + } +} + +impl Sink> for SqliteStorage +where + Args: Send + Sync + 'static, + Encode: Codec, + Encode::Error: std::error::Error + Send + Sync + 'static, + Encode::Error: Into, +{ + type Error = sqlx::Error; + + fn poll_ready(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn start_send(mut self: Pin<&mut Self>, item: SqliteTask) -> Result<(), Self::Error> { + // Add the item to the buffer + self.project() + .sink + .buffer + .push(item.try_map(|s| Encode::encode(&s).map_err(|e| sqlx::Error::Encode(e.into())))?); + Ok(()) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut this = self.project(); + + // If there's no existing future and buffer is empty, we're done + if this.sink.flush_future.is_none() && this.sink.buffer.is_empty() { + return Poll::Ready(Ok(())); + } + + // Create the future only if we don't have one and there's work to do + if this.sink.flush_future.is_none() && !this.sink.buffer.is_empty() { + let pool = this.pool.clone(); + let config = this.config.clone(); + let buffer = std::mem::take(&mut this.sink.buffer); + let sink_fut = push_tasks(pool, config, buffer); + this.sink.flush_future = Some(Box::pin(sink_fut)); + } + + // Poll the existing future + if let Some(mut fut) = this.sink.flush_future.take() { + match fut.as_mut().poll(cx) { + Poll::Ready(Ok(())) => { + // Future completed successfully, don't put it back + Poll::Ready(Ok(())) + } + Poll::Ready(Err(e)) => { + // Future completed with error, don't put it back + Poll::Ready(Err(e)) + } + Poll::Pending => { + // Future is still pending, put it back and return Pending + this.sink.flush_future = Some(fut); + Poll::Pending + } + } + } else { + // No future and no work to do + Poll::Ready(Ok(())) + } + } + + fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.poll_flush(cx) + } +} diff --git a/packages/apalis-sql/src/sqlite/sqlite_old.rs b/src/sqlite_old.rs similarity index 100% rename from packages/apalis-sql/src/sqlite/sqlite_old.rs rename to src/sqlite_old.rs From b14fbaab9c7693104dab1a43c5e754564dc86d19 Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Tue, 23 Sep 2025 12:26:54 +0300 Subject: [PATCH 109/121] get backends working --- queries/task/fetch_next.sql | 2 +- queries/task/fetch_shared.sql | 26 +++++ src/context.rs | 31 ++++-- src/fetcher.rs | 2 +- src/from_row.rs | 14 +-- src/hook.rs | 85 +++++++--------- src/lib.rs | 77 +++++++-------- src/shared.rs | 180 +++++++++++++++++++--------------- src/sink.rs | 2 +- 9 files changed, 233 insertions(+), 186 deletions(-) create mode 100644 queries/task/fetch_shared.sql diff --git a/queries/task/fetch_next.sql b/queries/task/fetch_next.sql index daf8858..91a176e 100644 --- a/queries/task/fetch_next.sql +++ b/queries/task/fetch_next.sql @@ -11,7 +11,7 @@ WHERE AND status = 'Pending' AND lock_by IS NULL AND (run_at IS NULL OR run_at <= CURRENT_TIMESTAMP) - ORDER BY priority DESC, id ASC + ORDER BY priority DESC, run_at ASC, id ASC LIMIT ?3 ) RETURNING * diff --git a/queries/task/fetch_shared.sql b/queries/task/fetch_shared.sql new file mode 100644 index 0000000..0a073a6 --- /dev/null +++ b/queries/task/fetch_shared.sql @@ -0,0 +1,26 @@ +UPDATE Jobs +SET + status = 'Running', + lock_at = CURRENT_TIMESTAMP +WHERE ROWID IN ( + SELECT ROWID + FROM Jobs + WHERE job_type IN ( + SELECT value FROM json_each(?1) + ) + AND status = 'Pending' + AND lock_by IS NULL + AND ( + run_at IS NULL + OR run_at <= CURRENT_TIMESTAMP + ) + AND ROWID IN ( + SELECT value FROM json_each(?2) + ) + ORDER BY + priority DESC, + run_at ASC, + id ASC + LIMIT ?3 +) +RETURNING *; diff --git a/src/context.rs b/src/context.rs index f112560..973a45f 100644 --- a/src/context.rs +++ b/src/context.rs @@ -19,6 +19,7 @@ pub struct SqliteContext { lock_by: Option, done_at: Option, priority: i32, + namespace: Option, } impl Default for SqliteContext { @@ -37,12 +38,14 @@ impl SqliteContext { last_result: None, lock_by: None, priority: 0, + namespace: None, } } /// Set the number of attempts - pub fn set_max_attempts(&mut self, max_attempts: i32) { + pub fn with_max_attempts(mut self, max_attempts: i32) -> Self { self.max_attempts = max_attempts; + self } /// Gets the maximum attempts for a job. Default 25 @@ -56,8 +59,9 @@ impl SqliteContext { } /// Set the time a job was done - pub fn set_done_at(&mut self, done_at: Option) { + pub fn with_done_at(mut self, done_at: Option) -> Self { self.done_at = done_at; + self } /// Get the time a job was locked @@ -66,8 +70,9 @@ impl SqliteContext { } /// Set the lock_at value - pub fn set_lock_at(&mut self, lock_at: Option) { + pub fn with_lock_at(mut self, lock_at: Option) -> Self { self.lock_at = lock_at; + self } /// Get the time a job was locked @@ -76,8 +81,9 @@ impl SqliteContext { } /// Set `lock_by` - pub fn set_lock_by(&mut self, lock_by: Option) { + pub fn with_lock_by(mut self, lock_by: Option) -> Self { self.lock_by = lock_by; + self } /// Get the time a job was locked @@ -86,19 +92,30 @@ impl SqliteContext { } /// Set the last result - pub fn set_last_result(&mut self, result: Option) { + pub fn with_last_result(mut self, result: Option) -> Self { self.last_result = result; + self } /// Set the job priority. Larger values will run sooner. Default is 0. - pub fn set_priority(&mut self, priority: i32) { - self.priority = priority + pub fn with_priority(mut self, priority: i32) -> Self { + self.priority = priority; + self } /// Get the job priority pub fn priority(&self) -> i32 { self.priority } + + pub fn namespace(&self) -> &Option { + &self.namespace + } + + pub fn with_namespace(mut self, namespace: String) -> Self { + self.namespace = Some(namespace); + self + } } impl FromRequest> for SqliteContext { diff --git a/src/fetcher.rs b/src/fetcher.rs index 74be18a..3f2aea2 100644 --- a/src/fetcher.rs +++ b/src/fetcher.rs @@ -118,7 +118,7 @@ where type Item = Result>, sqlx::Error>; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let mut this = self.get_mut(); + let this = self.get_mut(); if this.delay_stream.is_none() { let strategy = this .config diff --git a/src/from_row.rs b/src/from_row.rs index e336634..2e740ba 100644 --- a/src/from_row.rs +++ b/src/from_row.rs @@ -33,13 +33,13 @@ impl TaskRow { where D::Error: std::error::Error + Send + Sync + 'static, { - let mut ctx = SqliteContext::default(); - ctx.set_lock_by(self.lock_by); - ctx.set_max_attempts(self.max_attempts.unwrap_or(25) as i32); - ctx.set_done_at(self.done_at); - ctx.set_last_result(self.last_error); - ctx.set_priority(self.priority.unwrap_or(0) as i32); - ctx.set_lock_at(self.lock_at); + let ctx = SqliteContext::default() + .with_done_at(self.done_at) + .with_lock_by(self.lock_by) + .with_max_attempts(self.max_attempts.unwrap_or(25) as i32) + .with_last_result(self.last_error) + .with_priority(self.priority.unwrap_or(0) as i32) + .with_lock_at(self.lock_at); let args = D::decode(&self.job).map_err(|e| sqlx::Error::Decode(e.into()))?; let task = TaskBuilder::new(args) .with_ctx(ctx) diff --git a/src/hook.rs b/src/hook.rs index 1d3869e..09fc98d 100644 --- a/src/hook.rs +++ b/src/hook.rs @@ -1,19 +1,39 @@ -use futures::channel::mpsc::{self, Receiver}; -use futures::{SinkExt, StreamExt}; +use futures::channel::mpsc::{self, Receiver, UnboundedReceiver}; +use futures::{SinkExt, Stream, StreamExt}; use libsqlite3_sys::{sqlite3, sqlite3_update_hook}; use sqlx::Executor; use sqlx::SqlitePool; use std::ffi::{CStr, c_void}; use std::os::raw::{c_char, c_int}; +use std::pin::Pin; +use std::task::{Context, Poll}; #[derive(Debug)] pub struct DbEvent { - pub op: &'static str, + op: &'static str, db_name: String, - pub table_name: String, + table_name: String, rowid: i64, } +impl DbEvent { + pub fn operation(&self) -> &'static str { + self.op + } + + pub fn db_name(&self) -> &str { + &self.db_name + } + + pub fn table_name(&self) -> &str { + &self.table_name + } + + pub fn rowid(&self) -> i64 { + self.rowid + } +} + // Callback for SQLite update hook pub(crate) extern "C" fn update_hook_callback( arg: *mut c_void, @@ -37,7 +57,7 @@ pub(crate) extern "C" fn update_hook_callback( let tx = &mut *(arg as *mut mpsc::UnboundedSender); // Ignore send errors (receiver closed) - let _ = tx.send(DbEvent { + let _ = tx.start_send(DbEvent { op: op_str, db_name: db, table_name: table, @@ -46,52 +66,19 @@ pub(crate) extern "C" fn update_hook_callback( } } - -pub struct PushListener { - pub rx: Receiver, +pub struct HookListener { + rx: UnboundedReceiver, } - -#[cfg(test)] -async fn run_it() -> Result<(), sqlx::Error> { - let (tx, mut rx) = mpsc::channel::(10); - - let pool = SqlitePool::connect(":memory:").await.unwrap(); - - let mut conn = pool.acquire().await?; - - // Create table - conn.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)") - .await?; - - // Get raw sqlite3* handle - let handle: *mut sqlite3 = conn.lock_handle().await.unwrap().as_raw_handle().as_ptr(); - - // Put sender in a Box so it has a stable memory address - let tx_box = Box::new(tx); - let tx_ptr = Box::into_raw(tx_box) as *mut c_void; - - unsafe { - sqlite3_update_hook(handle, Some(update_hook_callback), tx_ptr); +impl HookListener { + pub fn new(rx: UnboundedReceiver) -> Self { + Self { rx } } +} - // Spawn a task to process events asynchronously - tokio::spawn(async move { - while let Some(event) = rx.next().await { - println!("Got DB event: {:?}", event); - } - }); - - // These will trigger the hook - conn.execute("INSERT INTO users (name) VALUES ('Alice')") - .await?; - conn.execute("INSERT INTO users (name) VALUES ('Bob')") - .await?; - conn.execute("UPDATE users SET name = 'Charlie' WHERE id = 1") - .await?; - conn.execute("DELETE FROM users WHERE id = 2").await?; - - // Let events flush - // tokio::time::sleep(std::time::Duration::from_millis(100)).await; +impl Stream for HookListener { + type Item = DbEvent; - Ok(()) + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.rx.poll_next_unpin(cx) + } } diff --git a/src/lib.rs b/src/lib.rs index ff2cda0..4c321a4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,8 +15,9 @@ use futures::{ future::ready, stream::{self, BoxStream, select}, }; -use serde_json::Value; +use libsqlite3_sys::{sqlite3, sqlite3_update_hook}; use sqlx::{Pool, Sqlite, SqlitePool}; +use std::ffi::c_void; use ulid::Ulid; use crate::{ @@ -24,7 +25,7 @@ use crate::{ config::Config, context::SqliteContext, fetcher::{SqliteFetcher, fetch_next}, - hook::{DbEvent, PushListener, update_hook_callback}, + hook::{DbEvent, HookListener, update_hook_callback}, sink::SqliteSink, }; @@ -34,13 +35,16 @@ mod context; mod fetcher; mod from_row; mod hook; +mod shared; mod sink; -// mod shared; pub type SqliteTask = Task; type DefaultFetcher = PhantomData>>; +const INSERT_OPERATION: &'static str = "INSERT"; +const JOBS_TABLE: &'static str = "Jobs"; + #[pin_project::pin_project] pub struct SqliteStorage, Fetcher = DefaultFetcher> { pool: Pool, @@ -141,15 +145,11 @@ impl SqliteStorage { pub async fn new_with_hook( pool: &Pool, config: &Config, - ) -> SqliteStorage, PushListener> { - use libsqlite3_sys::{sqlite3, sqlite3_update_hook}; - use std::ffi::{CStr, c_void}; - use std::os::raw::{c_char, c_int}; - + ) -> SqliteStorage, HookListener> { let pool = pool.clone(); - let (tx, rx) = mpsc::channel::(10); + let (tx, rx) = mpsc::unbounded::(); - let listener = PushListener { rx }; + let listener = HookListener::new(rx); let mut conn = pool.acquire().await.unwrap(); // Get raw sqlite3* handle @@ -214,7 +214,7 @@ where } } -impl Backend for SqliteStorage +impl Backend for SqliteStorage where Args: Send + 'static + Unpin, Decode: Codec + Send + 'static, @@ -241,7 +241,7 @@ where worker_type, worker.clone(), Utc::now().timestamp(), - "SqliteStorage", + "SqliteStorageWithHook", ); stream::once(fut).boxed() } @@ -258,11 +258,8 @@ where SqliteFetcher::new(&self.pool, &self.config, &worker); let lazy_fetcher = self .fetcher - .rx - .inspect(|res| { - println!("Received DB Event: {:?}", res); - }) - .filter(|a| ready(a.op == "UPDATE" && a.table_name == "Jobs")) + .filter(|a| ready(a.operation() == INSERT_OPERATION && a.table_name() == JOBS_TABLE)) + .ready_chunks(self.config.buffer_size()) .then(move |_| fetch_next::(pool.clone(), config.clone(), worker.clone())) .flat_map(|res| match res { Ok(tasks) => stream::iter(tasks).map(Ok).boxed(), @@ -273,7 +270,7 @@ where Err(e) => Err(e), }); - select(eager_fetcher, lazy_fetcher).boxed() + select(lazy_fetcher, eager_fetcher).boxed() } } @@ -317,7 +314,7 @@ mod tests { use apalis_core::{ backend::poll_strategy::{IntervalStrategy, StrategyBuilder}, error::BoxDynError, - worker::{builder::WorkerBuilder, event::Event, ext::event_listener::EventListenerExt}, + worker::builder::WorkerBuilder, }; use futures::SinkExt; @@ -336,7 +333,7 @@ mod tests { start += 1; let task = Task::builder(start) .run_after(Duration::from_secs(1)) - // .with_ctx(SqliteContext::new().with_priority(1)) + .with_ctx(SqliteContext::new().with_priority(1)) .build(); Ok(task) }) @@ -368,27 +365,29 @@ mod tests { let config = Config::default() .with_poll_interval(lazy_strategy) .set_buffer_size(5); - let mut backend = SqliteStorage::new_with_hook(&pool, &config).await; - - // SqliteStorage::setup(&backend.pool).await.unwrap(); - let mut start = 0; - - let mut items = stream::repeat_with(move || { - start += 1; - let task = Task::builder(start) - .run_after(Duration::from_secs(2)) - // .with_ctx(SqliteContext::new().with_priority(1)) - .build(); - Ok(task) - }) - .take(ITEMS); - backend.send_all(&mut items).await.unwrap(); - - println!("Starting worker at {}", Local::now()); + let backend = SqliteStorage::new_with_hook(&pool, &config).await; + + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(2)).await; + let mut start = 0; + + let items = stream::repeat_with(move || { + start += 1; + let task = Task::builder(serde_json::to_string(&start).unwrap()) + .run_after(Duration::from_secs(1)) + .with_ctx(SqliteContext::new().with_priority(start)) + .build(); + task + }) + .take(ITEMS) + .collect::>() + .await; + sink::push_tasks(pool, config, items).await.unwrap(); + }); async fn send_reminder(item: usize, wrk: WorkerContext) -> Result<(), BoxDynError> { - println!("Processed item: {}", item); - if ITEMS == item { + // Priority is in reverse order + if item == 1 { apalis_core::timer::sleep(Duration::from_secs(1)).await; wrk.stop().unwrap(); } diff --git a/src/shared.rs b/src/shared.rs index 0369e47..854a898 100644 --- a/src/shared.rs +++ b/src/shared.rs @@ -1,5 +1,7 @@ use std::{ + cmp::max, collections::{HashMap, HashSet}, + ffi::c_void, future::ready, marker::PhantomData, pin::Pin, @@ -8,8 +10,12 @@ use std::{ }; use crate::{ - Config, InsertEvent, SqliteTask, PostgresStorage, ack::SqliteAck, context::SqliteContext, - fetcher::SqliteFetcher, register, + Config, INSERT_OPERATION, JOBS_TABLE, SqliteStorage, SqliteTask, + ack::SqliteAck, + context::SqliteContext, + fetcher::SqliteFetcher, + hook::{DbEvent, update_hook_callback}, + register, }; use crate::{from_row::TaskRow, sink::SqliteSink}; use apalis_core::{ @@ -29,58 +35,97 @@ use futures::{ lock::Mutex, stream::{self, BoxStream, select}, }; -use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use libsqlite3_sys::{sqlite3, sqlite3_update_hook}; use serde_json::Value; -use sqlx::{SqlitePool, postgres::SqliteListener}; +use sqlx::SqlitePool; use ulid::Ulid; -pub struct SharedPostgresStorage> { +pub struct SharedSqliteStorage { pool: SqlitePool, - registry: Arc>>>, + registry: Arc>>>>, drive: Shared>, - _marker: PhantomData<(Compact, Codec)>, } -impl SharedPostgresStorage { +impl SharedSqliteStorage { pub fn new(pool: SqlitePool) -> Self { - let registry: Arc>>> = + let registry: Arc>>>> = Arc::new(Mutex::new(HashMap::default())); let p = pool.clone(); let instances = registry.clone(); Self { pool, drive: async move { - let mut listener = SqliteListener::connect_with(&p).await.unwrap(); - listener.listen("apalis::job::insert").await.unwrap(); - listener - .into_stream() - .filter_map(|notification| { - let instances = instances.clone(); - async move { - let Sqlite_notification = notification.ok()?; - let payload = Sqlite_notification.payload(); - let ev: InsertEvent = serde_json::from_str(payload).ok()?; - let instances = instances.lock().await; - if instances.get(&ev.job_type).is_some() { - return Some(ev); + let (tx, rx) = mpsc::unbounded::(); + + let mut conn = p.acquire().await.unwrap(); + // Get raw sqlite3* handle + let handle: *mut sqlite3 = + conn.lock_handle().await.unwrap().as_raw_handle().as_ptr(); + + // Put sender in a Box so it has a stable memory address + let tx_box = Box::new(tx); + let tx_ptr = Box::into_raw(tx_box) as *mut c_void; + + unsafe { + sqlite3_update_hook(handle, Some(update_hook_callback), tx_ptr); + } + + rx.filter(|a| { + ready(a.operation() == INSERT_OPERATION && a.table_name() == JOBS_TABLE) + }) + .ready_chunks(instances.try_lock().map(|r| r.len()).unwrap_or(10)) + .then(|events| { + let row_ids = events.iter().map(|e| e.rowid()).collect::>(); + let instances = instances.clone(); + let pool = p.clone(); + async move { + let instances = instances.lock().await; + let job_types = serde_json::to_string( + &instances.keys().cloned().collect::>(), + ) + .unwrap(); + let row_ids = serde_json::to_string(&row_ids).unwrap(); + let mut tx = pool.begin().await?; + let buffer_size = max(10, instances.len()) as i32; + let res: Vec<_> = sqlx::query_file_as!( + TaskRow, + "queries/task/fetch_shared.sql", + job_types, + row_ids, + buffer_size, + ) + .fetch(&mut *tx) + .map(|r| { + Ok::<_, sqlx::Error>(r?.try_into_task::, String>()?) + }) + .try_collect() + .await?; + tx.commit().await?; + Ok::<_, sqlx::Error>(res) + } + }) + .for_each(|r| async { + match r { + Ok(tasks) => { + let mut instances = instances.lock().await; + for task in tasks { + if let Some(tx) = instances.get_mut( + task.parts.ctx.namespace().as_ref().expect("Namespace must be set"), + ) { + let _ = tx.send(task).await; + } } - None } - }) - .for_each(|ev| { - let instances = instances.clone(); - async move { - let mut instances = instances.lock().await; - let sender = instances.get_mut(&ev.job_type).unwrap(); - sender.send(ev.id).await.unwrap(); + Err(e) => { + log::error!("Error fetching tasks: {:?}", e); } - }) - .await; + } + }) + .await; } .boxed() .shared(), registry, - _marker: PhantomData, } } } @@ -90,8 +135,8 @@ pub enum SharedPostgresError { RegistryLocked, } -impl MakeShared for SharedPostgresStorage { - type Backend = PostgresStorage; +impl MakeShared for SharedSqliteStorage { + type Backend = SqliteStorage, SharedFetcher>; type Config = Config; type MakeError = SharedPostgresError; fn make_shared(&mut self) -> Result @@ -115,8 +160,7 @@ impl MakeShared for SharedPostgresStorage MakeShared for SharedPostgresStorage { poller: Shared>, - receiver: Receiver, + receiver: Receiver>, } -impl Stream for SharedFetcher { - type Item = TaskId; +impl Stream for SharedFetcher { + type Item = SqliteTask; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); @@ -146,10 +192,10 @@ impl Stream for SharedFetcher { } } -impl Backend for PostgresStorage +impl Backend for SqliteStorage where Args: Send + 'static + Unpin, - Decode: Codec + 'static + Unpin + Send, + Decode: Codec + 'static + Unpin + Send, Decode::Error: std::error::Error + Send + Sync + 'static, { type IdType = Ulid; @@ -173,7 +219,7 @@ where worker_type, worker.clone(), Utc::now().timestamp(), - "SharedPostgresStorage", + "SharedSqliteStorage", ); stream::once(fut).boxed() } @@ -183,42 +229,20 @@ where } fn poll(self, worker: &WorkerContext) -> Self::Stream { - let pool = self.pool.clone(); - let worker_id = worker.name().to_owned(); + // TODO: Update lock_by field in the database to worker.name() + let _worker_id = worker.name().to_owned(); let lazy_fetcher = self .fetcher - .map(|t| t.to_string()) - .ready_chunks(self.config.buffer_size()) - .then(move |ids| { - let pool = pool.clone(); - let worker_id = worker_id.clone(); - async move { - let mut tx = pool.begin().await?; - let res: Vec<_> = sqlx::query_file_as!( - TaskRow, - "queries/task/lock_by_id.sql", - &ids, - &worker_id - ) - .fetch(&mut *tx) - .map(|r| Ok(Some(r?.try_into_task::()?))) - .collect() - .await; - tx.commit().await?; - Ok::<_, sqlx::Error>(res) - } + .map(|t| { + t.try_map(|args| Decode::decode(&args).map_err(|e| sqlx::Error::Decode(e.into()))) }) .flat_map(|vec| match vec { - Ok(vec) => stream::iter(vec.into_iter().map(|res| match res { - Ok(t) => Ok(t), - Err(e) => Err(e), - })) - .boxed(), + Ok(task) => stream::iter(vec![Ok(Some(task))]).boxed(), Err(e) => stream::once(ready(Err(e))).boxed(), }) .boxed(); - let eager_fetcher = StreamExt::boxed(SqliteFetcher::::new( + let eager_fetcher = StreamExt::boxed(SqliteFetcher::::new( &self.pool, &self.config, worker, @@ -245,10 +269,8 @@ mod tests { #[tokio::test] async fn basic_worker() { - let pool = SqlitePool::connect("postgres://postgres:postgres@localhost/apalis_dev") - .await - .unwrap(); - let mut store = SharedPostgresStorage::new(pool); + let pool = SqlitePool::connect("sqlite://./test.db").await.unwrap(); + let mut store = SharedSqliteStorage::new(pool); let mut map_store = store.make_shared().unwrap(); @@ -256,11 +278,7 @@ mod tests { let task = Task::builder(99u32) .run_after(Duration::from_secs(2)) - .with_ctx({ - let mut ctx = SqliteContext::default(); - ctx.set_priority(1); - ctx - }) + .with_ctx(SqliteContext::new().with_priority(1)) .build(); map_store diff --git a/src/sink.rs b/src/sink.rs index e813950..6fc9ae4 100644 --- a/src/sink.rs +++ b/src/sink.rs @@ -37,7 +37,7 @@ impl Clone for SqliteSink { } } -async fn push_tasks( +pub(crate) async fn push_tasks( pool: SqlitePool, cfg: Config, buffer: Vec>, From 7942b38c784634e72027b887c66818cfa89661a8 Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Tue, 23 Sep 2025 13:05:45 +0300 Subject: [PATCH 110/121] fix: improve shared setup --- src/lib.rs | 15 ++++++----- src/shared.rs | 71 ++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 62 insertions(+), 24 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 4c321a4..65640c0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -195,7 +195,7 @@ where fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { let worker_type = self.config.namespace().to_owned(); - let fut = register( + let fut = heartbeat( self.pool.clone(), worker_type, worker.clone(), @@ -236,7 +236,7 @@ where fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { let worker_type = self.config.namespace().to_owned(); - let fut = register( + let fut = heartbeat( self.pool.clone(), worker_type, worker.clone(), @@ -274,7 +274,7 @@ where } } -pub(crate) async fn register( +pub(crate) async fn heartbeat( pool: SqlitePool, worker_type: String, worker: WorkerContext, @@ -323,10 +323,11 @@ mod tests { #[tokio::test] async fn basic_worker() { const ITEMS: usize = 10; - let pool = SqlitePool::connect("sqlite://./test.db").await.unwrap(); + let pool = SqlitePool::connect(":memory:").await.unwrap(); + SqliteStorage::setup(&pool).await.unwrap(); + let mut backend = SqliteStorage::new(&pool); - // SqliteStorage::setup(&backend.pool).await.unwrap(); let mut start = 0; let mut items = stream::repeat_with(move || { @@ -358,7 +359,9 @@ mod tests { #[tokio::test] async fn hooked_worker() { const ITEMS: usize = 20; - let pool = SqlitePool::connect("sqlite://./test.db").await.unwrap(); + let pool = SqlitePool::connect(":memory:").await.unwrap(); + SqliteStorage::setup(&pool).await.unwrap(); + let lazy_strategy = StrategyBuilder::new() .apply(IntervalStrategy::new(Duration::from_secs(5))) .build(); diff --git a/src/shared.rs b/src/shared.rs index 854a898..17133a1 100644 --- a/src/shared.rs +++ b/src/shared.rs @@ -14,8 +14,8 @@ use crate::{ ack::SqliteAck, context::SqliteContext, fetcher::SqliteFetcher, + heartbeat, hook::{DbEvent, update_hook_callback}, - register, }; use crate::{from_row::TaskRow, sink::SqliteSink}; use apalis_core::{ @@ -37,7 +37,7 @@ use futures::{ }; use libsqlite3_sys::{sqlite3, sqlite3_update_hook}; use serde_json::Value; -use sqlx::SqlitePool; +use sqlx::{SqlitePool, pool}; use ulid::Ulid; pub struct SharedSqliteStorage { @@ -110,7 +110,11 @@ impl SharedSqliteStorage { let mut instances = instances.lock().await; for task in tasks { if let Some(tx) = instances.get_mut( - task.parts.ctx.namespace().as_ref().expect("Namespace must be set"), + task.parts + .ctx + .namespace() + .as_ref() + .expect("Namespace must be set"), ) { let _ = tx.send(task).await; } @@ -194,8 +198,8 @@ impl Stream for SharedFetcher { impl Backend for SqliteStorage where - Args: Send + 'static + Unpin, - Decode: Codec + 'static + Unpin + Send, + Args: Send + 'static + Unpin + Sync, + Decode: Codec + 'static + Unpin + Send + Sync, Decode::Error: std::error::Error + Send + Sync + 'static, { type IdType = Ulid; @@ -214,14 +218,24 @@ where fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { let worker_type = self.config.namespace().to_owned(); - let fut = register( - self.pool.clone(), - worker_type, - worker.clone(), - Utc::now().timestamp(), - "SharedSqliteStorage", - ); - stream::once(fut).boxed() + let keep_alive = *self.config.keep_alive(); + let pool = self.pool.clone(); + let worker = worker.clone(); + + stream::unfold((), move |()| async move { + apalis_core::timer::sleep(keep_alive).await; + Some(((), ())) + }) + .then(move |_| { + heartbeat( + pool.clone(), + worker_type.clone(), + worker.clone(), + Utc::now().timestamp(), + "SharedSqliteStorage", + ) + }) + .boxed() } fn middleware(&self) -> Self::Layer { @@ -231,6 +245,24 @@ where fn poll(self, worker: &WorkerContext) -> Self::Stream { // TODO: Update lock_by field in the database to worker.name() let _worker_id = worker.name().to_owned(); + let pool = self.pool.clone(); + let worker = worker.clone(); + let worker_type = self.config.namespace().to_owned(); + // Initial registration heartbeat + // This ensures that the worker is registered before fetching any tasks + // This also ensures that the worker is marked as alive in case it crashes + // before fetching any tasks + // Subsequent heartbeats are handled in the heartbeat stream + let init = heartbeat( + pool.clone(), + worker_type.clone(), + worker.clone(), + Utc::now().timestamp(), + "SharedSqliteStorage", + ); + let starter = stream::once(init) + .filter_map(|s| future::ready(s.ok().map(|_| Ok(None::>)))) + .boxed(); let lazy_fetcher = self .fetcher .map(|t| { @@ -245,9 +277,9 @@ where let eager_fetcher = StreamExt::boxed(SqliteFetcher::::new( &self.pool, &self.config, - worker, + &worker, )); - select(lazy_fetcher, eager_fetcher).boxed() + starter.chain(select(lazy_fetcher, eager_fetcher)).boxed() } } @@ -269,7 +301,10 @@ mod tests { #[tokio::test] async fn basic_worker() { - let pool = SqlitePool::connect("sqlite://./test.db").await.unwrap(); + let pool = SqlitePool::connect(":memory:").await.unwrap(); + + SqliteStorage::setup(&pool).await.unwrap(); + let mut store = SharedSqliteStorage::new(pool); let mut map_store = store.make_shared().unwrap(); @@ -289,7 +324,7 @@ mod tests { async fn send_reminder( _: T, - task_id: TaskId, + _task_id: TaskId, wrk: WorkerContext, ) -> Result<(), BoxDynError> { tokio::time::sleep(Duration::from_secs(2)).await; @@ -303,6 +338,6 @@ mod tests { let map_worker = WorkerBuilder::new("rango-tango-1") .backend(map_store) .build(send_reminder); - let res = tokio::try_join!(int_worker.run(), map_worker.run()).unwrap(); + tokio::try_join!(int_worker.run(), map_worker.run()).unwrap(); } } From a783bca882e2800c0d4edf6250bcc24efc73499a Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Tue, 23 Sep 2025 13:32:25 +0300 Subject: [PATCH 111/121] cliipy:lint --- src/ack.rs | 5 ++--- src/config.rs | 1 - src/context.rs | 7 +------ src/fetcher.rs | 7 ++----- src/from_row.rs | 2 -- src/hook.rs | 7 ++----- src/lib.rs | 14 ++++++++------ src/shared.rs | 19 +++++++++---------- src/sink.rs | 16 ++++++++-------- 9 files changed, 32 insertions(+), 46 deletions(-) diff --git a/src/ack.rs b/src/ack.rs index 8e733a1..7a32007 100644 --- a/src/ack.rs +++ b/src/ack.rs @@ -1,9 +1,8 @@ use apalis_core::{ error::BoxDynError, task::{Parts, status::Status}, - worker::{self, ext::ack::Acknowledge}, + worker::ext::ack::Acknowledge, }; -use chrono::Utc; use futures::{FutureExt, future::BoxFuture}; use serde::Serialize; use sqlx::SqlitePool; @@ -29,7 +28,7 @@ impl Acknowledge for SqliteAck { res: &Result, parts: &Parts, ) -> Self::Future { - let task_id = parts.task_id.clone(); + let task_id = parts.task_id; let worker_id = parts.ctx.lock_by().clone(); let response = serde_json::to_string(&res.as_ref().map_err(|e| e.to_string())); diff --git a/src/config.rs b/src/config.rs index cf99233..9f45262 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,7 +1,6 @@ use std::time::Duration; use apalis_core::backend::poll_strategy::{IntervalStrategy, MultiStrategy, StrategyBuilder}; -use sqlx::sqlite::SqliteConnectOptions; #[derive(Debug, Clone)] pub struct Config { diff --git a/src/context.rs b/src/context.rs index 973a45f..8688f5a 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,13 +1,8 @@ use std::convert::Infallible; -use apalis_core::{ - task::{Task, status::Status}, - task_fn::FromRequest, -}; +use apalis_core::task_fn::FromRequest; -use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use ulid::Ulid; use crate::SqliteTask; diff --git a/src/fetcher.rs b/src/fetcher.rs index 3f2aea2..cd3c0cb 100644 --- a/src/fetcher.rs +++ b/src/fetcher.rs @@ -4,7 +4,6 @@ use std::{ pin::Pin, sync::{Arc, atomic::AtomicUsize}, task::{Context, Poll}, - time::{Duration, Instant}, }; use apalis_core::{ @@ -13,16 +12,14 @@ use apalis_core::{ poll_strategy::{PollContext, PollStrategyExt}, }, task::Task, - timer::Delay, worker::context::WorkerContext, }; use futures::{ - Future, FutureExt, StreamExt, + FutureExt, StreamExt, future::BoxFuture, - stream::{self, BoxStream, Stream}, + stream::Stream, }; use pin_project::pin_project; -use serde_json::Value; use sqlx::{Pool, Sqlite, SqlitePool}; use ulid::Ulid; diff --git a/src/from_row.rs b/src/from_row.rs index 2e740ba..148f51e 100644 --- a/src/from_row.rs +++ b/src/from_row.rs @@ -4,8 +4,6 @@ use apalis_core::{ backend::codec::Codec, task::{attempt::Attempt, builder::TaskBuilder, status::Status, task_id::TaskId}, }; -use chrono::{DateTime, Utc}; -use serde_json::{Map, Value}; use crate::{SqliteTask, context::SqliteContext}; diff --git a/src/hook.rs b/src/hook.rs index 09fc98d..4b9a96d 100644 --- a/src/hook.rs +++ b/src/hook.rs @@ -1,8 +1,5 @@ -use futures::channel::mpsc::{self, Receiver, UnboundedReceiver}; -use futures::{SinkExt, Stream, StreamExt}; -use libsqlite3_sys::{sqlite3, sqlite3_update_hook}; -use sqlx::Executor; -use sqlx::SqlitePool; +use futures::channel::mpsc::{self, UnboundedReceiver}; +use futures::{Stream, StreamExt}; use std::ffi::{CStr, c_void}; use std::os::raw::{c_char, c_int}; use std::pin::Pin; diff --git a/src/lib.rs b/src/lib.rs index 65640c0..2757cfa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,10 +22,8 @@ use ulid::Ulid; use crate::{ ack::SqliteAck, - config::Config, - context::SqliteContext, fetcher::{SqliteFetcher, fetch_next}, - hook::{DbEvent, HookListener, update_hook_callback}, + hook::update_hook_callback, sink::SqliteSink, }; @@ -33,17 +31,21 @@ mod ack; mod config; mod context; mod fetcher; -mod from_row; +pub mod from_row; mod hook; mod shared; mod sink; pub type SqliteTask = Task; +pub use config::Config; +pub use context::SqliteContext; +pub use hook::{DbEvent, HookListener}; +pub use shared::{SharedPostgresError, SharedSqliteStorage}; type DefaultFetcher = PhantomData>>; -const INSERT_OPERATION: &'static str = "INSERT"; -const JOBS_TABLE: &'static str = "Jobs"; +const INSERT_OPERATION: &str = "INSERT"; +const JOBS_TABLE: &str = "Jobs"; #[pin_project::pin_project] pub struct SqliteStorage, Fetcher = DefaultFetcher> { diff --git a/src/shared.rs b/src/shared.rs index 17133a1..e23b838 100644 --- a/src/shared.rs +++ b/src/shared.rs @@ -24,7 +24,6 @@ use apalis_core::{ codec::{Codec, json::JsonCodec}, shared::MakeShared, }, - task::{Task, task_id::TaskId}, worker::{context::WorkerContext, ext::ack::AcknowledgeLayer}, }; use chrono::Utc; @@ -36,8 +35,7 @@ use futures::{ stream::{self, BoxStream, select}, }; use libsqlite3_sys::{sqlite3, sqlite3_update_hook}; -use serde_json::Value; -use sqlx::{SqlitePool, pool}; +use sqlx::SqlitePool; use ulid::Ulid; pub struct SharedSqliteStorage { @@ -95,9 +93,7 @@ impl SharedSqliteStorage { buffer_size, ) .fetch(&mut *tx) - .map(|r| { - Ok::<_, sqlx::Error>(r?.try_into_task::, String>()?) - }) + .map(|r| r?.try_into_task::, String>()) .try_collect() .await?; tx.commit().await?; @@ -121,7 +117,7 @@ impl SharedSqliteStorage { } } Err(e) => { - log::error!("Error fetching tasks: {:?}", e); + log::error!("Error fetching tasks: {e:?}"); } } }) @@ -133,9 +129,11 @@ impl SharedSqliteStorage { } } } -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum SharedPostgresError { + #[error("Namespace {0} already exists")] NamespaceExists(String), + #[error("Could not acquire registry lock")] RegistryLocked, } @@ -158,7 +156,7 @@ impl MakeShared for SharedSqliteStorage { .registry .try_lock() .ok_or(SharedPostgresError::RegistryLocked)?; - if let Some(_) = r.insert(config.namespace().to_owned(), tx) { + if r.insert(config.namespace().to_owned(), tx).is_some() { return Err(SharedPostgresError::NamespaceExists( config.namespace().to_owned(), )); @@ -186,7 +184,7 @@ pub struct SharedFetcher { impl Stream for SharedFetcher { type Item = SqliteTask; - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); // Keep the poller alive by polling it, but ignoring the output let _ = this.poller.poll_unpin(cx); @@ -292,6 +290,7 @@ mod tests { use apalis_core::{ backend::{TaskSink, memory::MemoryStorage}, error::BoxDynError, + task::{Task, task_id::TaskId}, worker::{builder::WorkerBuilder, event::Event, ext::event_listener::EventListenerExt}, }; diff --git a/src/sink.rs b/src/sink.rs index 6fc9ae4..722dfdb 100644 --- a/src/sink.rs +++ b/src/sink.rs @@ -1,5 +1,5 @@ use std::{ - pin::{self, Pin}, + pin::Pin, task::{Context, Poll}, }; @@ -7,21 +7,21 @@ use apalis_core::{ backend::codec::{Codec, json::JsonCodec}, error::BoxDynError, }; -use chrono::DateTime; use futures::Sink; -use serde_json::Value; use sqlx::SqlitePool; use ulid::Ulid; use crate::{SqliteStorage, SqliteTask, config::Config}; +type FlushFuture = Pin> + Send>>; + #[pin_project::pin_project] pub struct SqliteSink> { pool: SqlitePool, config: Config, buffer: Vec>, #[pin] - flush_future: Option> + Send>>>, + flush_future: Option, _marker: std::marker::PhantomData<(Args, Codec)>, } @@ -52,8 +52,8 @@ pub(crate) async fn push_tasks( .map(|id| id.to_string()) .unwrap_or(Ulid::new().to_string()); let run_at = task.parts.run_at as i64; - let max_attempts = task.parts.ctx.max_attempts() as i32; - let priority = task.parts.ctx.priority() as i32; + let max_attempts = task.parts.ctx.max_attempts(); + let priority = task.parts.ctx.priority(); let args = task.args; sqlx::query_file!( "queries/task/sink.sql", @@ -86,7 +86,7 @@ impl SqliteSink { impl Sink> for SqliteStorage where - Args: Send + Sync + 'static, + Args: Send + Sync + 'static, Encode: Codec, Encode::Error: std::error::Error + Send + Sync + 'static, Encode::Error: Into, @@ -97,7 +97,7 @@ where Poll::Ready(Ok(())) } - fn start_send(mut self: Pin<&mut Self>, item: SqliteTask) -> Result<(), Self::Error> { + fn start_send(self: Pin<&mut Self>, item: SqliteTask) -> Result<(), Self::Error> { // Add the item to the buffer self.project() .sink From c81da9a60b28308477cead5d2ce302334b3c628c Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Tue, 23 Sep 2025 13:42:41 +0300 Subject: [PATCH 112/121] cliipy:lint --- src/ack.rs | 7 ++++--- src/fetcher.rs | 6 +----- src/from_row.rs | 4 ++++ src/lib.rs | 8 ++++---- src/shared.rs | 8 +++----- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/ack.rs b/src/ack.rs index 7a32007..8cf89dc 100644 --- a/src/ack.rs +++ b/src/ack.rs @@ -1,5 +1,5 @@ use apalis_core::{ - error::BoxDynError, + error::{AbortError, BoxDynError}, task::{Parts, status::Status}, worker::ext::ack::Acknowledge, }; @@ -70,8 +70,9 @@ pub fn calculate_status( ) -> Status { match &res { Ok(_) => Status::Done, - Err(e) => match &e { - e if parts.ctx.max_attempts() as usize <= parts.attempt.current() => Status::Killed, + Err(e) => match e { + _ if parts.ctx.max_attempts() as usize <= parts.attempt.current() => Status::Killed, + e if e.downcast_ref::().is_some() => Status::Killed, _ => Status::Failed, }, } diff --git a/src/fetcher.rs b/src/fetcher.rs index cd3c0cb..91be745 100644 --- a/src/fetcher.rs +++ b/src/fetcher.rs @@ -14,11 +14,7 @@ use apalis_core::{ task::Task, worker::context::WorkerContext, }; -use futures::{ - FutureExt, StreamExt, - future::BoxFuture, - stream::Stream, -}; +use futures::{FutureExt, future::BoxFuture, stream::Stream}; use pin_project::pin_project; use sqlx::{Pool, Sqlite, SqlitePool}; use ulid::Ulid; diff --git a/src/from_row.rs b/src/from_row.rs index 148f51e..e29c56c 100644 --- a/src/from_row.rs +++ b/src/from_row.rs @@ -37,6 +37,10 @@ impl TaskRow { .with_max_attempts(self.max_attempts.unwrap_or(25) as i32) .with_last_result(self.last_error) .with_priority(self.priority.unwrap_or(0) as i32) + .with_namespace( + self.job_type + .ok_or(sqlx::Error::ColumnNotFound("job_type".to_owned()))?, + ) .with_lock_at(self.lock_at); let args = D::decode(&self.job).map_err(|e| sqlx::Error::Decode(e.into()))?; let task = TaskBuilder::new(args) diff --git a/src/lib.rs b/src/lib.rs index 2757cfa..ad1e24b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -309,7 +309,7 @@ pub(crate) async fn heartbeat( #[cfg(test)] mod tests { - use std::{collections::HashMap, str::FromStr, time::Duration}; + use std::time::Duration; use chrono::Local; @@ -378,11 +378,11 @@ mod tests { let items = stream::repeat_with(move || { start += 1; - let task = Task::builder(serde_json::to_string(&start).unwrap()) + + Task::builder(serde_json::to_string(&start).unwrap()) .run_after(Duration::from_secs(1)) .with_ctx(SqliteContext::new().with_priority(start)) - .build(); - task + .build() }) .take(ITEMS) .collect::>() diff --git a/src/shared.rs b/src/shared.rs index e23b838..52b61e8 100644 --- a/src/shared.rs +++ b/src/shared.rs @@ -283,15 +283,13 @@ where #[cfg(test)] mod tests { - use std::{str::FromStr, time::Duration}; - - use chrono::Local; + use std::time::Duration; use apalis_core::{ - backend::{TaskSink, memory::MemoryStorage}, + backend::TaskSink, error::BoxDynError, task::{Task, task_id::TaskId}, - worker::{builder::WorkerBuilder, event::Event, ext::event_listener::EventListenerExt}, + worker::builder::WorkerBuilder, }; use crate::context::SqliteContext; From 8af385e61892f360a17cb79881178fc7ced00ae8 Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Tue, 23 Sep 2025 13:44:35 +0300 Subject: [PATCH 113/121] cargo: sqlx prep --- ...0543bde1cbc72f692187d8a7fef06e952f513.json | 86 +++++++++++++++++++ ...c479d318a98f4891ed46ff8588bb4e85615b.json} | 4 +- 2 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 .sqlx/query-1beef6dd1673e83142d565dcd7a0543bde1cbc72f692187d8a7fef06e952f513.json rename .sqlx/{query-0819e39ed31495e0c70a6dd549b10e5c52e64d5289ca829e2a0135a14801b930.json => query-8e68049c1005dd0981469dac442cc479d318a98f4891ed46ff8588bb4e85615b.json} (87%) diff --git a/.sqlx/query-1beef6dd1673e83142d565dcd7a0543bde1cbc72f692187d8a7fef06e952f513.json b/.sqlx/query-1beef6dd1673e83142d565dcd7a0543bde1cbc72f692187d8a7fef06e952f513.json new file mode 100644 index 0000000..36daba6 --- /dev/null +++ b/.sqlx/query-1beef6dd1673e83142d565dcd7a0543bde1cbc72f692187d8a7fef06e952f513.json @@ -0,0 +1,86 @@ +{ + "db_name": "SQLite", + "query": "UPDATE Jobs\nSET\n status = 'Running',\n lock_at = CURRENT_TIMESTAMP\nWHERE ROWID IN (\n SELECT ROWID\n FROM Jobs\n WHERE job_type IN (\n SELECT value FROM json_each(?1)\n )\n AND status = 'Pending'\n AND lock_by IS NULL\n AND (\n run_at IS NULL\n OR run_at <= CURRENT_TIMESTAMP\n )\n AND ROWID IN (\n SELECT value FROM json_each(?2)\n )\n ORDER BY\n priority DESC,\n run_at ASC,\n id ASC\n LIMIT ?3\n)\nRETURNING *;\n", + "describe": { + "columns": [ + { + "name": "job", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "job_type", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "status", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "attempts", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "max_attempts", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "run_at", + "ordinal": 6, + "type_info": "Integer" + }, + { + "name": "last_error", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "lock_at", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "lock_by", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "done_at", + "ordinal": 10, + "type_info": "Integer" + }, + { + "name": "priority", + "ordinal": 11, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 3 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + false + ] + }, + "hash": "1beef6dd1673e83142d565dcd7a0543bde1cbc72f692187d8a7fef06e952f513" +} diff --git a/.sqlx/query-0819e39ed31495e0c70a6dd549b10e5c52e64d5289ca829e2a0135a14801b930.json b/.sqlx/query-8e68049c1005dd0981469dac442cc479d318a98f4891ed46ff8588bb4e85615b.json similarity index 87% rename from .sqlx/query-0819e39ed31495e0c70a6dd549b10e5c52e64d5289ca829e2a0135a14801b930.json rename to .sqlx/query-8e68049c1005dd0981469dac442cc479d318a98f4891ed46ff8588bb4e85615b.json index 426e55e..b5b03b4 100644 --- a/.sqlx/query-0819e39ed31495e0c70a6dd549b10e5c52e64d5289ca829e2a0135a14801b930.json +++ b/.sqlx/query-8e68049c1005dd0981469dac442cc479d318a98f4891ed46ff8588bb4e85615b.json @@ -1,6 +1,6 @@ { "db_name": "SQLite", - "query": "UPDATE Jobs\nSET\n status = 'Running',\n lock_by = ?1,\n lock_at = CURRENT_TIMESTAMP\nWHERE\n ROWID IN (\n SELECT ROWID\n FROM Jobs\n WHERE job_type = ?2\n AND status = 'Pending'\n AND lock_by IS NULL\n ORDER BY priority DESC, id ASC\n LIMIT ?3\n )\nRETURNING *\n", + "query": "UPDATE Jobs\nSET\n status = 'Running',\n lock_by = ?1,\n lock_at = CURRENT_TIMESTAMP\nWHERE\n ROWID IN (\n SELECT ROWID\n FROM Jobs\n WHERE job_type = ?2\n AND status = 'Pending'\n AND lock_by IS NULL\n AND (run_at IS NULL OR run_at <= CURRENT_TIMESTAMP)\n ORDER BY priority DESC, run_at ASC, id ASC\n LIMIT ?3\n )\nRETURNING *\n", "describe": { "columns": [ { @@ -82,5 +82,5 @@ false ] }, - "hash": "0819e39ed31495e0c70a6dd549b10e5c52e64d5289ca829e2a0135a14801b930" + "hash": "8e68049c1005dd0981469dac442cc479d318a98f4891ed46ff8588bb4e85615b" } From cfeb7dbaea7ebb10a2fe9f3519ed5cef1d55a10f Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Fri, 26 Sep 2025 12:06:20 +0300 Subject: [PATCH 114/121] improvements on queries --- Cargo.lock | 4 +- Cargo.toml | 4 +- queries/{task => backend}/fetch_next.sql | 0 .../fetch_next_shared.sql} | 0 .../register.sql => backend/heartbeat.sql} | 0 queries/backend/list_jobs.sql | 12 +++ queries/backend/list_workers.sql | 10 +++ queries/backend/queue_length.sql | 12 +++ queries/backend/reenqueue_orphaned.sql | 0 queries/backend/stats.sql | 11 +++ queries/backend/vacuum.sql | 11 +++ queries/task/find_by_id.sql | 8 ++ queries/task/kill.sql | 9 ++ queries/task/reschedule.sql | 11 +++ queries/task/retry.sql | 11 +++ queries/task/update_by_id.sql | 12 +++ src/fetcher.rs | 2 +- src/lib.rs | 3 +- src/shared.rs | 2 +- src/traits/list_workers.rs | 89 +++++++++++++++++++ src/traits/mod.rs | 1 + 21 files changed, 205 insertions(+), 7 deletions(-) rename queries/{task => backend}/fetch_next.sql (100%) rename queries/{task/fetch_shared.sql => backend/fetch_next_shared.sql} (100%) rename queries/{worker/register.sql => backend/heartbeat.sql} (100%) create mode 100644 queries/backend/list_jobs.sql create mode 100644 queries/backend/list_workers.sql create mode 100644 queries/backend/queue_length.sql create mode 100644 queries/backend/reenqueue_orphaned.sql create mode 100644 queries/backend/stats.sql create mode 100644 queries/backend/vacuum.sql create mode 100644 queries/task/find_by_id.sql create mode 100644 queries/task/kill.sql create mode 100644 queries/task/reschedule.sql create mode 100644 queries/task/retry.sql create mode 100644 queries/task/update_by_id.sql create mode 100644 src/traits/list_workers.rs create mode 100644 src/traits/mod.rs diff --git a/Cargo.lock b/Cargo.lock index cd9bd23..93e7e68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,9 +34,9 @@ dependencies = [ [[package]] name = "apalis-core" -version = "1.0.0-alpha.2" +version = "1.0.0-alpha.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4096618accabcf371a44e1ef9b9739cf3a2ba5878a6e64ef6ef78a1f035e8cb" +checksum = "647491a3c4be9cc79a806f5df91f3434dbf7201bf400fafdf76c14b78b3cc7fa" dependencies = [ "futures-channel", "futures-core", diff --git a/Cargo.toml b/Cargo.toml index f18c300..7d0947e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ features = ["chrono", "sqlite"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" -apalis-core = { version = "1.0.0-alpha.2", default-features = false, features = [ +apalis-core = { version = "1.0.0-alpha.3", default-features = false, features = [ "sleep", "json", ] } @@ -40,7 +40,7 @@ ulid = { version = "1" } [dev-dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } -apalis-core = { version = "1.0.0-alpha.2", features = ["test-utils"] } +apalis-core = { version = "1.0.0-alpha.3", features = ["test-utils"] } apalis-sqlite = { path = "." , features = ["migrate", "tokio-comp"] } [package.metadata.docs.rs] diff --git a/queries/task/fetch_next.sql b/queries/backend/fetch_next.sql similarity index 100% rename from queries/task/fetch_next.sql rename to queries/backend/fetch_next.sql diff --git a/queries/task/fetch_shared.sql b/queries/backend/fetch_next_shared.sql similarity index 100% rename from queries/task/fetch_shared.sql rename to queries/backend/fetch_next_shared.sql diff --git a/queries/worker/register.sql b/queries/backend/heartbeat.sql similarity index 100% rename from queries/worker/register.sql rename to queries/backend/heartbeat.sql diff --git a/queries/backend/list_jobs.sql b/queries/backend/list_jobs.sql new file mode 100644 index 0000000..bf0a3f5 --- /dev/null +++ b/queries/backend/list_jobs.sql @@ -0,0 +1,12 @@ +SELECT + * +FROM + Jobs +WHERE + status = ?1 + AND job_type = ?2 +ORDER BY + done_at DESC, + run_at DESC +LIMIT + ?3 OFFSET ?4 diff --git a/queries/backend/list_workers.sql b/queries/backend/list_workers.sql new file mode 100644 index 0000000..27558c7 --- /dev/null +++ b/queries/backend/list_workers.sql @@ -0,0 +1,10 @@ +SELECT + * +FROM + Workers +WHERE + worker_type = ?1 +ORDER BY + last_seen DESC +LIMIT + ?2 OFFSET ?3 diff --git a/queries/backend/queue_length.sql b/queries/backend/queue_length.sql new file mode 100644 index 0000000..0dd2c31 --- /dev/null +++ b/queries/backend/queue_length.sql @@ -0,0 +1,12 @@ +Select + COUNT(*) AS count +FROM + Jobs +WHERE + ( + status = 'Pending' + OR ( + status = 'Failed' + AND attempts < max_attempts + ) + ) diff --git a/queries/backend/reenqueue_orphaned.sql b/queries/backend/reenqueue_orphaned.sql new file mode 100644 index 0000000..e69de29 diff --git a/queries/backend/stats.sql b/queries/backend/stats.sql new file mode 100644 index 0000000..68d2da2 --- /dev/null +++ b/queries/backend/stats.sql @@ -0,0 +1,11 @@ +SELECT + COUNT(*) AS total, + SUM(CASE WHEN status = 'Pending' THEN 1 ELSE 0 END) AS pending, + SUM(CASE WHEN status = 'Running' THEN 1 ELSE 0 END) AS running, + SUM(CASE WHEN status = 'Done' THEN 1 ELSE 0 END) AS done, + SUM(CASE WHEN status = 'Failed' THEN 1 ELSE 0 END) AS failed, + SUM(CASE WHEN status = 'Killed' THEN 1 ELSE 0 END) AS killed, + SUM(CASE WHEN status IN ('Done', 'Failed', 'Killed') THEN 1 ELSE 0 END) AS completed, + SUM(CASE WHEN status IN ('Pending', 'Running') THEN 1 ELSE 0 END) AS active +FROM Jobs +WHERE job_type = ?1 diff --git a/queries/backend/vacuum.sql b/queries/backend/vacuum.sql new file mode 100644 index 0000000..94050db --- /dev/null +++ b/queries/backend/vacuum.sql @@ -0,0 +1,11 @@ +Delete from + Jobs +where + status = 'Done' + OR status = 'Killed' + OR ( + status = 'Failed' + AND max_attempts <= attempts + ); + +VACUUM; diff --git a/queries/task/find_by_id.sql b/queries/task/find_by_id.sql new file mode 100644 index 0000000..dbbcaff --- /dev/null +++ b/queries/task/find_by_id.sql @@ -0,0 +1,8 @@ +SELECT + * +FROM + Jobs +WHERE + id = ?1 +LIMIT + 1; diff --git a/queries/task/kill.sql b/queries/task/kill.sql new file mode 100644 index 0000000..4a6d6ed --- /dev/null +++ b/queries/task/kill.sql @@ -0,0 +1,9 @@ +UPDATE + Jobs +SET + status = 'Killed', + done_at = strftime('%s', 'now'), + last_error = ?3 +WHERE + id = ?1 + AND lock_by = ?2 diff --git a/queries/task/reschedule.sql b/queries/task/reschedule.sql new file mode 100644 index 0000000..6870a91 --- /dev/null +++ b/queries/task/reschedule.sql @@ -0,0 +1,11 @@ +UPDATE + Jobs +SET + status = 'Failed', + attempts = attempts + 1, + done_at = NULL, + lock_by = NULL, + lock_at = NULL, + run_at = ?2 +WHERE + id = ?1 diff --git a/queries/task/retry.sql b/queries/task/retry.sql new file mode 100644 index 0000000..f46aaef --- /dev/null +++ b/queries/task/retry.sql @@ -0,0 +1,11 @@ +UPDATE + Jobs +SET + status = 'Pending', + attempt = attempt + 1, + run_at = ?1, + done_at = NULL, + lock_by = NULL +WHERE + id = ?2 + AND lock_by = ?3 diff --git a/queries/task/update_by_id.sql b/queries/task/update_by_id.sql new file mode 100644 index 0000000..f4c1ae7 --- /dev/null +++ b/queries/task/update_by_id.sql @@ -0,0 +1,12 @@ +UPDATE + Jobs +SET + status = ?1, + attempts = ?2, + done_at = ?3, + lock_by = ?4, + lock_at = ?5, + last_error = ?6, + priority = ?7 +WHERE + id = ?8 diff --git a/src/fetcher.rs b/src/fetcher.rs index 91be745..c021fe0 100644 --- a/src/fetcher.rs +++ b/src/fetcher.rs @@ -34,7 +34,7 @@ where let worker = worker.name().to_string(); sqlx::query_file_as!( TaskRow, - "queries/task/fetch_next.sql", + "queries/backend/fetch_next.sql", worker, job_type, buffer_size diff --git a/src/lib.rs b/src/lib.rs index ad1e24b..2134ac8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,6 +35,7 @@ pub mod from_row; mod hook; mod shared; mod sink; +mod traits; pub type SqliteTask = Task; pub use config::Config; @@ -289,7 +290,7 @@ pub(crate) async fn heartbeat( let svc = worker.get_service().to_owned(); let worker = worker.name().to_owned(); let res = sqlx::query_file!( - "queries/worker/register.sql", + "queries/backend/heartbeat.sql", worker, worker_type, backend_type, diff --git a/src/shared.rs b/src/shared.rs index 52b61e8..2a216d6 100644 --- a/src/shared.rs +++ b/src/shared.rs @@ -87,7 +87,7 @@ impl SharedSqliteStorage { let buffer_size = max(10, instances.len()) as i32; let res: Vec<_> = sqlx::query_file_as!( TaskRow, - "queries/task/fetch_shared.sql", + "queries/backend/fetch_next_shared.sql", job_types, row_ids, buffer_size, diff --git a/src/traits/list_workers.rs b/src/traits/list_workers.rs new file mode 100644 index 0000000..4011c07 --- /dev/null +++ b/src/traits/list_workers.rs @@ -0,0 +1,89 @@ +use apalis_core::{ + backend::{Filter, ListTasks, ListWorkers, RunningWorker, codec::json::JsonCodec}, + task::status::Status, +}; +use futures::TryFutureExt; +use serde::{Serialize, de::DeserializeOwned}; + +use crate::{SqliteStorage, SqliteTask, from_row::TaskRow}; + +struct Worker { + id: String, + #[allow(unused)] + worker_type: String, + storage_name: String, + layers: Option, + last_seen: i64, +} + +impl ListWorkers for SqliteStorage +where + Args: Send + 'static + Unpin + Serialize + DeserializeOwned, +{ + fn list_workers(&self) -> impl Future, Self::Error>> + Send { + let namespace = self.config.namespace().to_string(); + let pool = self.pool.clone(); + let limit = 100; + let offset = 0; + let pid = std::process::id() as i64; + async move { + let workers = sqlx::query_file_as!( + Worker, + "queries/backend/list_workers.sql", + namespace, + limit, + offset + ) + .fetch_all(&pool) + .map_ok(|w| { + w.into_iter() + .map(|w| RunningWorker { + id: w.id, + pid: pid as u32, + hostname: w.storage_name, + started_at: 0, + last_heartbeat: w.last_seen as u64, + service: w.layers.unwrap_or_default(), + }) + .collect() + }) + .await?; + Ok(workers) + } + } +} + +impl ListTasks + for SqliteStorage +{ + fn list_tasks( + &self, + filter: &Filter, + ) -> impl Future>, Self::Error>> + Send { + let namespace = self.config.namespace().to_string(); + let pool = self.pool.clone(); + let limit = filter.limit() as i32; + let offset = filter.offset() as i32; + let status = filter + .status + .as_ref() + .unwrap_or(&Status::Pending) + .to_string(); + async move { + let tasks = sqlx::query_file_as!( + TaskRow, + "queries/backend/list_jobs.sql", + status, + namespace, + limit, + offset + ) + .fetch_all(&pool) + .await? + .into_iter() + .map(|r| r.try_into_task::, _>()) + .collect::, _>>()?; + Ok(tasks) + } + } +} diff --git a/src/traits/mod.rs b/src/traits/mod.rs new file mode 100644 index 0000000..158ee35 --- /dev/null +++ b/src/traits/mod.rs @@ -0,0 +1 @@ +mod list_workers; From 044188fb0b21ba6a00fc8fb903421faea5bf096e Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Fri, 17 Oct 2025 10:20:30 +0300 Subject: [PATCH 115/121] checkpoint: features working --- Cargo.lock | 6 +- Cargo.toml | 10 +- ... 20250313213411_add_job_priority copy.sql} | 0 migrations/20251013233016_stats_indexes.sql | 22 + queries/backend/fetch_next.sql | 13 +- queries/backend/fetch_next_shared.sql | 6 +- queries/backend/heartbeat.sql | 7 - queries/backend/keep_alive.sql | 6 + queries/backend/list_all_jobs.sql | 11 + queries/backend/list_all_workers.sql | 8 + queries/backend/list_queues.sql | 386 ++++++++++++++++ queries/backend/overview.sql | 421 +++++++++++++++++ queries/backend/overview_by_queue.sql | 433 ++++++++++++++++++ queries/backend/reenqueue_orphaned.sql | 24 + queries/backend/register_worker.sql | 18 + queries/backend/stats.sql | 16 +- queries/task/ack.sql | 2 +- queries/task/lock.sql | 16 + src/ack.rs | 100 +++- src/config.rs | 57 ++- src/context.rs | 12 +- src/fetcher.rs | 18 +- src/from_row.rs | 43 +- src/hook.rs | 21 +- src/lib.rs | 197 +++++--- src/reenqueue_orphaned.rs | 53 +++ src/shared.rs | 53 +-- src/sink.rs | 24 +- src/traits/fetch_by_id.rs | 0 src/traits/list_queues.rs | 0 src/traits/list_tasks.rs | 0 src/traits/list_workers.rs | 244 +++++++++- src/traits/metrics.rs | 0 33 files changed, 2048 insertions(+), 179 deletions(-) rename migrations/{20250313213411_add_job_priority.sql => 20250313213411_add_job_priority copy.sql} (100%) create mode 100644 migrations/20251013233016_stats_indexes.sql delete mode 100644 queries/backend/heartbeat.sql create mode 100644 queries/backend/keep_alive.sql create mode 100644 queries/backend/list_all_jobs.sql create mode 100644 queries/backend/list_all_workers.sql create mode 100644 queries/backend/list_queues.sql create mode 100644 queries/backend/overview.sql create mode 100644 queries/backend/overview_by_queue.sql create mode 100644 queries/backend/register_worker.sql create mode 100644 queries/task/lock.sql create mode 100644 src/reenqueue_orphaned.rs create mode 100644 src/traits/fetch_by_id.rs create mode 100644 src/traits/list_queues.rs create mode 100644 src/traits/list_tasks.rs create mode 100644 src/traits/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index 93e7e68..41ea2aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,8 +35,7 @@ dependencies = [ [[package]] name = "apalis-core" version = "1.0.0-alpha.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "647491a3c4be9cc79a806f5df91f3434dbf7201bf400fafdf76c14b78b3cc7fa" +source = "git+https://github.com/geofmureithi/apalis.git?branch=chore%2Ftraits-expansion#d9c4b2319f010dda9003ae07b36d724228e8ac3c" dependencies = [ "futures-channel", "futures-core", @@ -68,6 +67,8 @@ dependencies = [ "sqlx", "thiserror", "tokio", + "tower-layer", + "tower-service", "ulid", ] @@ -2239,6 +2240,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ "rand 0.9.2", + "serde", "web-time", ] diff --git a/Cargo.toml b/Cargo.toml index 7d0947e..af7e533 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ serde_json = "1" apalis-core = { version = "1.0.0-alpha.3", default-features = false, features = [ "sleep", "json", -] } +], git = "https://github.com/geofmureithi/apalis.git", branch = "chore/traits-expansion" } log = "0.4.21" futures = "0.3.30" tokio = { version = "1", features = ["rt", "net"], optional = true } @@ -36,12 +36,14 @@ chrono = { version = "0.4", features = ["serde"] } thiserror = "2.0.0" pin-project = "1.1.10" libsqlite3-sys = "0.30.1" -ulid = { version = "1" } +ulid = { version = "1", features = ["serde"] } +tower-layer = "0.3.3" +tower-service = "0.3.3" [dev-dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } -apalis-core = { version = "1.0.0-alpha.3", features = ["test-utils"] } -apalis-sqlite = { path = "." , features = ["migrate", "tokio-comp"] } +apalis-core = { version = "1.0.0-alpha.3", features = ["test-utils"], git = "https://github.com/geofmureithi/apalis.git", branch = "chore/traits-expansion" } +apalis-sqlite = { path = ".", features = ["migrate", "tokio-comp"] } [package.metadata.docs.rs] # defines the configuration attribute `docsrs` diff --git a/migrations/20250313213411_add_job_priority.sql b/migrations/20250313213411_add_job_priority copy.sql similarity index 100% rename from migrations/20250313213411_add_job_priority.sql rename to migrations/20250313213411_add_job_priority copy.sql diff --git a/migrations/20251013233016_stats_indexes.sql b/migrations/20251013233016_stats_indexes.sql new file mode 100644 index 0000000..9c029b9 --- /dev/null +++ b/migrations/20251013233016_stats_indexes.sql @@ -0,0 +1,22 @@ +CREATE INDEX IF NOT EXISTS idx_jobs_job_type_status_run_at ON Jobs(job_type, status, run_at); + +CREATE INDEX IF NOT EXISTS idx_jobs_status_run_at ON Jobs(status, run_at); + +CREATE INDEX IF NOT EXISTS idx_jobs_run_at_status ON Jobs(run_at, status); + +CREATE INDEX IF NOT EXISTS idx_jobs_completed_done_at ON Jobs(status, done_at, run_at) +WHERE + status IN ('Done', 'Failed', 'Killed') + AND done_at IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_jobs_pending ON Jobs(run_at) +WHERE + status = 'Pending'; + +CREATE INDEX IF NOT EXISTS idx_jobs_running ON Jobs(run_at) +WHERE + status = 'Running'; + +CREATE INDEX IF NOT EXISTS idx_jobs_job_type_run_at ON Jobs(job_type, run_at); + +CREATE INDEX IF NOT EXISTS idx_jobs_job_type_covering ON Jobs(job_type, status, run_at, done_at); diff --git a/queries/backend/fetch_next.sql b/queries/backend/fetch_next.sql index 91a176e..f2552f9 100644 --- a/queries/backend/fetch_next.sql +++ b/queries/backend/fetch_next.sql @@ -1,16 +1,19 @@ UPDATE Jobs SET - status = 'Running', + status = 'Queued', lock_by = ?1, - lock_at = CURRENT_TIMESTAMP + lock_at = strftime('%s', 'now') WHERE ROWID IN ( SELECT ROWID FROM Jobs WHERE job_type = ?2 - AND status = 'Pending' - AND lock_by IS NULL - AND (run_at IS NULL OR run_at <= CURRENT_TIMESTAMP) + AND ( + (status = 'Pending' AND lock_by IS NULL) + OR + (status = 'Failed' AND attempts < max_attempts) + ) + AND (run_at IS NULL OR run_at <= strftime('%s', 'now')) ORDER BY priority DESC, run_at ASC, id ASC LIMIT ?3 ) diff --git a/queries/backend/fetch_next_shared.sql b/queries/backend/fetch_next_shared.sql index 0a073a6..80fd356 100644 --- a/queries/backend/fetch_next_shared.sql +++ b/queries/backend/fetch_next_shared.sql @@ -1,7 +1,7 @@ UPDATE Jobs SET - status = 'Running', - lock_at = CURRENT_TIMESTAMP + status = 'Queued', + lock_at = strftime('%s', 'now') WHERE ROWID IN ( SELECT ROWID FROM Jobs @@ -12,7 +12,7 @@ WHERE ROWID IN ( AND lock_by IS NULL AND ( run_at IS NULL - OR run_at <= CURRENT_TIMESTAMP + OR run_at <= strftime('%s', 'now') ) AND ROWID IN ( SELECT value FROM json_each(?2) diff --git a/queries/backend/heartbeat.sql b/queries/backend/heartbeat.sql deleted file mode 100644 index 54f2da4..0000000 --- a/queries/backend/heartbeat.sql +++ /dev/null @@ -1,7 +0,0 @@ -INSERT INTO - Workers (id, worker_type, storage_name, layers, last_seen) -VALUES - ($1, $2, $3, $4, $5) ON CONFLICT (id) DO -UPDATE -SET - last_seen = EXCLUDED.last_seen diff --git a/queries/backend/keep_alive.sql b/queries/backend/keep_alive.sql new file mode 100644 index 0000000..a7984a9 --- /dev/null +++ b/queries/backend/keep_alive.sql @@ -0,0 +1,6 @@ +UPDATE + Workers +SET + last_seen = strftime('%s', 'now') +WHERE + id = $1 AND worker_type = $2; diff --git a/queries/backend/list_all_jobs.sql b/queries/backend/list_all_jobs.sql new file mode 100644 index 0000000..ad6c050 --- /dev/null +++ b/queries/backend/list_all_jobs.sql @@ -0,0 +1,11 @@ +SELECT + * +FROM + Jobs +WHERE + status = ?1 +ORDER BY + done_at DESC, + run_at DESC +LIMIT + ?2 OFFSET ?3 diff --git a/queries/backend/list_all_workers.sql b/queries/backend/list_all_workers.sql new file mode 100644 index 0000000..6215402 --- /dev/null +++ b/queries/backend/list_all_workers.sql @@ -0,0 +1,8 @@ +SELECT + * +FROM + Workers +ORDER BY + last_seen DESC +LIMIT + ?1 OFFSET ?2 diff --git a/queries/backend/list_queues.sql b/queries/backend/list_queues.sql new file mode 100644 index 0000000..5515254 --- /dev/null +++ b/queries/backend/list_queues.sql @@ -0,0 +1,386 @@ +WITH queue_stats AS ( + SELECT + job_type, + json_group_array( + json_object( + 'title', + statistic, + 'stat_type', + type, + 'value', + value, + 'priority', + priority + ) + ) as stats + FROM + ( + SELECT + job_type, + 1 AS priority, + 'Number' AS type, + 'RUNNING_JOBS' AS statistic, + CAST( + SUM( + CASE + WHEN status = 'Running' THEN 1 + ELSE 0 + END + ) AS TEXT + ) AS value + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 1, + 'Number', + 'PENDING_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Pending' THEN 1 + ELSE 0 + END + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 1, + 'Number', + 'FAILED_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Failed' THEN 1 + ELSE 0 + END + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 2, + 'Number', + 'ACTIVE_JOBS', + CAST( + SUM( + CASE + WHEN status IN ('Pending', 'Queued', 'Running') THEN 1 + ELSE 0 + END + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 2, + 'Number', + 'STALE_RUNNING_JOBS', + CAST(COUNT(*) AS TEXT) + FROM + Jobs + WHERE + status = 'Running' + AND run_at < strftime('%s', 'now', '-1 hour') + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 2, + 'Percentage', + 'KILL_RATE', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 3, + 'Number', + 'JOBS_PAST_HOUR', + CAST(COUNT(*) AS TEXT) + FROM + Jobs + WHERE + run_at >= strftime('%s', 'now', '-1 hour') + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 3, + 'Number', + 'JOBS_TODAY', + CAST(COUNT(*) AS TEXT) + FROM + Jobs + WHERE + date(run_at, 'unixepoch') = date('now') + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 3, + 'Number', + 'KILLED_JOBS_TODAY', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS TEXT + ) + FROM + Jobs + WHERE + date(run_at, 'unixepoch') = date('now') + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 3, + 'Decimal', + 'AVG_JOBS_PER_MINUTE_PAST_HOUR', + CAST(ROUND(COUNT(*) / 60.0, 2) AS TEXT) + FROM + Jobs + WHERE + run_at >= strftime('%s', 'now', '-1 hour') + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 4, + 'Number', + 'TOTAL_JOBS', + CAST(COUNT(*) AS TEXT) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 4, + 'Number', + 'DONE_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 4, + 'Number', + 'KILLED_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 4, + 'Percentage', + 'SUCCESS_RATE', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 5, + 'Decimal', + 'AVG_JOB_DURATION_MINS', + CAST(ROUND(AVG((done_at - run_at) / 60.0), 2) AS TEXT) + FROM + Jobs + WHERE + status IN ('Done', 'Failed', 'Killed') + AND done_at IS NOT NULL + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 5, + 'Decimal', + 'LONGEST_RUNNING_JOB_MINS', + CAST( + ROUND( + MAX( + CASE + WHEN status = 'Running' THEN (strftime('%s', 'now') - run_at) / 60.0 + ELSE 0 + END + ), + 2 + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 6, + 'Number', + 'JOBS_PAST_7_DAYS', + CAST(COUNT(*) AS TEXT) + FROM + Jobs + WHERE + run_at >= strftime('%s', 'now', '-7 days') + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 8, + 'Timestamp', + 'MOST_RECENT_JOB', + CAST(MAX(run_at) AS TEXT) + FROM + Jobs + GROUP BY + job_type + ORDER BY + job_type, + priority, + statistic + ) + GROUP BY + job_type +), +all_job_types AS ( + SELECT + worker_type AS job_type + FROM + Workers + UNION + SELECT + job_type + FROM + Jobs +) +SELECT + jt.job_type as name, + COALESCE(qs.stats, '[]') as stats, + COALESCE( + ( + SELECT + json_group_array(DISTINCT lock_by) + FROM + Jobs + WHERE + job_type = jt.job_type + AND lock_by IS NOT NULL + ), + '[]' + ) as workers, + COALESCE( + ( + SELECT + json_group_array(daily_count) + FROM + ( + SELECT + COUNT(*) as daily_count + FROM + Jobs + WHERE + job_type = jt.job_type + AND run_at >= strftime('%s', 'now', '-7 days') + GROUP BY + date(run_at, 'unixepoch') + ORDER BY + date(run_at, 'unixepoch') + ) + ), + '[]' + ) as activity +FROM + all_job_types jt + LEFT JOIN queue_stats qs ON jt.job_type = qs.job_type +ORDER BY + name; diff --git a/queries/backend/overview.sql b/queries/backend/overview.sql new file mode 100644 index 0000000..6963dc7 --- /dev/null +++ b/queries/backend/overview.sql @@ -0,0 +1,421 @@ +SELECT + 1 AS priority, + 'Number' AS type, + 'RUNNING_JOBS' AS statistic, + CAST( + SUM( + CASE + WHEN status = 'Running' THEN 1 + ELSE 0 + END + ) AS REAL + ) AS value +FROM + Jobs +UNION +ALL +SELECT + 1, + 'Number', + 'PENDING_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Pending' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 2, + 'Number', + 'FAILED_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Failed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 2, + 'Number', + 'ACTIVE_JOBS', + CAST( + SUM( + CASE + WHEN status IN ('Pending', 'Running', 'Queued') THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 2, + 'Number', + 'STALE_RUNNING_JOBS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + status = 'Running' + AND run_at < strftime('%s', 'now', '-1 hour') +UNION +ALL +SELECT + 2, + 'Percentage', + 'KILL_RATE', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 3, + 'Number', + 'JOBS_PAST_HOUR', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-1 hour') +UNION +ALL +SELECT + 3, + 'Number', + 'JOBS_TODAY', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + date(run_at, 'unixepoch') = date('now') +UNION +ALL +SELECT + 3, + 'Number', + 'KILLED_JOBS_TODAY', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + date(run_at, 'unixepoch') = date('now') +UNION +ALL +SELECT + 3, + 'Decimal', + 'AVG_JOBS_PER_MINUTE_PAST_HOUR', + CAST(ROUND(COUNT(*) / 60.0, 2) AS REAL) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-1 hour') +UNION +ALL +SELECT + 4, + 'Number', + 'TOTAL_JOBS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +UNION +ALL +SELECT + 4, + 'Number', + 'DONE_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 4, + 'Number', + 'COMPLETED_JOBS', + CAST( + SUM( + CASE + WHEN status IN ('Done', 'Failed', 'Killed') THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 4, + 'Number', + 'KILLED_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 4, + 'Percentage', + 'SUCCESS_RATE', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 5, + 'Decimal', + 'AVG_JOB_DURATION_MINS', + CAST(ROUND(AVG((done_at - run_at) / 60.0), 2) AS REAL) +FROM + Jobs +WHERE + status IN ('Done', 'Failed', 'Killed') + AND done_at IS NOT NULL +UNION +ALL +SELECT + 5, + 'Decimal', + 'LONGEST_RUNNING_JOB_MINS', + CAST( + ROUND( + MAX( + CASE + WHEN status = 'Running' THEN (strftime('%s', 'now') - run_at) / 60.0 + ELSE 0 + END + ), + 2 + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 5, + 'Number', + 'QUEUE_BACKLOG', + CAST( + SUM( + CASE + WHEN status = 'Pending' + AND run_at <= strftime('%s', 'now') THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 6, + 'Number', + 'JOBS_PAST_24_HOURS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-1 day') +UNION +ALL +SELECT + 6, + 'Number', + 'JOBS_PAST_7_DAYS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-7 days') +UNION +ALL +SELECT + 6, + 'Number', + 'KILLED_JOBS_PAST_7_DAYS', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-7 days') +UNION +ALL +SELECT + 6, + 'Percentage', + 'SUCCESS_RATE_PAST_24H', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS REAL + ) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-1 day') +UNION +ALL +SELECT + 7, + 'Decimal', + 'AVG_JOBS_PER_HOUR_PAST_24H', + CAST(ROUND(COUNT(*) / 24.0, 2) AS REAL) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-1 day') +UNION +ALL +SELECT + 7, + 'Decimal', + 'AVG_JOBS_PER_DAY_PAST_7D', + CAST(ROUND(COUNT(*) / 7.0, 2) AS REAL) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-7 days') +UNION +ALL +SELECT + 8, + 'Timestamp', + 'MOST_RECENT_JOB', + CAST(MAX(run_at) AS REAL) +FROM + Jobs +UNION +ALL +SELECT + 8, + 'Timestamp', + 'OLDEST_PENDING_JOB', + CAST(MIN(run_at) AS REAL) +FROM + Jobs +WHERE + status = 'Pending' + AND run_at <= strftime('%s', 'now') +UNION +ALL +SELECT + 8, + 'Number', + 'PEAK_HOUR_JOBS', + CAST(MAX(hourly_count) AS REAL) +FROM + ( + SELECT + COUNT(*) as hourly_count + FROM + Jobs + WHERE + run_at >= strftime('%s', 'now', '-1 day') + GROUP BY + strftime('%H', run_at, 'unixepoch') + ) +UNION +ALL +SELECT + 9, + 'Number', + 'DB_PAGE_SIZE', + CAST(page_size AS REAL) +FROM + pragma_page_size() +UNION +ALL +SELECT + 9, + 'Number', + 'DB_PAGE_COUNT', + CAST(page_count AS REAL) +FROM + pragma_page_count() +UNION +ALL +SELECT + 9, + 'Number', + 'DB_SIZE', + CAST(page_size * page_count AS REAL) +FROM + pragma_page_size(), + pragma_page_count() +ORDER BY + priority, + statistic; diff --git a/queries/backend/overview_by_queue.sql b/queries/backend/overview_by_queue.sql new file mode 100644 index 0000000..8252712 --- /dev/null +++ b/queries/backend/overview_by_queue.sql @@ -0,0 +1,433 @@ +SELECT + 1 AS priority, + 'Number' AS type, + 'RUNNING_JOBS' AS statistic, + CAST( + SUM( + CASE + WHEN status = 'Running' THEN 1 + ELSE 0 + END + ) AS REAL + ) AS value +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 1, + 'Number', + 'PENDING_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Pending' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 2, + 'Number', + 'FAILED_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Failed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 2, + 'Number', + 'ACTIVE_JOBS', + CAST( + SUM( + CASE + WHEN status IN ('Pending', 'Queued', 'Running') THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 2, + 'Number', + 'STALE_RUNNING_JOBS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND status = 'Running' + AND run_at < strftime('%s', 'now', '-1 hour') +UNION +ALL +SELECT + 2, + 'Percentage', + 'KILL_RATE', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 3, + 'Number', + 'JOBS_PAST_HOUR', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-1 hour') +UNION +ALL +SELECT + 3, + 'Number', + 'JOBS_TODAY', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND date(run_at, 'unixepoch') = date('now') +UNION +ALL +SELECT + 3, + 'Number', + 'KILLED_JOBS_TODAY', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 + AND date(run_at, 'unixepoch') = date('now') +UNION +ALL +SELECT + 3, + 'Decimal', + 'AVG_JOBS_PER_MINUTE_PAST_HOUR', + CAST(ROUND(COUNT(*) / 60.0, 2) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-1 hour') +UNION +ALL +SELECT + 4, + 'Number', + 'TOTAL_JOBS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 4, + 'Number', + 'DONE_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 4, + 'Number', + 'COMPLETED_JOBS', + CAST( + SUM( + CASE + WHEN status IN ('Done', 'Failed', 'Killed') THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 4, + 'Number', + 'KILLED_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 4, + 'Percentage', + 'SUCCESS_RATE', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 5, + 'Decimal', + 'AVG_JOB_DURATION_MINS', + CAST(ROUND(AVG((done_at - run_at) / 60.0), 2) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND status IN ('Done', 'Failed', 'Killed') + AND done_at IS NOT NULL +UNION +ALL +SELECT + 5, + 'Decimal', + 'LONGEST_RUNNING_JOB_MINS', + CAST( + ROUND( + MAX( + CASE + WHEN status = 'Running' THEN (strftime('%s', 'now') - run_at) / 60.0 + ELSE 0 + END + ), + 2 + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 5, + 'Number', + 'QUEUE_BACKLOG', + CAST( + SUM( + CASE + WHEN status = 'Pending' + AND run_at <= strftime('%s', 'now') THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 6, + 'Number', + 'JOBS_PAST_24_HOURS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-1 day') +UNION +ALL +SELECT + 6, + 'Number', + 'JOBS_PAST_7_DAYS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-7 days') +UNION +ALL +SELECT + 6, + 'Number', + 'KILLED_JOBS_PAST_7_DAYS', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-7 days') +UNION +ALL +SELECT + 6, + 'Percentage', + 'SUCCESS_RATE_PAST_24H', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-1 day') +UNION +ALL +SELECT + 7, + 'Decimal', + 'AVG_JOBS_PER_HOUR_PAST_24H', + CAST(ROUND(COUNT(*) / 24.0, 2) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-1 day') +UNION +ALL +SELECT + 7, + 'Decimal', + 'AVG_JOBS_PER_DAY_PAST_7D', + CAST(ROUND(COUNT(*) / 7.0, 2) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-7 days') +UNION +ALL +SELECT + 8, + 'Timestamp', + 'MOST_RECENT_JOB', + CAST(MAX(run_at) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 8, + 'Timestamp', + 'OLDEST_PENDING_JOB', + CAST(MIN(run_at) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND status = 'Pending' + AND run_at <= strftime('%s', 'now') +UNION +ALL +SELECT + 8, + 'Number', + 'PEAK_HOUR_JOBS', + CAST(MAX(hourly_count) AS REAL) +FROM + ( + SELECT + COUNT(*) as hourly_count + FROM + Jobs + WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-1 day') + GROUP BY + strftime('%H', run_at, 'unixepoch') + ) +ORDER BY + priority, + statistic; diff --git a/queries/backend/reenqueue_orphaned.sql b/queries/backend/reenqueue_orphaned.sql index e69de29..87c4621 100644 --- a/queries/backend/reenqueue_orphaned.sql +++ b/queries/backend/reenqueue_orphaned.sql @@ -0,0 +1,24 @@ +UPDATE + Jobs +SET + status = "Pending", + done_at = NULL, + lock_by = NULL, + lock_at = NULL, + attempts = attempts + 1, + last_error = '{"Err": "Re-enqueued due to worker heartbeat timeout."}' +WHERE + id IN ( + SELECT + Jobs.id + FROM + Jobs + INNER JOIN Workers ON lock_by = Workers.id + WHERE + ( + status = "Running" + OR status = "Queued" + ) + AND strftime('%s', 'now') - Workers.last_seen >= ?1 + AND Workers.worker_type = ?2 + ); diff --git a/queries/backend/register_worker.sql b/queries/backend/register_worker.sql new file mode 100644 index 0000000..81fe501 --- /dev/null +++ b/queries/backend/register_worker.sql @@ -0,0 +1,18 @@ +INSERT INTO + Workers (id, worker_type, storage_name, layers, last_seen) +SELECT + ?1, + ?2, + ?3, + ?4, + strftime('%s', 'now') +WHERE + NOT EXISTS ( + SELECT + 1 + FROM + Workers + WHERE + id = ?1 + AND strftime('%s', 'now') - last_seen >= ?5 + ); diff --git a/queries/backend/stats.sql b/queries/backend/stats.sql index 68d2da2..5ee7796 100644 --- a/queries/backend/stats.sql +++ b/queries/backend/stats.sql @@ -1,11 +1,11 @@ SELECT - COUNT(*) AS total, - SUM(CASE WHEN status = 'Pending' THEN 1 ELSE 0 END) AS pending, - SUM(CASE WHEN status = 'Running' THEN 1 ELSE 0 END) AS running, - SUM(CASE WHEN status = 'Done' THEN 1 ELSE 0 END) AS done, - SUM(CASE WHEN status = 'Failed' THEN 1 ELSE 0 END) AS failed, - SUM(CASE WHEN status = 'Killed' THEN 1 ELSE 0 END) AS killed, - SUM(CASE WHEN status IN ('Done', 'Failed', 'Killed') THEN 1 ELSE 0 END) AS completed, - SUM(CASE WHEN status IN ('Pending', 'Running') THEN 1 ELSE 0 END) AS active + CAST(COUNT(*) AS INTEGER) AS total, + CAST(SUM(CASE WHEN status = 'Pending' THEN 1 ELSE 0 END) AS INTEGER) AS pending, + CAST(SUM(CASE WHEN status = 'Running' THEN 1 ELSE 0 END) AS INTEGER) AS running, + CAST(SUM(CASE WHEN status = 'Done' THEN 1 ELSE 0 END) AS INTEGER) AS done, + CAST(SUM(CASE WHEN status = 'Failed' THEN 1 ELSE 0 END) AS INTEGER) AS failed, + CAST(SUM(CASE WHEN status = 'Killed' THEN 1 ELSE 0 END) AS INTEGER) AS killed, + CAST(SUM(CASE WHEN status IN ('Done', 'Failed', 'Killed') THEN 1 ELSE 0 END) AS INTEGER) AS completed, + CAST(SUM(CASE WHEN status IN ('Pending', 'Running') THEN 1 ELSE 0 END) AS INTEGER) AS active FROM Jobs WHERE job_type = ?1 diff --git a/queries/task/ack.sql b/queries/task/ack.sql index 337f9a0..5b91344 100644 --- a/queries/task/ack.sql +++ b/queries/task/ack.sql @@ -4,7 +4,7 @@ SET status = ?4, attempts = ?2, last_error = ?3, - done_at = CURRENT_TIMESTAMP + done_at = strftime('%s', 'now') WHERE id = ?1 AND lock_by = ?5 diff --git a/queries/task/lock.sql b/queries/task/lock.sql new file mode 100644 index 0000000..d2807b6 --- /dev/null +++ b/queries/task/lock.sql @@ -0,0 +1,16 @@ +UPDATE + Jobs +SET + status = 'Running', + lock_at = strftime('%s', 'now'), + lock_by = ?2 +WHERE + id = ?1 + AND ( + status = 'Queued' + OR status = 'Pending' + OR ( + status = 'Failed' + AND attempts < max_attempts + ) + ) diff --git a/src/ack.rs b/src/ack.rs index 8cf89dc..afea114 100644 --- a/src/ack.rs +++ b/src/ack.rs @@ -1,14 +1,16 @@ use apalis_core::{ error::{AbortError, BoxDynError}, - task::{Parts, status::Status}, - worker::ext::ack::Acknowledge, + task::{self, Parts, status::Status}, + worker::{context::WorkerContext, ext::ack::Acknowledge}, }; use futures::{FutureExt, future::BoxFuture}; use serde::Serialize; use sqlx::SqlitePool; +use tower_layer::Layer; +use tower_service::Service; use ulid::Ulid; -use crate::context::SqliteContext; +use crate::{SqliteTask, context::SqliteContext}; #[derive(Clone)] pub struct SqliteAck { @@ -32,11 +34,12 @@ impl Acknowledge for SqliteAck { let worker_id = parts.ctx.lock_by().clone(); let response = serde_json::to_string(&res.as_ref().map_err(|e| e.to_string())); - let status = calculate_status(parts, res).to_string(); + let status = calculate_status(parts, res); + parts.status.store(status.clone()); let attempt = parts.attempt.current() as i32; let pool = self.pool.clone(); let res = response.map_err(|e| sqlx::Error::Decode(e.into())); - + let status = status.to_string(); async move { let task_id = task_id .ok_or(sqlx::Error::ColumnNotFound("TASK_ID_FOR_ACK".to_owned()))? @@ -77,3 +80,90 @@ pub fn calculate_status( }, } } + +pub async fn lock_task( + pool: &SqlitePool, + task_id: &Ulid, + worker_id: &str, +) -> Result<(), sqlx::Error> { + let task_id = task_id.to_string(); + let res = sqlx::query_file!("queries/task/lock.sql", task_id, worker_id) + .execute(pool) + .await?; + + if res.rows_affected() == 0 { + return Err(sqlx::Error::RowNotFound); + } + Ok(()) +} + +pub struct LockTaskLayer { + pool: SqlitePool, +} + +impl LockTaskLayer { + pub fn new(pool: SqlitePool) -> Self { + Self { pool } + } +} + +impl Layer for LockTaskLayer { + type Service = LockTaskService; + + fn layer(&self, inner: S) -> Self::Service { + LockTaskService { + inner, + pool: self.pool.clone(), + } + } +} + +pub struct LockTaskService { + inner: S, + pool: SqlitePool, +} + +impl Service> for LockTaskService +where + S: Service> + Send + 'static, + S::Future: Send + 'static, + S::Error: Into, + Args: Send + 'static, +{ + type Response = S::Response; + type Error = BoxDynError; + type Future = BoxFuture<'static, Result>; + + fn poll_ready( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.inner.poll_ready(cx).map_err(|e| e.into()) + } + + fn call(&mut self, req: SqliteTask) -> Self::Future { + let pool = self.pool.clone(); + let worker_id = req + .parts + .data + .get::() + .map(|w| w.name().to_owned()) + .unwrap(); + let parts = &req.parts; + let task_id = match &parts.task_id { + Some(id) => id.inner().clone(), + None => { + return async { + Err(sqlx::Error::ColumnNotFound("TASK_ID_FOR_LOCK".to_owned()).into()) + } + .boxed(); + } + }; + let fut = self.inner.call(req); + async move { + lock_task(&pool, &task_id, &worker_id).await?; + fut.await.map_err(|e| e.into()) + } + .boxed() + } +} diff --git a/src/config.rs b/src/config.rs index 9f45262..7afa72e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,6 +1,13 @@ use std::time::Duration; -use apalis_core::backend::poll_strategy::{IntervalStrategy, MultiStrategy, StrategyBuilder}; +use apalis_core::backend::{ + Backend, ConfigExt, + poll_strategy::{BackoffConfig, IntervalStrategy, MultiStrategy, StrategyBuilder}, + queue::Queue, +}; +use ulid::Ulid; + +use crate::{SqliteContext, SqliteStorage}; #[derive(Debug, Clone)] pub struct Config { @@ -8,7 +15,7 @@ pub struct Config { buffer_size: usize, poll_strategy: MultiStrategy, reenqueue_orphaned_after: Duration, - namespace: String, + queue: Queue, ack: bool, } @@ -18,19 +25,25 @@ impl Default for Config { keep_alive: Duration::from_secs(30), buffer_size: 10, poll_strategy: StrategyBuilder::new() - .apply(IntervalStrategy::new(Duration::from_millis(100))) + .apply( + IntervalStrategy::new(Duration::from_millis(100)) + .with_backoff(BackoffConfig::default()), + ) .build(), reenqueue_orphaned_after: Duration::from_secs(300), // 5 minutes - namespace: String::from("apalis::sqlite"), + queue: Queue::from("default"), ack: true, } } } impl Config { - /// Create a new config with a jobs namespace - pub fn new(namespace: &str) -> Self { - Config::default().set_namespace(namespace) + /// Create a new config with a jobs queue + pub fn new(queue: &str) -> Self { + Config { + queue: Queue::from(queue), + ..Default::default() + } } /// Interval between database poll queries @@ -57,14 +70,6 @@ impl Config { self } - /// Set the namespace to consume and push jobs to - /// - /// Defaults to "apalis::sql" - pub fn set_namespace(mut self, namespace: &str) -> Self { - self.namespace = namespace.to_string(); - self - } - /// Gets a reference to the keep_alive duration. pub fn keep_alive(&self) -> &Duration { &self.keep_alive @@ -90,14 +95,14 @@ impl Config { &mut self.poll_strategy } - /// Gets a reference to the namespace. - pub fn namespace(&self) -> &String { - &self.namespace + /// Gets a reference to the queue. + pub fn queue(&self) -> &Queue { + &self.queue } - /// Gets a mutable reference to the namespace. - pub fn namespace_mut(&mut self) -> &mut String { - &mut self.namespace + /// Gets a mutable reference to the queue. + pub fn queue_mut(&mut self) -> &mut Queue { + &mut self.queue } /// Gets the reenqueue_orphaned_after duration. @@ -128,3 +133,13 @@ impl Config { self } } + +impl ConfigExt for SqliteStorage +where + SqliteStorage: + Backend, +{ + fn get_queue(&self) -> Queue { + self.config.queue.clone() + } +} diff --git a/src/context.rs b/src/context.rs index 8688f5a..c1c22ff 100644 --- a/src/context.rs +++ b/src/context.rs @@ -14,7 +14,7 @@ pub struct SqliteContext { lock_by: Option, done_at: Option, priority: i32, - namespace: Option, + queue: Option, } impl Default for SqliteContext { @@ -33,7 +33,7 @@ impl SqliteContext { last_result: None, lock_by: None, priority: 0, - namespace: None, + queue: None, } } @@ -103,12 +103,12 @@ impl SqliteContext { self.priority } - pub fn namespace(&self) -> &Option { - &self.namespace + pub fn queue(&self) -> &Option { + &self.queue } - pub fn with_namespace(mut self, namespace: String) -> Self { - self.namespace = Some(namespace); + pub fn with_queue(mut self, queue: String) -> Self { + self.queue = Some(queue); self } } diff --git a/src/fetcher.rs b/src/fetcher.rs index c021fe0..7316541 100644 --- a/src/fetcher.rs +++ b/src/fetcher.rs @@ -29,7 +29,7 @@ pub(crate) async fn fetch_next>( where D::Error: std::error::Error + Send + Sync + 'static, { - let job_type = config.namespace(); + let job_type = config.queue().to_string(); let buffer_size = config.buffer_size() as i32; let worker = worker.name().to_string(); sqlx::query_file_as!( @@ -54,8 +54,14 @@ enum StreamState { Empty, } -#[pin_project] +/// Dispatcher for fetching tasks from a SQLite backend via [SqlitePollFetcher] pub struct SqliteFetcher> { + pub _marker: PhantomData<(Args, Compact, Decode)>, +} + +/// Polling-based fetcher for retrieving tasks from a SQLite backend +#[pin_project] +pub struct SqlitePollFetcher> { pool: SqlitePool, config: Config, wrk: WorkerContext, @@ -69,7 +75,7 @@ pub struct SqliteFetcher> { prev_count: Arc, } -impl Clone for SqliteFetcher { +impl Clone for SqlitePollFetcher { fn clone(&self) -> Self { Self { pool: self.pool.clone(), @@ -83,7 +89,7 @@ impl Clone for SqliteFetcher { } } -impl SqliteFetcher { +impl SqlitePollFetcher { pub fn new(pool: &Pool, config: &Config, wrk: &WorkerContext) -> Self where Decode: Codec + 'static, @@ -101,7 +107,7 @@ impl SqliteFetcher { } } -impl Stream for SqliteFetcher +impl Stream for SqlitePollFetcher where Decode::Error: std::error::Error + Send + Sync + 'static, Args: Send + 'static + Unpin, @@ -191,7 +197,7 @@ where } } -impl SqliteFetcher { +impl SqlitePollFetcher { pub fn take_pending(&mut self) -> VecDeque> { match &mut self.state { StreamState::Buffered(tasks) => std::mem::take(tasks), diff --git a/src/from_row.rs b/src/from_row.rs index e29c56c..b10a362 100644 --- a/src/from_row.rs +++ b/src/from_row.rs @@ -37,7 +37,7 @@ impl TaskRow { .with_max_attempts(self.max_attempts.unwrap_or(25) as i32) .with_last_result(self.last_error) .with_priority(self.priority.unwrap_or(0) as i32) - .with_namespace( + .with_queue( self.job_type .ok_or(sqlx::Error::ColumnNotFound("job_type".to_owned()))?, ) @@ -71,4 +71,45 @@ impl TaskRow { ); Ok(task.build()) } + pub fn try_into_task_compact(self) -> Result, sqlx::Error> { + let ctx = SqliteContext::default() + .with_done_at(self.done_at) + .with_lock_by(self.lock_by) + .with_max_attempts(self.max_attempts.unwrap_or(25) as i32) + .with_last_result(self.last_error) + .with_priority(self.priority.unwrap_or(0) as i32) + .with_queue( + self.job_type + .ok_or(sqlx::Error::ColumnNotFound("job_type".to_owned()))?, + ) + .with_lock_at(self.lock_at); + + let task = TaskBuilder::new(self.job) + .with_ctx(ctx) + .with_attempt(Attempt::new_with_value( + self.attempts + .ok_or(sqlx::Error::ColumnNotFound("attempts".to_owned()))? + as usize, + )) + .with_status( + self.status + .ok_or(sqlx::Error::ColumnNotFound("status".to_owned())) + .and_then(|s| { + Status::from_str(&s).map_err(|e| sqlx::Error::Decode(e.into())) + })?, + ) + .with_task_id( + self.id + .ok_or(sqlx::Error::ColumnNotFound("task_id".to_owned())) + .and_then(|s| { + TaskId::from_str(&s).map_err(|e| sqlx::Error::Decode(e.into())) + })?, + ) + .run_at_timestamp( + self.run_at + .ok_or(sqlx::Error::ColumnNotFound("run_at".to_owned()))? + as u64, + ); + Ok(task.build()) + } } diff --git a/src/hook.rs b/src/hook.rs index 4b9a96d..6495d7f 100644 --- a/src/hook.rs +++ b/src/hook.rs @@ -3,6 +3,7 @@ use futures::{Stream, StreamExt}; use std::ffi::{CStr, c_void}; use std::os::raw::{c_char, c_int}; use std::pin::Pin; +use std::sync::{Arc, Mutex}; use std::task::{Context, Poll}; #[derive(Debug)] @@ -50,6 +51,14 @@ pub(crate) extern "C" fn update_hook_callback( let db = CStr::from_ptr(db_name).to_string_lossy().to_string(); let table = CStr::from_ptr(table_name).to_string_lossy().to_string(); + log::debug!( + "DB Event - Operation: {}, DB: {}, Table: {}, RowID: {}", + op_str, + db, + table, + rowid + ); + // Recover sender from raw pointer let tx = &mut *(arg as *mut mpsc::UnboundedSender); @@ -63,12 +72,15 @@ pub(crate) extern "C" fn update_hook_callback( } } +#[derive(Debug, Clone)] pub struct HookListener { - rx: UnboundedReceiver, + rx: Arc>>, } impl HookListener { pub fn new(rx: UnboundedReceiver) -> Self { - Self { rx } + Self { + rx: Arc::new(Mutex::new(rx)), + } } } @@ -76,6 +88,9 @@ impl Stream for HookListener { type Item = DbEvent; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.rx.poll_next_unpin(cx) + self.rx + .try_lock() + .map(|mut rx| rx.poll_next_unpin(cx)) + .unwrap_or(Poll::Pending) } } diff --git a/src/lib.rs b/src/lib.rs index 2134ac8..a53239d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,22 +8,23 @@ use apalis_core::{ task::Task, worker::{context::WorkerContext, ext::ack::AcknowledgeLayer}, }; -use chrono::{DateTime, Utc}; use futures::{ - StreamExt, + FutureExt, StreamExt, TryStreamExt, channel::mpsc, future::ready, stream::{self, BoxStream, select}, }; use libsqlite3_sys::{sqlite3, sqlite3_update_hook}; -use sqlx::{Pool, Sqlite, SqlitePool}; +use sqlx::{Pool, Sqlite}; use std::ffi::c_void; +use tower_layer::Stack; use ulid::Ulid; use crate::{ - ack::SqliteAck, - fetcher::{SqliteFetcher, fetch_next}, + ack::{LockTaskLayer, SqliteAck}, + fetcher::{SqliteFetcher, SqlitePollFetcher, fetch_next}, hook::update_hook_callback, + reenqueue_orphaned::{reenqueue_orphaned, reenqueue_orphaned_stream}, sink::SqliteSink, }; @@ -33,6 +34,7 @@ mod context; mod fetcher; pub mod from_row; mod hook; +mod reenqueue_orphaned; mod shared; mod sink; mod traits; @@ -42,8 +44,9 @@ pub use config::Config; pub use context::SqliteContext; pub use hook::{DbEvent, HookListener}; pub use shared::{SharedPostgresError, SharedSqliteStorage}; +pub use sqlx::SqlitePool; -type DefaultFetcher = PhantomData>>; +type DefaultFetcher = SqliteFetcher>; const INSERT_OPERATION: &str = "INSERT"; const JOBS_TABLE: &str = "Jobs"; @@ -112,14 +115,16 @@ impl SqliteStorage<()> { impl SqliteStorage { /// Create a new SqliteStorage pub fn new(pool: &Pool) -> Self { - let config = Config::default(); + let config = Config::new(std::any::type_name::()); Self { pool: pool.clone(), job_type: PhantomData, sink: SqliteSink::new(pool, &config), config, codec: PhantomData, - fetcher: PhantomData, + fetcher: fetcher::SqliteFetcher { + _marker: PhantomData, + }, } } @@ -130,7 +135,9 @@ impl SqliteStorage { config: config.clone(), codec: PhantomData, sink: SqliteSink::new(pool, config), - fetcher: PhantomData, + fetcher: fetcher::SqliteFetcher { + _marker: PhantomData, + }, } } @@ -141,7 +148,9 @@ impl SqliteStorage { config: config.clone(), codec: PhantomData, sink: SqliteSink::new(pool, config), - fetcher: PhantomData, + fetcher: fetcher::SqliteFetcher { + _marker: PhantomData, + }, } } @@ -176,89 +185,139 @@ impl SqliteStorage { } } -impl Backend for SqliteStorage +impl SqliteStorage { + pub fn config(&self) -> &Config { + &self.config + } +} + +impl Backend for SqliteStorage where Args: Send + 'static + Unpin, - Decode: Codec + 'static, + Decode: Codec + 'static + Send, Decode::Error: std::error::Error + Send + Sync + 'static, { + type Args = Args; type IdType = Ulid; type Context = SqliteContext; type Codec = Decode; + type Compact = String; + type Error = sqlx::Error; - type Stream = SqliteFetcher; + type Stream = TaskStream, sqlx::Error>; type Beat = BoxStream<'static, Result<(), sqlx::Error>>; - type Layer = AcknowledgeLayer; + type Layer = Stack>; fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { - let worker_type = self.config.namespace().to_owned(); - let fut = heartbeat( + let pool = self.pool.clone(); + let config = self.config.clone(); + let worker = worker.clone(); + let keep_alive = stream::unfold((), move |_| { + let register = keep_alive(pool.clone(), config.clone(), worker.clone()); + let interval = apalis_core::timer::Delay::new(*config.keep_alive()); + interval.then(move |_| register.map(|res| Some((res, ())))) + }); + let reenqueue = reenqueue_orphaned_stream( self.pool.clone(), - worker_type, - worker.clone(), - Utc::now().timestamp(), - "SqliteStorage", - ); - stream::once(fut).boxed() + self.config.clone(), + *self.config.keep_alive(), + ) + .map_ok(|_| ()); + futures::stream::select(keep_alive, reenqueue).boxed() } fn middleware(&self) -> Self::Layer { - AcknowledgeLayer::new(SqliteAck::new(self.pool.clone())) + let lock = LockTaskLayer::new(self.pool.clone()); + let ack = AcknowledgeLayer::new(SqliteAck::new(self.pool.clone())); + Stack::new(lock, ack) } fn poll(self, worker: &WorkerContext) -> Self::Stream { - SqliteFetcher::new(&self.pool, &self.config, worker) + let worker_type = self.config.queue().to_owned(); + let fut = initial_heartbeat( + self.pool.clone(), + self.config().clone(), + worker.clone(), + "SqliteStorage", + ); + let register = stream::once(fut.map(|_| Ok(None))); + register + .chain(SqlitePollFetcher::::new( + &self.pool, + &self.config, + worker, + )) + .boxed() } } -impl Backend for SqliteStorage +impl Backend for SqliteStorage where Args: Send + 'static + Unpin, Decode: Codec + Send + 'static, Decode::Error: std::error::Error + Send + Sync + 'static, { + type Args = Args; type IdType = Ulid; type Context = SqliteContext; type Codec = Decode; + type Compact = String; + type Error = sqlx::Error; type Stream = TaskStream, sqlx::Error>; type Beat = BoxStream<'static, Result<(), sqlx::Error>>; - type Layer = AcknowledgeLayer; + type Layer = Stack>; fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { - let worker_type = self.config.namespace().to_owned(); - let fut = heartbeat( + let worker_type = self.config.queue().to_owned(); + let pool = self.pool.clone(); + let config = self.config.clone(); + let worker = worker.clone(); + let keep_alive = stream::unfold((), move |_| { + let register = keep_alive(pool.clone(), config.clone(), worker.clone()); + let interval = apalis_core::timer::Delay::new(*config.keep_alive()); + interval.then(move |_| register.map(|res| Some((res, ())))) + }); + let reenqueue = reenqueue_orphaned_stream( self.pool.clone(), - worker_type, - worker.clone(), - Utc::now().timestamp(), - "SqliteStorageWithHook", - ); - stream::once(fut).boxed() + self.config.clone(), + *self.config.keep_alive(), + ) + .map_ok(|_| ()); + futures::stream::select(keep_alive, reenqueue).boxed() } fn middleware(&self) -> Self::Layer { - AcknowledgeLayer::new(SqliteAck::new(self.pool.clone())) + let lock = LockTaskLayer::new(self.pool.clone()); + let ack = AcknowledgeLayer::new(SqliteAck::new(self.pool.clone())); + Stack::new(lock, ack) } fn poll(self, worker: &WorkerContext) -> Self::Stream { let pool = self.pool.clone(); let config = self.config.clone(); let worker = worker.clone(); - let eager_fetcher: SqliteFetcher = - SqliteFetcher::new(&self.pool, &self.config, &worker); + let register_worker = initial_heartbeat( + self.pool.clone(), + self.config.clone(), + worker.clone(), + "SqliteStorageWithHook", + ); + let register_worker = stream::once(register_worker.map(|_| Ok(None))); + let eager_fetcher: SqlitePollFetcher = + SqlitePollFetcher::new(&self.pool, &self.config, &worker); let lazy_fetcher = self .fetcher .filter(|a| ready(a.operation() == INSERT_OPERATION && a.table_name() == JOBS_TABLE)) @@ -273,35 +332,65 @@ where Err(e) => Err(e), }); - select(lazy_fetcher, eager_fetcher).boxed() + register_worker + .chain(select(lazy_fetcher, eager_fetcher)) + .boxed() } } -pub(crate) async fn heartbeat( +pub async fn initial_heartbeat( pool: SqlitePool, - worker_type: String, + config: Config, + worker: WorkerContext, + storage_type: &str, +) -> Result<(), sqlx::Error> { + reenqueue_orphaned(pool.clone(), config.clone()).await?; + register_worker(pool, config, worker, storage_type).await?; + Ok(()) +} + +pub(crate) async fn keep_alive( + pool: SqlitePool, + config: Config, worker: WorkerContext, - last_seen: i64, - backend_type: &str, ) -> Result<(), sqlx::Error> { - let last_seen = DateTime::from_timestamp(last_seen, 0).ok_or(sqlx::Error::Io( - std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid Timestamp"), - ))?; - let svc = worker.get_service().to_owned(); let worker = worker.name().to_owned(); + let queue = config.queue().to_string(); + let res = sqlx::query_file!("queries/backend/keep_alive.sql", worker, queue) + .execute(&pool) + .await?; + if res.rows_affected() == 1 { + return Err(sqlx::Error::Io(std::io::Error::new( + std::io::ErrorKind::NotFound, + "WORKER_DOES_NOT_EXIST", + ))); + } + Ok(()) +} + +pub(crate) async fn register_worker( + pool: SqlitePool, + config: Config, + worker: WorkerContext, + storage_type: &str, +) -> Result<(), sqlx::Error> { + let worker_id = worker.name().to_owned(); + let queue = config.queue().to_string(); + let layers = worker.get_service().to_owned(); + let keep_alive = config.keep_alive().as_secs() as i64; let res = sqlx::query_file!( - "queries/backend/heartbeat.sql", - worker, - worker_type, - backend_type, - svc, - last_seen + "queries/backend/register_worker.sql", + worker_id, + queue, + storage_type, + layers, + keep_alive, ) .execute(&pool) .await?; if res.rows_affected() == 0 { return Err(sqlx::Error::Io(std::io::Error::new( - std::io::ErrorKind::AddrInUse, + std::io::ErrorKind::AlreadyExists, "WORKER_ALREADY_EXISTS", ))); } @@ -368,7 +457,7 @@ mod tests { let lazy_strategy = StrategyBuilder::new() .apply(IntervalStrategy::new(Duration::from_secs(5))) .build(); - let config = Config::default() + let config = Config::new("rango-tango-queue") .with_poll_interval(lazy_strategy) .set_buffer_size(5); let backend = SqliteStorage::new_with_hook(&pool, &config).await; diff --git a/src/reenqueue_orphaned.rs b/src/reenqueue_orphaned.rs new file mode 100644 index 0000000..bedf22b --- /dev/null +++ b/src/reenqueue_orphaned.rs @@ -0,0 +1,53 @@ +use std::time::Duration; + +use futures::{FutureExt, Stream, stream}; +use serde_json::json; +use sqlx::SqlitePool; + +use crate::Config; + +pub fn reenqueue_orphaned( + pool: SqlitePool, + config: Config, +) -> impl Future> + Send { + let dead_for = config.reenqueue_orphaned_after().as_secs() as i64; + let queue = config.queue().to_string(); + async move { + match sqlx::query_file!("queries/backend/reenqueue_orphaned.sql", dead_for, queue,) + .execute(&pool) + .await + { + Ok(res) => { + if res.rows_affected() > 0 { + log::info!( + "Re-enqueued {} orphaned tasks that were being processed by dead workers", + res.rows_affected() + ); + } + return Ok(res.rows_affected()); + } + Err(e) => { + log::error!("Failed to re-enqueue orphaned tasks: {}", e); + return Err(e); + } + } + } +} + +pub fn reenqueue_orphaned_stream( + pool: SqlitePool, + config: Config, + interval: Duration, +) -> impl Stream> + Send { + let config = config.clone(); + stream::unfold((), move |_| { + let pool = pool.clone(); + let config = config.clone(); + let interval = apalis_core::timer::Delay::new(interval); + let fut = async move { + interval.await; + reenqueue_orphaned(pool, config).await + }; + fut.map(|res| Some((res, ()))) + }) +} diff --git a/src/shared.rs b/src/shared.rs index 2a216d6..cd787cd 100644 --- a/src/shared.rs +++ b/src/shared.rs @@ -11,11 +11,12 @@ use std::{ use crate::{ Config, INSERT_OPERATION, JOBS_TABLE, SqliteStorage, SqliteTask, - ack::SqliteAck, + ack::{LockTaskLayer, SqliteAck}, + config, context::SqliteContext, - fetcher::SqliteFetcher, - heartbeat, + fetcher::SqlitePollFetcher, hook::{DbEvent, update_hook_callback}, + initial_heartbeat, keep_alive, }; use crate::{from_row::TaskRow, sink::SqliteSink}; use apalis_core::{ @@ -36,6 +37,7 @@ use futures::{ }; use libsqlite3_sys::{sqlite3, sqlite3_update_hook}; use sqlx::SqlitePool; +use tower_layer::Stack; use ulid::Ulid; pub struct SharedSqliteStorage { @@ -108,7 +110,7 @@ impl SharedSqliteStorage { if let Some(tx) = instances.get_mut( task.parts .ctx - .namespace() + .queue() .as_ref() .expect("Namespace must be set"), ) { @@ -156,9 +158,9 @@ impl MakeShared for SharedSqliteStorage { .registry .try_lock() .ok_or(SharedPostgresError::RegistryLocked)?; - if r.insert(config.namespace().to_owned(), tx).is_some() { + if r.insert(config.queue().to_string(), tx).is_some() { return Err(SharedPostgresError::NamespaceExists( - config.namespace().to_owned(), + config.queue().to_string(), )); } let sink = SqliteSink::new(&self.pool, &config); @@ -194,12 +196,14 @@ impl Stream for SharedFetcher { } } -impl Backend for SqliteStorage +impl Backend for SqliteStorage where Args: Send + 'static + Unpin + Sync, Decode: Codec + 'static + Unpin + Send + Sync, Decode::Error: std::error::Error + Send + Sync + 'static, { + type Args = Args; + type IdType = Ulid; type Error = sqlx::Error; @@ -210,52 +214,45 @@ where type Codec = Decode; + type Compact = String; + type Context = SqliteContext; - type Layer = AcknowledgeLayer; + type Layer = Stack>; fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { - let worker_type = self.config.namespace().to_owned(); - let keep_alive = *self.config.keep_alive(); + let keep_alive_interval = *self.config.keep_alive(); let pool = self.pool.clone(); let worker = worker.clone(); + let config = self.config.clone(); stream::unfold((), move |()| async move { - apalis_core::timer::sleep(keep_alive).await; + apalis_core::timer::sleep(keep_alive_interval).await; Some(((), ())) }) - .then(move |_| { - heartbeat( - pool.clone(), - worker_type.clone(), - worker.clone(), - Utc::now().timestamp(), - "SharedSqliteStorage", - ) - }) + .then(move |_| keep_alive(pool.clone(), config.clone(), worker.clone())) .boxed() } fn middleware(&self) -> Self::Layer { - AcknowledgeLayer::new(SqliteAck::new(self.pool.clone())) + let lock = LockTaskLayer::new(self.pool.clone()); + let ack = AcknowledgeLayer::new(SqliteAck::new(self.pool.clone())); + Stack::new(lock, ack) } fn poll(self, worker: &WorkerContext) -> Self::Stream { - // TODO: Update lock_by field in the database to worker.name() - let _worker_id = worker.name().to_owned(); let pool = self.pool.clone(); let worker = worker.clone(); - let worker_type = self.config.namespace().to_owned(); + let worker_type = self.config.queue().to_owned(); // Initial registration heartbeat // This ensures that the worker is registered before fetching any tasks // This also ensures that the worker is marked as alive in case it crashes // before fetching any tasks // Subsequent heartbeats are handled in the heartbeat stream - let init = heartbeat( + let init = initial_heartbeat( pool.clone(), - worker_type.clone(), + self.config.clone(), worker.clone(), - Utc::now().timestamp(), "SharedSqliteStorage", ); let starter = stream::once(init) @@ -272,7 +269,7 @@ where }) .boxed(); - let eager_fetcher = StreamExt::boxed(SqliteFetcher::::new( + let eager_fetcher = StreamExt::boxed(SqlitePollFetcher::::new( &self.pool, &self.config, &worker, diff --git a/src/sink.rs b/src/sink.rs index 722dfdb..dc9950f 100644 --- a/src/sink.rs +++ b/src/sink.rs @@ -1,5 +1,6 @@ use std::{ pin::Pin, + sync::Arc, task::{Context, Poll}, }; @@ -7,13 +8,16 @@ use apalis_core::{ backend::codec::{Codec, json::JsonCodec}, error::BoxDynError, }; -use futures::Sink; +use futures::{ + FutureExt, Sink, TryFutureExt, + future::{BoxFuture, Shared}, +}; use sqlx::SqlitePool; use ulid::Ulid; use crate::{SqliteStorage, SqliteTask, config::Config}; -type FlushFuture = Pin> + Send>>; +type FlushFuture = BoxFuture<'static, Result<(), Arc>>; #[pin_project::pin_project] pub struct SqliteSink> { @@ -21,7 +25,7 @@ pub struct SqliteSink> { config: Config, buffer: Vec>, #[pin] - flush_future: Option, + flush_future: Option>, _marker: std::marker::PhantomData<(Args, Codec)>, } @@ -41,9 +45,8 @@ pub(crate) async fn push_tasks( pool: SqlitePool, cfg: Config, buffer: Vec>, -) -> Result<(), sqlx::Error> { +) -> Result<(), Arc> { let mut tx = pool.begin().await?; - let job_type = cfg.namespace(); for task in buffer { let id = task @@ -55,6 +58,11 @@ pub(crate) async fn push_tasks( let max_attempts = task.parts.ctx.max_attempts(); let priority = task.parts.ctx.priority(); let args = task.args; + // Use specified queue if specified, otherwise use default + let job_type = match task.parts.queue { + Some(ref queue) => queue.to_string(), + None => cfg.queue().to_string(), + }; sqlx::query_file!( "queries/task/sink.sql", args, @@ -120,19 +128,19 @@ where let config = this.config.clone(); let buffer = std::mem::take(&mut this.sink.buffer); let sink_fut = push_tasks(pool, config, buffer); - this.sink.flush_future = Some(Box::pin(sink_fut)); + this.sink.flush_future = Some((Box::pin(sink_fut) as FlushFuture).shared()); } // Poll the existing future if let Some(mut fut) = this.sink.flush_future.take() { - match fut.as_mut().poll(cx) { + match fut.poll_unpin(cx) { Poll::Ready(Ok(())) => { // Future completed successfully, don't put it back Poll::Ready(Ok(())) } Poll::Ready(Err(e)) => { // Future completed with error, don't put it back - Poll::Ready(Err(e)) + Poll::Ready(Err(Arc::into_inner(e).unwrap())) } Poll::Pending => { // Future is still pending, put it back and return Pending diff --git a/src/traits/fetch_by_id.rs b/src/traits/fetch_by_id.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/traits/list_queues.rs b/src/traits/list_queues.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/traits/list_tasks.rs b/src/traits/list_tasks.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/traits/list_workers.rs b/src/traits/list_workers.rs index 4011c07..c5bf19f 100644 --- a/src/traits/list_workers.rs +++ b/src/traits/list_workers.rs @@ -1,36 +1,44 @@ use apalis_core::{ - backend::{Filter, ListTasks, ListWorkers, RunningWorker, codec::json::JsonCodec}, - task::status::Status, + backend::{ + Backend, FetchById, Filter, ListAllTasks, ListQueues, ListTasks, ListWorkers, Metrics, + QueueInfo, RunningWorker, StatType, Statistic, + codec::{Codec, json::JsonCodec}, + }, + task::{Task, status::Status, task_id::TaskId}, }; use futures::TryFutureExt; use serde::{Serialize, de::DeserializeOwned}; +use sqlx::Decode; +use ulid::Ulid; -use crate::{SqliteStorage, SqliteTask, from_row::TaskRow}; +use crate::{SqliteContext, SqliteStorage, SqliteTask, from_row::TaskRow}; struct Worker { id: String, - #[allow(unused)] worker_type: String, storage_name: String, layers: Option, last_seen: i64, } -impl ListWorkers for SqliteStorage +impl ListWorkers for SqliteStorage where - Args: Send + 'static + Unpin + Serialize + DeserializeOwned, + SqliteStorage: + Backend, { - fn list_workers(&self) -> impl Future, Self::Error>> + Send { - let namespace = self.config.namespace().to_string(); + fn list_workers( + &self, + queue: &str, + ) -> impl Future, Self::Error>> + Send { + let queue = queue.to_string(); let pool = self.pool.clone(); let limit = 100; let offset = 0; - let pid = std::process::id() as i64; async move { let workers = sqlx::query_file_as!( Worker, "queries/backend/list_workers.sql", - namespace, + queue, limit, offset ) @@ -39,11 +47,42 @@ where w.into_iter() .map(|w| RunningWorker { id: w.id, - pid: pid as u32, - hostname: w.storage_name, + backend: w.storage_name, started_at: 0, last_heartbeat: w.last_seen as u64, - service: w.layers.unwrap_or_default(), + layers: w.layers.unwrap_or_default(), + queue: w.worker_type, + }) + .collect() + }) + .await?; + Ok(workers) + } + } + + fn list_all_workers( + &self, + ) -> impl Future, Self::Error>> + Send { + let pool = self.pool.clone(); + let limit = 100; + let offset = 0; + async move { + let workers = sqlx::query_file_as!( + Worker, + "queries/backend/list_all_workers.sql", + limit, + offset + ) + .fetch_all(&pool) + .map_ok(|w| { + w.into_iter() + .map(|w| RunningWorker { + id: w.id, + backend: w.storage_name, + started_at: 0, + last_heartbeat: w.last_seen as u64, + layers: w.layers.unwrap_or_default(), + queue: w.worker_type, }) .collect() }) @@ -53,14 +92,18 @@ where } } -impl ListTasks - for SqliteStorage +impl ListTasks for SqliteStorage +where + SqliteStorage: + Backend, + Args: Serialize + DeserializeOwned, { fn list_tasks( &self, + queue: &str, filter: &Filter, ) -> impl Future>, Self::Error>> + Send { - let namespace = self.config.namespace().to_string(); + let queue = queue.to_string(); let pool = self.pool.clone(); let limit = filter.limit() as i32; let offset = filter.offset() as i32; @@ -74,7 +117,7 @@ impl ListTasks ListTasks ListAllTasks for SqliteStorage +where + SqliteStorage: + Backend, +{ + fn list_all_tasks( + &self, + filter: &Filter, + ) -> impl Future< + Output = Result>, Self::Error>, + > + Send { + let status = filter + .status + .as_ref() + .map(|s| s.to_string()) + .unwrap_or(Status::Pending.to_string()); + let pool = self.pool.clone(); + let limit = filter.limit() as i32; + let offset = filter.offset() as i32; + async move { + let tasks = sqlx::query_file_as!( + TaskRow, + "queries/backend/list_all_jobs.sql", + status, + limit, + offset + ) + .fetch_all(&pool) + .await? + .into_iter() + .map(|r| r.try_into_task_compact()) + .collect::, _>>()?; + Ok(tasks) + } + } +} + +struct QueueInfoRow { + name: String, + stats: String, // JSON string + workers: String, // JSON string + activity: String, // JSON string +} + +impl From for QueueInfo { + fn from(row: QueueInfoRow) -> Self { + QueueInfo { + name: row.name, + stats: serde_json::from_str(&row.stats).unwrap(), + workers: serde_json::from_str(&row.workers).unwrap(), + activity: serde_json::from_str(&row.activity).unwrap(), + } + } +} + +impl ListQueues for SqliteStorage +where + SqliteStorage: + Backend, +{ + fn list_queues(&self) -> impl Future, Self::Error>> + Send { + let pool = self.pool.clone(); + + async move { + let queues = sqlx::query_file_as!(QueueInfoRow, "queries/backend/list_queues.sql") + .fetch_all(&pool) + .await? + .into_iter() + .map(QueueInfo::from) + .collect(); + Ok(queues) + } + } +} + +fn stat_type_from_string(s: &str) -> StatType { + match s { + "Number" => StatType::Number, + "Decimal" => StatType::Decimal, + "Percentage" => StatType::Percentage, + "Timestamp" => StatType::Timestamp, + _ => StatType::Number, // default fallback + } +} +struct StatisticRow { + /// The priority of the statistic (lower number means higher priority) + pub priority: i64, + /// The statistics type + pub r#type: String, + /// Overall statistics of the backend + pub statistic: String, + /// The value of the statistic + pub value: Option, +} + +impl Metrics for SqliteStorage +where + SqliteStorage: + Backend, +{ + fn global(&self) -> impl Future, Self::Error>> + Send { + let pool = self.pool.clone(); + async move { + let rec = sqlx::query_file_as!(StatisticRow, "queries/backend/overview.sql") + .fetch_all(&pool) + .await? + .into_iter() + .map(|r| Statistic { + priority: Some(r.priority as u64), + stat_type: stat_type_from_string(&r.r#type), + title: r.statistic, + value: r.value.unwrap_or_default().to_string(), + }) + .collect(); + Ok(rec) + } + } + fn fetch_by_queue( + &self, + queue_id: &str, + ) -> impl Future, Self::Error>> + Send { + let pool = self.pool.clone(); + let queue_id = queue_id.to_string(); + async move { + let rec = sqlx::query_file_as!( + StatisticRow, + "queries/backend/overview_by_queue.sql", + queue_id + ) + .fetch_all(&pool) + .await? + .into_iter() + .map(|r| Statistic { + priority: Some(r.priority as u64), + stat_type: stat_type_from_string(&r.r#type), + title: r.statistic, + value: r.value.unwrap_or_default().to_string(), + }) + .collect(); + Ok(rec) + } + } +} + +impl FetchById for SqliteStorage +where + SqliteStorage: + Backend, + Args: Serialize + DeserializeOwned, +{ + fn fetch_by_id( + &mut self, + id: &TaskId, + ) -> impl Future>, Self::Error>> + Send { + let pool = self.pool.clone(); + let id = id.to_string(); + async move { + let task = sqlx::query_file_as!(TaskRow, "queries/task/find_by_id.sql", id) + .fetch_optional(&pool) + .await? + .map(|r| r.try_into_task::, _>()) + .transpose()?; + Ok(task) + } + } +} diff --git a/src/traits/metrics.rs b/src/traits/metrics.rs new file mode 100644 index 0000000..e69de29 From a7620e68c07e75ebe3bb3faebaaaeb503e8b4255 Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Fri, 17 Oct 2025 12:52:59 +0300 Subject: [PATCH 116/121] checkpoint: refactor --- README.md | 117 +++++++++- src/ack.rs | 2 +- src/{hook.rs => callback.rs} | 21 +- src/fetcher.rs | 2 +- src/lib.rs | 284 ++++++++++++++-------- src/queries/fetch_by_id.rs | 31 +++ src/queries/keep_alive.rs | 50 ++++ src/queries/list_queues.rs | 42 ++++ src/queries/list_tasks.rs | 84 +++++++ src/queries/list_workers.rs | 84 +++++++ src/queries/metrics.rs | 64 +++++ src/queries/mod.rs | 20 ++ src/{ => queries}/reenqueue_orphaned.rs | 1 - src/queries/register_worker.rs | 33 +++ src/shared.rs | 5 +- src/sink.rs | 4 +- src/traits/fetch_by_id.rs | 0 src/traits/list_queues.rs | 0 src/traits/list_tasks.rs | 0 src/traits/list_workers.rs | 299 ------------------------ src/traits/metrics.rs | 0 src/traits/mod.rs | 1 - 22 files changed, 719 insertions(+), 425 deletions(-) rename src/{hook.rs => callback.rs} (84%) create mode 100644 src/queries/fetch_by_id.rs create mode 100644 src/queries/keep_alive.rs create mode 100644 src/queries/list_queues.rs create mode 100644 src/queries/list_tasks.rs create mode 100644 src/queries/list_workers.rs create mode 100644 src/queries/metrics.rs create mode 100644 src/queries/mod.rs rename src/{ => queries}/reenqueue_orphaned.rs (98%) create mode 100644 src/queries/register_worker.rs delete mode 100644 src/traits/fetch_by_id.rs delete mode 100644 src/traits/list_queues.rs delete mode 100644 src/traits/list_tasks.rs delete mode 100644 src/traits/list_workers.rs delete mode 100644 src/traits/metrics.rs delete mode 100644 src/traits/mod.rs diff --git a/README.md b/README.md index 11e3580..631a220 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,117 @@ # apalis-sqlite -Background task processing for rust using apalis and sqlite + +Background task processing for Rust using Apalis and SQLite. + +## Features + +- **Reliable job queue** using SQLite as the backend. +- **Multiple storage types**: standard polling and event-driven (hooked) storage. +- **Custom codecs** for serializing/deserializing job arguments. +- **Heartbeat and orphaned job re-enqueueing** for robust job processing. +- **Integration with Apalis workers and middleware.** + +## Storage Types + +- [`SqliteStorage`]: Standard polling-based storage. +- [`SqliteStorageWithHook`]: Event-driven storage using SQLite update hooks for low-latency job fetching. +- [`SharedSqliteStorage`]: Shared storage for multiple job types. + +The naming is designed to clearly indicate the storage mechanism and its capabilities, but under the hood the result is the `SqliteStorage` struct with different configurations. + +## Examples + +### Basic Worker Example + +```rust,no_run +#[tokio::main] +async fn main() { + let pool = SqlitePool::connect(":memory:").await.unwrap(); + SqliteStorage::setup(&pool).await.unwrap(); + let mut backend = SqliteStorage::new(&pool); // With default config + + let mut start = 0; + let mut items = stream::repeat_with(move || { + start += 1; + let task = Task::builder(start) + .run_after(Duration::from_secs(1)) + .with_ctx(SqliteContext::new().with_priority(1)) + .build(); + Ok(task) + }) + .take(10); + backend.send_all(&mut items).await.unwrap(); + + async fn send_reminder(item: usize, wrk: WorkerContext) -> Result<(), BoxDynError> { + Ok(()) + } + + let worker = apalis_core::worker::builder::WorkerBuilder::new("worker-1") + .backend(backend) + .build(send_reminder); + worker.run().await.unwrap(); +} +``` + +### Hooked Worker Example (Event-driven) + +```rust,no_run + +#[tokio::main] +async fn main() { + let pool = SqlitePool::connect(":memory:").await.unwrap(); + SqliteStorage::setup(&pool).await.unwrap(); + + let lazy_strategy = StrategyBuilder::new() + .apply(IntervalStrategy::new(Duration::from_secs(5))) + .build(); + let config = Config::new("queue") + .with_poll_interval(lazy_strategy) + .set_buffer_size(5); + let backend = SqliteStorage::new_with_callback(&pool, &config).await; + + tokio::spawn({ + let pool = pool.clone(); + let config = config.clone(); + async move { + tokio::time::sleep(Duration::from_secs(2)).await; + let mut start = 0; + let items = stream::repeat_with(move || { + start += 1; + Task::builder(start) + .run_after(Duration::from_secs(1)) + .with_ctx(SqliteContext::new().with_priority(start)) + .build() + }) + .take(20) + .collect::>() + .await; + apalis_sqlite::sink::push_tasks(pool, config, items).await.unwrap(); + } + }); + + async fn send_reminder(item: usize, wrk: WorkerContext) -> Result<(), BoxDynError> { + Ok(()) + } + + let worker = apalis_core::worker::builder::WorkerBuilder::new("worker-2") + .backend(backend) + .build(send_reminder); + worker.run().await.unwrap(); +} +``` + +## Migrations + +If the `migrate` feature is enabled, you can run built-in migrations with: + +```rust,no_run +use sqlx::SqlitePool; +#[tokio::main] async fn main() { + let pool = SqlitePool::connect(":memory:").await.unwrap(); + apalis_sqlite::SqliteStorage::setup(&pool).await.unwrap(); +} +``` + +## License + +Licensed under either of Apache License, Version 2.0 or MIT license at your option. diff --git a/src/ack.rs b/src/ack.rs index afea114..875e0c1 100644 --- a/src/ack.rs +++ b/src/ack.rs @@ -1,6 +1,6 @@ use apalis_core::{ error::{AbortError, BoxDynError}, - task::{self, Parts, status::Status}, + task::{Parts, status::Status}, worker::{context::WorkerContext, ext::ack::Acknowledge}, }; use futures::{FutureExt, future::BoxFuture}; diff --git a/src/hook.rs b/src/callback.rs similarity index 84% rename from src/hook.rs rename to src/callback.rs index 6495d7f..eeecc63 100644 --- a/src/hook.rs +++ b/src/callback.rs @@ -3,7 +3,6 @@ use futures::{Stream, StreamExt}; use std::ffi::{CStr, c_void}; use std::os::raw::{c_char, c_int}; use std::pin::Pin; -use std::sync::{Arc, Mutex}; use std::task::{Context, Poll}; #[derive(Debug)] @@ -73,24 +72,22 @@ pub(crate) extern "C" fn update_hook_callback( } #[derive(Debug, Clone)] -pub struct HookListener { - rx: Arc>>, +pub struct HookCallbackListener; + +#[derive(Debug)] +pub struct CallbackListener { + rx: UnboundedReceiver, } -impl HookListener { +impl CallbackListener { pub fn new(rx: UnboundedReceiver) -> Self { - Self { - rx: Arc::new(Mutex::new(rx)), - } + Self { rx } } } -impl Stream for HookListener { +impl Stream for CallbackListener { type Item = DbEvent; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.rx - .try_lock() - .map(|mut rx| rx.poll_next_unpin(cx)) - .unwrap_or(Poll::Pending) + self.rx.poll_next_unpin(cx) } } diff --git a/src/fetcher.rs b/src/fetcher.rs index 7316541..d7df328 100644 --- a/src/fetcher.rs +++ b/src/fetcher.rs @@ -21,7 +21,7 @@ use ulid::Ulid; use crate::{SqliteTask, config::Config, context::SqliteContext, from_row::TaskRow}; -pub(crate) async fn fetch_next>( +pub async fn fetch_next>( pool: SqlitePool, config: Config, worker: WorkerContext, diff --git a/src/lib.rs b/src/lib.rs index a53239d..617c05d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,141 @@ +//! # apalis-sqlite +//! +//! Background task processing for rust using apalis and sqlite. +//! +//! ## Features +//! +//! - **Reliable job queue** using SQLite as the backend. +//! - **Multiple storage types**: standard polling and event-driven (hooked) storage. +//! - **Custom codecs** for serializing/deserializing job arguments. +//! - **Heartbeat and orphaned job re-enqueueing** for robust job processing. +//! - **Integration with apalis workers and middleware.** +//! +//! ## Storage Types +//! +//! - [`SqliteStorage`]: Standard polling-based storage. +//! - [`SqliteStorageWithHook`]: Event-driven storage using SQLite update hooks for low-latency job fetching. +//! - [`SharedSqliteStorage`]: Shared storage for multiple job types. +//! +//! The naming is designed to clearly indicate the storage mechanism and its capabilities, but under the hood its the result is the `SqliteStorage` struct with different configurations. +//! +//! ## Examples +//! +//! ### Basic Worker Example +//! +//! ```rust,no_run +//! use apalis_sqlite::{SqliteStorage, SqliteContext}; +//! use apalis_core::task::Task; +//! use apalis_core::worker::context::WorkerContext; +//! use sqlx::SqlitePool; +//! use futures::stream; +//! use std::time::Duration; +//! +//! #[tokio::main] +//! async fn main() { +//! let pool = SqlitePool::connect(":memory:").await.unwrap(); +//! SqliteStorage::setup(&pool).await.unwrap(); +//! let mut backend = SqliteStorage::new(&pool); +//! +//! let mut start = 0; +//! let mut items = stream::repeat_with(move || { +//! start += 1; +//! let task = Task::builder(start) +//! .run_after(Duration::from_secs(1)) +//! .with_ctx(SqliteContext::new().with_priority(1)) +//! .build(); +//! Ok(task) +//! }) +//! .take(10); +//! backend.send_all(&mut items).await.unwrap(); +//! +//! async fn send_reminder(item: usize, wrk: WorkerContext) -> Result<(), BoxDynError> { +//! # if item == 10 { +//! # wrk.stop().unwrap(); +//! # } +//! Ok(()) +//! } +//! +//! let worker = apalis_core::worker::builder::WorkerBuilder::new("worker-1") +//! .backend(backend) +//! .build(send_reminder); +//! worker.run().await.unwrap(); +//! } +//! ``` +//! +//! ### Hooked Worker Example (Event-driven) +//! +//! ```rust,no_run +//! use apalis_sqlite::{SqliteStorage, SqliteContext, Config}; +//! use apalis_core::task::Task; +//! use apalis_core::worker::context::WorkerContext; +//! use apalis_core::backend::poll_strategy::{IntervalStrategy, StrategyBuilder}; +//! use sqlx::SqlitePool; +//! use futures::stream; +//! use std::time::Duration; +//! +//! #[tokio::main] +//! async fn main() { +//! let pool = SqlitePool::connect(":memory:").await.unwrap(); +//! SqliteStorage::setup(&pool).await.unwrap(); +//! +//! let lazy_strategy = StrategyBuilder::new() +//! .apply(IntervalStrategy::new(Duration::from_secs(5))) +//! .build(); +//! let config = Config::new("queue") +//! .with_poll_interval(lazy_strategy) +//! .set_buffer_size(5); +//! let backend = SqliteStorage::new_with_callback(&pool, &config); +//! +//! tokio::spawn({ +//! let pool = pool.clone(); +//! let config = config.clone(); +//! async move { +//! tokio::time::sleep(Duration::from_secs(2)).await; +//! let mut start = 0; +//! let items = stream::repeat_with(move || { +//! start += 1; +//! Task::builder(start) +//! .run_after(Duration::from_secs(1)) +//! .with_ctx(SqliteContext::new().with_priority(start)) +//! .build() +//! }) +//! .take(20) +//! .collect::>() +//! .await; +//! apalis_sqlite::sink::push_tasks(pool, config, items).await.unwrap(); +//! } +//! }); +//! +//! async fn send_reminder(item: usize, wrk: WorkerContext) -> Result<(), BoxDynError> { +//! if item == 1 { +//! apalis_core::timer::sleep(Duration::from_secs(1)).await; +//! wrk.stop().unwrap(); +//! } +//! Ok(()) +//! } +//! +//! let worker = apalis_core::worker::builder::WorkerBuilder::new("worker-2") +//! .backend(backend) +//! .build(send_reminder); +//! worker.run().await.unwrap(); +//! } +//! ``` +//! +//! ## Migrations +//! +//! If the `migrate` feature is enabled, you can run built-in migrations with: +//! +//! ```rust,no_run +//! # use sqlx::SqlitePool; +//! # #[tokio::main] async fn main() { +//! let pool = SqlitePool::connect(":memory:").await.unwrap(); +//! apalis_sqlite::SqliteStorage::setup(&pool).await.unwrap(); +//! # } +//! ``` +//! +//! ## License +//! +//! Licensed under either of Apache License, Version 2.0 or MIT license at your option. use std::{fmt, marker::PhantomData}; use apalis_core::{ @@ -9,7 +147,7 @@ use apalis_core::{ worker::{context::WorkerContext, ext::ack::AcknowledgeLayer}, }; use futures::{ - FutureExt, StreamExt, TryStreamExt, + FutureExt, StreamExt, TryFutureExt, TryStreamExt, channel::mpsc, future::ready, stream::{self, BoxStream, select}, @@ -22,27 +160,29 @@ use ulid::Ulid; use crate::{ ack::{LockTaskLayer, SqliteAck}, + callback::{HookCallbackListener, update_hook_callback}, fetcher::{SqliteFetcher, SqlitePollFetcher, fetch_next}, - hook::update_hook_callback, - reenqueue_orphaned::{reenqueue_orphaned, reenqueue_orphaned_stream}, + queries::{ + keep_alive::{initial_heartbeat, keep_alive, keep_alive_stream}, + reenqueue_orphaned::reenqueue_orphaned_stream, + }, sink::SqliteSink, }; mod ack; +mod callback; mod config; mod context; -mod fetcher; +pub mod fetcher; pub mod from_row; -mod hook; -mod reenqueue_orphaned; +pub mod queries; mod shared; mod sink; -mod traits; pub type SqliteTask = Task; +pub use callback::{CallbackListener, DbEvent}; pub use config::Config; pub use context::SqliteContext; -pub use hook::{DbEvent, HookListener}; pub use shared::{SharedPostgresError, SharedSqliteStorage}; pub use sqlx::SqlitePool; @@ -154,33 +294,17 @@ impl SqliteStorage { } } - pub async fn new_with_hook( + pub fn new_with_callback( pool: &Pool, config: &Config, - ) -> SqliteStorage, HookListener> { - let pool = pool.clone(); - let (tx, rx) = mpsc::unbounded::(); - - let listener = HookListener::new(rx); - - let mut conn = pool.acquire().await.unwrap(); - // Get raw sqlite3* handle - let handle: *mut sqlite3 = conn.lock_handle().await.unwrap().as_raw_handle().as_ptr(); - - // Put sender in a Box so it has a stable memory address - let tx_box = Box::new(tx); - let tx_ptr = Box::into_raw(tx_box) as *mut c_void; - - unsafe { - sqlite3_update_hook(handle, Some(update_hook_callback), tx_ptr); - } + ) -> SqliteStorage, HookCallbackListener> { SqliteStorage { pool: pool.clone(), job_type: PhantomData, config: config.clone(), codec: PhantomData, sink: SqliteSink::new(&pool, config), - fetcher: listener, + fetcher: HookCallbackListener, } } } @@ -218,11 +342,7 @@ where let pool = self.pool.clone(); let config = self.config.clone(); let worker = worker.clone(); - let keep_alive = stream::unfold((), move |_| { - let register = keep_alive(pool.clone(), config.clone(), worker.clone()); - let interval = apalis_core::timer::Delay::new(*config.keep_alive()); - interval.then(move |_| register.map(|res| Some((res, ())))) - }); + let keep_alive = keep_alive_stream(pool, config, worker); let reenqueue = reenqueue_orphaned_stream( self.pool.clone(), self.config.clone(), @@ -239,7 +359,6 @@ where } fn poll(self, worker: &WorkerContext) -> Self::Stream { - let worker_type = self.config.queue().to_owned(); let fut = initial_heartbeat( self.pool.clone(), self.config().clone(), @@ -257,7 +376,7 @@ where } } -impl Backend for SqliteStorage +impl Backend for SqliteStorage where Args: Send + 'static + Unpin, Decode: Codec + Send + 'static, @@ -281,15 +400,10 @@ where type Layer = Stack>; fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { - let worker_type = self.config.queue().to_owned(); let pool = self.pool.clone(); let config = self.config.clone(); let worker = worker.clone(); - let keep_alive = stream::unfold((), move |_| { - let register = keep_alive(pool.clone(), config.clone(), worker.clone()); - let interval = apalis_core::timer::Delay::new(*config.keep_alive()); - interval.then(move |_| register.map(|res| Some((res, ())))) - }); + let keep_alive = keep_alive_stream(pool, config, worker); let reenqueue = reenqueue_orphaned_stream( self.pool.clone(), self.config.clone(), @@ -306,6 +420,10 @@ where } fn poll(self, worker: &WorkerContext) -> Self::Stream { + let (tx, rx) = mpsc::unbounded::(); + + let listener = CallbackListener::new(rx); + let pool = self.pool.clone(); let config = self.config.clone(); let worker = worker.clone(); @@ -315,11 +433,30 @@ where worker.clone(), "SqliteStorageWithHook", ); - let register_worker = stream::once(register_worker.map(|_| Ok(None))); + let p = pool.clone(); + let register_worker = stream::once( + register_worker + .and_then(|_| async move { + // This is still a little tbd, but the idea is to test the update hook + let mut conn = p.acquire().await?; + // Get raw sqlite3* handle + let handle: *mut sqlite3 = + conn.lock_handle().await.unwrap().as_raw_handle().as_ptr(); + + // Put sender in a Box so it has a stable memory address + let tx_box = Box::new(tx); + let tx_ptr = Box::into_raw(tx_box) as *mut c_void; + + unsafe { + sqlite3_update_hook(handle, Some(update_hook_callback), tx_ptr); + } + Ok(()) + }) + .map(|_| Ok(None)), + ); let eager_fetcher: SqlitePollFetcher = SqlitePollFetcher::new(&self.pool, &self.config, &worker); - let lazy_fetcher = self - .fetcher + let lazy_fetcher = listener .filter(|a| ready(a.operation() == INSERT_OPERATION && a.table_name() == JOBS_TABLE)) .ready_chunks(self.config.buffer_size()) .then(move |_| fetch_next::(pool.clone(), config.clone(), worker.clone())) @@ -338,65 +475,6 @@ where } } -pub async fn initial_heartbeat( - pool: SqlitePool, - config: Config, - worker: WorkerContext, - storage_type: &str, -) -> Result<(), sqlx::Error> { - reenqueue_orphaned(pool.clone(), config.clone()).await?; - register_worker(pool, config, worker, storage_type).await?; - Ok(()) -} - -pub(crate) async fn keep_alive( - pool: SqlitePool, - config: Config, - worker: WorkerContext, -) -> Result<(), sqlx::Error> { - let worker = worker.name().to_owned(); - let queue = config.queue().to_string(); - let res = sqlx::query_file!("queries/backend/keep_alive.sql", worker, queue) - .execute(&pool) - .await?; - if res.rows_affected() == 1 { - return Err(sqlx::Error::Io(std::io::Error::new( - std::io::ErrorKind::NotFound, - "WORKER_DOES_NOT_EXIST", - ))); - } - Ok(()) -} - -pub(crate) async fn register_worker( - pool: SqlitePool, - config: Config, - worker: WorkerContext, - storage_type: &str, -) -> Result<(), sqlx::Error> { - let worker_id = worker.name().to_owned(); - let queue = config.queue().to_string(); - let layers = worker.get_service().to_owned(); - let keep_alive = config.keep_alive().as_secs() as i64; - let res = sqlx::query_file!( - "queries/backend/register_worker.sql", - worker_id, - queue, - storage_type, - layers, - keep_alive, - ) - .execute(&pool) - .await?; - if res.rows_affected() == 0 { - return Err(sqlx::Error::Io(std::io::Error::new( - std::io::ErrorKind::AlreadyExists, - "WORKER_ALREADY_EXISTS", - ))); - } - Ok(()) -} - #[cfg(test)] mod tests { use std::time::Duration; @@ -460,7 +538,7 @@ mod tests { let config = Config::new("rango-tango-queue") .with_poll_interval(lazy_strategy) .set_buffer_size(5); - let backend = SqliteStorage::new_with_hook(&pool, &config).await; + let backend = SqliteStorage::new_with_callback(&pool, &config); tokio::spawn(async move { tokio::time::sleep(Duration::from_secs(2)).await; diff --git a/src/queries/fetch_by_id.rs b/src/queries/fetch_by_id.rs new file mode 100644 index 0000000..ad488ba --- /dev/null +++ b/src/queries/fetch_by_id.rs @@ -0,0 +1,31 @@ +use apalis_core::{ + backend::{Backend, FetchById, codec::json::JsonCodec}, + task::task_id::TaskId, +}; +use serde::{Serialize, de::DeserializeOwned}; +use ulid::Ulid; + +use crate::{SqliteContext, SqliteStorage, SqliteTask, from_row::TaskRow}; + +impl FetchById for SqliteStorage +where + SqliteStorage: + Backend, + Args: Serialize + DeserializeOwned, +{ + fn fetch_by_id( + &mut self, + id: &TaskId, + ) -> impl Future>, Self::Error>> + Send { + let pool = self.pool.clone(); + let id = id.to_string(); + async move { + let task = sqlx::query_file_as!(TaskRow, "queries/task/find_by_id.sql", id) + .fetch_optional(&pool) + .await? + .map(|r| r.try_into_task::, _>()) + .transpose()?; + Ok(task) + } + } +} diff --git a/src/queries/keep_alive.rs b/src/queries/keep_alive.rs new file mode 100644 index 0000000..39aa027 --- /dev/null +++ b/src/queries/keep_alive.rs @@ -0,0 +1,50 @@ +use apalis_core::worker::context::WorkerContext; +use futures::{FutureExt, Stream, stream}; +use sqlx::SqlitePool; + +use crate::{ + Config, + queries::{reenqueue_orphaned::reenqueue_orphaned, register_worker::register_worker}, +}; + +pub async fn keep_alive( + pool: SqlitePool, + config: Config, + worker: WorkerContext, +) -> Result<(), sqlx::Error> { + let worker = worker.name().to_owned(); + let queue = config.queue().to_string(); + let res = sqlx::query_file!("queries/backend/keep_alive.sql", worker, queue) + .execute(&pool) + .await?; + if res.rows_affected() == 1 { + return Err(sqlx::Error::Io(std::io::Error::new( + std::io::ErrorKind::NotFound, + "WORKER_DOES_NOT_EXIST", + ))); + } + Ok(()) +} + +pub async fn initial_heartbeat( + pool: SqlitePool, + config: Config, + worker: WorkerContext, + storage_type: &str, +) -> Result<(), sqlx::Error> { + reenqueue_orphaned(pool.clone(), config.clone()).await?; + register_worker(pool, config, worker, storage_type).await?; + Ok(()) +} + +pub fn keep_alive_stream( + pool: SqlitePool, + config: Config, + worker: WorkerContext, +) -> impl Stream> + Send { + stream::unfold((), move |_| { + let register = keep_alive(pool.clone(), config.clone(), worker.clone()); + let interval = apalis_core::timer::Delay::new(*config.keep_alive()); + interval.then(move |_| register.map(|res| Some((res, ())))) + }) +} diff --git a/src/queries/list_queues.rs b/src/queries/list_queues.rs new file mode 100644 index 0000000..a21dbab --- /dev/null +++ b/src/queries/list_queues.rs @@ -0,0 +1,42 @@ +use apalis_core::backend::{Backend, ListQueues, QueueInfo}; +use ulid::Ulid; + +use crate::{SqliteContext, SqliteStorage}; + +struct QueueInfoRow { + name: String, + stats: String, // JSON string + workers: String, // JSON string + activity: String, // JSON string +} + +impl From for QueueInfo { + fn from(row: QueueInfoRow) -> Self { + QueueInfo { + name: row.name, + stats: serde_json::from_str(&row.stats).unwrap(), + workers: serde_json::from_str(&row.workers).unwrap(), + activity: serde_json::from_str(&row.activity).unwrap(), + } + } +} + +impl ListQueues for SqliteStorage +where + SqliteStorage: + Backend, +{ + fn list_queues(&self) -> impl Future, Self::Error>> + Send { + let pool = self.pool.clone(); + + async move { + let queues = sqlx::query_file_as!(QueueInfoRow, "queries/backend/list_queues.sql") + .fetch_all(&pool) + .await? + .into_iter() + .map(QueueInfo::from) + .collect(); + Ok(queues) + } + } +} diff --git a/src/queries/list_tasks.rs b/src/queries/list_tasks.rs new file mode 100644 index 0000000..ffa3951 --- /dev/null +++ b/src/queries/list_tasks.rs @@ -0,0 +1,84 @@ +use apalis_core::{ + backend::{Backend, Filter, ListAllTasks, ListTasks, codec::json::JsonCodec}, + task::{Task, status::Status}, +}; +use serde::{Serialize, de::DeserializeOwned}; +use ulid::Ulid; + +use crate::{SqliteContext, SqliteStorage, SqliteTask, from_row::TaskRow}; + +impl ListTasks for SqliteStorage +where + SqliteStorage: + Backend, + Args: Serialize + DeserializeOwned, +{ + fn list_tasks( + &self, + queue: &str, + filter: &Filter, + ) -> impl Future>, Self::Error>> + Send { + let queue = queue.to_string(); + let pool = self.pool.clone(); + let limit = filter.limit() as i32; + let offset = filter.offset() as i32; + let status = filter + .status + .as_ref() + .unwrap_or(&Status::Pending) + .to_string(); + async move { + let tasks = sqlx::query_file_as!( + TaskRow, + "queries/backend/list_jobs.sql", + status, + queue, + limit, + offset + ) + .fetch_all(&pool) + .await? + .into_iter() + .map(|r| r.try_into_task::, _>()) + .collect::, _>>()?; + Ok(tasks) + } + } +} + +impl ListAllTasks for SqliteStorage +where + SqliteStorage: + Backend, +{ + fn list_all_tasks( + &self, + filter: &Filter, + ) -> impl Future< + Output = Result>, Self::Error>, + > + Send { + let status = filter + .status + .as_ref() + .map(|s| s.to_string()) + .unwrap_or(Status::Pending.to_string()); + let pool = self.pool.clone(); + let limit = filter.limit() as i32; + let offset = filter.offset() as i32; + async move { + let tasks = sqlx::query_file_as!( + TaskRow, + "queries/backend/list_all_jobs.sql", + status, + limit, + offset + ) + .fetch_all(&pool) + .await? + .into_iter() + .map(|r| r.try_into_task_compact()) + .collect::, _>>()?; + Ok(tasks) + } + } +} diff --git a/src/queries/list_workers.rs b/src/queries/list_workers.rs new file mode 100644 index 0000000..a5b6d1e --- /dev/null +++ b/src/queries/list_workers.rs @@ -0,0 +1,84 @@ +use apalis_core::backend::{Backend, ListWorkers, RunningWorker}; +use futures::TryFutureExt; +use ulid::Ulid; + +use crate::{SqliteContext, SqliteStorage}; + +struct Worker { + id: String, + worker_type: String, + storage_name: String, + layers: Option, + last_seen: i64, +} + +impl ListWorkers for SqliteStorage +where + SqliteStorage: + Backend, +{ + fn list_workers( + &self, + queue: &str, + ) -> impl Future, Self::Error>> + Send { + let queue = queue.to_string(); + let pool = self.pool.clone(); + let limit = 100; + let offset = 0; + async move { + let workers = sqlx::query_file_as!( + Worker, + "queries/backend/list_workers.sql", + queue, + limit, + offset + ) + .fetch_all(&pool) + .map_ok(|w| { + w.into_iter() + .map(|w| RunningWorker { + id: w.id, + backend: w.storage_name, + started_at: 0, + last_heartbeat: w.last_seen as u64, + layers: w.layers.unwrap_or_default(), + queue: w.worker_type, + }) + .collect() + }) + .await?; + Ok(workers) + } + } + + fn list_all_workers( + &self, + ) -> impl Future, Self::Error>> + Send { + let pool = self.pool.clone(); + let limit = 100; + let offset = 0; + async move { + let workers = sqlx::query_file_as!( + Worker, + "queries/backend/list_all_workers.sql", + limit, + offset + ) + .fetch_all(&pool) + .map_ok(|w| { + w.into_iter() + .map(|w| RunningWorker { + id: w.id, + backend: w.storage_name, + started_at: 0, + last_heartbeat: w.last_seen as u64, + layers: w.layers.unwrap_or_default(), + queue: w.worker_type, + }) + .collect() + }) + .await?; + Ok(workers) + } + } +} diff --git a/src/queries/metrics.rs b/src/queries/metrics.rs new file mode 100644 index 0000000..aa1515f --- /dev/null +++ b/src/queries/metrics.rs @@ -0,0 +1,64 @@ +use apalis_core::backend::{Backend, Metrics, Statistic}; +use ulid::Ulid; + +use crate::{SqliteContext, SqliteStorage}; + +struct StatisticRow { + /// The priority of the statistic (lower number means higher priority) + pub priority: i64, + /// The statistics type + pub r#type: String, + /// Overall statistics of the backend + pub statistic: String, + /// The value of the statistic + pub value: Option, +} + +impl Metrics for SqliteStorage +where + SqliteStorage: + Backend, +{ + fn global(&self) -> impl Future, Self::Error>> + Send { + let pool = self.pool.clone(); + async move { + let rec = sqlx::query_file_as!(StatisticRow, "queries/backend/overview.sql") + .fetch_all(&pool) + .await? + .into_iter() + .map(|r| Statistic { + priority: Some(r.priority as u64), + stat_type: super::stat_type_from_string(&r.r#type), + title: r.statistic, + value: r.value.unwrap_or_default().to_string(), + }) + .collect(); + Ok(rec) + } + } + fn fetch_by_queue( + &self, + queue_id: &str, + ) -> impl Future, Self::Error>> + Send { + let pool = self.pool.clone(); + let queue_id = queue_id.to_string(); + async move { + let rec = sqlx::query_file_as!( + StatisticRow, + "queries/backend/overview_by_queue.sql", + queue_id + ) + .fetch_all(&pool) + .await? + .into_iter() + .map(|r| Statistic { + priority: Some(r.priority as u64), + stat_type: super::stat_type_from_string(&r.r#type), + title: r.statistic, + value: r.value.unwrap_or_default().to_string(), + }) + .collect(); + Ok(rec) + } + } +} diff --git a/src/queries/mod.rs b/src/queries/mod.rs new file mode 100644 index 0000000..52ef448 --- /dev/null +++ b/src/queries/mod.rs @@ -0,0 +1,20 @@ +use apalis_core::backend::StatType; + +pub mod fetch_by_id; +pub mod list_queues; +pub mod list_tasks; +pub mod list_workers; +pub mod metrics; +pub mod reenqueue_orphaned; +pub mod register_worker; +pub mod keep_alive; + +fn stat_type_from_string(s: &str) -> StatType { + match s { + "Number" => StatType::Number, + "Decimal" => StatType::Decimal, + "Percentage" => StatType::Percentage, + "Timestamp" => StatType::Timestamp, + _ => StatType::Number, // default fallback + } +} diff --git a/src/reenqueue_orphaned.rs b/src/queries/reenqueue_orphaned.rs similarity index 98% rename from src/reenqueue_orphaned.rs rename to src/queries/reenqueue_orphaned.rs index bedf22b..93788da 100644 --- a/src/reenqueue_orphaned.rs +++ b/src/queries/reenqueue_orphaned.rs @@ -1,7 +1,6 @@ use std::time::Duration; use futures::{FutureExt, Stream, stream}; -use serde_json::json; use sqlx::SqlitePool; use crate::Config; diff --git a/src/queries/register_worker.rs b/src/queries/register_worker.rs new file mode 100644 index 0000000..59d1e79 --- /dev/null +++ b/src/queries/register_worker.rs @@ -0,0 +1,33 @@ +use apalis_core::worker::context::WorkerContext; +use sqlx::SqlitePool; + +use crate::Config; + +pub async fn register_worker( + pool: SqlitePool, + config: Config, + worker: WorkerContext, + storage_type: &str, +) -> Result<(), sqlx::Error> { + let worker_id = worker.name().to_owned(); + let queue = config.queue().to_string(); + let layers = worker.get_service().to_owned(); + let keep_alive = config.keep_alive().as_secs() as i64; + let res = sqlx::query_file!( + "queries/backend/register_worker.sql", + worker_id, + queue, + storage_type, + layers, + keep_alive, + ) + .execute(&pool) + .await?; + if res.rows_affected() == 0 { + return Err(sqlx::Error::Io(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "WORKER_ALREADY_EXISTS", + ))); + } + Ok(()) +} diff --git a/src/shared.rs b/src/shared.rs index cd787cd..68f4000 100644 --- a/src/shared.rs +++ b/src/shared.rs @@ -12,10 +12,9 @@ use std::{ use crate::{ Config, INSERT_OPERATION, JOBS_TABLE, SqliteStorage, SqliteTask, ack::{LockTaskLayer, SqliteAck}, - config, + callback::{DbEvent, update_hook_callback}, context::SqliteContext, fetcher::SqlitePollFetcher, - hook::{DbEvent, update_hook_callback}, initial_heartbeat, keep_alive, }; use crate::{from_row::TaskRow, sink::SqliteSink}; @@ -27,7 +26,6 @@ use apalis_core::{ }, worker::{context::WorkerContext, ext::ack::AcknowledgeLayer}, }; -use chrono::Utc; use futures::{ FutureExt, SinkExt, Stream, StreamExt, TryStreamExt, channel::mpsc::{self, Receiver, Sender}, @@ -243,7 +241,6 @@ where fn poll(self, worker: &WorkerContext) -> Self::Stream { let pool = self.pool.clone(); let worker = worker.clone(); - let worker_type = self.config.queue().to_owned(); // Initial registration heartbeat // This ensures that the worker is registered before fetching any tasks // This also ensures that the worker is marked as alive in case it crashes diff --git a/src/sink.rs b/src/sink.rs index dc9950f..590c3f5 100644 --- a/src/sink.rs +++ b/src/sink.rs @@ -9,7 +9,7 @@ use apalis_core::{ error::BoxDynError, }; use futures::{ - FutureExt, Sink, TryFutureExt, + FutureExt, Sink, future::{BoxFuture, Shared}, }; use sqlx::SqlitePool; @@ -41,7 +41,7 @@ impl Clone for SqliteSink { } } -pub(crate) async fn push_tasks( +pub async fn push_tasks( pool: SqlitePool, cfg: Config, buffer: Vec>, diff --git a/src/traits/fetch_by_id.rs b/src/traits/fetch_by_id.rs deleted file mode 100644 index e69de29..0000000 diff --git a/src/traits/list_queues.rs b/src/traits/list_queues.rs deleted file mode 100644 index e69de29..0000000 diff --git a/src/traits/list_tasks.rs b/src/traits/list_tasks.rs deleted file mode 100644 index e69de29..0000000 diff --git a/src/traits/list_workers.rs b/src/traits/list_workers.rs deleted file mode 100644 index c5bf19f..0000000 --- a/src/traits/list_workers.rs +++ /dev/null @@ -1,299 +0,0 @@ -use apalis_core::{ - backend::{ - Backend, FetchById, Filter, ListAllTasks, ListQueues, ListTasks, ListWorkers, Metrics, - QueueInfo, RunningWorker, StatType, Statistic, - codec::{Codec, json::JsonCodec}, - }, - task::{Task, status::Status, task_id::TaskId}, -}; -use futures::TryFutureExt; -use serde::{Serialize, de::DeserializeOwned}; -use sqlx::Decode; -use ulid::Ulid; - -use crate::{SqliteContext, SqliteStorage, SqliteTask, from_row::TaskRow}; - -struct Worker { - id: String, - worker_type: String, - storage_name: String, - layers: Option, - last_seen: i64, -} - -impl ListWorkers for SqliteStorage -where - SqliteStorage: - Backend, -{ - fn list_workers( - &self, - queue: &str, - ) -> impl Future, Self::Error>> + Send { - let queue = queue.to_string(); - let pool = self.pool.clone(); - let limit = 100; - let offset = 0; - async move { - let workers = sqlx::query_file_as!( - Worker, - "queries/backend/list_workers.sql", - queue, - limit, - offset - ) - .fetch_all(&pool) - .map_ok(|w| { - w.into_iter() - .map(|w| RunningWorker { - id: w.id, - backend: w.storage_name, - started_at: 0, - last_heartbeat: w.last_seen as u64, - layers: w.layers.unwrap_or_default(), - queue: w.worker_type, - }) - .collect() - }) - .await?; - Ok(workers) - } - } - - fn list_all_workers( - &self, - ) -> impl Future, Self::Error>> + Send { - let pool = self.pool.clone(); - let limit = 100; - let offset = 0; - async move { - let workers = sqlx::query_file_as!( - Worker, - "queries/backend/list_all_workers.sql", - limit, - offset - ) - .fetch_all(&pool) - .map_ok(|w| { - w.into_iter() - .map(|w| RunningWorker { - id: w.id, - backend: w.storage_name, - started_at: 0, - last_heartbeat: w.last_seen as u64, - layers: w.layers.unwrap_or_default(), - queue: w.worker_type, - }) - .collect() - }) - .await?; - Ok(workers) - } - } -} - -impl ListTasks for SqliteStorage -where - SqliteStorage: - Backend, - Args: Serialize + DeserializeOwned, -{ - fn list_tasks( - &self, - queue: &str, - filter: &Filter, - ) -> impl Future>, Self::Error>> + Send { - let queue = queue.to_string(); - let pool = self.pool.clone(); - let limit = filter.limit() as i32; - let offset = filter.offset() as i32; - let status = filter - .status - .as_ref() - .unwrap_or(&Status::Pending) - .to_string(); - async move { - let tasks = sqlx::query_file_as!( - TaskRow, - "queries/backend/list_jobs.sql", - status, - queue, - limit, - offset - ) - .fetch_all(&pool) - .await? - .into_iter() - .map(|r| r.try_into_task::, _>()) - .collect::, _>>()?; - Ok(tasks) - } - } -} - -impl ListAllTasks for SqliteStorage -where - SqliteStorage: - Backend, -{ - fn list_all_tasks( - &self, - filter: &Filter, - ) -> impl Future< - Output = Result>, Self::Error>, - > + Send { - let status = filter - .status - .as_ref() - .map(|s| s.to_string()) - .unwrap_or(Status::Pending.to_string()); - let pool = self.pool.clone(); - let limit = filter.limit() as i32; - let offset = filter.offset() as i32; - async move { - let tasks = sqlx::query_file_as!( - TaskRow, - "queries/backend/list_all_jobs.sql", - status, - limit, - offset - ) - .fetch_all(&pool) - .await? - .into_iter() - .map(|r| r.try_into_task_compact()) - .collect::, _>>()?; - Ok(tasks) - } - } -} - -struct QueueInfoRow { - name: String, - stats: String, // JSON string - workers: String, // JSON string - activity: String, // JSON string -} - -impl From for QueueInfo { - fn from(row: QueueInfoRow) -> Self { - QueueInfo { - name: row.name, - stats: serde_json::from_str(&row.stats).unwrap(), - workers: serde_json::from_str(&row.workers).unwrap(), - activity: serde_json::from_str(&row.activity).unwrap(), - } - } -} - -impl ListQueues for SqliteStorage -where - SqliteStorage: - Backend, -{ - fn list_queues(&self) -> impl Future, Self::Error>> + Send { - let pool = self.pool.clone(); - - async move { - let queues = sqlx::query_file_as!(QueueInfoRow, "queries/backend/list_queues.sql") - .fetch_all(&pool) - .await? - .into_iter() - .map(QueueInfo::from) - .collect(); - Ok(queues) - } - } -} - -fn stat_type_from_string(s: &str) -> StatType { - match s { - "Number" => StatType::Number, - "Decimal" => StatType::Decimal, - "Percentage" => StatType::Percentage, - "Timestamp" => StatType::Timestamp, - _ => StatType::Number, // default fallback - } -} -struct StatisticRow { - /// The priority of the statistic (lower number means higher priority) - pub priority: i64, - /// The statistics type - pub r#type: String, - /// Overall statistics of the backend - pub statistic: String, - /// The value of the statistic - pub value: Option, -} - -impl Metrics for SqliteStorage -where - SqliteStorage: - Backend, -{ - fn global(&self) -> impl Future, Self::Error>> + Send { - let pool = self.pool.clone(); - async move { - let rec = sqlx::query_file_as!(StatisticRow, "queries/backend/overview.sql") - .fetch_all(&pool) - .await? - .into_iter() - .map(|r| Statistic { - priority: Some(r.priority as u64), - stat_type: stat_type_from_string(&r.r#type), - title: r.statistic, - value: r.value.unwrap_or_default().to_string(), - }) - .collect(); - Ok(rec) - } - } - fn fetch_by_queue( - &self, - queue_id: &str, - ) -> impl Future, Self::Error>> + Send { - let pool = self.pool.clone(); - let queue_id = queue_id.to_string(); - async move { - let rec = sqlx::query_file_as!( - StatisticRow, - "queries/backend/overview_by_queue.sql", - queue_id - ) - .fetch_all(&pool) - .await? - .into_iter() - .map(|r| Statistic { - priority: Some(r.priority as u64), - stat_type: stat_type_from_string(&r.r#type), - title: r.statistic, - value: r.value.unwrap_or_default().to_string(), - }) - .collect(); - Ok(rec) - } - } -} - -impl FetchById for SqliteStorage -where - SqliteStorage: - Backend, - Args: Serialize + DeserializeOwned, -{ - fn fetch_by_id( - &mut self, - id: &TaskId, - ) -> impl Future>, Self::Error>> + Send { - let pool = self.pool.clone(); - let id = id.to_string(); - async move { - let task = sqlx::query_file_as!(TaskRow, "queries/task/find_by_id.sql", id) - .fetch_optional(&pool) - .await? - .map(|r| r.try_into_task::, _>()) - .transpose()?; - Ok(task) - } - } -} diff --git a/src/traits/metrics.rs b/src/traits/metrics.rs deleted file mode 100644 index e69de29..0000000 diff --git a/src/traits/mod.rs b/src/traits/mod.rs deleted file mode 100644 index 158ee35..0000000 --- a/src/traits/mod.rs +++ /dev/null @@ -1 +0,0 @@ -mod list_workers; From 3171686e636c36f7be80607c5011f9e64bca9d65 Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Fri, 17 Oct 2025 17:12:46 +0300 Subject: [PATCH 117/121] add bytes support --- ...bc68adca35acef6ce80e58bad3b70c3fd6d97.json | 12 - ...2acf90e2a3710690f398f9ffaa71e180f1d9e.json | 38 + ...1eaa74232f6f6f3121c7086a0b138195e9b0d.json | 86 ++ ...d6063f186242ca1808cfae9d6477c7d8687b.json} | 8 +- ...9af4519adf779487e4d2e8b7137b79db1b5e7.json | 12 + ...4dd6e8a3e7ea31ef8d622534a9fa0093c31e2.json | 50 + ...29a245d0bb2b84595f87a883c60d6f120cf63.json | 38 + ...0f03d75403970fc8347d0f017f04d746b2b98.json | 86 ++ ...8edce306564754651c4098f8dfe693195f6ca.json | 38 + ...5a78685df4c262c2a13eafd9803c3672dcd10.json | 50 + ...5ad7b0a3ddc713341adcbbc959021082c8e9.json} | 4 +- ...5653f4fe06d10361c555f8ade4b65589520bc.json | 86 ++ ...a9f1318e707f1ee1acd8662952d0e0e67200b.json | 12 + ...b7ec3470a737fa5174e7a2aa16dcf16fc5576.json | 12 + ...0f217b575f0ca464a5faddd0fe2d51793983.json} | 8 +- ...b72909f4e44f49d068fb64c0b40a0b017f22e.json | 12 + Cargo.toml | 10 +- .../20251017141210_initial_bytes_storage.sql | 63 ++ .../20251017162501_worker_started_at.sql | 4 + .../20220530084123_jobs_workers.sql | 0 .../20250313213411_add_job_priority.sql} | 0 .../20251013233016_stats_indexes.sql | 0 .../json/20251017150712_rename_last_error.sql | 1 + .../json/20251017162501_worker_started_at.sql | 4 + queries/backend/reenqueue_orphaned.sql | 2 +- queries/backend/register_worker.sql | 3 +- queries/task/ack.sql | 2 +- queries/task/kill.sql | 2 +- queries/task/update_by_id.sql | 2 +- src/config.rs | 4 +- src/fetcher.rs | 19 +- src/from_row.rs | 14 +- src/lib.rs | 83 +- src/queries/fetch_by_id.rs | 12 +- src/queries/keep_alive.rs | 2 +- src/queries/list_queues.rs | 4 +- src/queries/list_tasks.rs | 14 +- src/queries/list_workers.rs | 9 +- src/queries/metrics.rs | 4 +- src/shared.rs | 36 +- src/sink.rs | 15 +- src/sqlite_old.rs | 897 ------------------ src/traits/list_workers.rs | 117 +++ 43 files changed, 859 insertions(+), 1016 deletions(-) delete mode 100644 .sqlx/query-0be6db0bdde33835113a74e4749bc68adca35acef6ce80e58bad3b70c3fd6d97.json create mode 100644 .sqlx/query-0c51f68a6dc3a43da8d955530e02acf90e2a3710690f398f9ffaa71e180f1d9e.json create mode 100644 .sqlx/query-1cd760004a2341bbded38a4fa431eaa74232f6f6f3121c7086a0b138195e9b0d.json rename .sqlx/{query-8e68049c1005dd0981469dac442cc479d318a98f4891ed46ff8588bb4e85615b.json => query-40a420986a37c9db2fc35b161092d6063f186242ca1808cfae9d6477c7d8687b.json} (71%) create mode 100644 .sqlx/query-41198057d67f5386d366f332c349af4519adf779487e4d2e8b7137b79db1b5e7.json create mode 100644 .sqlx/query-41ce7e67750c304b39bb88def9c4dd6e8a3e7ea31ef8d622534a9fa0093c31e2.json create mode 100644 .sqlx/query-42042a81a76b1f366a6f421acc329a245d0bb2b84595f87a883c60d6f120cf63.json create mode 100644 .sqlx/query-61843a18bffdee192cd01f1537f0f03d75403970fc8347d0f017f04d746b2b98.json create mode 100644 .sqlx/query-75c53a93669a7ac2141b1bb62af8edce306564754651c4098f8dfe693195f6ca.json create mode 100644 .sqlx/query-7bd7de027caa72c93f7f89bd24a5a78685df4c262c2a13eafd9803c3672dcd10.json rename .sqlx/{query-cfc8346ce97a0f65e057b439787acaa0ee3ad0de2ad6453554e95be9df4e487e.json => query-90406ca41cb79e68f66a300730f85ad7b0a3ddc713341adcbbc959021082c8e9.json} (50%) create mode 100644 .sqlx/query-a43052e877930a522d6a63789935653f4fe06d10361c555f8ade4b65589520bc.json create mode 100644 .sqlx/query-b44b1058179869ec68916630635a9f1318e707f1ee1acd8662952d0e0e67200b.json create mode 100644 .sqlx/query-d2a2123b8ede2014f106836cf41b7ec3470a737fa5174e7a2aa16dcf16fc5576.json rename .sqlx/{query-1beef6dd1673e83142d565dcd7a0543bde1cbc72f692187d8a7fef06e952f513.json => query-d7aefe54cd7388c208fff5b946390f217b575f0ca464a5faddd0fe2d51793983.json} (66%) create mode 100644 .sqlx/query-f4e22035b9e6b6c274b326c7a25b72909f4e44f49d068fb64c0b40a0b017f22e.json create mode 100644 migrations/bytes/20251017141210_initial_bytes_storage.sql create mode 100644 migrations/bytes/20251017162501_worker_started_at.sql rename migrations/{ => json}/20220530084123_jobs_workers.sql (100%) rename migrations/{20250313213411_add_job_priority copy.sql => json/20250313213411_add_job_priority.sql} (100%) rename migrations/{ => json}/20251013233016_stats_indexes.sql (100%) create mode 100644 migrations/json/20251017150712_rename_last_error.sql create mode 100644 migrations/json/20251017162501_worker_started_at.sql delete mode 100644 src/sqlite_old.rs create mode 100644 src/traits/list_workers.rs diff --git a/.sqlx/query-0be6db0bdde33835113a74e4749bc68adca35acef6ce80e58bad3b70c3fd6d97.json b/.sqlx/query-0be6db0bdde33835113a74e4749bc68adca35acef6ce80e58bad3b70c3fd6d97.json deleted file mode 100644 index e914ad4..0000000 --- a/.sqlx/query-0be6db0bdde33835113a74e4749bc68adca35acef6ce80e58bad3b70c3fd6d97.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "INSERT INTO\n Workers (id, worker_type, storage_name, layers, last_seen)\nVALUES\n ($1, $2, $3, $4, $5) ON CONFLICT (id) DO\nUPDATE\nSET\n last_seen = EXCLUDED.last_seen\n", - "describe": { - "columns": [], - "parameters": { - "Right": 5 - }, - "nullable": [] - }, - "hash": "0be6db0bdde33835113a74e4749bc68adca35acef6ce80e58bad3b70c3fd6d97" -} diff --git a/.sqlx/query-0c51f68a6dc3a43da8d955530e02acf90e2a3710690f398f9ffaa71e180f1d9e.json b/.sqlx/query-0c51f68a6dc3a43da8d955530e02acf90e2a3710690f398f9ffaa71e180f1d9e.json new file mode 100644 index 0000000..c9b9a22 --- /dev/null +++ b/.sqlx/query-0c51f68a6dc3a43da8d955530e02acf90e2a3710690f398f9ffaa71e180f1d9e.json @@ -0,0 +1,38 @@ +{ + "db_name": "SQLite", + "query": "SELECT\n 1 AS priority,\n 'Number' AS type,\n 'RUNNING_JOBS' AS statistic,\n CAST(\n SUM(\n CASE\n WHEN status = 'Running' THEN 1\n ELSE 0\n END\n ) AS REAL\n ) AS value\nFROM\n Jobs\nUNION\nALL\nSELECT\n 1,\n 'Number',\n 'PENDING_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Pending' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 2,\n 'Number',\n 'FAILED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Failed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 2,\n 'Number',\n 'ACTIVE_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status IN ('Pending', 'Running', 'Queued') THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 2,\n 'Number',\n 'STALE_RUNNING_JOBS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n status = 'Running'\n AND run_at < strftime('%s', 'now', '-1 hour')\nUNION\nALL\nSELECT\n 2,\n 'Percentage',\n 'KILL_RATE',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 3,\n 'Number',\n 'JOBS_PAST_HOUR',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-1 hour')\nUNION\nALL\nSELECT\n 3,\n 'Number',\n 'JOBS_TODAY',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n date(run_at, 'unixepoch') = date('now')\nUNION\nALL\nSELECT\n 3,\n 'Number',\n 'KILLED_JOBS_TODAY',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n date(run_at, 'unixepoch') = date('now')\nUNION\nALL\nSELECT\n 3,\n 'Decimal',\n 'AVG_JOBS_PER_MINUTE_PAST_HOUR',\n CAST(ROUND(COUNT(*) / 60.0, 2) AS REAL)\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-1 hour')\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'TOTAL_JOBS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'DONE_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'COMPLETED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status IN ('Done', 'Failed', 'Killed') THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'KILLED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 4,\n 'Percentage',\n 'SUCCESS_RATE',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 5,\n 'Decimal',\n 'AVG_JOB_DURATION_MINS',\n CAST(ROUND(AVG((done_at - run_at) / 60.0), 2) AS REAL)\nFROM\n Jobs\nWHERE\n status IN ('Done', 'Failed', 'Killed')\n AND done_at IS NOT NULL\nUNION\nALL\nSELECT\n 5,\n 'Decimal',\n 'LONGEST_RUNNING_JOB_MINS',\n CAST(\n ROUND(\n MAX(\n CASE\n WHEN status = 'Running' THEN (strftime('%s', 'now') - run_at) / 60.0\n ELSE 0\n END\n ),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 5,\n 'Number',\n 'QUEUE_BACKLOG',\n CAST(\n SUM(\n CASE\n WHEN status = 'Pending'\n AND run_at <= strftime('%s', 'now') THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 6,\n 'Number',\n 'JOBS_PAST_24_HOURS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-1 day')\nUNION\nALL\nSELECT\n 6,\n 'Number',\n 'JOBS_PAST_7_DAYS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-7 days')\nUNION\nALL\nSELECT\n 6,\n 'Number',\n 'KILLED_JOBS_PAST_7_DAYS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-7 days')\nUNION\nALL\nSELECT\n 6,\n 'Percentage',\n 'SUCCESS_RATE_PAST_24H',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-1 day')\nUNION\nALL\nSELECT\n 7,\n 'Decimal',\n 'AVG_JOBS_PER_HOUR_PAST_24H',\n CAST(ROUND(COUNT(*) / 24.0, 2) AS REAL)\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-1 day')\nUNION\nALL\nSELECT\n 7,\n 'Decimal',\n 'AVG_JOBS_PER_DAY_PAST_7D',\n CAST(ROUND(COUNT(*) / 7.0, 2) AS REAL)\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-7 days')\nUNION\nALL\nSELECT\n 8,\n 'Timestamp',\n 'MOST_RECENT_JOB',\n CAST(MAX(run_at) AS REAL)\nFROM\n Jobs\nUNION\nALL\nSELECT\n 8,\n 'Timestamp',\n 'OLDEST_PENDING_JOB',\n CAST(MIN(run_at) AS REAL)\nFROM\n Jobs\nWHERE\n status = 'Pending'\n AND run_at <= strftime('%s', 'now')\nUNION\nALL\nSELECT\n 8,\n 'Number',\n 'PEAK_HOUR_JOBS',\n CAST(MAX(hourly_count) AS REAL)\nFROM\n (\n SELECT\n COUNT(*) as hourly_count\n FROM\n Jobs\n WHERE\n run_at >= strftime('%s', 'now', '-1 day')\n GROUP BY\n strftime('%H', run_at, 'unixepoch')\n )\nUNION\nALL\nSELECT\n 9,\n 'Number',\n 'DB_PAGE_SIZE',\n CAST(page_size AS REAL)\nFROM\n pragma_page_size()\nUNION\nALL\nSELECT\n 9,\n 'Number',\n 'DB_PAGE_COUNT',\n CAST(page_count AS REAL)\nFROM\n pragma_page_count()\nUNION\nALL\nSELECT\n 9,\n 'Number',\n 'DB_SIZE',\n CAST(page_size * page_count AS REAL)\nFROM\n pragma_page_size(),\n pragma_page_count()\nORDER BY\n priority,\n statistic;\n", + "describe": { + "columns": [ + { + "name": "priority", + "ordinal": 0, + "type_info": "Integer" + }, + { + "name": "type", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "statistic", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "value", + "ordinal": 3, + "type_info": "Float" + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "0c51f68a6dc3a43da8d955530e02acf90e2a3710690f398f9ffaa71e180f1d9e" +} diff --git a/.sqlx/query-1cd760004a2341bbded38a4fa431eaa74232f6f6f3121c7086a0b138195e9b0d.json b/.sqlx/query-1cd760004a2341bbded38a4fa431eaa74232f6f6f3121c7086a0b138195e9b0d.json new file mode 100644 index 0000000..90bd90a --- /dev/null +++ b/.sqlx/query-1cd760004a2341bbded38a4fa431eaa74232f6f6f3121c7086a0b138195e9b0d.json @@ -0,0 +1,86 @@ +{ + "db_name": "SQLite", + "query": "SELECT\n *\nFROM\n Jobs\nWHERE\n status = ?1\n AND job_type = ?2\nORDER BY\n done_at DESC,\n run_at DESC\nLIMIT\n ?3 OFFSET ?4\n", + "describe": { + "columns": [ + { + "name": "job", + "ordinal": 0, + "type_info": "Blob" + }, + { + "name": "id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "job_type", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "status", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "attempts", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "max_attempts", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "run_at", + "ordinal": 6, + "type_info": "Integer" + }, + { + "name": "last_result", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "lock_at", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "lock_by", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "done_at", + "ordinal": 10, + "type_info": "Integer" + }, + { + "name": "priority", + "ordinal": 11, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 4 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + false + ] + }, + "hash": "1cd760004a2341bbded38a4fa431eaa74232f6f6f3121c7086a0b138195e9b0d" +} diff --git a/.sqlx/query-8e68049c1005dd0981469dac442cc479d318a98f4891ed46ff8588bb4e85615b.json b/.sqlx/query-40a420986a37c9db2fc35b161092d6063f186242ca1808cfae9d6477c7d8687b.json similarity index 71% rename from .sqlx/query-8e68049c1005dd0981469dac442cc479d318a98f4891ed46ff8588bb4e85615b.json rename to .sqlx/query-40a420986a37c9db2fc35b161092d6063f186242ca1808cfae9d6477c7d8687b.json index b5b03b4..ac61445 100644 --- a/.sqlx/query-8e68049c1005dd0981469dac442cc479d318a98f4891ed46ff8588bb4e85615b.json +++ b/.sqlx/query-40a420986a37c9db2fc35b161092d6063f186242ca1808cfae9d6477c7d8687b.json @@ -1,12 +1,12 @@ { "db_name": "SQLite", - "query": "UPDATE Jobs\nSET\n status = 'Running',\n lock_by = ?1,\n lock_at = CURRENT_TIMESTAMP\nWHERE\n ROWID IN (\n SELECT ROWID\n FROM Jobs\n WHERE job_type = ?2\n AND status = 'Pending'\n AND lock_by IS NULL\n AND (run_at IS NULL OR run_at <= CURRENT_TIMESTAMP)\n ORDER BY priority DESC, run_at ASC, id ASC\n LIMIT ?3\n )\nRETURNING *\n", + "query": "SELECT\n *\nFROM\n Jobs\nWHERE\n status = ?1\nORDER BY\n done_at DESC,\n run_at DESC\nLIMIT\n ?2 OFFSET ?3\n", "describe": { "columns": [ { "name": "job", "ordinal": 0, - "type_info": "Text" + "type_info": "Blob" }, { "name": "id", @@ -39,7 +39,7 @@ "type_info": "Integer" }, { - "name": "last_error", + "name": "last_result", "ordinal": 7, "type_info": "Text" }, @@ -82,5 +82,5 @@ false ] }, - "hash": "8e68049c1005dd0981469dac442cc479d318a98f4891ed46ff8588bb4e85615b" + "hash": "40a420986a37c9db2fc35b161092d6063f186242ca1808cfae9d6477c7d8687b" } diff --git a/.sqlx/query-41198057d67f5386d366f332c349af4519adf779487e4d2e8b7137b79db1b5e7.json b/.sqlx/query-41198057d67f5386d366f332c349af4519adf779487e4d2e8b7137b79db1b5e7.json new file mode 100644 index 0000000..f0c561c --- /dev/null +++ b/.sqlx/query-41198057d67f5386d366f332c349af4519adf779487e4d2e8b7137b79db1b5e7.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "UPDATE\n Jobs\nSET\n status = 'Running',\n lock_at = strftime('%s', 'now'),\n lock_by = ?2\nWHERE\n id = ?1\n AND (\n status = 'Queued'\n OR status = 'Pending'\n OR (\n status = 'Failed'\n AND attempts < max_attempts\n )\n )\n", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + }, + "hash": "41198057d67f5386d366f332c349af4519adf779487e4d2e8b7137b79db1b5e7" +} diff --git a/.sqlx/query-41ce7e67750c304b39bb88def9c4dd6e8a3e7ea31ef8d622534a9fa0093c31e2.json b/.sqlx/query-41ce7e67750c304b39bb88def9c4dd6e8a3e7ea31ef8d622534a9fa0093c31e2.json new file mode 100644 index 0000000..5c528f8 --- /dev/null +++ b/.sqlx/query-41ce7e67750c304b39bb88def9c4dd6e8a3e7ea31ef8d622534a9fa0093c31e2.json @@ -0,0 +1,50 @@ +{ + "db_name": "SQLite", + "query": "SELECT\n *\nFROM\n Workers\nORDER BY\n last_seen DESC\nLIMIT\n ?1 OFFSET ?2\n", + "describe": { + "columns": [ + { + "name": "id", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "worker_type", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "storage_name", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "layers", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "last_seen", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "started_at", + "ordinal": 5, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + false, + false, + false, + true, + false, + true + ] + }, + "hash": "41ce7e67750c304b39bb88def9c4dd6e8a3e7ea31ef8d622534a9fa0093c31e2" +} diff --git a/.sqlx/query-42042a81a76b1f366a6f421acc329a245d0bb2b84595f87a883c60d6f120cf63.json b/.sqlx/query-42042a81a76b1f366a6f421acc329a245d0bb2b84595f87a883c60d6f120cf63.json new file mode 100644 index 0000000..9aef9ee --- /dev/null +++ b/.sqlx/query-42042a81a76b1f366a6f421acc329a245d0bb2b84595f87a883c60d6f120cf63.json @@ -0,0 +1,38 @@ +{ + "db_name": "SQLite", + "query": "WITH queue_stats AS (\n SELECT\n job_type,\n json_group_array(\n json_object(\n 'title',\n statistic,\n 'stat_type',\n type,\n 'value',\n value,\n 'priority',\n priority\n )\n ) as stats\n FROM\n (\n SELECT\n job_type,\n 1 AS priority,\n 'Number' AS type,\n 'RUNNING_JOBS' AS statistic,\n CAST(\n SUM(\n CASE\n WHEN status = 'Running' THEN 1\n ELSE 0\n END\n ) AS TEXT\n ) AS value\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 1,\n 'Number',\n 'PENDING_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Pending' THEN 1\n ELSE 0\n END\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 1,\n 'Number',\n 'FAILED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Failed' THEN 1\n ELSE 0\n END\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 2,\n 'Number',\n 'ACTIVE_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status IN ('Pending', 'Queued', 'Running') THEN 1\n ELSE 0\n END\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 2,\n 'Number',\n 'STALE_RUNNING_JOBS',\n CAST(COUNT(*) AS TEXT)\n FROM\n Jobs\n WHERE\n status = 'Running'\n AND run_at < strftime('%s', 'now', '-1 hour')\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 2,\n 'Percentage',\n 'KILL_RATE',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 3,\n 'Number',\n 'JOBS_PAST_HOUR',\n CAST(COUNT(*) AS TEXT)\n FROM\n Jobs\n WHERE\n run_at >= strftime('%s', 'now', '-1 hour')\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 3,\n 'Number',\n 'JOBS_TODAY',\n CAST(COUNT(*) AS TEXT)\n FROM\n Jobs\n WHERE\n date(run_at, 'unixepoch') = date('now')\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 3,\n 'Number',\n 'KILLED_JOBS_TODAY',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS TEXT\n )\n FROM\n Jobs\n WHERE\n date(run_at, 'unixepoch') = date('now')\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 3,\n 'Decimal',\n 'AVG_JOBS_PER_MINUTE_PAST_HOUR',\n CAST(ROUND(COUNT(*) / 60.0, 2) AS TEXT)\n FROM\n Jobs\n WHERE\n run_at >= strftime('%s', 'now', '-1 hour')\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 4,\n 'Number',\n 'TOTAL_JOBS',\n CAST(COUNT(*) AS TEXT)\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 4,\n 'Number',\n 'DONE_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 4,\n 'Number',\n 'KILLED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 4,\n 'Percentage',\n 'SUCCESS_RATE',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 5,\n 'Decimal',\n 'AVG_JOB_DURATION_MINS',\n CAST(ROUND(AVG((done_at - run_at) / 60.0), 2) AS TEXT)\n FROM\n Jobs\n WHERE\n status IN ('Done', 'Failed', 'Killed')\n AND done_at IS NOT NULL\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 5,\n 'Decimal',\n 'LONGEST_RUNNING_JOB_MINS',\n CAST(\n ROUND(\n MAX(\n CASE\n WHEN status = 'Running' THEN (strftime('%s', 'now') - run_at) / 60.0\n ELSE 0\n END\n ),\n 2\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 6,\n 'Number',\n 'JOBS_PAST_7_DAYS',\n CAST(COUNT(*) AS TEXT)\n FROM\n Jobs\n WHERE\n run_at >= strftime('%s', 'now', '-7 days')\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 8,\n 'Timestamp',\n 'MOST_RECENT_JOB',\n CAST(MAX(run_at) AS TEXT)\n FROM\n Jobs\n GROUP BY\n job_type\n ORDER BY\n job_type,\n priority,\n statistic\n )\n GROUP BY\n job_type\n),\nall_job_types AS (\n SELECT\n worker_type AS job_type\n FROM\n Workers\n UNION\n SELECT\n job_type\n FROM\n Jobs\n)\nSELECT\n jt.job_type as name,\n COALESCE(qs.stats, '[]') as stats,\n COALESCE(\n (\n SELECT\n json_group_array(DISTINCT lock_by)\n FROM\n Jobs\n WHERE\n job_type = jt.job_type\n AND lock_by IS NOT NULL\n ),\n '[]'\n ) as workers,\n COALESCE(\n (\n SELECT\n json_group_array(daily_count)\n FROM\n (\n SELECT\n COUNT(*) as daily_count\n FROM\n Jobs\n WHERE\n job_type = jt.job_type\n AND run_at >= strftime('%s', 'now', '-7 days')\n GROUP BY\n date(run_at, 'unixepoch')\n ORDER BY\n date(run_at, 'unixepoch')\n )\n ),\n '[]'\n ) as activity\nFROM\n all_job_types jt\n LEFT JOIN queue_stats qs ON jt.job_type = qs.job_type\nORDER BY\n name;\n", + "describe": { + "columns": [ + { + "name": "name", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "stats", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "workers", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "activity", + "ordinal": 3, + "type_info": "Text" + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "42042a81a76b1f366a6f421acc329a245d0bb2b84595f87a883c60d6f120cf63" +} diff --git a/.sqlx/query-61843a18bffdee192cd01f1537f0f03d75403970fc8347d0f017f04d746b2b98.json b/.sqlx/query-61843a18bffdee192cd01f1537f0f03d75403970fc8347d0f017f04d746b2b98.json new file mode 100644 index 0000000..d9bc816 --- /dev/null +++ b/.sqlx/query-61843a18bffdee192cd01f1537f0f03d75403970fc8347d0f017f04d746b2b98.json @@ -0,0 +1,86 @@ +{ + "db_name": "SQLite", + "query": "UPDATE Jobs\nSET\n status = 'Queued',\n lock_at = strftime('%s', 'now')\nWHERE ROWID IN (\n SELECT ROWID\n FROM Jobs\n WHERE job_type IN (\n SELECT value FROM json_each(?1)\n )\n AND status = 'Pending'\n AND lock_by IS NULL\n AND (\n run_at IS NULL\n OR run_at <= strftime('%s', 'now')\n )\n AND ROWID IN (\n SELECT value FROM json_each(?2)\n )\n ORDER BY\n priority DESC,\n run_at ASC,\n id ASC\n LIMIT ?3\n)\nRETURNING *;\n", + "describe": { + "columns": [ + { + "name": "job", + "ordinal": 0, + "type_info": "Blob" + }, + { + "name": "id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "job_type", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "status", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "attempts", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "max_attempts", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "run_at", + "ordinal": 6, + "type_info": "Integer" + }, + { + "name": "last_result", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "lock_at", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "lock_by", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "done_at", + "ordinal": 10, + "type_info": "Integer" + }, + { + "name": "priority", + "ordinal": 11, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 3 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + false + ] + }, + "hash": "61843a18bffdee192cd01f1537f0f03d75403970fc8347d0f017f04d746b2b98" +} diff --git a/.sqlx/query-75c53a93669a7ac2141b1bb62af8edce306564754651c4098f8dfe693195f6ca.json b/.sqlx/query-75c53a93669a7ac2141b1bb62af8edce306564754651c4098f8dfe693195f6ca.json new file mode 100644 index 0000000..2d8ca07 --- /dev/null +++ b/.sqlx/query-75c53a93669a7ac2141b1bb62af8edce306564754651c4098f8dfe693195f6ca.json @@ -0,0 +1,38 @@ +{ + "db_name": "SQLite", + "query": "SELECT\n 1 AS priority,\n 'Number' AS type,\n 'RUNNING_JOBS' AS statistic,\n CAST(\n SUM(\n CASE\n WHEN status = 'Running' THEN 1\n ELSE 0\n END\n ) AS REAL\n ) AS value\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 1,\n 'Number',\n 'PENDING_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Pending' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 2,\n 'Number',\n 'FAILED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Failed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 2,\n 'Number',\n 'ACTIVE_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status IN ('Pending', 'Queued', 'Running') THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 2,\n 'Number',\n 'STALE_RUNNING_JOBS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND status = 'Running'\n AND run_at < strftime('%s', 'now', '-1 hour')\nUNION\nALL\nSELECT\n 2,\n 'Percentage',\n 'KILL_RATE',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 3,\n 'Number',\n 'JOBS_PAST_HOUR',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-1 hour')\nUNION\nALL\nSELECT\n 3,\n 'Number',\n 'JOBS_TODAY',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND date(run_at, 'unixepoch') = date('now')\nUNION\nALL\nSELECT\n 3,\n 'Number',\n 'KILLED_JOBS_TODAY',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND date(run_at, 'unixepoch') = date('now')\nUNION\nALL\nSELECT\n 3,\n 'Decimal',\n 'AVG_JOBS_PER_MINUTE_PAST_HOUR',\n CAST(ROUND(COUNT(*) / 60.0, 2) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-1 hour')\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'TOTAL_JOBS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'DONE_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'COMPLETED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status IN ('Done', 'Failed', 'Killed') THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'KILLED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 4,\n 'Percentage',\n 'SUCCESS_RATE',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 5,\n 'Decimal',\n 'AVG_JOB_DURATION_MINS',\n CAST(ROUND(AVG((done_at - run_at) / 60.0), 2) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND status IN ('Done', 'Failed', 'Killed')\n AND done_at IS NOT NULL\nUNION\nALL\nSELECT\n 5,\n 'Decimal',\n 'LONGEST_RUNNING_JOB_MINS',\n CAST(\n ROUND(\n MAX(\n CASE\n WHEN status = 'Running' THEN (strftime('%s', 'now') - run_at) / 60.0\n ELSE 0\n END\n ),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 5,\n 'Number',\n 'QUEUE_BACKLOG',\n CAST(\n SUM(\n CASE\n WHEN status = 'Pending'\n AND run_at <= strftime('%s', 'now') THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 6,\n 'Number',\n 'JOBS_PAST_24_HOURS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-1 day')\nUNION\nALL\nSELECT\n 6,\n 'Number',\n 'JOBS_PAST_7_DAYS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-7 days')\nUNION\nALL\nSELECT\n 6,\n 'Number',\n 'KILLED_JOBS_PAST_7_DAYS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-7 days')\nUNION\nALL\nSELECT\n 6,\n 'Percentage',\n 'SUCCESS_RATE_PAST_24H',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-1 day')\nUNION\nALL\nSELECT\n 7,\n 'Decimal',\n 'AVG_JOBS_PER_HOUR_PAST_24H',\n CAST(ROUND(COUNT(*) / 24.0, 2) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-1 day')\nUNION\nALL\nSELECT\n 7,\n 'Decimal',\n 'AVG_JOBS_PER_DAY_PAST_7D',\n CAST(ROUND(COUNT(*) / 7.0, 2) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-7 days')\nUNION\nALL\nSELECT\n 8,\n 'Timestamp',\n 'MOST_RECENT_JOB',\n CAST(MAX(run_at) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 8,\n 'Timestamp',\n 'OLDEST_PENDING_JOB',\n CAST(MIN(run_at) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND status = 'Pending'\n AND run_at <= strftime('%s', 'now')\nUNION\nALL\nSELECT\n 8,\n 'Number',\n 'PEAK_HOUR_JOBS',\n CAST(MAX(hourly_count) AS REAL)\nFROM\n (\n SELECT\n COUNT(*) as hourly_count\n FROM\n Jobs\n WHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-1 day')\n GROUP BY\n strftime('%H', run_at, 'unixepoch')\n )\nORDER BY\n priority,\n statistic;\n", + "describe": { + "columns": [ + { + "name": "priority", + "ordinal": 0, + "type_info": "Integer" + }, + { + "name": "type", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "statistic", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "value", + "ordinal": 3, + "type_info": "Float" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "75c53a93669a7ac2141b1bb62af8edce306564754651c4098f8dfe693195f6ca" +} diff --git a/.sqlx/query-7bd7de027caa72c93f7f89bd24a5a78685df4c262c2a13eafd9803c3672dcd10.json b/.sqlx/query-7bd7de027caa72c93f7f89bd24a5a78685df4c262c2a13eafd9803c3672dcd10.json new file mode 100644 index 0000000..8be601e --- /dev/null +++ b/.sqlx/query-7bd7de027caa72c93f7f89bd24a5a78685df4c262c2a13eafd9803c3672dcd10.json @@ -0,0 +1,50 @@ +{ + "db_name": "SQLite", + "query": "SELECT\n *\nFROM\n Workers\nWHERE\n worker_type = ?1\nORDER BY\n last_seen DESC\nLIMIT\n ?2 OFFSET ?3\n", + "describe": { + "columns": [ + { + "name": "id", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "worker_type", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "storage_name", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "layers", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "last_seen", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "started_at", + "ordinal": 5, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 3 + }, + "nullable": [ + false, + false, + false, + true, + false, + true + ] + }, + "hash": "7bd7de027caa72c93f7f89bd24a5a78685df4c262c2a13eafd9803c3672dcd10" +} diff --git a/.sqlx/query-cfc8346ce97a0f65e057b439787acaa0ee3ad0de2ad6453554e95be9df4e487e.json b/.sqlx/query-90406ca41cb79e68f66a300730f85ad7b0a3ddc713341adcbbc959021082c8e9.json similarity index 50% rename from .sqlx/query-cfc8346ce97a0f65e057b439787acaa0ee3ad0de2ad6453554e95be9df4e487e.json rename to .sqlx/query-90406ca41cb79e68f66a300730f85ad7b0a3ddc713341adcbbc959021082c8e9.json index fea0ff3..6dbad87 100644 --- a/.sqlx/query-cfc8346ce97a0f65e057b439787acaa0ee3ad0de2ad6453554e95be9df4e487e.json +++ b/.sqlx/query-90406ca41cb79e68f66a300730f85ad7b0a3ddc713341adcbbc959021082c8e9.json @@ -1,6 +1,6 @@ { "db_name": "SQLite", - "query": "UPDATE\n Jobs\nSET\n status = ?4,\n attempts = ?2,\n last_error = ?3,\n done_at = CURRENT_TIMESTAMP\nWHERE\n id = ?1\n AND lock_by = ?5\n", + "query": "UPDATE\n Jobs\nSET\n status = ?4,\n attempts = ?2,\n last_result = ?3,\n done_at = strftime('%s', 'now')\nWHERE\n id = ?1\n AND lock_by = ?5\n", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "cfc8346ce97a0f65e057b439787acaa0ee3ad0de2ad6453554e95be9df4e487e" + "hash": "90406ca41cb79e68f66a300730f85ad7b0a3ddc713341adcbbc959021082c8e9" } diff --git a/.sqlx/query-a43052e877930a522d6a63789935653f4fe06d10361c555f8ade4b65589520bc.json b/.sqlx/query-a43052e877930a522d6a63789935653f4fe06d10361c555f8ade4b65589520bc.json new file mode 100644 index 0000000..f26492b --- /dev/null +++ b/.sqlx/query-a43052e877930a522d6a63789935653f4fe06d10361c555f8ade4b65589520bc.json @@ -0,0 +1,86 @@ +{ + "db_name": "SQLite", + "query": "SELECT\n *\nFROM\n Jobs\nWHERE\n id = ?1\nLIMIT\n 1;\n", + "describe": { + "columns": [ + { + "name": "job", + "ordinal": 0, + "type_info": "Blob" + }, + { + "name": "id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "job_type", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "status", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "attempts", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "max_attempts", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "run_at", + "ordinal": 6, + "type_info": "Integer" + }, + { + "name": "last_result", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "lock_at", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "lock_by", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "done_at", + "ordinal": 10, + "type_info": "Integer" + }, + { + "name": "priority", + "ordinal": 11, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + false + ] + }, + "hash": "a43052e877930a522d6a63789935653f4fe06d10361c555f8ade4b65589520bc" +} diff --git a/.sqlx/query-b44b1058179869ec68916630635a9f1318e707f1ee1acd8662952d0e0e67200b.json b/.sqlx/query-b44b1058179869ec68916630635a9f1318e707f1ee1acd8662952d0e0e67200b.json new file mode 100644 index 0000000..a656bce --- /dev/null +++ b/.sqlx/query-b44b1058179869ec68916630635a9f1318e707f1ee1acd8662952d0e0e67200b.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "INSERT INTO\n Workers (id, worker_type, storage_name, layers, last_seen, started_at)\nSELECT\n ?1,\n ?2,\n ?3,\n ?4,\n strftime('%s', 'now'),\n strftime('%s', 'now')\nWHERE\n NOT EXISTS (\n SELECT\n 1\n FROM\n Workers\n WHERE\n id = ?1\n AND strftime('%s', 'now') - last_seen >= ?5\n );\n", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + }, + "hash": "b44b1058179869ec68916630635a9f1318e707f1ee1acd8662952d0e0e67200b" +} diff --git a/.sqlx/query-d2a2123b8ede2014f106836cf41b7ec3470a737fa5174e7a2aa16dcf16fc5576.json b/.sqlx/query-d2a2123b8ede2014f106836cf41b7ec3470a737fa5174e7a2aa16dcf16fc5576.json new file mode 100644 index 0000000..d2a9814 --- /dev/null +++ b/.sqlx/query-d2a2123b8ede2014f106836cf41b7ec3470a737fa5174e7a2aa16dcf16fc5576.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "UPDATE\n Jobs\nSET\n status = \"Pending\",\n done_at = NULL,\n lock_by = NULL,\n lock_at = NULL,\n attempts = attempts + 1,\n last_result = '{\"Err\": \"Re-enqueued due to worker heartbeat timeout.\"}'\nWHERE\n id IN (\n SELECT\n Jobs.id\n FROM\n Jobs\n INNER JOIN Workers ON lock_by = Workers.id\n WHERE\n (\n status = \"Running\"\n OR status = \"Queued\"\n )\n AND strftime('%s', 'now') - Workers.last_seen >= ?1\n AND Workers.worker_type = ?2\n );\n", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + }, + "hash": "d2a2123b8ede2014f106836cf41b7ec3470a737fa5174e7a2aa16dcf16fc5576" +} diff --git a/.sqlx/query-1beef6dd1673e83142d565dcd7a0543bde1cbc72f692187d8a7fef06e952f513.json b/.sqlx/query-d7aefe54cd7388c208fff5b946390f217b575f0ca464a5faddd0fe2d51793983.json similarity index 66% rename from .sqlx/query-1beef6dd1673e83142d565dcd7a0543bde1cbc72f692187d8a7fef06e952f513.json rename to .sqlx/query-d7aefe54cd7388c208fff5b946390f217b575f0ca464a5faddd0fe2d51793983.json index 36daba6..fef32c7 100644 --- a/.sqlx/query-1beef6dd1673e83142d565dcd7a0543bde1cbc72f692187d8a7fef06e952f513.json +++ b/.sqlx/query-d7aefe54cd7388c208fff5b946390f217b575f0ca464a5faddd0fe2d51793983.json @@ -1,12 +1,12 @@ { "db_name": "SQLite", - "query": "UPDATE Jobs\nSET\n status = 'Running',\n lock_at = CURRENT_TIMESTAMP\nWHERE ROWID IN (\n SELECT ROWID\n FROM Jobs\n WHERE job_type IN (\n SELECT value FROM json_each(?1)\n )\n AND status = 'Pending'\n AND lock_by IS NULL\n AND (\n run_at IS NULL\n OR run_at <= CURRENT_TIMESTAMP\n )\n AND ROWID IN (\n SELECT value FROM json_each(?2)\n )\n ORDER BY\n priority DESC,\n run_at ASC,\n id ASC\n LIMIT ?3\n)\nRETURNING *;\n", + "query": "UPDATE Jobs\nSET\n status = 'Queued',\n lock_by = ?1,\n lock_at = strftime('%s', 'now')\nWHERE\n ROWID IN (\n SELECT ROWID\n FROM Jobs\n WHERE job_type = ?2\n AND (\n (status = 'Pending' AND lock_by IS NULL) \n OR \n (status = 'Failed' AND attempts < max_attempts)\n )\n AND (run_at IS NULL OR run_at <= strftime('%s', 'now'))\n ORDER BY priority DESC, run_at ASC, id ASC\n LIMIT ?3\n )\nRETURNING *\n", "describe": { "columns": [ { "name": "job", "ordinal": 0, - "type_info": "Text" + "type_info": "Blob" }, { "name": "id", @@ -39,7 +39,7 @@ "type_info": "Integer" }, { - "name": "last_error", + "name": "last_result", "ordinal": 7, "type_info": "Text" }, @@ -82,5 +82,5 @@ false ] }, - "hash": "1beef6dd1673e83142d565dcd7a0543bde1cbc72f692187d8a7fef06e952f513" + "hash": "d7aefe54cd7388c208fff5b946390f217b575f0ca464a5faddd0fe2d51793983" } diff --git a/.sqlx/query-f4e22035b9e6b6c274b326c7a25b72909f4e44f49d068fb64c0b40a0b017f22e.json b/.sqlx/query-f4e22035b9e6b6c274b326c7a25b72909f4e44f49d068fb64c0b40a0b017f22e.json new file mode 100644 index 0000000..44b9bdb --- /dev/null +++ b/.sqlx/query-f4e22035b9e6b6c274b326c7a25b72909f4e44f49d068fb64c0b40a0b017f22e.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "UPDATE\n Workers\nSET\n last_seen = strftime('%s', 'now')\nWHERE\n id = $1 AND worker_type = $2;\n", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + }, + "hash": "f4e22035b9e6b6c274b326c7a25b72909f4e44f49d068fb64c0b40a0b017f22e" +} diff --git a/Cargo.toml b/Cargo.toml index af7e533..8a7aca9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,12 +9,14 @@ license = "MIT" description = "Background task processing for rust using apalis and sqlite" [features] -default = ["migrate"] +default = ["migrate", "json"] migrate = ["sqlx/migrate", "sqlx/macros"] async-std-comp = ["async-std", "sqlx/runtime-async-std-rustls"] async-std-comp-native-tls = ["async-std", "sqlx/runtime-async-std-native-tls"] tokio-comp = ["tokio", "sqlx/runtime-tokio-rustls"] tokio-comp-native-tls = ["tokio", "sqlx/runtime-tokio-native-tls"] +bytes = [] +json = ["sqlx/json"] [dependencies.sqlx] version = "0.8.6" @@ -23,7 +25,7 @@ features = ["chrono", "sqlite"] [dependencies] serde = { version = "1", features = ["derive"] } -serde_json = "1" +serde_json = { version = "1" } apalis-core = { version = "1.0.0-alpha.3", default-features = false, features = [ "sleep", "json", @@ -42,7 +44,9 @@ tower-service = "0.3.3" [dev-dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } -apalis-core = { version = "1.0.0-alpha.3", features = ["test-utils"], git = "https://github.com/geofmureithi/apalis.git", branch = "chore/traits-expansion" } +apalis-core = { version = "1.0.0-alpha.3", features = [ + "test-utils", +], git = "https://github.com/geofmureithi/apalis.git", branch = "chore/traits-expansion" } apalis-sqlite = { path = ".", features = ["migrate", "tokio-comp"] } [package.metadata.docs.rs] diff --git a/migrations/bytes/20251017141210_initial_bytes_storage.sql b/migrations/bytes/20251017141210_initial_bytes_storage.sql new file mode 100644 index 0000000..97836d8 --- /dev/null +++ b/migrations/bytes/20251017141210_initial_bytes_storage.sql @@ -0,0 +1,63 @@ +CREATE TABLE IF NOT EXISTS Workers ( + id TEXT NOT NULL UNIQUE, + worker_type TEXT NOT NULL, + storage_name TEXT NOT NULL, + layers TEXT, + last_seen INTEGER NOT NULL DEFAULT (strftime('%s', 'now')) +); + +CREATE INDEX IF NOT EXISTS Idx ON Workers(id); + +CREATE INDEX IF NOT EXISTS WTIdx ON Workers(worker_type); + +CREATE INDEX IF NOT EXISTS LSIdx ON Workers(last_seen); + +CREATE TABLE IF NOT EXISTS Jobs ( + job BLOB NOT NULL, + id TEXT NOT NULL UNIQUE, + job_type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'Pending', + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 25, + run_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')), + last_result TEXT, + lock_at INTEGER, + lock_by TEXT, + done_at INTEGER, + FOREIGN KEY(lock_by) REFERENCES Workers(id) +); + +CREATE INDEX IF NOT EXISTS TIdx ON Jobs(id); + +CREATE INDEX IF NOT EXISTS SIdx ON Jobs(status); + +CREATE INDEX IF NOT EXISTS LIdx ON Jobs(lock_by); + +CREATE INDEX IF NOT EXISTS JTIdx ON Jobs(job_type); + + +ALTER TABLE Jobs +ADD priority INTEGER NOT NULL DEFAULT 0; + +CREATE INDEX IF NOT EXISTS idx_jobs_job_type_status_run_at ON Jobs(job_type, status, run_at); + +CREATE INDEX IF NOT EXISTS idx_jobs_status_run_at ON Jobs(status, run_at); + +CREATE INDEX IF NOT EXISTS idx_jobs_run_at_status ON Jobs(run_at, status); + +CREATE INDEX IF NOT EXISTS idx_jobs_completed_done_at ON Jobs(status, done_at, run_at) +WHERE + status IN ('Done', 'Failed', 'Killed') + AND done_at IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_jobs_pending ON Jobs(run_at) +WHERE + status = 'Pending'; + +CREATE INDEX IF NOT EXISTS idx_jobs_running ON Jobs(run_at) +WHERE + status = 'Running'; + +CREATE INDEX IF NOT EXISTS idx_jobs_job_type_run_at ON Jobs(job_type, run_at); + +CREATE INDEX IF NOT EXISTS idx_jobs_job_type_covering ON Jobs(job_type, status, run_at, done_at); diff --git a/migrations/bytes/20251017162501_worker_started_at.sql b/migrations/bytes/20251017162501_worker_started_at.sql new file mode 100644 index 0000000..1ef26e5 --- /dev/null +++ b/migrations/bytes/20251017162501_worker_started_at.sql @@ -0,0 +1,4 @@ +ALTER TABLE + Workers +ADD + COLUMN started_at INTEGER; diff --git a/migrations/20220530084123_jobs_workers.sql b/migrations/json/20220530084123_jobs_workers.sql similarity index 100% rename from migrations/20220530084123_jobs_workers.sql rename to migrations/json/20220530084123_jobs_workers.sql diff --git a/migrations/20250313213411_add_job_priority copy.sql b/migrations/json/20250313213411_add_job_priority.sql similarity index 100% rename from migrations/20250313213411_add_job_priority copy.sql rename to migrations/json/20250313213411_add_job_priority.sql diff --git a/migrations/20251013233016_stats_indexes.sql b/migrations/json/20251013233016_stats_indexes.sql similarity index 100% rename from migrations/20251013233016_stats_indexes.sql rename to migrations/json/20251013233016_stats_indexes.sql diff --git a/migrations/json/20251017150712_rename_last_error.sql b/migrations/json/20251017150712_rename_last_error.sql new file mode 100644 index 0000000..f04d95b --- /dev/null +++ b/migrations/json/20251017150712_rename_last_error.sql @@ -0,0 +1 @@ +ALTER TABLE Jobs RENAME COLUMN last_error TO last_result; diff --git a/migrations/json/20251017162501_worker_started_at.sql b/migrations/json/20251017162501_worker_started_at.sql new file mode 100644 index 0000000..1ef26e5 --- /dev/null +++ b/migrations/json/20251017162501_worker_started_at.sql @@ -0,0 +1,4 @@ +ALTER TABLE + Workers +ADD + COLUMN started_at INTEGER; diff --git a/queries/backend/reenqueue_orphaned.sql b/queries/backend/reenqueue_orphaned.sql index 87c4621..3e09593 100644 --- a/queries/backend/reenqueue_orphaned.sql +++ b/queries/backend/reenqueue_orphaned.sql @@ -6,7 +6,7 @@ SET lock_by = NULL, lock_at = NULL, attempts = attempts + 1, - last_error = '{"Err": "Re-enqueued due to worker heartbeat timeout."}' + last_result = '{"Err": "Re-enqueued due to worker heartbeat timeout."}' WHERE id IN ( SELECT diff --git a/queries/backend/register_worker.sql b/queries/backend/register_worker.sql index 81fe501..cfe59ab 100644 --- a/queries/backend/register_worker.sql +++ b/queries/backend/register_worker.sql @@ -1,10 +1,11 @@ INSERT INTO - Workers (id, worker_type, storage_name, layers, last_seen) + Workers (id, worker_type, storage_name, layers, last_seen, started_at) SELECT ?1, ?2, ?3, ?4, + strftime('%s', 'now'), strftime('%s', 'now') WHERE NOT EXISTS ( diff --git a/queries/task/ack.sql b/queries/task/ack.sql index 5b91344..3089f74 100644 --- a/queries/task/ack.sql +++ b/queries/task/ack.sql @@ -3,7 +3,7 @@ UPDATE SET status = ?4, attempts = ?2, - last_error = ?3, + last_result = ?3, done_at = strftime('%s', 'now') WHERE id = ?1 diff --git a/queries/task/kill.sql b/queries/task/kill.sql index 4a6d6ed..7bb6483 100644 --- a/queries/task/kill.sql +++ b/queries/task/kill.sql @@ -3,7 +3,7 @@ UPDATE SET status = 'Killed', done_at = strftime('%s', 'now'), - last_error = ?3 + last_result = ?3 WHERE id = ?1 AND lock_by = ?2 diff --git a/queries/task/update_by_id.sql b/queries/task/update_by_id.sql index f4c1ae7..15418d9 100644 --- a/queries/task/update_by_id.sql +++ b/queries/task/update_by_id.sql @@ -6,7 +6,7 @@ SET done_at = ?3, lock_by = ?4, lock_at = ?5, - last_error = ?6, + last_result = ?6, priority = ?7 WHERE id = ?8 diff --git a/src/config.rs b/src/config.rs index 7afa72e..fa8aeb2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -7,7 +7,7 @@ use apalis_core::backend::{ }; use ulid::Ulid; -use crate::{SqliteContext, SqliteStorage}; +use crate::{CompactType, SqliteContext, SqliteStorage}; #[derive(Debug, Clone)] pub struct Config { @@ -137,7 +137,7 @@ impl Config { impl ConfigExt for SqliteStorage where SqliteStorage: - Backend, + Backend, { fn get_queue(&self) -> Queue { self.config.queue.clone() diff --git a/src/fetcher.rs b/src/fetcher.rs index d7df328..02f87f1 100644 --- a/src/fetcher.rs +++ b/src/fetcher.rs @@ -8,7 +8,7 @@ use std::{ use apalis_core::{ backend::{ - codec::{Codec, json::JsonCodec}, + codec::{Codec, }, poll_strategy::{PollContext, PollStrategyExt}, }, task::Task, @@ -19,9 +19,9 @@ use pin_project::pin_project; use sqlx::{Pool, Sqlite, SqlitePool}; use ulid::Ulid; -use crate::{SqliteTask, config::Config, context::SqliteContext, from_row::TaskRow}; +use crate::{CompactType, SqliteTask, config::Config, context::SqliteContext, from_row::TaskRow}; -pub async fn fetch_next>( +pub async fn fetch_next>( pool: SqlitePool, config: Config, worker: WorkerContext, @@ -55,13 +55,14 @@ enum StreamState { } /// Dispatcher for fetching tasks from a SQLite backend via [SqlitePollFetcher] -pub struct SqliteFetcher> { +#[derive(Clone, Debug)] +pub struct SqliteFetcher { pub _marker: PhantomData<(Args, Compact, Decode)>, } /// Polling-based fetcher for retrieving tasks from a SQLite backend #[pin_project] -pub struct SqlitePollFetcher> { +pub struct SqlitePollFetcher { pool: SqlitePool, config: Config, wrk: WorkerContext, @@ -89,10 +90,10 @@ impl Clone for SqlitePollFetcher { } } -impl SqlitePollFetcher { +impl SqlitePollFetcher { pub fn new(pool: &Pool, config: &Config, wrk: &WorkerContext) -> Self where - Decode: Codec + 'static, + Decode: Codec + 'static, Decode::Error: std::error::Error + Send + Sync + 'static, { Self { @@ -107,11 +108,11 @@ impl SqlitePollFetcher { } } -impl Stream for SqlitePollFetcher +impl Stream for SqlitePollFetcher where Decode::Error: std::error::Error + Send + Sync + 'static, Args: Send + 'static + Unpin, - Decode: Codec + 'static, + Decode: Codec + 'static, // Compact: Unpin + Send + 'static, { type Item = Result>, sqlx::Error>; diff --git a/src/from_row.rs b/src/from_row.rs index b10a362..a08d351 100644 --- a/src/from_row.rs +++ b/src/from_row.rs @@ -5,18 +5,18 @@ use apalis_core::{ task::{attempt::Attempt, builder::TaskBuilder, status::Status, task_id::TaskId}, }; -use crate::{SqliteTask, context::SqliteContext}; +use crate::{CompactType, SqliteTask, context::SqliteContext}; #[derive(Debug)] pub(crate) struct TaskRow { - pub(crate) job: String, + pub(crate) job: CompactType, pub(crate) id: Option, pub(crate) job_type: Option, pub(crate) status: Option, pub(crate) attempts: Option, pub(crate) max_attempts: Option, pub(crate) run_at: Option, - pub(crate) last_error: Option, + pub(crate) last_result: Option, pub(crate) lock_at: Option, pub(crate) lock_by: Option, pub(crate) done_at: Option, @@ -25,7 +25,7 @@ pub(crate) struct TaskRow { } impl TaskRow { - pub fn try_into_task, Args>( + pub fn try_into_task, Args>( self, ) -> Result, sqlx::Error> where @@ -35,7 +35,7 @@ impl TaskRow { .with_done_at(self.done_at) .with_lock_by(self.lock_by) .with_max_attempts(self.max_attempts.unwrap_or(25) as i32) - .with_last_result(self.last_error) + .with_last_result(self.last_result) .with_priority(self.priority.unwrap_or(0) as i32) .with_queue( self.job_type @@ -71,12 +71,12 @@ impl TaskRow { ); Ok(task.build()) } - pub fn try_into_task_compact(self) -> Result, sqlx::Error> { + pub fn try_into_task_compact(self) -> Result, sqlx::Error> { let ctx = SqliteContext::default() .with_done_at(self.done_at) .with_lock_by(self.lock_by) .with_max_attempts(self.max_attempts.unwrap_or(25) as i32) - .with_last_result(self.last_error) + .with_last_result(self.last_result) .with_priority(self.priority.unwrap_or(0) as i32) .with_queue( self.job_type diff --git a/src/lib.rs b/src/lib.rs index 617c05d..1942bc8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -140,8 +140,7 @@ use std::{fmt, marker::PhantomData}; use apalis_core::{ backend::{ - Backend, TaskStream, - codec::{Codec, json::JsonCodec}, + codec::{json::JsonCodec, Codec }, Backend, TaskStream }, task::Task, worker::{context::WorkerContext, ext::ack::AcknowledgeLayer}, @@ -186,19 +185,23 @@ pub use context::SqliteContext; pub use shared::{SharedPostgresError, SharedSqliteStorage}; pub use sqlx::SqlitePool; -type DefaultFetcher = SqliteFetcher>; +#[cfg(feature = "json")] +pub type CompactType = serde_json::Value; + +#[cfg(feature = "bytes")] +pub type CompactType = Vec; const INSERT_OPERATION: &str = "INSERT"; const JOBS_TABLE: &str = "Jobs"; #[pin_project::pin_project] -pub struct SqliteStorage, Fetcher = DefaultFetcher> { +pub struct SqliteStorage { pool: Pool, job_type: PhantomData, config: Config, codec: PhantomData, #[pin] - sink: SqliteSink, + sink: SqliteSink, #[pin] fetcher: Fetcher, } @@ -227,7 +230,7 @@ impl Clone for SqliteStorage { } } -impl SqliteStorage<()> { +impl SqliteStorage<(), (), ()> { /// Perform migrations for storage #[cfg(feature = "migrate")] pub async fn setup(pool: &Pool) -> Result<(), sqlx::Error> { @@ -248,15 +251,29 @@ impl SqliteStorage<()> { /// Get sqlite migrations without running them #[cfg(feature = "migrate")] pub fn migrations() -> sqlx::migrate::Migrator { - sqlx::migrate!("./migrations") + if cfg!(feature = "bytes") && cfg!(not(feature = "json")) { + sqlx::migrate!("./migrations/bytes") + } else if cfg!(feature = "json") && cfg!(not(feature = "bytes")) { + sqlx::migrate!("./migrations/json") + } else { + panic!( + "One of either the 'json' or 'bytes' feature must be enabled for migrations to work." + ); + } } } -impl SqliteStorage { +impl SqliteStorage { /// Create a new SqliteStorage - pub fn new(pool: &Pool) -> Self { + pub fn new( + pool: &Pool, + ) -> SqliteStorage< + T, + JsonCodec, + fetcher::SqliteFetcher>, + > { let config = Config::new(std::any::type_name::()); - Self { + SqliteStorage { pool: pool.clone(), job_type: PhantomData, sink: SqliteSink::new(pool, &config), @@ -268,7 +285,10 @@ impl SqliteStorage { } } - pub fn new_with_codec(pool: &Pool, config: &Config) -> SqliteStorage { + pub fn new_with_codec( + pool: &Pool, + config: &Config, + ) -> SqliteStorage> { SqliteStorage { pool: pool.clone(), job_type: PhantomData, @@ -281,8 +301,15 @@ impl SqliteStorage { } } - pub fn new_with_config(pool: &Pool, config: &Config) -> Self { - Self { + pub fn new_with_config( + pool: &Pool, + config: &Config, + ) -> SqliteStorage< + T, + JsonCodec, + fetcher::SqliteFetcher>, + > { + SqliteStorage { pool: pool.clone(), job_type: PhantomData, config: config.clone(), @@ -297,7 +324,21 @@ impl SqliteStorage { pub fn new_with_callback( pool: &Pool, config: &Config, - ) -> SqliteStorage, HookCallbackListener> { + ) -> SqliteStorage, HookCallbackListener> { + SqliteStorage { + pool: pool.clone(), + job_type: PhantomData, + config: config.clone(), + codec: PhantomData, + sink: SqliteSink::new(&pool, config), + fetcher: HookCallbackListener, + } + } + + pub fn new_with_codec_callback( + pool: &Pool, + config: &Config, + ) -> SqliteStorage { SqliteStorage { pool: pool.clone(), job_type: PhantomData, @@ -315,10 +356,10 @@ impl SqliteStorage { } } -impl Backend for SqliteStorage +impl Backend for SqliteStorage> where Args: Send + 'static + Unpin, - Decode: Codec + 'static + Send, + Decode: Codec + 'static + Send, Decode::Error: std::error::Error + Send + Sync + 'static, { type Args = Args; @@ -328,7 +369,7 @@ where type Codec = Decode; - type Compact = String; + type Compact = CompactType; type Error = sqlx::Error; @@ -367,7 +408,7 @@ where ); let register = stream::once(fut.map(|_| Ok(None))); register - .chain(SqlitePollFetcher::::new( + .chain(SqlitePollFetcher::::new( &self.pool, &self.config, worker, @@ -379,7 +420,7 @@ where impl Backend for SqliteStorage where Args: Send + 'static + Unpin, - Decode: Codec + Send + 'static, + Decode: Codec + Send + 'static, Decode::Error: std::error::Error + Send + Sync + 'static, { type Args = Args; @@ -389,7 +430,7 @@ where type Codec = Decode; - type Compact = String; + type Compact = CompactType; type Error = sqlx::Error; @@ -454,7 +495,7 @@ where }) .map(|_| Ok(None)), ); - let eager_fetcher: SqlitePollFetcher = + let eager_fetcher: SqlitePollFetcher = SqlitePollFetcher::new(&self.pool, &self.config, &worker); let lazy_fetcher = listener .filter(|a| ready(a.operation() == INSERT_OPERATION && a.table_name() == JOBS_TABLE)) diff --git a/src/queries/fetch_by_id.rs b/src/queries/fetch_by_id.rs index ad488ba..04d5658 100644 --- a/src/queries/fetch_by_id.rs +++ b/src/queries/fetch_by_id.rs @@ -1,17 +1,17 @@ use apalis_core::{ - backend::{Backend, FetchById, codec::json::JsonCodec}, + backend::{Backend, FetchById, codec::Codec}, task::task_id::TaskId, }; -use serde::{Serialize, de::DeserializeOwned}; use ulid::Ulid; -use crate::{SqliteContext, SqliteStorage, SqliteTask, from_row::TaskRow}; +use crate::{CompactType, SqliteContext, SqliteStorage, SqliteTask, from_row::TaskRow}; impl FetchById for SqliteStorage where SqliteStorage: - Backend, - Args: Serialize + DeserializeOwned, + Backend, + D: Codec, + D::Error: std::error::Error + Send + Sync + 'static, { fn fetch_by_id( &mut self, @@ -23,7 +23,7 @@ where let task = sqlx::query_file_as!(TaskRow, "queries/task/find_by_id.sql", id) .fetch_optional(&pool) .await? - .map(|r| r.try_into_task::, _>()) + .map(|r| r.try_into_task::()) .transpose()?; Ok(task) } diff --git a/src/queries/keep_alive.rs b/src/queries/keep_alive.rs index 39aa027..9c7218b 100644 --- a/src/queries/keep_alive.rs +++ b/src/queries/keep_alive.rs @@ -17,7 +17,7 @@ pub async fn keep_alive( let res = sqlx::query_file!("queries/backend/keep_alive.sql", worker, queue) .execute(&pool) .await?; - if res.rows_affected() == 1 { + if res.rows_affected() == 0 { return Err(sqlx::Error::Io(std::io::Error::new( std::io::ErrorKind::NotFound, "WORKER_DOES_NOT_EXIST", diff --git a/src/queries/list_queues.rs b/src/queries/list_queues.rs index a21dbab..2be5499 100644 --- a/src/queries/list_queues.rs +++ b/src/queries/list_queues.rs @@ -1,7 +1,7 @@ use apalis_core::backend::{Backend, ListQueues, QueueInfo}; use ulid::Ulid; -use crate::{SqliteContext, SqliteStorage}; +use crate::{CompactType, SqliteContext, SqliteStorage}; struct QueueInfoRow { name: String, @@ -24,7 +24,7 @@ impl From for QueueInfo { impl ListQueues for SqliteStorage where SqliteStorage: - Backend, + Backend, { fn list_queues(&self) -> impl Future, Self::Error>> + Send { let pool = self.pool.clone(); diff --git a/src/queries/list_tasks.rs b/src/queries/list_tasks.rs index ffa3951..d86843b 100644 --- a/src/queries/list_tasks.rs +++ b/src/queries/list_tasks.rs @@ -1,17 +1,17 @@ use apalis_core::{ - backend::{Backend, Filter, ListAllTasks, ListTasks, codec::json::JsonCodec}, + backend::{Backend, Filter, ListAllTasks, ListTasks, codec::Codec}, task::{Task, status::Status}, }; -use serde::{Serialize, de::DeserializeOwned}; use ulid::Ulid; -use crate::{SqliteContext, SqliteStorage, SqliteTask, from_row::TaskRow}; +use crate::{CompactType, SqliteContext, SqliteStorage, SqliteTask, from_row::TaskRow}; impl ListTasks for SqliteStorage where SqliteStorage: - Backend, - Args: Serialize + DeserializeOwned, + Backend, + D: Codec, + D::Error: std::error::Error + Send + Sync + 'static, { fn list_tasks( &self, @@ -39,7 +39,7 @@ where .fetch_all(&pool) .await? .into_iter() - .map(|r| r.try_into_task::, _>()) + .map(|r| r.try_into_task::()) .collect::, _>>()?; Ok(tasks) } @@ -49,7 +49,7 @@ where impl ListAllTasks for SqliteStorage where SqliteStorage: - Backend, + Backend, { fn list_all_tasks( &self, diff --git a/src/queries/list_workers.rs b/src/queries/list_workers.rs index a5b6d1e..9dcedec 100644 --- a/src/queries/list_workers.rs +++ b/src/queries/list_workers.rs @@ -2,7 +2,7 @@ use apalis_core::backend::{Backend, ListWorkers, RunningWorker}; use futures::TryFutureExt; use ulid::Ulid; -use crate::{SqliteContext, SqliteStorage}; +use crate::{CompactType, SqliteContext, SqliteStorage}; struct Worker { id: String, @@ -10,12 +10,13 @@ struct Worker { storage_name: String, layers: Option, last_seen: i64, + started_at: Option, } impl ListWorkers for SqliteStorage where SqliteStorage: - Backend, + Backend, { fn list_workers( &self, @@ -39,7 +40,7 @@ where .map(|w| RunningWorker { id: w.id, backend: w.storage_name, - started_at: 0, + started_at: w.started_at.unwrap_or_default() as u64, last_heartbeat: w.last_seen as u64, layers: w.layers.unwrap_or_default(), queue: w.worker_type, @@ -70,7 +71,7 @@ where .map(|w| RunningWorker { id: w.id, backend: w.storage_name, - started_at: 0, + started_at: w.started_at.unwrap_or_default() as u64, last_heartbeat: w.last_seen as u64, layers: w.layers.unwrap_or_default(), queue: w.worker_type, diff --git a/src/queries/metrics.rs b/src/queries/metrics.rs index aa1515f..ec31493 100644 --- a/src/queries/metrics.rs +++ b/src/queries/metrics.rs @@ -1,7 +1,7 @@ use apalis_core::backend::{Backend, Metrics, Statistic}; use ulid::Ulid; -use crate::{SqliteContext, SqliteStorage}; +use crate::{CompactType, SqliteContext, SqliteStorage}; struct StatisticRow { /// The priority of the statistic (lower number means higher priority) @@ -17,7 +17,7 @@ struct StatisticRow { impl Metrics for SqliteStorage where SqliteStorage: - Backend, + Backend, { fn global(&self) -> impl Future, Self::Error>> + Send { let pool = self.pool.clone(); diff --git a/src/shared.rs b/src/shared.rs index 68f4000..7382f48 100644 --- a/src/shared.rs +++ b/src/shared.rs @@ -10,7 +10,7 @@ use std::{ }; use crate::{ - Config, INSERT_OPERATION, JOBS_TABLE, SqliteStorage, SqliteTask, + CompactType, Config, INSERT_OPERATION, JOBS_TABLE, SqliteStorage, SqliteTask, ack::{LockTaskLayer, SqliteAck}, callback::{DbEvent, update_hook_callback}, context::SqliteContext, @@ -19,11 +19,7 @@ use crate::{ }; use crate::{from_row::TaskRow, sink::SqliteSink}; use apalis_core::{ - backend::{ - Backend, TaskStream, - codec::{Codec, json::JsonCodec}, - shared::MakeShared, - }, + backend::{Backend, TaskStream, codec::Codec, shared::MakeShared}, worker::{context::WorkerContext, ext::ack::AcknowledgeLayer}, }; use futures::{ @@ -38,15 +34,16 @@ use sqlx::SqlitePool; use tower_layer::Stack; use ulid::Ulid; -pub struct SharedSqliteStorage { +pub struct SharedSqliteStorage { pool: SqlitePool, - registry: Arc>>>>, + registry: Arc>>>>, drive: Shared>, + _marker: PhantomData, } -impl SharedSqliteStorage { +impl> SharedSqliteStorage { pub fn new(pool: SqlitePool) -> Self { - let registry: Arc>>>> = + let registry: Arc>>>> = Arc::new(Mutex::new(HashMap::default())); let p = pool.clone(); let instances = registry.clone(); @@ -93,7 +90,7 @@ impl SharedSqliteStorage { buffer_size, ) .fetch(&mut *tx) - .map(|r| r?.try_into_task::, String>()) + .map(|r| r?.try_into_task_compact()) .try_collect() .await?; tx.commit().await?; @@ -126,6 +123,7 @@ impl SharedSqliteStorage { .boxed() .shared(), registry, + _marker: PhantomData, } } } @@ -137,8 +135,10 @@ pub enum SharedPostgresError { RegistryLocked, } -impl MakeShared for SharedSqliteStorage { - type Backend = SqliteStorage, SharedFetcher>; +impl> MakeShared + for SharedSqliteStorage +{ + type Backend = SqliteStorage>; type Config = Config; type MakeError = SharedPostgresError; fn make_shared(&mut self) -> Result @@ -176,7 +176,7 @@ impl MakeShared for SharedSqliteStorage { } } -pub struct SharedFetcher { +pub struct SharedFetcher { poller: Shared>, receiver: Receiver>, } @@ -194,10 +194,10 @@ impl Stream for SharedFetcher { } } -impl Backend for SqliteStorage +impl Backend for SqliteStorage> where Args: Send + 'static + Unpin + Sync, - Decode: Codec + 'static + Unpin + Send + Sync, + Decode: Codec + 'static + Unpin + Send + Sync, Decode::Error: std::error::Error + Send + Sync + 'static, { type Args = Args; @@ -212,7 +212,7 @@ where type Codec = Decode; - type Compact = String; + type Compact = CompactType; type Context = SqliteContext; @@ -266,7 +266,7 @@ where }) .boxed(); - let eager_fetcher = StreamExt::boxed(SqlitePollFetcher::::new( + let eager_fetcher = StreamExt::boxed(SqlitePollFetcher::::new( &self.pool, &self.config, &worker, diff --git a/src/sink.rs b/src/sink.rs index 590c3f5..95453ff 100644 --- a/src/sink.rs +++ b/src/sink.rs @@ -4,10 +4,7 @@ use std::{ task::{Context, Poll}, }; -use apalis_core::{ - backend::codec::{Codec, json::JsonCodec}, - error::BoxDynError, -}; +use apalis_core::backend::codec::Codec; use futures::{ FutureExt, Sink, future::{BoxFuture, Shared}, @@ -15,12 +12,12 @@ use futures::{ use sqlx::SqlitePool; use ulid::Ulid; -use crate::{SqliteStorage, SqliteTask, config::Config}; +use crate::{CompactType, SqliteStorage, SqliteTask, config::Config}; type FlushFuture = BoxFuture<'static, Result<(), Arc>>; #[pin_project::pin_project] -pub struct SqliteSink> { +pub struct SqliteSink { pool: SqlitePool, config: Config, buffer: Vec>, @@ -44,10 +41,9 @@ impl Clone for SqliteSink { pub async fn push_tasks( pool: SqlitePool, cfg: Config, - buffer: Vec>, + buffer: Vec>, ) -> Result<(), Arc> { let mut tx = pool.begin().await?; - for task in buffer { let id = task .parts @@ -95,9 +91,8 @@ impl SqliteSink { impl Sink> for SqliteStorage where Args: Send + Sync + 'static, - Encode: Codec, + Encode: Codec, Encode::Error: std::error::Error + Send + Sync + 'static, - Encode::Error: Into, { type Error = sqlx::Error; diff --git a/src/sqlite_old.rs b/src/sqlite_old.rs deleted file mode 100644 index d3f6cf3..0000000 --- a/src/sqlite_old.rs +++ /dev/null @@ -1,897 +0,0 @@ -use crate::context::SqlContext; -use crate::{calculate_status, Config, SqlError}; -use apalis_core::backend::{BackendExpose, Stat, WorkerState}; -use apalis_core::codec::json::JsonCodec; -use apalis_core::error::Error; -use apalis_core::layers::{Ack, AckLayer}; -use apalis_core::poller::controller::Controller; -use apalis_core::poller::stream::BackendStream; -use apalis_core::poller::Poller; -use apalis_core::response::Response; -use apalis_core::storage::Storage; -use apalis_core::task::namespace::Namespace; -use apalis_core::task::task_id::TaskId; -use apalis_core::task::{Metadata, RequestStream, State, Task}; -use apalis_core::worker::{Event, SimpleWorker, WorkerContext, WorkerId}; -use apalis_core::{backend::Backend, codec::Codec}; -use async_stream::try_stream; -use chrono::{DateTime, Utc}; -use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; -use log::error; -use serde::{de::DeserializeOwned, Serialize}; -use sqlx::{Pool, Row, Sqlite}; -use std::any::type_name; -use std::convert::TryInto; -use std::fmt::Debug; -use std::sync::Arc; -use std::{fmt, io}; -use std::{marker::PhantomData, time::Duration}; - -use crate::from_row::SqlRequest; - -pub use sqlx::sqlite::SqlitePool; - -/// Represents a [Storage] that persists to Sqlite -// #[derive(Debug)] -pub struct SqliteStorage> { - pool: Pool, - job_type: PhantomData, - controller: Controller, - config: Config, - codec: PhantomData, -} - -impl fmt::Debug for SqliteStorage { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SqliteStorage") - .field("pool", &self.pool) - .field("job_type", &"PhantomData") - .field("controller", &self.controller) - .field("config", &self.config) - .field("codec", &std::any::type_name::()) - .finish() - } -} - -impl Clone for SqliteStorage { - fn clone(&self) -> Self { - SqliteStorage { - pool: self.pool.clone(), - job_type: PhantomData, - controller: self.controller.clone(), - config: self.config.clone(), - codec: self.codec, - } - } -} - -impl SqliteStorage<()> { - /// Perform migrations for storage - #[cfg(feature = "migrate")] - pub async fn setup(pool: &Pool) -> Result<(), sqlx::Error> { - sqlx::query("PRAGMA journal_mode = 'WAL';") - .execute(pool) - .await?; - sqlx::query("PRAGMA temp_store = 2;").execute(pool).await?; - sqlx::query("PRAGMA synchronous = NORMAL;") - .execute(pool) - .await?; - sqlx::query("PRAGMA cache_size = 64000;") - .execute(pool) - .await?; - Self::migrations().run(pool).await?; - Ok(()) - } - - /// Get sqlite migrations without running them - #[cfg(feature = "migrate")] - pub fn migrations() -> sqlx::migrate::Migrator { - sqlx::migrate!("migrations/sqlite") - } -} - -impl SqliteStorage { - /// Create a new instance - pub fn new(pool: SqlitePool) -> Self { - Self { - pool, - job_type: PhantomData, - controller: Controller::new(), - config: Config::new(type_name::()), - codec: PhantomData, - } - } - - /// Create a new instance with a custom config - pub fn new_with_config(pool: SqlitePool, config: Config) -> Self { - Self { - pool, - job_type: PhantomData, - controller: Controller::new(), - config, - codec: PhantomData, - } - } -} -impl SqliteStorage { - /// Keeps a storage notified that the worker is still alive manually - pub async fn keep_alive_at( - &mut self, - worker: &WorkerContext, - last_seen: i64, - ) -> Result<(), sqlx::Error> { - let worker_type = self.config.namespace.clone(); - let storage_name = std::any::type_name::(); - let query = "INSERT INTO Workers (id, worker_type, storage_name, layers, last_seen) - VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (id) DO - UPDATE SET last_seen = EXCLUDED.last_seen"; - sqlx::query(query) - .bind(worker.id().to_string()) - .bind(worker_type) - .bind(storage_name) - .bind(worker.get_service()) - .bind(last_seen) - .execute(&self.pool) - .await?; - Ok(()) - } - - /// Expose the pool for other functionality, eg custom migrations - pub fn pool(&self) -> &Pool { - &self.pool - } - - /// Get the config used by the storage - pub fn get_config(&self) -> &Config { - &self.config - } -} - -impl SqliteStorage { - /// Expose the code used - pub fn codec(&self) -> &PhantomData { - &self.codec - } -} - -async fn fetch_next( - pool: &Pool, - worker_id: &WorkerId, - id: String, - config: &Config, -) -> Result>, sqlx::Error> { - let now: i64 = Utc::now().timestamp(); - let update_query = "UPDATE Jobs SET status = 'Running', lock_by = ?2, lock_at = ?3 WHERE id = ?1 AND job_type = ?4 AND status = 'Pending' AND lock_by IS NULL RETURNING *"; - let job: Option> = sqlx::query_as(update_query) - .bind(id.to_string()) - .bind(worker_id.to_string()) - .bind(now) - .bind(config.namespace.clone()) - .fetch_optional(pool) - .await?; - - if job.is_none() { - // Retry path. - let job: Option> = - sqlx::query_as("SELECT * FROM Jobs WHERE id = ?1 AND lock_by = ?2 AND job_type = ?3") - .bind(id.to_string()) - .bind(worker_id.to_string()) - .bind(config.namespace.clone()) - .fetch_optional(pool) - .await?; - - return Ok(job); - } - - Ok(job) -} - -impl SqliteStorage -where - T: DeserializeOwned + Send + Unpin, - C: Codec, -{ - fn stream_jobs( - &self, - worker: &WorkerContext, - interval: Duration, - buffer_size: usize, - ) -> impl Stream>, sqlx::Error>> { - let pool = self.pool.clone(); - let worker = worker.clone(); - let config = self.config.clone(); - let namespace = Namespace(self.config.namespace.clone()); - try_stream! { - loop { - apalis_core::sleep(interval).await; - if !worker.is_ready() { - continue; - } - let worker_id = worker.id(); - let job_type = &config.namespace; - let fetch_query = "SELECT id FROM Jobs - WHERE (status = 'Pending' OR (status = 'Failed' AND attempts < max_attempts)) AND run_at < ?1 AND job_type = ?2 ORDER BY priority DESC LIMIT ?3"; - let now: i64 = Utc::now().timestamp(); - let ids: Vec<(String,)> = sqlx::query_as(fetch_query) - .bind(now) - .bind(job_type) - .bind(i64::try_from(buffer_size).map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?) - .fetch_all(&pool) - .await?; - - if ids.is_empty() { - yield None::>; - } else { - for id in ids { - let res = fetch_next(&pool, worker_id, id.0, &config).await?; - yield match res { - None => None::>, - Some(job) => { - let (req, parts) = job.req.take_parts(); - let args = C::decode(req) - .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; - let mut req = Request::new_with_parts(args, parts); - req.parts.namespace = Some(namespace.clone()); - Some(req) - } - }; - - } - } - } - } - } -} - -impl Storage for SqliteStorage -where - T: Serialize + DeserializeOwned + Send + 'static + Unpin + Sync, - C: Codec + Send + 'static + Sync, - C::Error: std::error::Error + Send + Sync + 'static, -{ - type Job = T; - - type Error = sqlx::Error; - - type Context = SqlContext; - - type Compact = String; - - async fn push_request( - &mut self, - job: Task, - ) -> Result, Self::Error> { - let query = "INSERT INTO Jobs VALUES (?1, ?2, ?3, 'Pending', 0, ?4, strftime('%s','now'), NULL, NULL, NULL, NULL, ?5)"; - let (task, parts) = job.take(); - let raw = C::encode(&task) - .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; - let job_type = self.config.namespace.clone(); - sqlx::query(query) - .bind(raw) - .bind(parts.task_id.to_string()) - .bind(job_type.to_string()) - .bind(parts.context.max_attempts()) - .bind(parts.context.priority()) - .execute(&self.pool) - .await?; - Ok(parts) - } - - async fn push_raw_request( - &mut self, - job: Task, - ) -> Result, Self::Error> { - let query = "INSERT INTO Jobs VALUES (?1, ?2, ?3, 'Pending', 0, ?4, strftime('%s','now'), NULL, NULL, NULL, NULL, ?5)"; - let (task, parts) = job.take(); - let raw = C::encode(&task) - .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; - let job_type = self.config.namespace.clone(); - sqlx::query(query) - .bind(raw) - .bind(parts.task_id.to_string()) - .bind(job_type.to_string()) - .bind(parts.context.max_attempts()) - .bind(parts.context.priority()) - .execute(&self.pool) - .await?; - Ok(parts) - } - - async fn schedule_request( - &mut self, - req: Task, - on: i64, - ) -> Result, Self::Error> { - let query = - "INSERT INTO Jobs VALUES (?1, ?2, ?3, 'Pending', 0, ?4, ?5, NULL, NULL, NULL, NULL, ?6)"; - let id = &req.meta.task_id; - let job = C::encode(&req.args) - .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; - let job_type = self.config.namespace.clone(); - sqlx::query(query) - .bind(job) - .bind(id.to_string()) - .bind(job_type) - .bind(req.meta.context.max_attempts()) - .bind(on) - .bind(req.meta.context.priority()) - .execute(&self.pool) - .await?; - Ok(req.meta) - } - - async fn fetch_by_id( - &mut self, - job_id: &TaskId, - ) -> Result>, Self::Error> { - let fetch_query = "SELECT * FROM Jobs WHERE id = ?1"; - let res: Option> = sqlx::query_as(fetch_query) - .bind(job_id.to_string()) - .fetch_optional(&self.pool) - .await?; - match res { - None => Ok(None), - Some(job) => Ok(Some({ - let (req, parts) = job.req.take(); - let args = C::decode(req) - .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; - - let mut req: Task = Task::new_with_parts(args, parts); - req.meta.namespace = Some(Namespace(self.config.namespace.clone())); - req - })), - } - } - - async fn len(&mut self) -> Result { - let query = "Select COUNT(*) AS count FROM Jobs WHERE (status = 'Pending' OR (status = 'Failed' AND attempts < max_attempts))"; - let record = sqlx::query(query).fetch_one(&self.pool).await?; - record.try_get("count") - } - - async fn reschedule( - &mut self, - job: Task, - wait: Duration, - ) -> Result<(), Self::Error> { - let task_id = job.meta.task_id; - - let wait: i64 = wait - .as_secs() - .try_into() - .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; - - let query = - "UPDATE Jobs SET status = 'Failed', done_at = NULL, lock_by = NULL, lock_at = NULL, run_at = ?2 WHERE id = ?1"; - let now: i64 = Utc::now().timestamp(); - let wait_until = now + wait; - - sqlx::query(query) - .bind(task_id.to_string()) - .bind(wait_until) - .execute(self.pool()) - .await?; - Ok(()) - } - - async fn update(&mut self, job: Task) -> Result<(), Self::Error> { - let ctx = job.meta.context; - let status = ctx.status().to_string(); - let attempts = job.meta.attempt; - let done_at = *ctx.done_at(); - let lock_by = ctx.lock_by().clone(); - let lock_at = *ctx.lock_at(); - let last_error = ctx.last_error().clone(); - let priority = *ctx.priority(); - let job_id = job.meta.task_id; - let query = - "UPDATE Jobs SET status = ?1, attempts = ?2, done_at = ?3, lock_by = ?4, lock_at = ?5, last_error = ?6, priority = ?7 WHERE id = ?8"; - sqlx::query(query) - .bind(status.to_owned()) - .bind::( - attempts - .current() - .try_into() - .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?, - ) - .bind(done_at) - .bind(lock_by.map(|w| w.name().to_string())) - .bind(lock_at) - .bind(last_error) - .bind(priority) - .bind(job_id.to_string()) - .execute(self.pool()) - .await?; - Ok(()) - } - - async fn is_empty(&mut self) -> Result { - self.len().map_ok(|c| c == 0).await - } - - async fn vacuum(&mut self) -> Result { - let query = "Delete from Jobs where status='Done'"; - let record = sqlx::query(query).execute(&self.pool).await?; - Ok(record.rows_affected().try_into().unwrap_or_default()) - } -} - -impl SqliteStorage { - /// Puts the job instantly back into the queue - /// Another Worker may consume - pub async fn retry( - &mut self, - worker_id: &WorkerId, - job_id: &TaskId, - ) -> Result<(), sqlx::Error> { - let query = - "UPDATE Jobs SET status = 'Pending', done_at = NULL, lock_by = NULL WHERE id = ?1 AND lock_by = ?2"; - sqlx::query(query) - .bind(job_id.to_string()) - .bind(worker_id.to_string()) - .execute(self.pool()) - .await?; - Ok(()) - } - - /// Kill a job - pub async fn kill(&mut self, worker_id: &WorkerId, job_id: &TaskId) -> Result<(), sqlx::Error> { - let query = - "UPDATE Jobs SET status = 'Killed', done_at = strftime('%s','now') WHERE id = ?1 AND lock_by = ?2"; - sqlx::query(query) - .bind(job_id.to_string()) - .bind(worker_id.to_string()) - .execute(self.pool()) - .await?; - Ok(()) - } - - /// Add jobs that workers have disappeared to the queue - pub async fn reenqueue_orphaned( - &self, - count: i32, - dead_since: DateTime, - ) -> Result<(), sqlx::Error> { - let job_type = self.config.namespace.clone(); - let query = r#"Update Jobs - SET status = "Pending", done_at = NULL, lock_by = NULL, lock_at = NULL, attempts = attempts + 1, last_error ="Job was abandoned" - WHERE id in - (SELECT Jobs.id from Jobs INNER join Workers ON lock_by = Workers.id - WHERE status= "Running" AND workers.last_seen < ?1 - AND Workers.worker_type = ?2 ORDER BY lock_at ASC LIMIT ?3);"#; - - sqlx::query(query) - .bind(dead_since.timestamp()) - .bind(job_type) - .bind(count) - .execute(self.pool()) - .await?; - Ok(()) - } -} - -/// Errors that can occur while polling an SQLite database. -#[derive(thiserror::Error, Debug)] -pub enum SqlitePollError { - /// Error during a keep-alive heartbeat. - #[error("Encountered an error during KeepAlive heartbeat: `{0}`")] - KeepAliveError(sqlx::Error), - - /// Error during re-enqueuing orphaned tasks. - #[error("Encountered an error during ReenqueueOrphaned heartbeat: `{0}`")] - ReenqueueOrphanedError(sqlx::Error), -} - -impl Backend> for SqliteStorage -where - C: Codec + Send + 'static + Sync, - C::Error: std::error::Error + 'static + Send + Sync, - T: Serialize + DeserializeOwned + Sync + Send + Unpin + 'static, -{ - type Stream = BackendStream>>; - type Layer = AckLayer, T, SqlContext, C>; - - type Codec = JsonCodec; - - fn poll(mut self, worker: &WorkerContext) -> Poller { - let layer = AckLayer::new(self.clone()); - let config = self.config.clone(); - let controller = self.controller.clone(); - let stream = self - .stream_jobs(worker, config.poll_interval, config.buffer_size) - .map_err(|e| Error::SourceError(Arc::new(Box::new(e)))); - let stream = BackendStream::new(stream.boxed(), controller); - let requeue_storage = self.clone(); - let w = worker.clone(); - let heartbeat = async move { - // Lets reenqueue any jobs that belonged to this worker in case of a death - if let Err(e) = self - .reenqueue_orphaned((config.buffer_size * 10) as i32, Utc::now()) - .await - { - w.emit(Event::Error(Box::new( - SqlitePollError::ReenqueueOrphanedError(e), - ))); - } - loop { - let now: i64 = Utc::now().timestamp(); - if let Err(e) = self.keep_alive_at(&w, now).await { - w.emit(Event::Error(Box::new(SqlitePollError::KeepAliveError(e)))); - } - apalis_core::sleep(Duration::from_secs(30)).await; - } - } - .boxed(); - let w = worker.clone(); - let reenqueue_beat = async move { - loop { - let dead_since = Utc::now() - - chrono::Duration::from_std(config.reenqueue_orphaned_after).unwrap(); - if let Err(e) = requeue_storage - .reenqueue_orphaned( - config - .buffer_size - .try_into() - .expect("could not convert usize to i32"), - dead_since, - ) - .await - { - w.emit(Event::Error(Box::new( - SqlitePollError::ReenqueueOrphanedError(e), - ))); - } - apalis_core::sleep(config.poll_interval).await; - } - }; - Poller::new_with_layer( - stream, - async { - futures::join!(heartbeat, reenqueue_beat); - }, - layer, - ) - } -} - -impl Ack for SqliteStorage { - type Context = SqlContext; - type AckError = sqlx::Error; - async fn ack(&mut self, ctx: &Self::Context, res: &Response) -> Result<(), sqlx::Error> { - let pool = self.pool.clone(); - let query = - "UPDATE Jobs SET status = ?4, attempts = ?5, done_at = strftime('%s','now'), last_error = ?3 WHERE id = ?1 AND lock_by = ?2"; - let result = serde_json::to_string(&res.inner.as_ref().map_err(|r| r.to_string())) - .map_err(|e| sqlx::Error::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; - sqlx::query(query) - .bind(res.task_id.to_string()) - .bind( - ctx.lock_by() - .as_ref() - .expect("Task is not locked") - .to_string(), - ) - .bind(result) - .bind(calculate_status(ctx, res).to_string()) - .bind(res.attempt.current() as u32) - .execute(&pool) - .await?; - Ok(()) - } -} - -impl BackendExpose - for SqliteStorage> -{ - type Request = Task>; - type Error = SqlError; - async fn stats(&self) -> Result { - let fetch_query = "SELECT - COUNT(1) FILTER (WHERE status = 'Pending') AS pending, - COUNT(1) FILTER (WHERE status = 'Running') AS running, - COUNT(1) FILTER (WHERE status = 'Done') AS done, - COUNT(1) FILTER (WHERE status = 'Failed') AS failed, - COUNT(1) FILTER (WHERE status = 'Killed') AS killed - FROM Jobs WHERE job_type = ?"; - - let res: (i64, i64, i64, i64, i64) = sqlx::query_as(fetch_query) - .bind(self.get_config().namespace()) - .fetch_one(self.pool()) - .await?; - - Ok(Stat { - pending: res.0.try_into()?, - running: res.1.try_into()?, - dead: res.4.try_into()?, - failed: res.3.try_into()?, - success: res.2.try_into()?, - }) - } - - async fn list_jobs( - &self, - status: &State, - page: i32, - ) -> Result, Self::Error> { - let status = status.to_string(); - let fetch_query = "SELECT * FROM Jobs WHERE status = ? AND job_type = ? ORDER BY done_at DESC, run_at DESC LIMIT 10 OFFSET ?"; - let res: Vec> = sqlx::query_as(fetch_query) - .bind(status) - .bind(self.get_config().namespace()) - .bind(((page - 1) * 10).to_string()) - .fetch_all(self.pool()) - .await?; - Ok(res - .into_iter() - .map(|j| { - let (req, ctx) = j.req.take(); - let req = JsonCodec::::decode(req).unwrap(); - Task::new_with_ctx(req, ctx) - }) - .collect()) - } - - async fn list_workers(&self) -> Result>, Self::Error> { - let fetch_query = - "SELECT id, layers, last_seen FROM Workers WHERE worker_type = ? ORDER BY last_seen DESC LIMIT 20 OFFSET ?"; - let res: Vec<(String, String, i64)> = sqlx::query_as(fetch_query) - .bind(self.get_config().namespace()) - .bind(0) - .fetch_all(self.pool()) - .await?; - Ok(res - .into_iter() - .map(|w| SimpleWorker::new(WorkerId::new(w.0), WorkerState::new::(w.1))) - .collect()) - } -} - -#[cfg(test)] -mod tests { - - use crate::sql_storage_tests; - - use super::*; - use apalis_core::task::State; - use chrono::Utc; - use email_service::example_good_email; - use email_service::Email; - use futures::StreamExt; - - use apalis_core::generic_storage_test; - use apalis_core::test_utils::apalis_test_service_fn; - use apalis_core::test_utils::TestWrapper; - - generic_storage_test!(setup); - sql_storage_tests!(setup::, SqliteStorage, Email); - - /// migrate DB and return a storage instance. - async fn setup() -> SqliteStorage { - // Because connections cannot be shared across async runtime - // (different runtimes are created for each test), - // we don't share the storage and tests must be run sequentially. - let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); - SqliteStorage::setup(&pool) - .await - .expect("failed to migrate DB"); - let config = Config::new("apalis::test"); - let storage = SqliteStorage::::new_with_config(pool, config); - - storage - } - - #[tokio::test] - async fn test_inmemory_sqlite_worker() { - let mut sqlite = setup().await; - sqlite - .push(Email { - subject: "Test Subject".to_string(), - to: "example@sqlite".to_string(), - text: "Some Text".to_string(), - }) - .await - .expect("Unable to push job"); - let len = sqlite.len().await.expect("Could not fetch the jobs count"); - assert_eq!(len, 1); - } - - async fn consume_one( - storage: &mut SqliteStorage, - worker: &WorkerContext, - ) -> Task { - let mut stream = storage - .stream_jobs(worker, std::time::Duration::from_secs(10), 1) - .boxed(); - stream - .next() - .await - .expect("stream is empty") - .expect("failed to poll job") - .expect("no job is pending") - } - - async fn register_worker_at( - storage: &mut SqliteStorage, - last_seen: i64, - ) -> WorkerContext { - let worker_id = WorkerId::new("test-worker"); - - let worker = SimpleWorker::new(worker_id, Default::default()); - storage - .keep_alive_at(&worker, last_seen) - .await - .expect("failed to register worker"); - worker.start(); - worker - } - - async fn register_worker(storage: &mut SqliteStorage) -> WorkerContext { - register_worker_at(storage, Utc::now().timestamp()).await - } - - async fn push_email(storage: &mut SqliteStorage, email: Email) { - storage.push(email).await.expect("failed to push a job"); - } - - async fn get_job( - storage: &mut SqliteStorage, - job_id: &TaskId, - ) -> Task { - storage - .fetch_by_id(job_id) - .await - .expect("failed to fetch job by id") - .expect("no job found by id") - } - - #[tokio::test] - async fn test_consume_last_pushed_job() { - let mut storage = setup().await; - let worker = register_worker(&mut storage).await; - - push_email(&mut storage, example_good_email()).await; - let len = storage.len().await.expect("Could not fetch the jobs count"); - assert_eq!(len, 1); - - let job = consume_one(&mut storage, &worker).await; - let ctx = job.meta.context; - assert_eq!(*ctx.status(), State::Running); - assert_eq!(*ctx.lock_by(), Some(worker.id().clone())); - assert!(ctx.lock_at().is_some()); - } - - #[tokio::test] - async fn test_acknowledge_job() { - let mut storage = setup().await; - let worker = register_worker(&mut storage).await; - - push_email(&mut storage, example_good_email()).await; - let job = consume_one(&mut storage, &worker).await; - let job_id = &job.meta.task_id; - let ctx = &job.meta.context; - let res = 1usize; - storage - .ack( - ctx, - &Response::success(res, job_id.clone(), job.meta.attempt.clone()), - ) - .await - .expect("failed to acknowledge the job"); - - let job = get_job(&mut storage, job_id).await; - let ctx = job.meta.context; - assert_eq!(*ctx.status(), State::Done); - assert!(ctx.done_at().is_some()); - } - - #[tokio::test] - async fn test_kill_job() { - let mut storage = setup().await; - - push_email(&mut storage, example_good_email()).await; - - let worker = register_worker(&mut storage).await; - - let job = consume_one(&mut storage, &worker).await; - let job_id = &job.meta.task_id; - - storage - .kill(&worker.id(), job_id) - .await - .expect("failed to kill job"); - - let job = get_job(&mut storage, job_id).await; - let ctx = job.meta.context; - assert_eq!(*ctx.status(), State::Killed); - assert!(ctx.done_at().is_some()); - } - - #[tokio::test] - async fn test_heartbeat_renqueueorphaned_pulse_last_seen_6min() { - let mut storage = setup().await; - - push_email(&mut storage, example_good_email()).await; - - let six_minutes_ago = Utc::now() - Duration::from_secs(6 * 60); - - let five_minutes_ago = Utc::now() - Duration::from_secs(5 * 60); - let worker = register_worker_at(&mut storage, six_minutes_ago.timestamp()).await; - - let job = consume_one(&mut storage, &worker).await; - let job_id = &job.meta.task_id; - storage - .reenqueue_orphaned(1, five_minutes_ago) - .await - .expect("failed to heartbeat"); - let job = get_job(&mut storage, job_id).await; - let ctx = &job.meta.context; - assert_eq!(*ctx.status(), State::Pending); - assert!(ctx.done_at().is_none()); - assert!(ctx.lock_by().is_none()); - assert!(ctx.lock_at().is_none()); - assert_eq!(*ctx.last_error(), Some("Job was abandoned".to_owned())); - assert_eq!(job.meta.attempt.current(), 1); - - let job = consume_one(&mut storage, &worker).await; - let ctx = &job.meta.context; - // Simulate worker - job.meta.attempt.increment(); - storage - .ack( - ctx, - &Response::new(Ok("success".to_owned()), job_id.clone(), job.meta.attempt), - ) - .await - .unwrap(); - //end simulate worker - - let job = get_job(&mut storage, &job_id).await; - let ctx = &job.meta.context; - assert_eq!(*ctx.status(), State::Done); - assert_eq!(*ctx.lock_by(), Some(worker.id().clone())); - assert!(ctx.lock_at().is_some()); - assert_eq!(*ctx.last_error(), Some("{\"Ok\":\"success\"}".to_owned())); - assert_eq!(job.meta.attempt.current(), 2); - } - - #[tokio::test] - async fn test_heartbeat_renqueueorphaned_pulse_last_seen_4min() { - let mut storage = setup().await; - - push_email(&mut storage, example_good_email()).await; - - let six_minutes_ago = Utc::now() - Duration::from_secs(6 * 60); - let four_minutes_ago = Utc::now() - Duration::from_secs(4 * 60); - let worker = register_worker_at(&mut storage, four_minutes_ago.timestamp()).await; - - let job = consume_one(&mut storage, &worker).await; - let job_id = job.meta.task_id; - storage - .reenqueue_orphaned(1, six_minutes_ago) - .await - .expect("failed to heartbeat"); - - let job = get_job(&mut storage, &job_id).await; - let ctx = &job.meta.context; - - // Simulate worker - job.meta.attempt.increment(); - storage - .ack( - ctx, - &Response::new(Ok("success".to_owned()), job_id.clone(), job.meta.attempt), - ) - .await - .unwrap(); - //end simulate worker - - let job = get_job(&mut storage, &job_id).await; - let ctx = &job.meta.context; - assert_eq!(*ctx.status(), State::Done); - assert_eq!(*ctx.lock_by(), Some(worker.id().clone())); - assert!(ctx.lock_at().is_some()); - assert_eq!(*ctx.last_error(), Some("{\"Ok\":\"success\"}".to_owned())); - assert_eq!(job.meta.attempt.current(), 1); - } -} diff --git a/src/traits/list_workers.rs b/src/traits/list_workers.rs new file mode 100644 index 0000000..f565670 --- /dev/null +++ b/src/traits/list_workers.rs @@ -0,0 +1,117 @@ +use apalis_core::{ + backend::{ + Backend, FetchById, Filter, ListQueues, ListTasks, ListWorkers, Metrics, QueueInfo, + RunningWorker, + codec::{Codec, }, + }, + task::{Task, status::Status, task_id::TaskId}, +}; +use futures::TryFutureExt; +use serde::{Serialize, de::DeserializeOwned}; +use sqlx::Decode; +use ulid::Ulid; + +use crate::{SqliteContext, SqliteStorage, SqliteTask, from_row::TaskRow}; + +struct Worker { + id: String, + worker_type: String, + storage_name: String, + layers: Option, + last_seen: i64, +} + +impl ListWorkers for SqliteStorage +where + SqliteStorage: + Backend, +{ + fn list_workers( + &self, + queue: &str, + ) -> impl Future, Self::Error>> + Send { + let queue = queue.to_string(); + let pool = self.pool.clone(); + let limit = 100; + let offset = 0; + async move { + let workers = sqlx::query_file_as!( + Worker, + "queries/backend/list_workers.sql", + queue, + limit, + offset + ) + .fetch_all(&pool) + .map_ok(|w| { + w.into_iter() + .map(|w| RunningWorker { + id: w.id, + backend: w.storage_name, + started_at: 0, + last_heartbeat: w.last_seen as u64, + layers: w.layers.unwrap_or_default(), + queue: w.worker_type, + }) + .collect() + }) + .await?; + Ok(workers) + } + } + + fn list_all_workers( + &self, + ) -> impl Future, Self::Error>> + Send { + let pool = self.pool.clone(); + let limit = 100; + let offset = 0; + async move { + let workers = sqlx::query_file_as!( + Worker, + "queries/backend/list_all_workers.sql", + limit, + offset + ) + .fetch_all(&pool) + .map_ok(|w| { + w.into_iter() + .map(|w| RunningWorker { + id: w.id, + backend: w.storage_name, + started_at: 0, + last_heartbeat: w.last_seen as u64, + layers: w.layers.unwrap_or_default(), + queue: w.worker_type, + }) + .collect() + }) + .await?; + Ok(workers) + } + } +} + + +impl FetchById for SqliteStorage +where + SqliteStorage: + Backend, + Args: Serialize + DeserializeOwned, +{ + fn fetch_by_id( + &mut self, + id: &TaskId, + ) -> impl Future>, Self::Error>> + Send { + let pool = self.pool.clone(); + let id = id.to_string(); + async move { + let task = sqlx::query_file_as!(TaskRow, "queries/task/find_by_id.sql", id) + .fetch_optional(&pool) + .await? + .map(|r| r.try_into_task::, _>()) + .transpose()?; + Ok(task) + } + } +} From 04f0e9ee505bdd4307510c7c36f9cc0c19fc1e66 Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Fri, 17 Oct 2025 20:34:31 +0300 Subject: [PATCH 118/121] chore: ready for CI --- ...1eaa74232f6f6f3121c7086a0b138195e9b0d.json | 2 +- ...2d6063f186242ca1808cfae9d6477c7d8687b.json | 2 +- ...0f03d75403970fc8347d0f017f04d746b2b98.json | 2 +- ...5653f4fe06d10361c555f8ade4b65589520bc.json | 2 +- ...90f217b575f0ca464a5faddd0fe2d51793983.json | 2 +- Cargo.lock | 1 + Cargo.toml | 1 + README.md | 4 +- src/lib.rs | 54 ++++---- src/shared.rs | 13 +- src/traits/list_workers.rs | 117 ------------------ 11 files changed, 51 insertions(+), 149 deletions(-) delete mode 100644 src/traits/list_workers.rs diff --git a/.sqlx/query-1cd760004a2341bbded38a4fa431eaa74232f6f6f3121c7086a0b138195e9b0d.json b/.sqlx/query-1cd760004a2341bbded38a4fa431eaa74232f6f6f3121c7086a0b138195e9b0d.json index 90bd90a..92182f9 100644 --- a/.sqlx/query-1cd760004a2341bbded38a4fa431eaa74232f6f6f3121c7086a0b138195e9b0d.json +++ b/.sqlx/query-1cd760004a2341bbded38a4fa431eaa74232f6f6f3121c7086a0b138195e9b0d.json @@ -6,7 +6,7 @@ { "name": "job", "ordinal": 0, - "type_info": "Blob" + "type_info": "Text" }, { "name": "id", diff --git a/.sqlx/query-40a420986a37c9db2fc35b161092d6063f186242ca1808cfae9d6477c7d8687b.json b/.sqlx/query-40a420986a37c9db2fc35b161092d6063f186242ca1808cfae9d6477c7d8687b.json index ac61445..bab3c7a 100644 --- a/.sqlx/query-40a420986a37c9db2fc35b161092d6063f186242ca1808cfae9d6477c7d8687b.json +++ b/.sqlx/query-40a420986a37c9db2fc35b161092d6063f186242ca1808cfae9d6477c7d8687b.json @@ -6,7 +6,7 @@ { "name": "job", "ordinal": 0, - "type_info": "Blob" + "type_info": "Text" }, { "name": "id", diff --git a/.sqlx/query-61843a18bffdee192cd01f1537f0f03d75403970fc8347d0f017f04d746b2b98.json b/.sqlx/query-61843a18bffdee192cd01f1537f0f03d75403970fc8347d0f017f04d746b2b98.json index d9bc816..194ba6c 100644 --- a/.sqlx/query-61843a18bffdee192cd01f1537f0f03d75403970fc8347d0f017f04d746b2b98.json +++ b/.sqlx/query-61843a18bffdee192cd01f1537f0f03d75403970fc8347d0f017f04d746b2b98.json @@ -6,7 +6,7 @@ { "name": "job", "ordinal": 0, - "type_info": "Blob" + "type_info": "Text" }, { "name": "id", diff --git a/.sqlx/query-a43052e877930a522d6a63789935653f4fe06d10361c555f8ade4b65589520bc.json b/.sqlx/query-a43052e877930a522d6a63789935653f4fe06d10361c555f8ade4b65589520bc.json index f26492b..cb5c3e6 100644 --- a/.sqlx/query-a43052e877930a522d6a63789935653f4fe06d10361c555f8ade4b65589520bc.json +++ b/.sqlx/query-a43052e877930a522d6a63789935653f4fe06d10361c555f8ade4b65589520bc.json @@ -6,7 +6,7 @@ { "name": "job", "ordinal": 0, - "type_info": "Blob" + "type_info": "Text" }, { "name": "id", diff --git a/.sqlx/query-d7aefe54cd7388c208fff5b946390f217b575f0ca464a5faddd0fe2d51793983.json b/.sqlx/query-d7aefe54cd7388c208fff5b946390f217b575f0ca464a5faddd0fe2d51793983.json index fef32c7..d5f2525 100644 --- a/.sqlx/query-d7aefe54cd7388c208fff5b946390f217b575f0ca464a5faddd0fe2d51793983.json +++ b/.sqlx/query-d7aefe54cd7388c208fff5b946390f217b575f0ca464a5faddd0fe2d51793983.json @@ -6,7 +6,7 @@ { "name": "job", "ordinal": 0, - "type_info": "Blob" + "type_info": "Text" }, { "name": "id", diff --git a/Cargo.lock b/Cargo.lock index 41ea2aa..d6c1308 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,6 +57,7 @@ dependencies = [ "apalis-core", "apalis-sqlite", "async-std", + "bytes", "chrono", "futures", "libsqlite3-sys", diff --git a/Cargo.toml b/Cargo.toml index 8a7aca9..d8da991 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ libsqlite3-sys = "0.30.1" ulid = { version = "1", features = ["serde"] } tower-layer = "0.3.3" tower-service = "0.3.3" +bytes = "1.1.0" [dev-dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } diff --git a/README.md b/README.md index 631a220..f0166c3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Background task processing for Rust using Apalis and SQLite. - **Reliable job queue** using SQLite as the backend. - **Multiple storage types**: standard polling and event-driven (hooked) storage. -- **Custom codecs** for serializing/deserializing job arguments. +- **Custom codecs** for serializing/deserializing job arguments as bytes and json. - **Heartbeat and orphaned job re-enqueueing** for robust job processing. - **Integration with Apalis workers and middleware.** @@ -77,7 +77,7 @@ async fn main() { let mut start = 0; let items = stream::repeat_with(move || { start += 1; - Task::builder(start) + Task::builder(serde_json::to_value(&start).unwrap()) .run_after(Duration::from_secs(1)) .with_ctx(SqliteContext::new().with_priority(start)) .build() diff --git a/src/lib.rs b/src/lib.rs index 1942bc8..1b2be94 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,14 +22,17 @@ //! //! ### Basic Worker Example //! -//! ```rust,no_run -//! use apalis_sqlite::{SqliteStorage, SqliteContext}; -//! use apalis_core::task::Task; -//! use apalis_core::worker::context::WorkerContext; -//! use sqlx::SqlitePool; -//! use futures::stream; -//! use std::time::Duration; -//! +//! ```rust +//! # use apalis_sqlite::{SqliteStorage, SqliteContext}; +//! # use apalis_core::task::Task; +//! # use apalis_core::worker::context::WorkerContext; +//! # use sqlx::SqlitePool; +//! # use futures::stream; +//! # use std::time::Duration; +//! # use apalis_core::error::BoxDynError; +//! # use futures::StreamExt; +//! # use futures::SinkExt; +//! # use apalis_core::worker::builder::WorkerBuilder; //! #[tokio::main] //! async fn main() { //! let pool = SqlitePool::connect(":memory:").await.unwrap(); @@ -55,7 +58,7 @@ //! Ok(()) //! } //! -//! let worker = apalis_core::worker::builder::WorkerBuilder::new("worker-1") +//! let worker = WorkerBuilder::new("worker-1") //! .backend(backend) //! .build(send_reminder); //! worker.run().await.unwrap(); @@ -65,13 +68,17 @@ //! ### Hooked Worker Example (Event-driven) //! //! ```rust,no_run -//! use apalis_sqlite::{SqliteStorage, SqliteContext, Config}; -//! use apalis_core::task::Task; -//! use apalis_core::worker::context::WorkerContext; -//! use apalis_core::backend::poll_strategy::{IntervalStrategy, StrategyBuilder}; -//! use sqlx::SqlitePool; -//! use futures::stream; -//! use std::time::Duration; +//! # use apalis_sqlite::{SqliteStorage, SqliteContext, Config}; +//! # use apalis_core::task::Task; +//! # use apalis_core::worker::context::WorkerContext; +//! # use apalis_core::backend::poll_strategy::{IntervalStrategy, StrategyBuilder}; +//! # use sqlx::SqlitePool; +//! # use futures::stream; +//! # use std::time::Duration; +//! # use apalis_core::error::BoxDynError; +//! # use futures::StreamExt; +//! # use futures::SinkExt; +//! # use apalis_core::worker::builder::WorkerBuilder; //! //! #[tokio::main] //! async fn main() { @@ -94,7 +101,7 @@ //! let mut start = 0; //! let items = stream::repeat_with(move || { //! start += 1; -//! Task::builder(start) +//! Task::builder(serde_json::to_string(&start).unwrap()) //! .run_after(Duration::from_secs(1)) //! .with_ctx(SqliteContext::new().with_priority(start)) //! .build() @@ -102,6 +109,7 @@ //! .take(20) //! .collect::>() //! .await; +//! // push encoded tasks //! apalis_sqlite::sink::push_tasks(pool, config, items).await.unwrap(); //! } //! }); @@ -114,7 +122,7 @@ //! Ok(()) //! } //! -//! let worker = apalis_core::worker::builder::WorkerBuilder::new("worker-2") +//! let worker = WorkerBuilder::new("worker-2") //! .backend(backend) //! .build(send_reminder); //! worker.run().await.unwrap(); @@ -127,9 +135,10 @@ //! //! ```rust,no_run //! # use sqlx::SqlitePool; +//! # use apalis_sqlite::SqliteStorage; //! # #[tokio::main] async fn main() { //! let pool = SqlitePool::connect(":memory:").await.unwrap(); -//! apalis_sqlite::SqliteStorage::setup(&pool).await.unwrap(); +//! SqliteStorage::setup(&pool).await.unwrap(); //! # } //! ``` //! @@ -140,7 +149,8 @@ use std::{fmt, marker::PhantomData}; use apalis_core::{ backend::{ - codec::{json::JsonCodec, Codec }, Backend, TaskStream + Backend, TaskStream, + codec::{Codec, json::JsonCodec}, }, task::Task, worker::{context::WorkerContext, ext::ack::AcknowledgeLayer}, @@ -176,7 +186,7 @@ pub mod fetcher; pub mod from_row; pub mod queries; mod shared; -mod sink; +pub mod sink; pub type SqliteTask = Task; pub use callback::{CallbackListener, DbEvent}; @@ -186,7 +196,7 @@ pub use shared::{SharedPostgresError, SharedSqliteStorage}; pub use sqlx::SqlitePool; #[cfg(feature = "json")] -pub type CompactType = serde_json::Value; +pub type CompactType = String; #[cfg(feature = "bytes")] pub type CompactType = Vec; diff --git a/src/shared.rs b/src/shared.rs index 7382f48..4ca3844 100644 --- a/src/shared.rs +++ b/src/shared.rs @@ -19,7 +19,11 @@ use crate::{ }; use crate::{from_row::TaskRow, sink::SqliteSink}; use apalis_core::{ - backend::{Backend, TaskStream, codec::Codec, shared::MakeShared}, + backend::{ + Backend, TaskStream, + codec::{Codec, json::JsonCodec}, + shared::MakeShared, + }, worker::{context::WorkerContext, ext::ack::AcknowledgeLayer}, }; use futures::{ @@ -41,13 +45,16 @@ pub struct SharedSqliteStorage { _marker: PhantomData, } -impl> SharedSqliteStorage { +impl SharedSqliteStorage> { pub fn new(pool: SqlitePool) -> Self { + SharedSqliteStorage::new_with_codec(pool) + } + pub fn new_with_codec(pool: SqlitePool) -> SharedSqliteStorage { let registry: Arc>>>> = Arc::new(Mutex::new(HashMap::default())); let p = pool.clone(); let instances = registry.clone(); - Self { + SharedSqliteStorage { pool, drive: async move { let (tx, rx) = mpsc::unbounded::(); diff --git a/src/traits/list_workers.rs b/src/traits/list_workers.rs deleted file mode 100644 index f565670..0000000 --- a/src/traits/list_workers.rs +++ /dev/null @@ -1,117 +0,0 @@ -use apalis_core::{ - backend::{ - Backend, FetchById, Filter, ListQueues, ListTasks, ListWorkers, Metrics, QueueInfo, - RunningWorker, - codec::{Codec, }, - }, - task::{Task, status::Status, task_id::TaskId}, -}; -use futures::TryFutureExt; -use serde::{Serialize, de::DeserializeOwned}; -use sqlx::Decode; -use ulid::Ulid; - -use crate::{SqliteContext, SqliteStorage, SqliteTask, from_row::TaskRow}; - -struct Worker { - id: String, - worker_type: String, - storage_name: String, - layers: Option, - last_seen: i64, -} - -impl ListWorkers for SqliteStorage -where - SqliteStorage: - Backend, -{ - fn list_workers( - &self, - queue: &str, - ) -> impl Future, Self::Error>> + Send { - let queue = queue.to_string(); - let pool = self.pool.clone(); - let limit = 100; - let offset = 0; - async move { - let workers = sqlx::query_file_as!( - Worker, - "queries/backend/list_workers.sql", - queue, - limit, - offset - ) - .fetch_all(&pool) - .map_ok(|w| { - w.into_iter() - .map(|w| RunningWorker { - id: w.id, - backend: w.storage_name, - started_at: 0, - last_heartbeat: w.last_seen as u64, - layers: w.layers.unwrap_or_default(), - queue: w.worker_type, - }) - .collect() - }) - .await?; - Ok(workers) - } - } - - fn list_all_workers( - &self, - ) -> impl Future, Self::Error>> + Send { - let pool = self.pool.clone(); - let limit = 100; - let offset = 0; - async move { - let workers = sqlx::query_file_as!( - Worker, - "queries/backend/list_all_workers.sql", - limit, - offset - ) - .fetch_all(&pool) - .map_ok(|w| { - w.into_iter() - .map(|w| RunningWorker { - id: w.id, - backend: w.storage_name, - started_at: 0, - last_heartbeat: w.last_seen as u64, - layers: w.layers.unwrap_or_default(), - queue: w.worker_type, - }) - .collect() - }) - .await?; - Ok(workers) - } - } -} - - -impl FetchById for SqliteStorage -where - SqliteStorage: - Backend, - Args: Serialize + DeserializeOwned, -{ - fn fetch_by_id( - &mut self, - id: &TaskId, - ) -> impl Future>, Self::Error>> + Send { - let pool = self.pool.clone(); - let id = id.to_string(); - async move { - let task = sqlx::query_file_as!(TaskRow, "queries/task/find_by_id.sql", id) - .fetch_optional(&pool) - .await? - .map(|r| r.try_into_task::, _>()) - .transpose()?; - Ok(task) - } - } -} From 510aaf9df4288085b3ff9ecad1c768116ec0960e Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Fri, 17 Oct 2025 21:17:23 +0300 Subject: [PATCH 119/121] lib cleanup --- .gitignore | 1 + Cargo.lock | 2825 ---------------------------------------------------- Cargo.toml | 8 +- 3 files changed, 4 insertions(+), 2830 deletions(-) delete mode 100644 Cargo.lock diff --git a/.gitignore b/.gitignore index 2e19174..1a7bd7d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ # will have compiled files and executables debug target +Cargo.lock # These are backup files generated by rustfmt **/*.rs.bk diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index d6c1308..0000000 --- a/Cargo.lock +++ /dev/null @@ -1,2825 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "apalis-core" -version = "1.0.0-alpha.3" -source = "git+https://github.com/geofmureithi/apalis.git?branch=chore%2Ftraits-expansion#d9c4b2319f010dda9003ae07b36d724228e8ac3c" -dependencies = [ - "futures-channel", - "futures-core", - "futures-sink", - "futures-timer", - "futures-util", - "pin-project", - "serde", - "serde_json", - "thiserror", - "tower-layer", - "tower-service", -] - -[[package]] -name = "apalis-sqlite" -version = "1.0.0-alpha.1" -dependencies = [ - "apalis-core", - "apalis-sqlite", - "async-std", - "bytes", - "chrono", - "futures", - "libsqlite3-sys", - "log", - "pin-project", - "serde", - "serde_json", - "sqlx", - "thiserror", - "tokio", - "tower-layer", - "tower-service", - "ulid", -] - -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener 2.5.3", - "futures-core", -] - -[[package]] -name = "async-channel" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" -dependencies = [ - "concurrent-queue", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-executor" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497c00e0fd83a72a79a39fcbd8e3e2f055d6f6c7e025f3b3d91f4f8e76527fb8" -dependencies = [ - "async-task", - "concurrent-queue", - "fastrand 2.3.0", - "futures-lite 2.6.1", - "pin-project-lite", - "slab", -] - -[[package]] -name = "async-global-executor" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" -dependencies = [ - "async-channel 2.5.0", - "async-executor", - "async-io 2.6.0", - "async-lock 3.4.1", - "blocking", - "futures-lite 2.6.1", - "once_cell", -] - -[[package]] -name = "async-io" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" -dependencies = [ - "async-lock 2.8.0", - "autocfg", - "cfg-if", - "concurrent-queue", - "futures-lite 1.13.0", - "log", - "parking", - "polling 2.8.0", - "rustix 0.37.28", - "slab", - "socket2 0.4.10", - "waker-fn", -] - -[[package]] -name = "async-io" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" -dependencies = [ - "autocfg", - "cfg-if", - "concurrent-queue", - "futures-io", - "futures-lite 2.6.1", - "parking", - "polling 3.11.0", - "rustix 1.1.2", - "slab", - "windows-sys 0.61.0", -] - -[[package]] -name = "async-lock" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" -dependencies = [ - "event-listener 2.5.3", -] - -[[package]] -name = "async-lock" -version = "3.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd03604047cee9b6ce9de9f70c6cd540a0520c813cbd49bae61f33ab80ed1dc" -dependencies = [ - "event-listener 5.4.1", - "event-listener-strategy", - "pin-project-lite", -] - -[[package]] -name = "async-std" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c8e079a4ab67ae52b7403632e4618815d6db36d2a010cfe41b02c1b1578f93b" -dependencies = [ - "async-channel 1.9.0", - "async-global-executor", - "async-io 2.6.0", - "async-lock 3.4.1", - "crossbeam-utils", - "futures-channel", - "futures-core", - "futures-io", - "futures-lite 2.6.1", - "gloo-timers", - "kv-log-macro", - "log", - "memchr", - "once_cell", - "pin-project-lite", - "pin-utils", - "slab", - "wasm-bindgen-futures", -] - -[[package]] -name = "async-task" -version = "4.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" - -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", -] - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "autocfg" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" - -[[package]] -name = "backtrace" -version = "0.3.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets 0.52.6", -] - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "base64ct" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" -dependencies = [ - "serde", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "blocking" -version = "1.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" -dependencies = [ - "async-channel 2.5.0", - "async-task", - "futures-io", - "futures-lite 2.6.1", - "piper", -] - -[[package]] -name = "bumpalo" -version = "3.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "bytes" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" - -[[package]] -name = "cc" -version = "1.2.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9" -dependencies = [ - "find-msvc-tools", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" - -[[package]] -name = "chrono" -version = "0.4.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" -dependencies = [ - "iana-time-zone", - "js-sys", - "num-traits", - "serde", - "wasm-bindgen", - "windows-link", -] - -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "crc" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - -[[package]] -name = "crossbeam-queue" -version = "0.3.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid", - "pem-rfc7468", - "zeroize", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", - "subtle", -] - -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "dotenvy" -version = "0.15.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" - -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" -dependencies = [ - "serde", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.61.0", -] - -[[package]] -name = "etcetera" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" -dependencies = [ - "cfg-if", - "home", - "windows-sys 0.48.0", -] - -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - -[[package]] -name = "event-listener" -version = "5.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" -dependencies = [ - "event-listener 5.4.1", - "pin-project-lite", -] - -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - -[[package]] -name = "fastrand" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - -[[package]] -name = "find-msvc-tools" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" - -[[package]] -name = "flume" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" -dependencies = [ - "futures-core", - "futures-sink", - "spin", -] - -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "futures" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" - -[[package]] -name = "futures-executor" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-intrusive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" -dependencies = [ - "futures-core", - "lock_api", - "parking_lot", -] - -[[package]] -name = "futures-io" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" - -[[package]] -name = "futures-lite" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" -dependencies = [ - "fastrand 1.9.0", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - -[[package]] -name = "futures-lite" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" -dependencies = [ - "fastrand 2.3.0", - "futures-core", - "futures-io", - "parking", - "pin-project-lite", -] - -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "futures-sink" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" - -[[package]] -name = "futures-task" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" - -[[package]] -name = "futures-timer" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" - -[[package]] -name = "futures-util" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", -] - -[[package]] -name = "getrandom" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasi 0.14.7+wasi-0.2.4", -] - -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - -[[package]] -name = "gloo-timers" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - -[[package]] -name = "hashbrown" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" - -[[package]] -name = "hashlink" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" -dependencies = [ - "hashbrown 0.15.5", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - -[[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hkdf" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" -dependencies = [ - "hmac", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "home" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "icu_collections" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" -dependencies = [ - "displaydoc", - "potential_utf", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" - -[[package]] -name = "icu_properties" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "potential_utf", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" - -[[package]] -name = "icu_provider" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" -dependencies = [ - "displaydoc", - "icu_locale_core", - "stable_deref_trait", - "tinystr", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "indexmap" -version = "2.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" -dependencies = [ - "equivalent", - "hashbrown 0.16.0", -] - -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi 0.3.9", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "io-uring" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" -dependencies = [ - "bitflags 2.9.4", - "cfg-if", - "libc", -] - -[[package]] -name = "itoa" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" - -[[package]] -name = "js-sys" -version = "0.3.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852f13bec5eba4ba9afbeb93fd7c13fe56147f055939ae21c43a29a0ecb2702e" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - -[[package]] -name = "kv-log-macro" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" -dependencies = [ - "log", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -dependencies = [ - "spin", -] - -[[package]] -name = "libc" -version = "0.2.175" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" - -[[package]] -name = "libm" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" - -[[package]] -name = "libredox" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" -dependencies = [ - "bitflags 2.9.4", - "libc", - "redox_syscall", -] - -[[package]] -name = "libsqlite3-sys" -version = "0.30.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "linux-raw-sys" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" - -[[package]] -name = "linux-raw-sys" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" - -[[package]] -name = "litemap" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" - -[[package]] -name = "lock_api" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" -dependencies = [ - "value-bag", -] - -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest", -] - -[[package]] -name = "memchr" -version = "2.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" - -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", -] - -[[package]] -name = "mio" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" -dependencies = [ - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.59.0", -] - -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - -[[package]] -name = "num-bigint-dig" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" -dependencies = [ - "byteorder", - "lazy_static", - "libm", - "num-integer", - "num-iter", - "num-traits", - "rand 0.8.5", - "smallvec", - "zeroize", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", - "libm", -] - -[[package]] -name = "object" -version = "0.36.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" - -[[package]] -name = "openssl" -version = "0.10.73" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" -dependencies = [ - "bitflags 2.9.4", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - -[[package]] -name = "openssl-sys" -version = "0.9.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - -[[package]] -name = "parking_lot" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets 0.52.6", -] - -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "pin-project" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "piper" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" -dependencies = [ - "atomic-waker", - "fastrand 2.3.0", - "futures-io", -] - -[[package]] -name = "pkcs1" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" -dependencies = [ - "der", - "pkcs8", - "spki", -] - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "polling" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" -dependencies = [ - "autocfg", - "bitflags 1.3.2", - "cfg-if", - "concurrent-queue", - "libc", - "log", - "pin-project-lite", - "windows-sys 0.48.0", -] - -[[package]] -name = "polling" -version = "3.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" -dependencies = [ - "cfg-if", - "concurrent-queue", - "hermit-abi 0.5.2", - "pin-project-lite", - "rustix 1.1.2", - "windows-sys 0.61.0", -] - -[[package]] -name = "potential_utf" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" -dependencies = [ - "zerovec", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "proc-macro2" -version = "1.0.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.16", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom 0.3.3", -] - -[[package]] -name = "redox_syscall" -version = "0.5.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" -dependencies = [ - "bitflags 2.9.4", -] - -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.16", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - -[[package]] -name = "rsa" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" -dependencies = [ - "const-oid", - "digest", - "num-bigint-dig", - "num-integer", - "num-traits", - "pkcs1", - "pkcs8", - "rand_core 0.6.4", - "signature", - "spki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" - -[[package]] -name = "rustix" -version = "0.37.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "519165d378b97752ca44bbe15047d5d3409e875f39327546b42ac81d7e18c1b6" -dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys 0.48.0", -] - -[[package]] -name = "rustix" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" -dependencies = [ - "bitflags 2.9.4", - "errno", - "libc", - "linux-raw-sys 0.11.0", - "windows-sys 0.61.0", -] - -[[package]] -name = "rustls" -version = "0.23.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd3c25631629d034ce7cd9940adc9d45762d46de2b0f57193c4443b92c6d4d40" -dependencies = [ - "once_cell", - "ring", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-pki-types" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" -dependencies = [ - "zeroize", -] - -[[package]] -name = "rustls-webpki" -version = "0.103.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8572f3c2cb9934231157b45499fc41e1f58c589fdfb81a844ba873265e80f8eb" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] -name = "rustversion" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" - -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - -[[package]] -name = "schannel" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" -dependencies = [ - "windows-sys 0.61.0", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.9.4", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "serde" -version = "1.0.226" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dca6411025b24b60bfa7ec1fe1f8e710ac09782dca409ee8237ba74b51295fd" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.226" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba2ba63999edb9dac981fb34b3e5c0d111a69b0924e253ed29d83f7c99e966a4" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.226" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8db53ae22f34573731bafa1db20f04027b2d25e02d8205921b569171699cdb33" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.145" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", - "serde_core", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "digest", - "rand_core 0.6.4", -] - -[[package]] -name = "slab" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" - -[[package]] -name = "smallvec" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -dependencies = [ - "serde", -] - -[[package]] -name = "socket2" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "socket2" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" -dependencies = [ - "libc", - "windows-sys 0.59.0", -] - -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] - -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der", -] - -[[package]] -name = "sqlx" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" -dependencies = [ - "sqlx-core", - "sqlx-macros", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", -] - -[[package]] -name = "sqlx-core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" -dependencies = [ - "async-io 1.13.0", - "async-std", - "base64", - "bytes", - "chrono", - "crc", - "crossbeam-queue", - "either", - "event-listener 5.4.1", - "futures-core", - "futures-intrusive", - "futures-io", - "futures-util", - "hashbrown 0.15.5", - "hashlink", - "indexmap", - "log", - "memchr", - "native-tls", - "once_cell", - "percent-encoding", - "rustls", - "serde", - "serde_json", - "sha2", - "smallvec", - "thiserror", - "tokio", - "tokio-stream", - "tracing", - "url", - "webpki-roots 0.26.11", -] - -[[package]] -name = "sqlx-macros" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" -dependencies = [ - "proc-macro2", - "quote", - "sqlx-core", - "sqlx-macros-core", - "syn", -] - -[[package]] -name = "sqlx-macros-core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" -dependencies = [ - "async-std", - "dotenvy", - "either", - "heck", - "hex", - "once_cell", - "proc-macro2", - "quote", - "serde", - "serde_json", - "sha2", - "sqlx-core", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", - "syn", - "tokio", - "url", -] - -[[package]] -name = "sqlx-mysql" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" -dependencies = [ - "atoi", - "base64", - "bitflags 2.9.4", - "byteorder", - "bytes", - "chrono", - "crc", - "digest", - "dotenvy", - "either", - "futures-channel", - "futures-core", - "futures-io", - "futures-util", - "generic-array", - "hex", - "hkdf", - "hmac", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "percent-encoding", - "rand 0.8.5", - "rsa", - "serde", - "sha1", - "sha2", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror", - "tracing", - "whoami", -] - -[[package]] -name = "sqlx-postgres" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" -dependencies = [ - "atoi", - "base64", - "bitflags 2.9.4", - "byteorder", - "chrono", - "crc", - "dotenvy", - "etcetera", - "futures-channel", - "futures-core", - "futures-util", - "hex", - "hkdf", - "hmac", - "home", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "rand 0.8.5", - "serde", - "serde_json", - "sha2", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror", - "tracing", - "whoami", -] - -[[package]] -name = "sqlx-sqlite" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" -dependencies = [ - "atoi", - "chrono", - "flume", - "futures-channel", - "futures-core", - "futures-executor", - "futures-intrusive", - "futures-util", - "libsqlite3-sys", - "log", - "percent-encoding", - "serde", - "serde_urlencoded", - "sqlx-core", - "thiserror", - "tracing", - "url", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "stringprep" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" -dependencies = [ - "unicode-bidi", - "unicode-normalization", - "unicode-properties", -] - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - -[[package]] -name = "syn" -version = "2.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tempfile" -version = "3.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84fa4d11fadde498443cca10fd3ac23c951f0dc59e080e9f4b93d4df4e4eea53" -dependencies = [ - "fastrand 2.3.0", - "getrandom 0.3.3", - "once_cell", - "rustix 1.1.2", - "windows-sys 0.61.0", -] - -[[package]] -name = "thiserror" -version = "2.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tinystr" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.47.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" -dependencies = [ - "backtrace", - "bytes", - "io-uring", - "libc", - "mio", - "pin-project-lite", - "slab", - "socket2 0.6.0", - "tokio-macros", - "windows-sys 0.59.0", -] - -[[package]] -name = "tokio-macros" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tokio-stream" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" -dependencies = [ - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tracing-core" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" -dependencies = [ - "once_cell", -] - -[[package]] -name = "typenum" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" - -[[package]] -name = "ulid" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" -dependencies = [ - "rand 0.9.2", - "serde", - "web-time", -] - -[[package]] -name = "unicode-bidi" -version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" - -[[package]] -name = "unicode-ident" -version = "1.0.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" - -[[package]] -name = "unicode-normalization" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-properties" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "url" -version = "2.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "value-bag" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943ce29a8a743eb10d6082545d861b24f9d1b160b7d741e0f2cdf726bec909c5" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "waker-fn" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasi" -version = "0.14.7+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" -dependencies = [ - "wasip2", -] - -[[package]] -name = "wasip2" -version = "1.0.1+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "wasite" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" - -[[package]] -name = "wasm-bindgen" -version = "0.2.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab10a69fbd0a177f5f649ad4d8d3305499c42bab9aef2f7ff592d0ec8f833819" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb702423545a6007bbc368fde243ba47ca275e549c8a28617f56f6ba53b1d1c" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.53" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0b221ff421256839509adbb55998214a70d829d3a28c69b4a6672e9d2a42f67" -dependencies = [ - "cfg-if", - "js-sys", - "once_cell", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc65f4f411d91494355917b605e1480033152658d71f722a90647f56a70c88a0" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc003a991398a8ee604a401e194b6b3a39677b3173d6e74495eb51b82e99a32" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "293c37f4efa430ca14db3721dfbe48d8c33308096bd44d80ebaa775ab71ba1cf" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "web-sys" -version = "0.3.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbe734895e869dc429d78c4b433f8d17d95f8d05317440b4fad5ab2d33e596dc" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki-roots" -version = "0.26.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" -dependencies = [ - "webpki-roots 1.0.2", -] - -[[package]] -name = "webpki-roots" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "whoami" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" -dependencies = [ - "libredox", - "wasite", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-core" -version = "0.62.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57fe7168f7de578d2d8a05b07fd61870d2e73b4020e9f49aa00da8471723497c" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-implement" -version = "0.60.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-interface" -version = "0.59.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-link" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" - -[[package]] -name = "windows-result" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7084dcc306f89883455a206237404d3eaf961e5bd7e0f312f7c91f57eb44167f" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7218c655a553b0bed4426cf54b20d7ba363ef543b52d515b3e48d7fd55318dda" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.61.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "wit-bindgen" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" - -[[package]] -name = "writeable" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" - -[[package]] -name = "yoke" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.8.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zerofrom" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zeroize" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" - -[[package]] -name = "zerotrie" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] diff --git a/Cargo.toml b/Cargo.toml index d8da991..61082ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,10 +26,10 @@ features = ["chrono", "sqlite"] [dependencies] serde = { version = "1", features = ["derive"] } serde_json = { version = "1" } -apalis-core = { version = "1.0.0-alpha.3", default-features = false, features = [ +apalis-core = { version = "1.0.0-alpha.4", default-features = false, features = [ "sleep", "json", -], git = "https://github.com/geofmureithi/apalis.git", branch = "chore/traits-expansion" } +] } log = "0.4.21" futures = "0.3.30" tokio = { version = "1", features = ["rt", "net"], optional = true } @@ -45,9 +45,7 @@ bytes = "1.1.0" [dev-dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } -apalis-core = { version = "1.0.0-alpha.3", features = [ - "test-utils", -], git = "https://github.com/geofmureithi/apalis.git", branch = "chore/traits-expansion" } +apalis-core = { version = "1.0.0-alpha.4", features = ["test-utils"] } apalis-sqlite = { path = ".", features = ["migrate", "tokio-comp"] } [package.metadata.docs.rs] From fd0657d82429100815c014140e17e8938f20d416 Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Fri, 17 Oct 2025 21:20:22 +0300 Subject: [PATCH 120/121] chore: minor fixes --- .github/dependabot.yml | 6 +++--- src/fetcher.rs | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4449dce..876c9ab 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,9 +9,9 @@ updates: time: "06:00" open-pull-requests-limit: 10 reviewers: - - "mureithinjuguna" + - "geofmureithi" assignees: - - "mureithinjuguna" + - "geofmureithi" commit-message: prefix: "deps" include: "scope" @@ -36,4 +36,4 @@ updates: include: "scope" labels: - "dependencies" - - "github-actions" \ No newline at end of file + - "github-actions" diff --git a/src/fetcher.rs b/src/fetcher.rs index 02f87f1..3490005 100644 --- a/src/fetcher.rs +++ b/src/fetcher.rs @@ -113,7 +113,6 @@ where Decode::Error: std::error::Error + Send + Sync + 'static, Args: Send + 'static + Unpin, Decode: Codec + 'static, - // Compact: Unpin + Send + 'static, { type Item = Result>, sqlx::Error>; From 496d169f7be34934dc5326824a7f7e048e57a2e9 Mon Sep 17 00:00:00 2001 From: geofmureithi Date: Fri, 17 Oct 2025 21:48:57 +0300 Subject: [PATCH 121/121] remove old files --- packages/apalis-sql/Cargo.toml | 56 ------------------- packages/apalis-sql/build.rs | 3 - .../sqlite/20220530084123_jobs_workers.sql | 36 ------------ .../20250313213411_add_job_priority.sql | 2 - 4 files changed, 97 deletions(-) delete mode 100644 packages/apalis-sql/Cargo.toml delete mode 100644 packages/apalis-sql/build.rs delete mode 100644 packages/apalis-sql/migrations/sqlite/20220530084123_jobs_workers.sql delete mode 100644 packages/apalis-sql/migrations/sqlite/20250313213411_add_job_priority.sql diff --git a/packages/apalis-sql/Cargo.toml b/packages/apalis-sql/Cargo.toml deleted file mode 100644 index 9a72d29..0000000 --- a/packages/apalis-sql/Cargo.toml +++ /dev/null @@ -1,56 +0,0 @@ -[package] -name = "apalis-sql" -version = "0.7.3" -authors = ["Njuguna Mureithi "] -edition.workspace = true -repository.workspace = true -readme = "../../README.md" - -license = "MIT" -description = "SQL Storage for apalis. Use sqlite, postgres and mysql for background job processing" - -[features] -default = ["migrate"] -postgres = ["sqlx/postgres", "sqlx/json"] -sqlite = ["sqlx/sqlite", "sqlx/json"] -mysql = ["sqlx/mysql", "sqlx/json", "sqlx/bigdecimal"] -migrate = ["sqlx/migrate", "sqlx/macros"] -async-std-comp = ["async-std", "sqlx/runtime-async-std-rustls"] -async-std-comp-native-tls = ["async-std", "sqlx/runtime-async-std-native-tls"] -tokio-comp = ["tokio", "sqlx/runtime-tokio-rustls"] -tokio-comp-native-tls = ["tokio", "sqlx/runtime-tokio-native-tls"] - -[dependencies.sqlx] -version = "0.8.1" -default-features = false -features = ["chrono"] - -[dependencies] -serde = { version = "1", features = ["derive"] } -serde_json = "1" -apalis-core = { path = "../../packages/apalis-core", version = "0.7.3", default-features = false, features = [ - "sleep", - "json", -] } -log = "0.4.21" -futures = "0.3.30" -async-stream = "0.3.5" -tokio = { version = "1", features = ["rt", "net"], optional = true } -futures-lite = "2.3.0" -async-std = { version = "1.13.0", optional = true } -chrono = { version = "0.4", features = ["serde"] } -thiserror = "2.0.0" - - -[dev-dependencies] -tokio = { version = "1", features = ["macros", "rt-multi-thread"] } -email-service = { path = "../../examples/email-service" } -apalis = { path = "../../", default-features = false } -once_cell = "1.19.0" -apalis-sql = { path = ".", features = ["tokio-comp"] } -apalis-core = { path = "../apalis-core", features = ["test-utils"] } - -[package.metadata.docs.rs] -# defines the configuration attribute `docsrs` -rustdoc-args = ["--cfg", "docsrs"] -all-features = true diff --git a/packages/apalis-sql/build.rs b/packages/apalis-sql/build.rs deleted file mode 100644 index 3a8149e..0000000 --- a/packages/apalis-sql/build.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("cargo:rerun-if-changed=migrations"); -} diff --git a/packages/apalis-sql/migrations/sqlite/20220530084123_jobs_workers.sql b/packages/apalis-sql/migrations/sqlite/20220530084123_jobs_workers.sql deleted file mode 100644 index 5883b48..0000000 --- a/packages/apalis-sql/migrations/sqlite/20220530084123_jobs_workers.sql +++ /dev/null @@ -1,36 +0,0 @@ -CREATE TABLE IF NOT EXISTS Workers ( - id TEXT NOT NULL UNIQUE, - worker_type TEXT NOT NULL, - storage_name TEXT NOT NULL, - layers TEXT, - last_seen INTEGER NOT NULL DEFAULT (strftime('%s', 'now')) -); - -CREATE INDEX IF NOT EXISTS Idx ON Workers(id); - -CREATE INDEX IF NOT EXISTS WTIdx ON Workers(worker_type); - -CREATE INDEX IF NOT EXISTS LSIdx ON Workers(last_seen); - -CREATE TABLE IF NOT EXISTS Jobs ( - job TEXT NOT NULL, - id TEXT NOT NULL UNIQUE, - job_type TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'Pending', - attempts INTEGER NOT NULL DEFAULT 0, - max_attempts INTEGER NOT NULL DEFAULT 25, - run_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')), - last_error TEXT, - lock_at INTEGER, - lock_by TEXT, - done_at INTEGER, - FOREIGN KEY(lock_by) REFERENCES Workers(id) -); - -CREATE INDEX IF NOT EXISTS TIdx ON Jobs(id); - -CREATE INDEX IF NOT EXISTS SIdx ON Jobs(status); - -CREATE INDEX IF NOT EXISTS LIdx ON Jobs(lock_by); - -CREATE INDEX IF NOT EXISTS JTIdx ON Jobs(job_type); \ No newline at end of file diff --git a/packages/apalis-sql/migrations/sqlite/20250313213411_add_job_priority.sql b/packages/apalis-sql/migrations/sqlite/20250313213411_add_job_priority.sql deleted file mode 100644 index 030fda7..0000000 --- a/packages/apalis-sql/migrations/sqlite/20250313213411_add_job_priority.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE Jobs -ADD priority INTEGER NOT NULL DEFAULT 0;