feat: custom script path for manual resolution (#111)

* feat: implement manual job resolution with configurable jobs file path and enhance stack trace parsing

* feat: enhance manual job resolution with custom jobs file path and improve error handling

* feat: add dashboard configuration options and enhance manual job resolution documentation

* feat: rename resolveScriptPath to resolveScriptPathForJob and update references

* feat: update manual-loader mock to include resolveScriptPath and improve error message clarity

* feat: update resolveScriptPath tests for cross-platform compatibility and improve error handling

* fix: improve error handling in afterAll cleanup for MANUAL_SCRIPT_TAG file

* refactor: move MANUAL_SCRIPT_TAG setup and teardown to manualJobResolution tests

* feat: change to better-sqlite3

* docs: update documentation to specify better-sqlite3 in SqliteBackend

Author: GiovaniGuizzo

package.json

@@ -80,7 +80,7 @@
     "semantic-release": "^24.2.7",
     "tailwindcss": "^4.1.13",
     "tsx": "^4.20.5",
-    "turbo": "^2.5.6",
+    "turbo": "^2.5.8",
     "typescript": "^5.9.2",
     "typescript-eslint": "^8.42.0",
     "vitepress": "^1.6.4",

packages/backends/sqlite/package.json

@@ -45,8 +45,8 @@
   "dependencies": {
     "@sidequest/backend": "workspace:*",
     "@sidequest/core": "workspace:*",
-    "knex": "^3.1.0",
-    "sqlite3": "^5.1.7"
+    "better-sqlite3": "^12.4.1",
+    "knex": "^3.1.0"
   },
   "devDependencies": {
     "@sidequest/backend-test": "workspace:*"

packages/backends/sqlite/src/sqlite-backend.ts

@@ -5,7 +5,7 @@ import { hostname } from "os";
 import path from "path";
 
 const defaultKnexConfig = {
-  client: "sqlite3",
+  client: "better-sqlite3",
   useNullAsDefault: true,
   migrations: {
     directory: path.join(import.meta.dirname, "..", "migrations"),
@@ -18,7 +18,7 @@ const defaultKnexConfig = {
  * Represents a backend implementation for SQLite databases using Knex.
  *
  * This class extends the `SQLBackend` and configures a Knex instance
- * for SQLite3, specifying the database file path, migration directory,
+ * for better-sqlite3, specifying the database file path, migration directory,
  * migration table name, and file extension for migration files.
  *
  * @example

packages/core/src/job/job.test.ts

@@ -1,7 +1,7 @@
 import { CompletedResult, RetryResult, SnoozeResult } from "@sidequest/core";
 import path from "path";
 import { pathToFileURL } from "url";
-import { Job, resolveScriptPath } from "./job";
+import { Job, resolveScriptPathForJob } from "./job";
 
 export class DummyJob extends Job {
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -116,31 +116,31 @@ describe("job.ts", () => {
   });
 });
 
-describe("resolveScriptPath", () => {
+describe("resolveScriptPathForJob", () => {
   it("should return the input if it is already a file URL", () => {
     const fileUrl = "file:///some/path/to/file.js";
-    expect(resolveScriptPath(fileUrl)).toBe(fileUrl);
+    expect(resolveScriptPathForJob(fileUrl)).toBe(fileUrl);
   });
 
   it("should convert an absolute path to a file URL", () => {
     const absPath = path.resolve("/tmp/test.js");
     const expected = pathToFileURL(absPath).href;
-    expect(resolveScriptPath(absPath)).toBe(expected);
+    expect(resolveScriptPathForJob(absPath)).toBe(expected);
   });
 
   it("should resolve a relative path to a file URL based on import.meta.dirname", () => {
     const relativePath = "test/fixtures/job-script.js";
     const baseDir = import.meta?.dirname ? import.meta.dirname : __dirname;
     const absPath = path.resolve(baseDir, relativePath);
     const expected = pathToFileURL(absPath).href;
-    expect(resolveScriptPath(relativePath)).toBe(expected);
+    expect(resolveScriptPathForJob(relativePath)).toBe(expected);
   });
 
   it("should handle relative paths with ./ and ../", () => {
     const relativePath = "../test/fixtures/job-script.js";
     const baseDir = import.meta?.dirname ? import.meta.dirname : __dirname;
     const absPath = path.resolve(baseDir, relativePath);
     const expected = pathToFileURL(absPath).href;
-    expect(resolveScriptPath(relativePath)).toBe(expected);
+    expect(resolveScriptPathForJob(relativePath)).toBe(expected);
   });
 });

packages/core/src/job/job.ts

@@ -4,6 +4,7 @@ import { pathToFileURL } from "url";
 import { logger } from "../logger";
 import { BackoffStrategy, ErrorData, JobData, JobState } from "../schema";
 import { toErrorData } from "../tools";
+import { parseStackTrace } from "../tools/stack-parser";
 import { CompletedResult, FailedResult, isJobResult, JobResult, RetryResult, SnoozeResult } from "../transitions";
 import { UniquenessConfig } from "../uniquiness";
 
@@ -250,30 +251,20 @@ export abstract class Job implements JobData {
  */
 async function buildPath(className: string) {
   const err = new Error();
-  let stackLines = err.stack?.split("\n") ?? [];
-  stackLines = stackLines.slice(1);
-  logger("Job").debug(`Resolving script file path. Stack lines: ${stackLines.join("\n")}`);
-  const filePaths = stackLines
-    .map((line) => {
-      const match = /(file:\/\/)?(((\/?)(\w:))?([/\\].+)):\d+:\d+/.exec(line);
-      if (match) {
-        return `${match[5] ?? ""}${match[6].replaceAll("\\", "/")}`;
-      }
-      return null;
-    })
-    .filter(Boolean);
+  logger("Job").debug(`Resolving script file path. Stack lines: ${err.stack}`);
+  const filePaths = parseStackTrace(err);
 
   for (const filePath of filePaths) {
-    const hasExported = await hasClassExported(filePath!, className);
+    const hasExported = await hasClassExported(filePath, className);
     if (hasExported) {
-      const relativePath = path.relative(import.meta.dirname, filePath!);
+      const relativePath = path.relative(import.meta.dirname, filePath);
       logger("Job").debug(`${filePath} exports class ${className}, relative path: ${relativePath}`);
       return relativePath.replaceAll("\\", "/");
     }
   }
 
   if (filePaths.length > 0) {
-    const relativePath = path.relative(import.meta.dirname, filePaths[0]!);
+    const relativePath = path.relative(import.meta.dirname, filePaths[0]);
     logger("Job").debug(`No class ${className} found in stack, returning first file path: ${relativePath}`);
     return relativePath.replaceAll("\\", "/");
   }
@@ -294,11 +285,11 @@ async function buildPath(className: string) {
  *
  * @example
  * ```typescript
- * const scriptUrl = resolveScriptPath("../../../examples/hello-job.js");
+ * const scriptUrl = resolveScriptPathForJob("../../../examples/hello-job.js");
  * const module = await import(scriptUrl);
  * ```
  */
-export function resolveScriptPath(relativePath: string): string {
+export function resolveScriptPathForJob(relativePath: string): string {
   // If it's already a file URL, return as-is
   if (relativePath.startsWith("file://")) {
     return relativePath;

packages/core/src/tools/index.ts

@@ -1,2 +1,3 @@
 export * from "./parse-error-data";
 export * from "./serialize-error";
+export * from "./stack-parser";

packages/core/src/tools/stack-parser.test.ts

@@ -0,0 +1,52 @@
+import { parseStackTrace } from "./stack-parser";
+
+describe("parseStackTrace", () => {
+  it("should parse stack trace with Windows file paths", () => {
+    const error = new Error("Test error");
+    error.stack = `Error: Test error
+        at function1 (C:\\Users\\test\\file.js:10:5)
+        at function2 (C:\\Projects\\app\\index.ts:25:12)`;
+
+    const result = parseStackTrace(error);
+    expect(result).toEqual(["C:/Users/test/file.js", "C:/Projects/app/index.ts"]);
+  });
+
+  it("should parse stack trace with Unix file paths", () => {
+    const error = new Error("Test error");
+    error.stack = `Error: Test error
+        at function1 (/home/user/file.js:10:5)
+        at function2 (/opt/app/index.ts:25:12)`;
+
+    const result = parseStackTrace(error);
+    expect(result).toEqual(["/home/user/file.js", "/opt/app/index.ts"]);
+  });
+
+  it("should parse stack trace with file:// protocol", () => {
+    const error = new Error("Test error");
+    error.stack = `Error: Test error
+        at function1 (file:///C:/Users/test/file.js:10:5)
+        at function2 (file:///home/user/app.ts:15:8)`;
+
+    const result = parseStackTrace(error);
+    expect(result).toEqual(["C:/Users/test/file.js", "/home/user/app.ts"]);
+  });
+
+  it("should handle mixed path formats", () => {
+    const error = new Error("Test error");
+    error.stack = `Error: Test error
+        at function1 (C:\\Windows\\file.js:10:5)
+        at function2 (/usr/local/file.ts:20:3)
+        at function3 (file:///D:/project/main.js:5:1)`;
+
+    const result = parseStackTrace(error);
+    expect(result).toEqual(["C:/Windows/file.js", "/usr/local/file.ts", "D:/project/main.js"]);
+  });
+
+  it("should return empty array when stack is undefined", () => {
+    const error = new Error("Test error");
+    error.stack = undefined;
+
+    const result = parseStackTrace(error);
+    expect(result).toEqual([]);
+  });
+});

packages/core/src/tools/stack-parser.ts

@@ -0,0 +1,25 @@
+/**
+ * Parses an error stack trace to extract file paths.
+ *
+ * @param err - The Error object containing the stack trace to parse
+ * @returns An array of normalized file paths extracted from the stack trace, with backslashes converted to forward slashes and null entries filtered out
+ *
+ * @example
+ * ```typescript
+ * const error = new Error('Something went wrong');
+ * const filePaths = parseStackTrace(error);
+ * console.log(filePaths); // ['C:/path/to/file.js', '/another/path/file.ts']
+ * ```
+ */
+export function parseStackTrace(err: Error): string[] {
+  const stackLines = err.stack?.split("\n") ?? [];
+  return stackLines
+    .map((line) => {
+      const match = /(file:\/\/)?(((\/?)(\w:))?([/\\].+)):\d+:\d+/.exec(line);
+      if (match) {
+        return `${match[5] ?? ""}${match[6].replaceAll("\\", "/")}`;
+      }
+      return undefined;
+    })
+    .filter(Boolean) as string[];
+}

packages/docs/engine/configuration.md

@@ -176,6 +176,9 @@ await Sidequest.start({
 | -------------------------------- | ------------------------------------------------------------------------------------------------------------------ | --------------------------- |
 | `backend.driver`                 | Backend driver package name (SQLite, Postgres, MySQL, MongoDB)                                                     | `@sidequest/sqlite-backend` |
 | `backend.config`                 | Backend-specific connection string or [Knex configuration object](https://knexjs.org/guide/#configuration-options) | `./sidequest.sqlite`        |
+| `dashboard.enabled`              | Whether to enable the dashboard web interface                                                                      | `true`                      |
+| `dashboard.port`                 | Port for the dashboard web interface                                                                               | `8678`                      |
+| `dashboard.auth`                 | Basic auth configuration with `user` and `password`. If omitted, no auth is required.                              | `undefined`                 |
 | `queues`                         | Array of queue configurations with name, concurrency, priority, and state                                          | `[]`                        |
 | `maxConcurrentJobs`              | Maximum number of jobs processed simultaneously across all queues                                                  | `10`                        |
 | `minThreads`                     | Minimum number of worker threads to use                                                                            | Number of CPU cores         |
@@ -192,6 +195,8 @@ await Sidequest.start({
 | `gracefulShutdown`               | Whether to enable graceful shutdown handling                                                                       | `true`                      |
 | `jobDefaults`                    | Default values for new jobs. Used while enqueueing                                                                 | `undefined`                 |
 | `queueDefaults`                  | Default values for auto-created queues                                                                             | `undefined`                 |
+| `manualJobResolution`            | Whether to manually resolve job classes. See [Manual Job Resolution](/jobs/manual-resolution.md)                   | `false`                     |
+| `jobsFilePath`                   | Optional path to the file where job classes are exported. Ignored if `manualJobResolution` is `false`.             | `undefined`                 |
 
 ::: danger
 If `auth` is not configured and `dashboard: true` is enabled in production, the dashboard will be publicly accessible. This is a security risk and **not recommended**.

packages/docs/jobs/manual-resolution.md

@@ -28,7 +28,7 @@ When you see `sidequest.jobs.js` as the job script, it indicates that the job cl
 When manual job resolution is enabled:
 
 1. **Job Enqueuing**: Jobs are enqueued with `script: "sidequest.jobs.js"` instead of specific file paths
-2. **Job Execution**: The runner looks for a `sidequest.jobs.js` file in the current directory or any parent directory
+2. **Job Execution**: The runner looks for a `sidequest.jobs.js` file in the current directory or any parent directory. If the `jobsFilePath` configuration is set, it uses that path instead.
 3. **Class Resolution**: Job classes are imported from the central registry file instead of individual script files
 
 ## Setting Up Manual Job Resolution
@@ -45,12 +45,13 @@ await Sidequest.start({
   backend: { driver: "@sidequest/sqlite-backend" },
   queues: [{ name: "default" }],
   manualJobResolution: true, // Enable manual job resolution
+  jobsFilePath: "./sidequest.jobs.js", // Optional: specify custom path. If not set, defaults to searching for sidequest.jobs.js in parent dirs
 });
 ```
 
 ### Step 2: Create the Job Registry
 
-Create a `sidequest.jobs.js` file in your project root (or any parent directory):
+Create a `sidequest.jobs.js` file in your project root (or any parent directory), or where you specified in `jobsFilePath`:
 
 ```javascript
 // sidequest.jobs.js
@@ -75,6 +76,8 @@ await Sidequest.build(ProcessImageJob).with(imageProcessor).enqueue("/path/to/im
 
 ## File Discovery
 
+### Finding `sidequest.jobs.js` when `jobsFilePath` is not set
+
 Sidequest searches for the `sidequest.jobs.js` file using the following strategy:
 
 1. **Current Working Directory**: Starts from `process.cwd()`
@@ -126,6 +129,24 @@ For example:
 In this case, since `sidequest.jobs.js` is at the `My Projects/` level, both worker projects can find it when they start up.
 If this file exports all job classes used by both projects, everything will work seamlessly.
 
+### Finding `sidequest.jobs.js` when `jobsFilePath` is set
+
+If you set the `jobsFilePath` configuration option, Sidequest will use that exact path to locate the `sidequest.jobs.js` file.
+This is useful if you want to place the file in a non-standard location or have multiple job registries for different environments.
+
+If you provide an absolute path or file URL, Sidequest will use that directly.
+However, if you provide a relative path, it will be resolved relative to the file that called `Sidequest.start` or `Sidequest.configure`.
+
+For example, if you provide `jobsFilePath: "./config/sidequest.jobs.js"` in your main application file which is located at `/app/src/server.js`, Sidequest will look for the jobs file at `/app/src/config/sidequest.jobs.js`.
+
+```text
+/app/
+└── src/
+    ├── server.js
+    └── config/
+        └── sidequest.jobs.js
+```
+
 ## Best Practices
 
 ### 1. Consistent Export Strategy