feat: add obey robots flag (#6)

* feat: add obey robots flag

* chore: upgrade tooling packages

* docs(changeset): *Features*
- (#5) Add obey_robots option

*Misc*
- Upgrade tooling packages

* RELEASING: Releasing 1 package(s)

Releases:
  @lightpanda/browser@1.1.0

[skip ci]

Author: nrigaudiere

package.json

@@ -24,9 +24,9 @@
     "publish-packages": "turbo run build lint && changeset version && changeset publish"
   },
   "devDependencies": {
-    "@changesets/changelog-github": "^0.5.1",
-    "@changesets/cli": "^2.29.5",
-    "turbo": "^2.5.4"
+    "@changesets/changelog-github": "^0.5.2",
+    "@changesets/cli": "^2.29.8",
+    "turbo": "^2.8.5"
   },
   "engines": {
     "node": "22.16",

packages/browser/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @lightpanda/browser
 
+## 1.1.0
+
+### Minor Changes
+
+- [#6](https://github.com/lightpanda-io/node-packages/pull/6) [`f64ce54`](https://github.com/lightpanda-io/node-packages/commit/f64ce54c8bbc4b7dd11d02c727f19281af705d1c) Thanks [@nrigaudiere](https://github.com/nrigaudiere)! - _Features_
+
+  - (#5) Add obey_robots option
+
+  _Misc_
+
+  - Upgrade tooling packages
+
 ## 1.0.1
 
 ### Patch Changes

packages/browser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightpanda/browser",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "description": "Lightpanda for Node.js",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",
@@ -38,10 +38,10 @@
   "devDependencies": {
     "@biomejs/biome": "^1.9.4",
     "@types/node": "22.15.32",
-    "@types/yargs": "^17.0.33",
-    "terser": "^5.43.1",
-    "tsup": "^8.5.0",
-    "typescript": "^5.8.3"
+    "@types/yargs": "^17.0.35",
+    "terser": "^5.46.0",
+    "tsup": "^8.5.1",
+    "typescript": "^5.9.3"
   },
   "dependencies": {
     "yargs": "^18.0.0"

packages/browser/src/fetch.ts

@@ -21,11 +21,13 @@ import { getExecutablePath, validateUrl } from './utils'
  * @type {object}
  * @property {boolean} dump - Export fetched output as string
  * @property {boolean} disableHostVerification - Disables host verification on all HTTP requests
+ * @property {boolean} obeyRobots - Fetches and obeys the robots.txt (if available) of the web pages we make requests towards.
  * @property {string} httpProxy - The HTTP proxy to use for all HTTP requests
  */
 export type LightpandaFetchOptions = {
   dump?: boolean
   disableHostVerification?: boolean
+  obeyRobots?: boolean
   httpProxy?: string
 }
 
@@ -40,7 +42,7 @@ const defaultOptions: LightpandaFetchOptions = {
  * @returns {Promise<Buffer | string>}
  */
 export const fetch = (url: string, options: LightpandaFetchOptions = defaultOptions) => {
-  const { dump, disableHostVerification, httpProxy } = options
+  const { dump, disableHostVerification, obeyRobots, httpProxy } = options
   validateUrl(url)
 
   if (httpProxy) {
@@ -53,6 +55,7 @@ export const fetch = (url: string, options: LightpandaFetchOptions = defaultOpti
       const flags = [
         { flag: '--dump', condition: dump },
         { flag: '--insecure_disable_tls_host_verification', condition: disableHostVerification },
+        { flag: '--obey_robots', condition: obeyRobots },
         { flag: `--http_proxy ${httpProxy}`, condition: httpProxy },
       ]
         .map(f => (f.condition ? f.flag : ''))

packages/browser/src/serve.ts

@@ -23,13 +23,15 @@ import { getExecutablePath, validatePort, validateUrl } from './utils'
  * @property {string} port - Port of the CDP server
  * @property {number} timeout - Inactivity timeout in seconds before disconnecting clients
  * @property {boolean} disableHostVerification - Disables host verification on all HTTP requests
+ * @property {boolean} obeyRobots - Fetches and obeys the robots.txt (if available) of the web pages we make requests towards.
  * @property {string} httpProxy - The HTTP proxy to use for all HTTP requests
  */
 export type LightpandaServeOptions = {
   host?: string
   port?: number
   timeout?: number
   disableHostVerification?: boolean
+  obeyRobots?: boolean
   httpProxy?: string
 }
 
@@ -44,7 +46,7 @@ const defaultOptions: LightpandaServeOptions = {
  * @returns {Promise<ChildProcessWithoutNullStreams>}
  */
 export const serve = (options: LightpandaServeOptions = defaultOptions) => {
-  const { host, port, timeout, disableHostVerification, httpProxy } = options
+  const { host, port, timeout, disableHostVerification, obeyRobots, httpProxy } = options
 
   if (port) {
     validatePort(port)
@@ -64,6 +66,11 @@ export const serve = (options: LightpandaServeOptions = defaultOptions) => {
         value: disableHostVerification,
         flagOnly: true,
       },
+      {
+        flag: '--obey_robots',
+        value: obeyRobots,
+        flagOnly: true,
+      },
       { flag: '--http_proxy', value: httpProxy },
     ]
       .flatMap(f => (f.value ? [f.flag, !f.flagOnly ? f.value.toString() : ''] : ''))