From 454bacd7d783e52cfd2fd4bc40e4b626671c0c2d Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Tue, 8 Apr 2025 20:38:19 +0300 Subject: [PATCH 01/13] fix: python project detection and project detection --- package.json | 12 +- src/commands/actors/pull.ts | 2 +- src/commands/actors/push.ts | 1 - src/commands/actors/rm.ts | 2 +- src/commands/builds/rm.ts | 2 +- src/commands/create.ts | 4 - src/commands/datasets/rm.ts | 2 +- src/commands/init.ts | 54 ++-- src/commands/key-value-stores/delete-value.ts | 2 +- src/commands/key-value-stores/rm.ts | 2 +- src/commands/run.ts | 5 - src/commands/runs/rm.ts | 2 +- src/lib/apify_command.ts | 2 +- src/lib/files.ts | 2 - src/lib/hooks/runtimes/javascript.ts | 79 ++++++ src/lib/hooks/runtimes/python.ts | 79 ++++++ src/lib/hooks/useActorConfig.ts | 225 ++++++++++++++++ src/lib/hooks/useCwdProject.ts | 187 +++++++++++++ src/lib/input_schema.ts | 25 +- src/lib/local_state.ts | 10 +- src/lib/secrets.ts | 12 +- src/lib/telemetry.ts | 13 +- src/lib/utils.ts | 248 +----------------- src/lib/{commands => utils}/confirm.ts | 16 +- yarn.lock | 147 +++++------ 25 files changed, 726 insertions(+), 409 deletions(-) create mode 100644 src/lib/hooks/runtimes/javascript.ts create mode 100644 src/lib/hooks/runtimes/python.ts create mode 100644 src/lib/hooks/useActorConfig.ts create mode 100644 src/lib/hooks/useCwdProject.ts rename src/lib/{commands => utils}/confirm.ts (67%) diff --git a/package.json b/package.json index a7f837598..5b63fb9b2 100644 --- a/package.json +++ b/package.json @@ -81,8 +81,8 @@ "cors": "~2.8.5", "detect-indent": "~7.0.1", "escape-string-regexp": "~5.0.0", + "execa": "^9.5.2", "express": "~4.21.0", - "fs-extra": "^11.2.0", "globby": "~14.1.0", "handlebars": "~4.7.8", "inquirer": "~12.5.0", @@ -90,7 +90,6 @@ "is-online": "~11.0.0", "istextorbinary": "~9.5.0", "jju": "~1.4.0", - "load-json-file": "~7.0.1", "lodash.clonedeep": "^4.5.0", "mime": "~4.0.4", "mixpanel": "~0.18.0", @@ -99,8 +98,7 @@ "rimraf": "~6.0.1", "semver": "~7.7.0", "tiged": "~2.12.7", - "underscore": "~1.13.7", - "write-json-file": "~6.0.0" + "which": "^5.0.0" }, "devDependencies": { "@apify/eslint-config": "^0.4.0", @@ -110,11 +108,12 @@ "@crawlee/types": "^3.11.1", "@cucumber/cucumber": "^11.0.0", "@oclif/test": "^4.0.8", - "@sapphire/result": "^2.6.6", + "@sapphire/result": "^2.7.2", "@types/adm-zip": "^0.5.5", "@types/archiver": "^6.0.2", "@types/chai": "^4.3.17", "@types/cors": "^2.8.17", + "@types/execa": "^2.0.2", "@types/express": "^4.17.21", "@types/fs-extra": "^11", "@types/inquirer": "^9.0.7", @@ -124,7 +123,7 @@ "@types/mime": "^4.0.0", "@types/node": "^22.0.0", "@types/semver": "^7.5.8", - "@types/underscore": "^1.11.15", + "@types/which": "^3.0.4", "@typescript-eslint/eslint-plugin": "^7.0.2", "@typescript-eslint/parser": "^7.0.2", "@yarnpkg/core": "^4.1.2", @@ -133,7 +132,6 @@ "cross-env": "^7.0.3", "eslint": "^8.57.0", "eslint-config-prettier": "^10.0.0", - "execa": "^9.5.2", "lint-staged": "^15.2.8", "mdast-util-from-markdown": "^2.0.2", "mock-stdin": "^1.0.0", diff --git a/src/commands/actors/pull.ts b/src/commands/actors/pull.ts index bd4b74c80..2edd8e9c2 100644 --- a/src/commands/actors/pull.ts +++ b/src/commands/actors/pull.ts @@ -12,7 +12,7 @@ import tiged from 'tiged'; import { ApifyCommand } from '../../lib/apify_command.js'; import { CommandExitCodes, LOCAL_CONFIG_PATH } from '../../lib/consts.js'; import { error, success } from '../../lib/outputs.js'; -import { getLocalConfigOrThrow, getLocalUserInfo, getLoggedClientOrThrow } from '../../lib/utils.js'; +import { getLocalUserInfo, getLoggedClientOrThrow } from '../../lib/utils.js'; const extractGitHubZip = async (url: string, directoryPath: string) => { const { data } = await axios.get(url, { responseType: 'arraybuffer' }); diff --git a/src/commands/actors/push.ts b/src/commands/actors/push.ts index f7c12739f..da7ad3ba7 100644 --- a/src/commands/actors/push.ts +++ b/src/commands/actors/push.ts @@ -19,7 +19,6 @@ import { createActZip, createSourceFiles, getActorLocalFilePaths, - getLocalConfigOrThrow, getLocalUserInfo, getLoggedClientOrThrow, outputJobLog, diff --git a/src/commands/actors/rm.ts b/src/commands/actors/rm.ts index cab233c41..bcb5c1b42 100644 --- a/src/commands/actors/rm.ts +++ b/src/commands/actors/rm.ts @@ -2,7 +2,7 @@ import { Args } from '@oclif/core'; import type { ApifyApiError } from 'apify-client'; import { ApifyCommand } from '../../lib/apify_command.js'; -import { confirmAction } from '../../lib/commands/confirm.js'; +import { confirmAction } from '../../lib/utils/confirm.js'; import { error, info, success } from '../../lib/outputs.js'; import { getLoggedClientOrThrow } from '../../lib/utils.js'; diff --git a/src/commands/builds/rm.ts b/src/commands/builds/rm.ts index e03c7ac7b..fc9ddbe40 100644 --- a/src/commands/builds/rm.ts +++ b/src/commands/builds/rm.ts @@ -2,7 +2,7 @@ import { Args } from '@oclif/core'; import type { ActorTaggedBuild, ApifyApiError } from 'apify-client'; import { ApifyCommand } from '../../lib/apify_command.js'; -import { confirmAction } from '../../lib/commands/confirm.js'; +import { confirmAction } from '../../lib/utils/confirm.js'; import { error, info, success } from '../../lib/outputs.js'; import { getLoggedClientOrThrow } from '../../lib/utils.js'; diff --git a/src/commands/create.ts b/src/commands/create.ts index edd6fc620..b17c51ff5 100644 --- a/src/commands/create.ts +++ b/src/commands/create.ts @@ -15,13 +15,9 @@ import { updateLocalJson } from '../lib/files.js'; import { createPrefilledInputFileFromInputSchema } from '../lib/input_schema.js'; import { error, info, success, warning } from '../lib/outputs.js'; import { - detectNodeVersion, - detectNpmVersion, - detectPythonVersion, downloadAndUnzip, getJsonFileContent, getNpmCmd, - getPythonCommand, isNodeVersionSupported, isPythonVersionSupported, setLocalConfig, diff --git a/src/commands/datasets/rm.ts b/src/commands/datasets/rm.ts index 18d046cca..4c2297dd2 100644 --- a/src/commands/datasets/rm.ts +++ b/src/commands/datasets/rm.ts @@ -3,7 +3,7 @@ import type { ApifyApiError } from 'apify-client'; import chalk from 'chalk'; import { ApifyCommand } from '../../lib/apify_command.js'; -import { confirmAction } from '../../lib/commands/confirm.js'; +import { confirmAction } from '../../lib/utils/confirm.js'; import { tryToGetDataset } from '../../lib/commands/storages.js'; import { error, info, success } from '../../lib/outputs.js'; import { getLoggedClientOrThrow } from '../../lib/utils.js'; diff --git a/src/commands/init.ts b/src/commands/init.ts index 07869c627..6c01b4e70 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -9,22 +9,15 @@ import { CommandExitCodes, DEFAULT_LOCAL_STORAGE_DIR, EMPTY_LOCAL_CONFIG, - LANGUAGE, LOCAL_CONFIG_PATH, PROJECT_TYPES, } from '../lib/consts.js'; +import { useActorConfig } from '../lib/hooks/useActorConfig.js'; +import { ProjectLanguage, useCwdProject } from '../lib/hooks/useCwdProject.js'; import { createPrefilledInputFileFromInputSchema } from '../lib/input_schema.js'; import { error, info, success, warning } from '../lib/outputs.js'; -import { ProjectAnalyzer } from '../lib/project_analyzer.js'; import { wrapScrapyProject } from '../lib/projects/scrapy/wrapScrapyProject.js'; -import { - detectLocalActorLanguage, - getLocalConfig, - getLocalConfigOrThrow, - setLocalConfig, - setLocalEnv, - validateActorName, -} from '../lib/utils.js'; +import { setLocalConfig, setLocalEnv, validateActorName } from '../lib/utils.js'; export class InitCommand extends ApifyCommand { static override description = @@ -53,26 +46,44 @@ export class InitCommand extends ApifyCommand { let { actorName } = this.args; const cwd = process.cwd(); - if (ProjectAnalyzer.getProjectType(cwd) === PROJECT_TYPES.SCRAPY) { + const projectResult = await useCwdProject(); + + // TODO: use direct .unwrap() once we migrate to yargs + if (projectResult.isErr()) { + error({ message: projectResult.unwrapErr().message }); + process.exit(1); + } + + const project = projectResult.unwrap(); + + if (project.type === ProjectLanguage.Scrapy) { info({ message: 'The current directory looks like a Scrapy project. Using automatic project wrapping.' }); this.telemetryData.actorWrapper = PROJECT_TYPES.SCRAPY; return wrapScrapyProject({ projectPath: cwd }); } - if (!this.flags.yes && detectLocalActorLanguage(cwd).language === LANGUAGE.UNKNOWN) { + if (!this.flags.yes && project.type === ProjectLanguage.Unknown) { warning({ message: 'The current directory does not look like a Node.js or Python project.' }); const { c } = await inquirer.prompt([{ name: 'c', message: 'Do you want to continue?', type: 'confirm' }]); if (!c) return; } - if (getLocalConfig(cwd)) { + const actorConfig = await useActorConfig({ cwd }); + + if (actorConfig.isOkAnd((cfg) => cfg.exists && !cfg.migrated)) { warning({ message: `Skipping creation of '${LOCAL_CONFIG_PATH}', the file already exists in the current directory.`, }); } else { + if (actorConfig.isErr()) { + error({ message: actorConfig.unwrapErr().message }); + process.exitCode = CommandExitCodes.InvalidActorJson; + return; + } + if (!actorName) { - let response = null; + let response = actorConfig.isOk() ? { actName: actorConfig.unwrap().config.name as string } : null; while (!response) { try { @@ -89,21 +100,8 @@ export class InitCommand extends ApifyCommand { ({ actName: actorName } = response); } - let existingLocalConfig: Record | undefined; - - try { - existingLocalConfig = await getLocalConfigOrThrow(cwd); - } catch (_error) { - const casted = _error as Error; - const cause = casted.cause as Error; - - error({ message: `${casted.message}\n ${cause.message}` }); - process.exitCode = CommandExitCodes.InvalidActorJson; - return; - } - // Migrate apify.json to .actor/actor.json - const localConfig = { ...EMPTY_LOCAL_CONFIG, ...existingLocalConfig }; + const localConfig = { ...EMPTY_LOCAL_CONFIG, ...actorConfig.unwrap().config }; await setLocalConfig(Object.assign(localConfig, { name: actorName }), cwd); } diff --git a/src/commands/key-value-stores/delete-value.ts b/src/commands/key-value-stores/delete-value.ts index f66402f78..c756301ff 100644 --- a/src/commands/key-value-stores/delete-value.ts +++ b/src/commands/key-value-stores/delete-value.ts @@ -3,7 +3,7 @@ import type { ApifyApiError } from 'apify-client'; import chalk from 'chalk'; import { ApifyCommand } from '../../lib/apify_command.js'; -import { confirmAction } from '../../lib/commands/confirm.js'; +import { confirmAction } from '../../lib/utils/confirm.js'; import { tryToGetKeyValueStore } from '../../lib/commands/storages.js'; import { error, info } from '../../lib/outputs.js'; import { getLoggedClientOrThrow } from '../../lib/utils.js'; diff --git a/src/commands/key-value-stores/rm.ts b/src/commands/key-value-stores/rm.ts index 1f0c48fbc..53536dc01 100644 --- a/src/commands/key-value-stores/rm.ts +++ b/src/commands/key-value-stores/rm.ts @@ -3,7 +3,7 @@ import type { ApifyApiError } from 'apify-client'; import chalk from 'chalk'; import { ApifyCommand } from '../../lib/apify_command.js'; -import { confirmAction } from '../../lib/commands/confirm.js'; +import { confirmAction } from '../../lib/utils/confirm.js'; import { tryToGetKeyValueStore } from '../../lib/commands/storages.js'; import { error, info, success } from '../../lib/outputs.js'; import { getLoggedClientOrThrow } from '../../lib/utils.js'; diff --git a/src/commands/run.ts b/src/commands/run.ts index 883515790..c381cf0bc 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -6,8 +6,6 @@ import process from 'node:process'; import { APIFY_ENV_VARS } from '@apify/consts'; import { validateInputSchema, validateInputUsingValidator } from '@apify/input_schema'; import { Flags } from '@oclif/core'; -import { ensureDir } from 'fs-extra'; -import { loadJsonFile } from 'load-json-file'; import mime from 'mime'; import { minVersion } from 'semver'; @@ -31,14 +29,11 @@ import { replaceSecretsValue } from '../lib/secrets.js'; import { Ajv, checkIfStorageIsEmpty, - detectLocalActorLanguage, - getLocalConfigOrThrow, getLocalInput, getLocalKeyValueStorePath, getLocalStorageDir, getLocalUserInfo, getNpmCmd, - getPythonCommand, isNodeVersionSupported, isPythonVersionSupported, purgeDefaultDataset, diff --git a/src/commands/runs/rm.ts b/src/commands/runs/rm.ts index 1ef3281fe..5c6afc995 100644 --- a/src/commands/runs/rm.ts +++ b/src/commands/runs/rm.ts @@ -3,7 +3,7 @@ import { Args } from '@oclif/core'; import type { ApifyApiError } from 'apify-client'; import { ApifyCommand } from '../../lib/apify_command.js'; -import { confirmAction } from '../../lib/commands/confirm.js'; +import { confirmAction } from '../../lib/utils/confirm.js'; import { error, info, success } from '../../lib/outputs.js'; import { getLoggedClientOrThrow } from '../../lib/utils.js'; diff --git a/src/lib/apify_command.ts b/src/lib/apify_command.ts index aed1d559d..ef67cb4fa 100644 --- a/src/lib/apify_command.ts +++ b/src/lib/apify_command.ts @@ -4,7 +4,7 @@ import { Command, type Interfaces, loadHelpClass } from '@oclif/core'; import { COMMANDS_WITHIN_ACTOR, LANGUAGE } from './consts.js'; import { maybeTrackTelemetry } from './telemetry.js'; -import { type KeysToCamelCase, argsToCamelCase, detectLocalActorLanguage } from './utils.js'; +import { type KeysToCamelCase, argsToCamelCase } from './utils.js'; import { detectInstallationType } from './version_check.js'; export type ApifyFlags = KeysToCamelCase>; diff --git a/src/lib/files.ts b/src/lib/files.ts index ffae7e026..5c3d6728c 100644 --- a/src/lib/files.ts +++ b/src/lib/files.ts @@ -2,9 +2,7 @@ import { existsSync, mkdirSync } from 'node:fs'; import { stat, unlink } from 'node:fs/promises'; import { join, sep } from 'node:path'; -import { loadJsonFile } from 'load-json-file'; import { rimraf } from 'rimraf'; -import { writeJsonFile } from 'write-json-file'; export const updateLocalJson = async ( jsonFilePath: string, diff --git a/src/lib/hooks/runtimes/javascript.ts b/src/lib/hooks/runtimes/javascript.ts new file mode 100644 index 000000000..05d0d300d --- /dev/null +++ b/src/lib/hooks/runtimes/javascript.ts @@ -0,0 +1,79 @@ +import { none, some, type Option } from '@sapphire/result'; +import { execa } from 'execa'; +import which from 'which'; + +import type { Runtime } from '../useCwdProject.js'; + +// Runtimes, in order of preference +const runtimesToCheck = { + node: ['--version'], + deno: ['eval', '"console.log(process.versions.node)"'], + bun: ['--eval', '"console.log(process.versions.node)"'], +}; + +async function getRuntimeVersion(runtimePath: string, args: string[]) { + try { + const result = await execa(runtimePath, args, { + shell: true, + windowsHide: true, + }); + + // No output -> issue or who knows + if (!result.stdout) { + return null; + } + + return result.stdout.trim().replace(/^v/, ''); + } catch { + return null; + } +} + +async function getNpmVersion(npmPath: string) { + const result = await execa(npmPath, ['--version'], { + shell: true, + windowsHide: true, + }); + + if (!result.stdout) { + return null; + } + + return result.stdout.trim().replace(/^v/, ''); +} + +export async function useJavaScriptRuntime(): Promise< + Option +> { + for (const [runtime, args] of Object.entries(runtimesToCheck) as [keyof typeof runtimesToCheck, string[]][]) { + try { + const runtimePath = await which(runtime); + + const version = await getRuntimeVersion(runtimePath, args); + + if (version) { + const res: Runtime & { runtimeShorthand: keyof typeof runtimesToCheck } = { + executablePath: runtimePath, + version, + runtimeShorthand: runtime, + }; + + // For npm, we also fetch the npm version + if (runtime === 'node') { + const npmPath = await which('npm').catch(() => null); + + if (npmPath) { + res.pmVersion = await getNpmVersion(npmPath); + res.pmPath = npmPath; + } + } + + return some(res); + } + } catch { + // Ignore errors + } + } + + return none; +} diff --git a/src/lib/hooks/runtimes/python.ts b/src/lib/hooks/runtimes/python.ts new file mode 100644 index 000000000..4a5e89d2d --- /dev/null +++ b/src/lib/hooks/runtimes/python.ts @@ -0,0 +1,79 @@ +import { access } from 'node:fs/promises'; +import { platform } from 'node:os'; +import { join } from 'node:path'; +import process from 'node:process'; + +import { none, some, type Option } from '@sapphire/result'; +import { execa } from 'execa'; +import which from 'which'; + +import type { Runtime } from '../useCwdProject.js'; + +async function getPythonVersion(runtimePath: string) { + try { + const result = await execa(runtimePath, ['-c', '"import platform; print(platform.python_version())"'], { + shell: true, + windowsHide: true, + }); + + // No output -> issue or who knows + if (!result.stdout) { + return null; + } + + return result.stdout.trim(); + } catch (ex) { + // const casted = ex as ExecaError; + + return null; + } +} + +export async function usePythonRuntime(cwd = process.cwd()): Promise> { + const isWindows = platform() === 'win32'; + + const pathParts = isWindows ? ['Scripts', 'python.exe'] : ['bin', 'python3']; + + let fullPythonVenvPath; + if (process.env.VIRTUAL_ENV) { + fullPythonVenvPath = join(process.env.VIRTUAL_ENV, ...pathParts); + } else { + fullPythonVenvPath = join(cwd, '.venv', ...pathParts); + } + + try { + await access(fullPythonVenvPath); + + const version = await getPythonVersion(fullPythonVenvPath); + + if (version) { + return some({ + executablePath: fullPythonVenvPath, + version, + }); + } + } catch { + // Ignore errors + } + + const fallbacks = ['python3', 'python', ...(isWindows ? ['python3.exe', 'python.exe'] : [])]; + + for (const fallback of fallbacks) { + try { + const fullPath = await which(fallback); + + const version = await getPythonVersion(fullPath); + + if (version) { + return some({ + executablePath: fullPath, + version, + }); + } + } catch { + // Continue + } + } + + return none; +} diff --git a/src/lib/hooks/useActorConfig.ts b/src/lib/hooks/useActorConfig.ts new file mode 100644 index 000000000..759a7d2fa --- /dev/null +++ b/src/lib/hooks/useActorConfig.ts @@ -0,0 +1,225 @@ +import { mkdir, rename, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import process from 'node:process'; +import { inspect } from 'node:util'; + +import { err, ok, type Result } from '@sapphire/result'; + +import { ACTOR_SPECIFICATION_VERSION, DEPRECATED_LOCAL_CONFIG_NAME } from '../consts.js'; +import { error, info, warning } from '../outputs.js'; +import { confirmAction } from '../utils/confirm.js'; +import { getJsonFileContent, getLocalConfigPath } from '../utils.js'; + +const getDeprecatedLocalConfigPath = (cwd: string) => join(cwd, DEPRECATED_LOCAL_CONFIG_NAME); + +export type ActorConfigError = { + message: string; + cause?: Error; + exists: boolean; + config: Record; +}; + +export interface ActorConfigResult { + exists: boolean; + migrated: boolean; + config: Record; +} + +export interface ActorConfigInput { + cwd?: string; + migrateConfig?: boolean; +} + +export async function useActorConfig({ + cwd = process.cwd(), + migrateConfig = true, +}: ActorConfigInput): Promise> { + const newConfigPath = getLocalConfigPath(cwd); + const deprecatedConfigPath = getDeprecatedLocalConfigPath(cwd); + + let config: Record | undefined; + let deprecatedConfig: Record | undefined; + + try { + config = getJsonFileContent(newConfigPath); + } catch (ex) { + return err({ + message: `Failed to read local config at path: '${newConfigPath}':`, + cause: ex as Error, + exists: false, + config: {}, + }); + } + + try { + deprecatedConfig = getJsonFileContent(deprecatedConfigPath); + } catch (ex) { + return err({ + message: `Failed to read local config at path: '${deprecatedConfigPath}':`, + cause: ex as Error, + exists: false, + config: {}, + }); + } + + // Handle cleanup of deprecated config, we just ignore the deprecated config if both exist + if (config && deprecatedConfig) { + await handleBothConfigVersionsFound(deprecatedConfigPath); + } + + if (!config && !deprecatedConfig) { + return ok({ exists: false, migrated: false, config: {} }); + } + + let migrated = false; + + if (!config && deprecatedConfig && migrateConfig) { + const migratedConfig = await handleMigrationFlow(deprecatedConfig, deprecatedConfigPath, newConfigPath); + + if (migratedConfig.isErr()) { + return err(migratedConfig.unwrapErr()); + } + + config = migratedConfig.unwrap(); + migrated = true; + } + + return ok({ exists: true, migrated, config: config || deprecatedConfig || {} }); +} + +async function handleBothConfigVersionsFound(deprecatedConfigPath: string) { + const confirmed = await confirmAction({ + type: 'actor config', + message: `The new version of Apify CLI uses the ".actor/actor.json" instead of the "apify.json" file. Since we have found both files in your Actor directory, "apify.json" will be renamed to "apify.json.deprecated". Going forward, all commands will use ".actor/actor.json". You can read about the differences between the old and the new config at https://github.com/apify/apify-cli/blob/master/MIGRATIONS.md. Do you want to continue?`, + }); + + // If users refuse to migrate, 🤷 + if (!confirmed) { + warning({ + message: + 'The "apify.json" file present in your Actor directory will be ignored, and the new ".actor/actor.json" file will be used instead. Please, either rename or remove the old file.', + }); + + return; + } + + try { + await rename(deprecatedConfigPath, `${deprecatedConfigPath}.deprecated`); + + info({ + message: `The "apify.json" file has been renamed to "apify.json.deprecated". The deprecated file is no longer used by the CLI or Apify Console. If you do not need it for some specific purpose, it can be safely deleted.`, + }); + } catch (ex) { + if (ex instanceof Error) { + error({ + message: `Failed to rename the deprecated "apify.json" file to "apify.json.deprecated".\n ${ex.message || ex}`, + }); + } else { + error({ + message: `Failed to rename the deprecated "apify.json" file to "apify.json.deprecated".\n ${inspect(ex, { showHidden: false })}`, + }); + } + } +} + +// Properties from apify.json file that will me migrated to Actor specs in .actor/actor.json +const MIGRATED_APIFY_JSON_PROPERTIES = ['name', 'version', 'buildTag']; + +async function handleMigrationFlow( + deprecatedConfig: Record, + deprecatedConfigPath: string, + configPath: string, +): Promise, ActorConfigError>> { + let migratedConfig = { ...deprecatedConfig }; + + if (typeof migratedConfig.version === 'object') { + migratedConfig = updateLocalConfigStructure(migratedConfig); + } + + migratedConfig = { + actorSpecification: ACTOR_SPECIFICATION_VERSION, + environmentVariables: deprecatedConfig?.env || undefined, + ...MIGRATED_APIFY_JSON_PROPERTIES.reduce( + (acc, prop) => { + acc[prop] = deprecatedConfig[prop]; + return acc; + }, + {} as Record, + ), + }; + + const confirmed = await confirmAction({ + type: 'actor config', + message: `The new version of Apify CLI uses the ".actor/actor.json" instead of the "apify.json" file. Your "apify.json" file will be automatically updated to the new format under ".actor/actor.json". The original file will be renamed by adding the ".deprecated" suffix. Do you want to continue?`, + }); + + if (!confirmed) { + return err({ + message: + 'Command can not run with old "apify.json" structure. Either let the CLI auto-update it or follow the guide on https://github.com/apify/apify-cli/blob/master/MIGRATIONS.md and update it manually.', + exists: true, + config: migratedConfig, + }); + } + + // If we fail to write the new config, its an error + + try { + await mkdir(dirname(configPath), { recursive: true }); + await writeFile(configPath, JSON.stringify(migratedConfig, null, '\t')); + } catch (ex) { + const casted = ex as Error; + + return err({ + message: `Failed to write the new "actor.json" file to path: '${configPath}'.\n ${casted.message || casted}`, + exists: true, + config: migratedConfig, + }); + } + + // but if we fail to rename the deprecated config, its a warning, cause future runs will ignore it + + try { + await rename(deprecatedConfigPath, `${deprecatedConfigPath}.deprecated`); + } catch (ex) { + const casted = ex as Error; + + warning({ + message: `Failed to rename the deprecated "apify.json" file to "apify.json.deprecated".\n ${casted.message || casted}`, + }); + } + + info({ + message: `The "apify.json" file has been migrated to ".actor/actor.json" and the original file renamed to "apify.json.deprecated". The deprecated file is no longer used by the CLI or Apify Console. If you do not need it for some specific purpose, it can be safely deleted. Do not forget to commit the new file to your Git repository.`, + }); + + return ok(migratedConfig); +} + +/** + * Migration for deprecated structure of apify.json to latest. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function updateLocalConfigStructure(localConfig: any) { + const updatedLocalConfig: Record = { + name: localConfig.name, + template: localConfig.template, + version: localConfig.version.versionNumber, + buildTag: localConfig.version.buildTag, + env: null, + }; + + if (localConfig.version.envVars?.length) { + const env = {} as Record; + + localConfig.version.envVars.forEach((envVar: { name: string; value: string }) => { + if (envVar.name && envVar.value) { + env[envVar.name] = envVar.value; + } + }); + + updatedLocalConfig.env = env; + } + + return updatedLocalConfig; +} diff --git a/src/lib/hooks/useCwdProject.ts b/src/lib/hooks/useCwdProject.ts new file mode 100644 index 000000000..b1ce606c2 --- /dev/null +++ b/src/lib/hooks/useCwdProject.ts @@ -0,0 +1,187 @@ +import { access, readFile } from 'node:fs/promises'; +import { basename, join } from 'node:path'; +import process from 'node:process'; + +import { err, ok, type Result } from '@sapphire/result'; + +import { useJavaScriptRuntime } from './runtimes/javascript.js'; +import { usePythonRuntime } from './runtimes/python.js'; +import { ScrapyProjectAnalyzer } from '../projects/scrapy/ScrapyProjectAnalyzer.js'; + +export enum ProjectLanguage { + JavaScript = 0, + Python = 1, + // Special handling for Scrapy projects + Scrapy = 2, + Unknown = 3, + // TODO: eventually when we support entrypoint config in actor json + // https://github.com/apify/apify-cli/issues/766 + StaticEntrypoint = 4, +} + +export interface Runtime { + executablePath: string; + version: string; + pmPath?: string | null; + pmVersion?: string | null; +} + +export interface Entrypoint { + path?: string; + script?: string; +} + +export interface CwdProject { + type: ProjectLanguage; + entrypoint?: Entrypoint; + runtime?: Runtime; +} + +export interface CwdProjectError { + message: string; +} + +export async function useCwdProject(cwd = process.cwd()): Promise> { + const project: CwdProject = { + type: ProjectLanguage.Unknown, + }; + + const check = async (): Promise | undefined> => { + const isScrapy = await checkScrapyProject(cwd); + + if (isScrapy) { + project.type = ProjectLanguage.Scrapy; + return; + } + + const isPython = await checkPythonProject(cwd); + + if (isPython) { + project.type = ProjectLanguage.Python; + + const runtime = await usePythonRuntime(cwd); + + if (runtime.isNone()) { + return err({ + message: 'Failed to detect Python runtime', + }); + } + + project.entrypoint = { + path: isPython, + }; + + project.runtime = runtime.unwrap(); + + return; + } + + const isNode = await checkNodeProject(cwd); + + if (isNode) { + project.type = ProjectLanguage.JavaScript; + + const runtime = await useJavaScriptRuntime(); + + if (runtime.isNone()) { + return err({ + message: 'Failed to detect JavaScript runtime', + }); + } + + project.runtime = runtime.unwrap(); + + if (isNode.type === 'file') { + project.entrypoint = { + path: isNode.path, + }; + } else if (isNode.type === 'script') { + project.entrypoint = { + script: isNode.script, + }; + } + + return; + } + + return ok(project); + }; + + const maybeErr = await check(); + + if (maybeErr?.isErr()) { + return maybeErr; + } + + return ok(project); +} + +async function checkNodeProject(cwd: string) { + const packageJsonPath = join(cwd, 'package.json'); + + try { + const rawString = await readFile(packageJsonPath, 'utf-8'); + + const pkg = JSON.parse(rawString); + + if (pkg.main) { + return { path: join(cwd, pkg.main), type: 'file' } as const; + } + + if (pkg.scripts?.start) { + return { type: 'script', script: pkg.scripts.start } as const; + } + } catch { + // Ignore missing package.json and try some common files + } + + const filesToCheck = [ + join(cwd, 'index.js'), + join(cwd, 'index.mjs'), + join(cwd, 'index.cjs'), + join(cwd, 'src', 'index.js'), + join(cwd, 'src', 'index.mjs'), + join(cwd, 'src', 'index.cjs'), + join(cwd, 'dist', 'index.js'), + join(cwd, 'dist', 'index.mjs'), + join(cwd, 'dist', 'index.cjs'), + ]; + + for (const path of filesToCheck) { + try { + await access(path); + return { path, type: 'file' } as const; + } catch { + // Ignore errors + } + } + + return null; +} + +async function checkPythonProject(cwd: string) { + const baseName = basename(cwd); + + const filesToCheck = [ + join(cwd, 'src', '__main__.py'), + join(cwd, '__main__.py'), + join(cwd, baseName, '__main__.py'), + join(cwd, baseName.replaceAll('-', '_').replaceAll(' ', '_'), '__main__.py'), + ]; + + for (const path of filesToCheck) { + try { + await access(path); + return path; + } catch { + // Ignore errors + } + } + + return null; +} + +async function checkScrapyProject(cwd: string) { + // TODO: maybe rewrite this to a newer format 🤷 + return ScrapyProjectAnalyzer.isApplicable(cwd); +} diff --git a/src/lib/input_schema.ts b/src/lib/input_schema.ts index 7773599fb..0cb53dd49 100644 --- a/src/lib/input_schema.ts +++ b/src/lib/input_schema.ts @@ -1,13 +1,11 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { existsSync } from 'node:fs'; +import { existsSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import process from 'node:process'; import { KEY_VALUE_STORE_KEYS } from '@apify/consts'; import { validateInputSchema } from '@apify/input_schema'; import deepClone from 'lodash.clonedeep'; -import _ from 'underscore'; -import { writeJsonFile } from 'write-json-file'; import { ACTOR_SPECIFICATION_FOLDER } from './consts.js'; import { warning } from './outputs.js'; @@ -90,10 +88,21 @@ export const createPrefilledInputFileFromInputSchema = async (actorFolderDir: st const validator = new Ajv({ strict: false }); validateInputSchema(validator, inputSchema); - inputFile = _.mapObject(inputSchema.properties as any, (fieldSchema) => - fieldSchema.type === 'boolean' || fieldSchema.editor === 'hidden' - ? fieldSchema.default - : fieldSchema.prefill, + // inputFile = _.mapObject(inputSchema.properties as any, (fieldSchema) => + // fieldSchema.type === 'boolean' || fieldSchema.editor === 'hidden' + // ? fieldSchema.default + // : fieldSchema.prefill, + // ); + inputFile = Object.entries(inputSchema.properties as any).reduce( + (acc, [key, fieldSchema]: [string, any]) => { + acc[key] = + fieldSchema.type === 'boolean' || fieldSchema.editor === 'hidden' + ? fieldSchema.default + : fieldSchema.prefill; + + return acc; + }, + {} as Record, ); } } catch (err) { @@ -105,7 +114,7 @@ export const createPrefilledInputFileFromInputSchema = async (actorFolderDir: st } finally { const keyValueStorePath = getLocalKeyValueStorePath(); const inputJsonPath = join(actorFolderDir, keyValueStorePath, `${KEY_VALUE_STORE_KEYS.INPUT}.json`); - await writeJsonFile(inputJsonPath, inputFile); + writeFileSync(inputJsonPath, JSON.stringify(inputFile, null, '\t')); } }; diff --git a/src/lib/local_state.ts b/src/lib/local_state.ts index 2f558d0e8..b449dd6ac 100644 --- a/src/lib/local_state.ts +++ b/src/lib/local_state.ts @@ -1,5 +1,4 @@ -import { loadJsonFileSync } from 'load-json-file'; -import { writeJsonFileSync } from 'write-json-file'; +import { readFileSync, writeFileSync } from 'node:fs'; import { STATE_FILE_PATH } from './consts.js'; @@ -9,7 +8,7 @@ import { STATE_FILE_PATH } from './consts.js'; */ export const getLocalState = () => { try { - return loadJsonFileSync>(STATE_FILE_PATH()) || {}; + return JSON.parse(readFileSync(STATE_FILE_PATH(), 'utf-8')) || {}; } catch (e) { return {}; } @@ -21,8 +20,5 @@ export const getLocalState = () => { */ export const extendLocalState = (data: Record) => { const state = getLocalState(); - writeJsonFileSync(STATE_FILE_PATH(), { - ...state, - ...data, - }); + writeFileSync(STATE_FILE_PATH(), JSON.stringify({ ...state, ...data }, null, '\t')); }; diff --git a/src/lib/secrets.ts b/src/lib/secrets.ts index 776aa5dcf..a0a7f22b4 100644 --- a/src/lib/secrets.ts +++ b/src/lib/secrets.ts @@ -1,6 +1,4 @@ -import { loadJsonFileSync } from 'load-json-file'; -import _ from 'underscore'; -import { writeJsonFileSync } from 'write-json-file'; +import { readFileSync, writeFileSync } from 'node:fs'; import { SECRETS_FILE_PATH } from './consts.js'; import { warning } from './outputs.js'; @@ -13,7 +11,7 @@ const MAX_ENV_VAR_VALUE_LENGTH = 50000; export const getSecretsFile = () => { try { - return loadJsonFileSync>(SECRETS_FILE_PATH()) || {}; + return JSON.parse(readFileSync(SECRETS_FILE_PATH(), 'utf-8')) || {}; } catch (e) { return {}; } @@ -21,7 +19,7 @@ export const getSecretsFile = () => { const writeSecretsFile = (secrets: Record) => { ensureApifyDirectory(SECRETS_FILE_PATH()); - writeJsonFileSync(SECRETS_FILE_PATH(), secrets); + writeFileSync(SECRETS_FILE_PATH(), JSON.stringify(secrets, null, '\t')); return secrets; }; @@ -30,10 +28,10 @@ export const addSecret = (name: string, value: string) => { if (secrets[name]) throw new Error(`Secret with name ${name} already exists. Call "apify secrets rm ${name}" to remove it.`); - if (!_.isString(name) || name.length > MAX_ENV_VAR_NAME_LENGTH) { + if (typeof name !== 'string' || name.length > MAX_ENV_VAR_NAME_LENGTH) { throw new Error(`Secret name has to be string with maximum length ${MAX_ENV_VAR_NAME_LENGTH}.`); } - if (!_.isString(value) || value.length > MAX_ENV_VAR_VALUE_LENGTH) { + if (typeof value !== 'string' || value.length > MAX_ENV_VAR_VALUE_LENGTH) { throw new Error(`Secret value has to be string with maximum length ${MAX_ENV_VAR_VALUE_LENGTH}.`); } diff --git a/src/lib/telemetry.ts b/src/lib/telemetry.ts index 43937aeb5..a12c4c31f 100644 --- a/src/lib/telemetry.ts +++ b/src/lib/telemetry.ts @@ -1,9 +1,8 @@ +import { readFileSync, writeFileSync } from 'node:fs'; import { promisify } from 'node:util'; import { cryptoRandomObjectId } from '@apify/utilities'; -import { loadJsonFile } from 'load-json-file'; import Mixpanel, { type PropertyDict } from 'mixpanel'; -import { writeJsonFile, writeJsonFileSync } from 'write-json-file'; import { MIXPANEL_TOKEN, TELEMETRY_FILE_PATH } from './consts.js'; import { info } from './outputs.js'; @@ -29,7 +28,7 @@ const createLocalDistinctId = () => `CLI:${cryptoRandomObjectId()}`; */ export const getOrCreateLocalDistinctId = async () => { try { - const telemetry = await loadJsonFile<{ distinctId: string }>(TELEMETRY_FILE_PATH()); + const telemetry = JSON.parse(readFileSync(TELEMETRY_FILE_PATH(), 'utf-8')); return telemetry.distinctId; } catch (e) { const userInfo = await getLocalUserInfo(); @@ -39,7 +38,7 @@ export const getOrCreateLocalDistinctId = async () => { info({ message: TELEMETRY_WARNING_TEXT }); ensureApifyDirectory(TELEMETRY_FILE_PATH()); - await writeJsonFile(TELEMETRY_FILE_PATH(), { distinctId }); + writeFileSync(TELEMETRY_FILE_PATH(), JSON.stringify({ distinctId }, null, '\t')); return distinctId; } @@ -48,9 +47,7 @@ export const getOrCreateLocalDistinctId = async () => { export const regenerateLocalDistinctId = () => { try { ensureApifyDirectory(TELEMETRY_FILE_PATH()); - writeJsonFileSync(TELEMETRY_FILE_PATH(), { - distinctId: createLocalDistinctId(), - }); + writeFileSync(TELEMETRY_FILE_PATH(), JSON.stringify({ distinctId: createLocalDistinctId() }, null, '\t')); } catch { // Ignore errors } @@ -89,7 +86,7 @@ export const useApifyIdentity = async (userId: string) => { try { const distinctId = getOrCreateLocalDistinctId(); ensureApifyDirectory(TELEMETRY_FILE_PATH()); - await writeJsonFile(TELEMETRY_FILE_PATH(), { distinctId: userId }); + writeFileSync(TELEMETRY_FILE_PATH(), JSON.stringify({ distinctId: userId }, null, '\t')); await maybeTrackTelemetry({ eventName: '$create_alias', eventData: { diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 47a74f7b9..fbd27dd52 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -1,13 +1,5 @@ -import { type SpawnSyncOptions, spawnSync } from 'node:child_process'; -import { - createWriteStream, - existsSync, - mkdirSync, - readFileSync, - readdirSync, - renameSync, - writeFileSync, -} from 'node:fs'; +import { createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { mkdir } from 'node:fs/promises'; import type { IncomingMessage } from 'node:http'; import { get } from 'node:https'; import { dirname, join } from 'node:path'; @@ -33,22 +25,16 @@ import archiver from 'archiver'; import { AxiosHeaders } from 'axios'; import escapeStringRegexp from 'escape-string-regexp'; import { globby } from 'globby'; -import inquirer from 'inquirer'; import { getEncoding } from 'istextorbinary'; -import { loadJsonFile, loadJsonFileSync } from 'load-json-file'; import { Mime } from 'mime'; import otherMimes from 'mime/types/other.js'; import standardMimes from 'mime/types/standard.js'; import { gte, minVersion, satisfies } from 'semver'; -import _ from 'underscore'; -import { writeJsonFile, writeJsonFileSync } from 'write-json-file'; import { - ACTOR_SPECIFICATION_VERSION, APIFY_CLIENT_DEFAULT_HEADERS, AUTH_FILE_PATH, DEFAULT_LOCAL_STORAGE_DIR, - DEPRECATED_LOCAL_CONFIG_NAME, GLOBAL_CONFIGS_FOLDER, INPUT_FILE_REG_EXP, LANGUAGE, @@ -59,7 +45,6 @@ import { SUPPORTED_NODEJS_VERSION, } from './consts.js'; import { deleteFile, ensureFolderExistsSync, rimrafPromised } from './files.js'; -import { info } from './outputs.js'; import { ProjectAnalyzer } from './project_analyzer.js'; import type { AuthJSON } from './types.js'; @@ -82,9 +67,6 @@ export const httpsGet = async (url: string) => { }); }; -// Properties from apify.json file that will me migrated to Actor specs in .actor/actor.json -const MIGRATED_APIFY_JSON_PROPERTIES = ['name', 'version', 'buildTag']; - export const getLocalStorageDir = () => { const envVar = APIFY_ENV_VARS.LOCAL_STORAGE_DIR; @@ -197,126 +179,29 @@ export const getLoggedClient = async (token?: string, apiBaseUrl?: string) => { // Always refresh Auth file ensureApifyDirectory(AUTH_FILE_PATH()); - await writeJsonFile(AUTH_FILE_PATH(), { token: apifyClient.token, ...userInfo }); + writeFileSync(AUTH_FILE_PATH(), JSON.stringify({ token: apifyClient.token, ...userInfo }, null, '\t')); return apifyClient; }; -const getLocalConfigPath = (cwd: string) => join(cwd, LOCAL_CONFIG_PATH); - -/** - * @deprecated Use getLocalConfigPath - */ -const getDeprecatedLocalConfigPath = (cwd: string) => join(cwd, DEPRECATED_LOCAL_CONFIG_NAME); +export const getLocalConfigPath = (cwd: string) => join(cwd, LOCAL_CONFIG_PATH); export const getJsonFileContent = >(filePath: string) => { if (!existsSync(filePath)) { return; } - return loadJsonFileSync(filePath); + return JSON.parse(readFileSync(filePath, { encoding: 'utf-8' })) as T; }; export const getLocalConfig = (cwd: string) => getJsonFileContent(getLocalConfigPath(cwd)); -/** - * @deprecated Use getLocalConfig - */ -const getDeprecatedLocalConfig = (cwd: string) => getJsonFileContent(getDeprecatedLocalConfigPath(cwd)); - -export const getLocalConfigOrThrow = async (cwd: string) => { - let localConfig: Record | undefined; - - try { - localConfig = getLocalConfig(cwd); - } catch (error) { - throw new Error(`Failed to read local config at path: '${LOCAL_CONFIG_PATH}':`, { cause: error }); - } - - let deprecatedLocalConfig: Record | undefined; - - try { - deprecatedLocalConfig = getDeprecatedLocalConfig(cwd); - } catch (error) { - throw new Error(`Failed to read local config at path: '${DEPRECATED_LOCAL_CONFIG_NAME}':`, { cause: error }); - } - - if (localConfig && deprecatedLocalConfig) { - const answer = await inquirer.prompt([ - { - name: 'isConfirm', - type: 'confirm', - - message: `The new version of Apify CLI uses the "${LOCAL_CONFIG_PATH}" instead of the "apify.json" file. Since we have found both files in your Actor directory, "apify.json" will be renamed to "apify.json.deprecated". Going forward, all commands will use "${LOCAL_CONFIG_PATH}". You can read about the differences between the old and the new config at https://github.com/apify/apify-cli/blob/master/MIGRATIONS.md. Do you want to continue?`, - }, - ]); - - if (!answer.isConfirm) { - throw new Error( - 'Command can not run with old "apify.json" file present in your Actor directory., Please, either rename or remove it.', - ); - } - try { - renameSync(getDeprecatedLocalConfigPath(cwd), `${getDeprecatedLocalConfigPath(cwd)}.deprecated`); - - info({ - message: `The "apify.json" file has been renamed to "apify.json.deprecated". The deprecated file is no longer used by the CLI or Apify Console. If you do not need it for some specific purpose, it can be safely deleted.`, - }); - } catch (e) { - throw new Error('Failed to rename deprecated "apify.json".'); - } - } - - if (!localConfig && !deprecatedLocalConfig) { - return {}; - } - - // If apify.json exists migrate it to .actor/actor.json - if (!localConfig && deprecatedLocalConfig) { - const answer = await inquirer.prompt([ - { - name: 'isConfirm', - type: 'confirm', - - message: `The new version of Apify CLI uses the "${LOCAL_CONFIG_PATH}" instead of the "apify.json" file. Your "apify.json" file will be automatically updated to the new format under "${LOCAL_CONFIG_PATH}". The original file will be renamed by adding the ".deprecated" suffix. Do you want to continue?`, - }, - ]); - if (!answer.isConfirm) { - throw new Error( - 'Command can not run with old apify.json structure. Either let the CLI auto-update it or follow the guide on https://github.com/apify/apify-cli/blob/master/MIGRATIONS.md and update it manually.', - ); - } - try { - // Check if apify.json contains old deprecated structure. If so, updates it. - if (_.isObject(deprecatedLocalConfig.version)) { - deprecatedLocalConfig = updateLocalConfigStructure(deprecatedLocalConfig); - } - - localConfig = { - actorSpecification: ACTOR_SPECIFICATION_VERSION, - environmentVariables: deprecatedLocalConfig?.env || undefined, - ..._.pick(deprecatedLocalConfig, MIGRATED_APIFY_JSON_PROPERTIES), - }; - - await writeJsonFile(getLocalConfigPath(cwd), localConfig); - renameSync(getDeprecatedLocalConfigPath(cwd), `${getDeprecatedLocalConfigPath(cwd)}.deprecated`); - - info({ - message: `The "apify.json" file has been migrated to "${LOCAL_CONFIG_PATH}" and the original file renamed to "apify.json.deprecated". The deprecated file is no longer used by the CLI or Apify Console. If you do not need it for some specific purpose, it can be safely deleted. Do not forget to commit the new file to your Git repository.`, - }); - } catch (e) { - throw new Error( - `Can not update "${LOCAL_CONFIG_PATH}" structure. Follow guide on https://github.com/apify/apify-cli/blob/master/MIGRATIONS.md and update it manually.`, - ); - } - } +export const setLocalConfig = async (localConfig: Record, actDir?: string) => { + const fullPath = join(actDir || process.cwd(), LOCAL_CONFIG_PATH); - return localConfig; -}; + await mkdir(dirname(fullPath), { recursive: true }); -export const setLocalConfig = async (localConfig: Record, actDir?: string) => { - actDir = actDir || process.cwd(); - writeJsonFileSync(join(actDir, LOCAL_CONFIG_PATH), localConfig); + writeFileSync(fullPath, JSON.stringify(localConfig, null, '\t')); }; const GITIGNORE_REQUIRED_CONTENTS = [getLocalStorageDir(), 'node_modules', '.venv']; @@ -581,30 +466,6 @@ export const checkIfStorageIsEmpty = async () => { return filesWithoutInput.length === 0; }; -/** - * Migration for deprecated structure of apify.json to latest. - */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const updateLocalConfigStructure = (localConfig: any) => { - const updatedLocalConfig: Record = { - name: localConfig.name, - template: localConfig.template, - version: localConfig.version.versionNumber, - buildTag: localConfig.version.buildTag, - env: null, - }; - - if (localConfig.version.envVars && localConfig.version.envVars.length) { - const env = {} as Record; - localConfig.version.envVars.forEach((envVar: { name: string; value: string }) => { - if (envVar.name && envVar.value) env[envVar.name] = envVar.value; - }); - - updatedLocalConfig.env = env; - } - return updatedLocalConfig; -}; - /** * Validates Actor name, if finds issue throws error. * @param actorName @@ -633,64 +494,10 @@ export const sanitizeActorName = (actorName: string) => { return sanitizedName.slice(0, ACTOR_NAME.MAX_LENGTH); }; -export const getPythonCommand = (directory: string) => { - const pythonVenvPath = /^win/.test(process.platform) ? 'Scripts/python.exe' : 'bin/python3'; - - let fullPythonVenvPath; - if (process.env.VIRTUAL_ENV) { - fullPythonVenvPath = join(process.env.VIRTUAL_ENV, pythonVenvPath); - } else { - fullPythonVenvPath = join(directory, '.venv', pythonVenvPath); - } - - if (existsSync(fullPythonVenvPath)) { - return fullPythonVenvPath; - } - - return /^win/.test(process.platform) ? 'python' : 'python3'; -}; - -const spawnOptions: SpawnSyncOptions = { shell: true, windowsHide: true }; - -export const detectPythonVersion = (directory: string) => { - const pythonCommand = getPythonCommand(directory); - try { - const spawnResult = spawnSync(pythonCommand, ['-c', '"import platform; print(platform.python_version())"'], { - ...spawnOptions, - encoding: 'utf-8', - }); - - if (!spawnResult.error && spawnResult.stdout) { - return spawnResult.stdout.trim(); - } - - return undefined; - } catch { - return undefined; - } -}; - export const isPythonVersionSupported = (installedPythonVersion: string) => { return satisfies(installedPythonVersion, `^${MINIMUM_SUPPORTED_PYTHON_VERSION}`); }; -export const detectNodeVersion = () => { - try { - const spawnResult = spawnSync('node', ['--version'], { - ...spawnOptions, - encoding: 'utf-8', - }); - - if (!spawnResult.error && spawnResult.stdout) { - return spawnResult.stdout.trim().replace(/^v/, ''); - } - - return undefined; - } catch { - return undefined; - } -}; - export const isNodeVersionSupported = (installedNodeVersion: string) => { // SUPPORTED_NODEJS_VERSION can be a version range, // we need to get the minimum supported version from that range to be able to compare them @@ -698,48 +505,11 @@ export const isNodeVersionSupported = (installedNodeVersion: string) => { return gte(installedNodeVersion, minimumSupportedNodeVersion); }; -export const detectNpmVersion = () => { - const npmCommand = getNpmCmd(); - try { - const spawnResult = spawnSync(npmCommand, ['--version'], { - ...spawnOptions, - encoding: 'utf-8', - }); - - if (!spawnResult.error && spawnResult.stdout) { - return spawnResult.stdout.trim().replace(/^v/, ''); - } - - return undefined; - } catch { - return undefined; - } -}; - export interface ActorLanguage { language: Language; languageVersion?: string; } -export const detectLocalActorLanguage = (cwd: string) => { - const isActorInNode = existsSync(join(cwd, 'package.json')); - const isActorInPython = - existsSync(join(cwd, 'src/__main__.py')) || ProjectAnalyzer.getProjectType(cwd) === PROJECT_TYPES.SCRAPY; - const result = {} as ActorLanguage; - - if (isActorInNode) { - result.language = LANGUAGE.NODEJS; - result.languageVersion = detectNodeVersion(); - } else if (isActorInPython) { - result.language = LANGUAGE.PYTHON; - result.languageVersion = detectPythonVersion(cwd); - } else { - result.language = LANGUAGE.UNKNOWN; - } - - return result; -}; - export const downloadAndUnzip = async ({ url, pathTo }: { url: string; pathTo: string }) => { const zipStream = await httpsGet(url); const chunks: Buffer[] = []; diff --git a/src/lib/commands/confirm.ts b/src/lib/utils/confirm.ts similarity index 67% rename from src/lib/commands/confirm.ts rename to src/lib/utils/confirm.ts index 5801eabf0..bafd56441 100644 --- a/src/lib/commands/confirm.ts +++ b/src/lib/utils/confirm.ts @@ -1,10 +1,10 @@ import inquirer, { type DistinctQuestion } from 'inquirer'; -const yesNoConfirmation = ({ type }: { type: string }) => +const yesNoConfirmation = ({ type, message }: { type: string; message?: string }) => ({ name: 'confirmed', type: 'confirm', - message: `Are you sure you want to delete this ${type}?`, + message: message ?? `Are you sure you want to delete this ${type}?`, default: false, }) as const satisfies DistinctQuestion; @@ -12,11 +12,12 @@ const inputValidation = ({ type, expectedValue, failureMessage = 'That is not the correct input!', -}: { type: string; expectedValue: string; failureMessage?: string }) => + message = `Are you sure you want to delete this ${type}? If so, please type in "${expectedValue}":`, +}: { type: string; expectedValue: string; failureMessage?: string; message?: string }) => ({ name: 'confirmed', type: 'input', - message: `Are you sure you want to delete this ${type}? If so, please type in "${expectedValue}":`, + message, validate(value) { if (value === expectedValue) { return true; @@ -30,9 +31,12 @@ export async function confirmAction({ expectedValue, type, failureMessage, -}: { type: string; expectedValue?: string; failureMessage?: string }): Promise { + message, +}: { type: string; expectedValue?: string; failureMessage?: string; message?: string }): Promise { const result = await inquirer.prompt<{ confirmed: boolean | string }>( - expectedValue ? inputValidation({ type, expectedValue, failureMessage }) : yesNoConfirmation({ type }), + expectedValue + ? inputValidation({ type, expectedValue, failureMessage, message }) + : yesNoConfirmation({ type, message }), ); if (typeof result.confirmed === 'boolean') { diff --git a/yarn.lock b/yarn.lock index 0cace03c9..b13ff9f64 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19,6 +19,13 @@ __metadata: languageName: node linkType: hard +"@apify/consts@npm:^2.38.0": + version: 2.38.0 + resolution: "@apify/consts@npm:2.38.0" + checksum: 10c0/4600d45674ded3d346474f02ec6d6b18831c059cbf83bbc3cf81a3976b1112b27dcb593e328432ede9b5509b05b7c3de6f7432e3b469f86847e3586da7b32422 + languageName: node + linkType: hard + "@apify/datastructures@npm:^2.0.0": version: 2.0.3 resolution: "@apify/datastructures@npm:2.0.3" @@ -96,6 +103,16 @@ __metadata: languageName: node linkType: hard +"@apify/log@npm:^2.5.14": + version: 2.5.14 + resolution: "@apify/log@npm:2.5.14" + dependencies: + "@apify/consts": "npm:^2.38.0" + ansi-colors: "npm:^4.1.1" + checksum: 10c0/542708d6049b299f1fec2ac1a57bc4651400dc9ae11834534253d440df8697c3786c3d856261053e5822fb07a7c3aa29a9a4eda621edc62ad67612826a6d05e7 + languageName: node + linkType: hard + "@apify/ps-tree@npm:^1.2.0": version: 1.2.0 resolution: "@apify/ps-tree@npm:1.2.0" @@ -130,7 +147,7 @@ __metadata: languageName: node linkType: hard -"@apify/utilities@npm:^2.13.0, @apify/utilities@npm:^2.15.1, @apify/utilities@npm:^2.15.3, @apify/utilities@npm:^2.7.10": +"@apify/utilities@npm:^2.13.0, @apify/utilities@npm:^2.15.3": version: 2.15.3 resolution: "@apify/utilities@npm:2.15.3" dependencies: @@ -140,6 +157,16 @@ __metadata: languageName: node linkType: hard +"@apify/utilities@npm:^2.15.1, @apify/utilities@npm:^2.7.10": + version: 2.15.1 + resolution: "@apify/utilities@npm:2.15.1" + dependencies: + "@apify/consts": "npm:^2.38.0" + "@apify/log": "npm:^2.5.14" + checksum: 10c0/1269731a571f3ff85da0829f6bcec5b4544f94b8792fedc7ba6914406b1e9c75ef928bb94ed02c691f02f05d401dc7cb3a0c9a5eea6b47f63296adc8fc5f4ac2 + languageName: node + linkType: hard + "@arcanis/slice-ansi@npm:^1.1.1": version: 1.1.1 resolution: "@arcanis/slice-ansi@npm:1.1.1" @@ -2176,7 +2203,7 @@ __metadata: languageName: node linkType: hard -"@sapphire/result@npm:^2.6.6": +"@sapphire/result@npm:^2.7.2": version: 2.7.2 resolution: "@sapphire/result@npm:2.7.2" checksum: 10c0/4d0e67363c7adcc5cc990c3bf3d600560c24c2d2ac1f62c7b5d1fbd639ce8a764f854c549210e13ef8a775fe931b951546bd82030036e2cb4034b77bdffacfdc @@ -2985,6 +3012,15 @@ __metadata: languageName: node linkType: hard +"@types/execa@npm:^2.0.2": + version: 2.0.2 + resolution: "@types/execa@npm:2.0.2" + dependencies: + execa: "npm:*" + checksum: 10c0/a27044cd623f189dc776ae0f53fe1f9569395a21c9d32a29ebcda77a891c176c73bef927185c8fc03d4f05481656ad70bcaab02a8919f02d40a0add75f843b05 + languageName: node + linkType: hard + "@types/express-serve-static-core@npm:^4.17.33": version: 4.19.6 resolution: "@types/express-serve-static-core@npm:4.19.6" @@ -3242,13 +3278,6 @@ __metadata: languageName: node linkType: hard -"@types/underscore@npm:^1.11.15": - version: 1.13.0 - resolution: "@types/underscore@npm:1.13.0" - checksum: 10c0/240d3f36f694e177b1896c464b1254249e64b51a2afc703a1dda61f0c544d6e3b081ac4d955fb057e873982a62a7ba78e3a0aaa252c9d766f6cbe5e85283bc04 - languageName: node - linkType: hard - "@types/unist@npm:*, @types/unist@npm:^3.0.0": version: 3.0.3 resolution: "@types/unist@npm:3.0.3" @@ -3270,6 +3299,13 @@ __metadata: languageName: node linkType: hard +"@types/which@npm:^3.0.4": + version: 3.0.4 + resolution: "@types/which@npm:3.0.4" + checksum: 10c0/036e4cb243ebfd5cf4893be2ab3b9a60a22368811c1f1c78fb8fc70cadc274024282d04b8d7c0948268372600003252d84e2d3a5e064014a543a5da235c5989d + languageName: node + linkType: hard + "@types/wrap-ansi@npm:^3.0.0": version: 3.0.0 resolution: "@types/wrap-ansi@npm:3.0.0" @@ -3896,12 +3932,13 @@ __metadata: "@oclif/test": "npm:^4.0.8" "@root/walk": "npm:~1.1.0" "@sapphire/duration": "npm:^1.1.2" - "@sapphire/result": "npm:^2.6.6" + "@sapphire/result": "npm:^2.7.2" "@sapphire/timestamp": "npm:^1.0.3" "@types/adm-zip": "npm:^0.5.5" "@types/archiver": "npm:^6.0.2" "@types/chai": "npm:^4.3.17" "@types/cors": "npm:^2.8.17" + "@types/execa": "npm:^2.0.2" "@types/express": "npm:^4.17.21" "@types/fs-extra": "npm:^11" "@types/inquirer": "npm:^9.0.7" @@ -3911,7 +3948,7 @@ __metadata: "@types/mime": "npm:^4.0.0" "@types/node": "npm:^22.0.0" "@types/semver": "npm:^7.5.8" - "@types/underscore": "npm:^1.11.15" + "@types/which": "npm:^3.0.4" "@typescript-eslint/eslint-plugin": "npm:^7.0.2" "@typescript-eslint/parser": "npm:^7.0.2" "@yarnpkg/core": "npm:^4.1.2" @@ -3934,7 +3971,6 @@ __metadata: eslint-config-prettier: "npm:^10.0.0" execa: "npm:^9.5.2" express: "npm:~4.21.0" - fs-extra: "npm:^11.2.0" globby: "npm:~14.1.0" handlebars: "npm:~4.7.8" inquirer: "npm:~12.5.0" @@ -3943,7 +3979,6 @@ __metadata: istextorbinary: "npm:~9.5.0" jju: "npm:~1.4.0" lint-staged: "npm:^15.2.8" - load-json-file: "npm:~7.0.1" lodash.clonedeep: "npm:^4.5.0" mdast-util-from-markdown: "npm:^2.0.2" mime: "npm:~4.0.4" @@ -3958,9 +3993,8 @@ __metadata: tiged: "npm:~2.12.7" tsx: "npm:^4.16.5" typescript: "npm:^5.5.4" - underscore: "npm:~1.13.7" vitest: "npm:^3.0.0" - write-json-file: "npm:~6.0.0" + which: "npm:^5.0.0" bin: apify: ./bin/run.js languageName: unknown @@ -6286,24 +6320,7 @@ __metadata: languageName: node linkType: hard -"execa@npm:^8.0.1": - version: 8.0.1 - resolution: "execa@npm:8.0.1" - dependencies: - cross-spawn: "npm:^7.0.3" - get-stream: "npm:^8.0.1" - human-signals: "npm:^5.0.0" - is-stream: "npm:^3.0.0" - merge-stream: "npm:^2.0.0" - npm-run-path: "npm:^5.1.0" - onetime: "npm:^6.0.0" - signal-exit: "npm:^4.1.0" - strip-final-newline: "npm:^3.0.0" - checksum: 10c0/2c52d8775f5bf103ce8eec9c7ab3059909ba350a5164744e9947ed14a53f51687c040a250bda833f906d1283aa8803975b84e6c8f7a7c42f99dc8ef80250d1af - languageName: node - linkType: hard - -"execa@npm:^9.5.2": +"execa@npm:*, execa@npm:^9.5.2": version: 9.5.2 resolution: "execa@npm:9.5.2" dependencies: @@ -6323,6 +6340,23 @@ __metadata: languageName: node linkType: hard +"execa@npm:^8.0.1": + version: 8.0.1 + resolution: "execa@npm:8.0.1" + dependencies: + cross-spawn: "npm:^7.0.3" + get-stream: "npm:^8.0.1" + human-signals: "npm:^5.0.0" + is-stream: "npm:^3.0.0" + merge-stream: "npm:^2.0.0" + npm-run-path: "npm:^5.1.0" + onetime: "npm:^6.0.0" + signal-exit: "npm:^4.1.0" + strip-final-newline: "npm:^3.0.0" + checksum: 10c0/2c52d8775f5bf103ce8eec9c7ab3059909ba350a5164744e9947ed14a53f51687c040a250bda833f906d1283aa8803975b84e6c8f7a7c42f99dc8ef80250d1af + languageName: node + linkType: hard + "expect-type@npm:^1.2.0": version: 1.2.1 resolution: "expect-type@npm:1.2.1" @@ -7871,7 +7905,7 @@ __metadata: languageName: node linkType: hard -"is-plain-obj@npm:^4.0.0, is-plain-obj@npm:^4.1.0": +"is-plain-obj@npm:^4.1.0": version: 4.1.0 resolution: "is-plain-obj@npm:4.1.0" checksum: 10c0/32130d651d71d9564dc88ba7e6fda0e91a1010a3694648e9f4f47bb6080438140696d3e3e15c741411d712e47ac9edc1a8a9de1fe76f3487b0d90be06ac9975e @@ -8341,13 +8375,6 @@ __metadata: languageName: node linkType: hard -"load-json-file@npm:~7.0.1": - version: 7.0.1 - resolution: "load-json-file@npm:7.0.1" - checksum: 10c0/7117459608a0b6329c7f78e6e1f541b3162dd901c29dd5af721fec8b270177d2e3d7999c971f344fff04daac368d052732e2c7146014bc84d15e0b636975e19a - languageName: node - linkType: hard - "locate-path@npm:^6.0.0": version: 6.0.0 resolution: "locate-path@npm:6.0.0" @@ -10876,15 +10903,6 @@ __metadata: languageName: node linkType: hard -"sort-keys@npm:^5.0.0": - version: 5.1.0 - resolution: "sort-keys@npm:5.1.0" - dependencies: - is-plain-obj: "npm:^4.0.0" - checksum: 10c0/fdb7aeb02368ad91b2ea947b59f3c95d80f8c71bbcb5741ebd55852994f54a129af3b3663b280951566fe5897de056428810dbb58c61db831e588c0ac110f2b0 - languageName: node - linkType: hard - "sort-object-keys@npm:^1.1.3": version: 1.1.3 resolution: "sort-object-keys@npm:1.1.3" @@ -11834,13 +11852,6 @@ __metadata: languageName: node linkType: hard -"underscore@npm:~1.13.7": - version: 1.13.7 - resolution: "underscore@npm:1.13.7" - checksum: 10c0/fad2b4aac48847674aaf3c30558f383399d4fdafad6dd02dd60e4e1b8103b52c5a9e5937e0cc05dacfd26d6a0132ed0410ab4258241240757e4a4424507471cd - languageName: node - linkType: hard - "undici-types@npm:~6.21.0": version: 6.21.0 resolution: "undici-types@npm:6.21.0" @@ -12378,28 +12389,6 @@ __metadata: languageName: node linkType: hard -"write-file-atomic@npm:^5.0.1": - version: 5.0.1 - resolution: "write-file-atomic@npm:5.0.1" - dependencies: - imurmurhash: "npm:^0.1.4" - signal-exit: "npm:^4.0.1" - checksum: 10c0/e8c850a8e3e74eeadadb8ad23c9d9d63e4e792bd10f4836ed74189ef6e996763959f1249c5650e232f3c77c11169d239cbfc8342fc70f3fe401407d23810505d - languageName: node - linkType: hard - -"write-json-file@npm:~6.0.0": - version: 6.0.0 - resolution: "write-json-file@npm:6.0.0" - dependencies: - detect-indent: "npm:^7.0.1" - is-plain-obj: "npm:^4.1.0" - sort-keys: "npm:^5.0.0" - write-file-atomic: "npm:^5.0.1" - checksum: 10c0/3f8f0caec7948d696b1e898512547bba7b63e99eb5b67ddb74cd9ea02aa0b88d48f5b710be3b26ac3ffa8c3e58c779bd610fb349d3cec6e8fb621596a8f85069 - languageName: node - linkType: hard - "ws@npm:^8.18.0": version: 8.18.1 resolution: "ws@npm:8.18.1" From be4981f6dd10961400f22fb226f1cad039da5098 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Sun, 13 Apr 2025 20:50:43 +0300 Subject: [PATCH 02/13] fix: run command cleanup --- src/commands/run.ts | 250 +++++++++++++++------------ src/lib/apify_command.ts | 21 ++- src/lib/exec.ts | 13 +- src/lib/files.ts | 7 +- src/lib/hooks/runtimes/javascript.ts | 29 +++- src/lib/hooks/runtimes/python.ts | 26 +++ src/lib/hooks/useActorConfig.ts | 26 ++- src/lib/hooks/useCwdProject.ts | 45 +++-- src/lib/hooks/useModuleVersion.ts | 113 ++++++++++++ src/lib/utils.ts | 11 +- 10 files changed, 387 insertions(+), 154 deletions(-) create mode 100644 src/lib/hooks/useModuleVersion.ts diff --git a/src/commands/run.ts b/src/commands/run.ts index c381cf0bc..bf3b6d6ea 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -1,5 +1,5 @@ import { existsSync, renameSync } from 'node:fs'; -import { stat, writeFile } from 'node:fs/promises'; +import { mkdir, readFile, stat, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import process from 'node:process'; @@ -14,17 +14,17 @@ import { getInputOverride } from '../lib/commands/resolve-input.js'; import { CommandExitCodes, DEFAULT_LOCAL_STORAGE_DIR, - LANGUAGE, LEGACY_LOCAL_STORAGE_DIR, - PROJECT_TYPES, + MINIMUM_SUPPORTED_PYTHON_VERSION, SUPPORTED_NODEJS_VERSION, } from '../lib/consts.js'; import { execWithLog } from '../lib/exec.js'; import { deleteFile } from '../lib/files.js'; +import { useActorConfig } from '../lib/hooks/useActorConfig.js'; +import { ProjectLanguage, useCwdProject } from '../lib/hooks/useCwdProject.js'; +import { useModuleVersion } from '../lib/hooks/useModuleVersion.js'; import { getAjvValidator, getDefaultsFromInputSchema, readInputSchema } from '../lib/input_schema.js'; import { error, info, warning } from '../lib/outputs.js'; -import { ProjectAnalyzer } from '../lib/project_analyzer.js'; -import { ScrapyProjectAnalyzer } from '../lib/projects/scrapy/ScrapyProjectAnalyzer.js'; import { replaceSecretsValue } from '../lib/secrets.js'; import { Ajv, @@ -33,7 +33,6 @@ import { getLocalKeyValueStorePath, getLocalStorageDir, getLocalUserInfo, - getNpmCmd, isNodeVersionSupported, isPythonVersionSupported, purgeDefaultDataset, @@ -100,58 +99,92 @@ export class RunCommand extends ApifyCommand { const { proxy, id: userId, token } = await getLocalUserInfo(); - let localConfig: Record; + const localConfigResult = await useActorConfig(); - try { - localConfig = (await getLocalConfigOrThrow(cwd))!; - } catch (_error) { - const casted = _error as Error; - const cause = casted.cause as Error; - - error({ message: `${casted.message}\n ${cause.message}` }); + if (localConfigResult.isErr()) { + error({ message: localConfigResult.unwrapErr().message }); process.exitCode = CommandExitCodes.InvalidActorJson; return; } - const packageJsonPath = join(cwd, 'package.json'); - const mainPyPath = join(cwd, 'src/__main__.py'); + const { config: localConfig } = localConfigResult.unwrap(); - const projectType = ProjectAnalyzer.getProjectType(cwd); const actualStoragePath = getLocalStorageDir(); - const packageJsonExists = existsSync(packageJsonPath); - const mainPyExists = existsSync(mainPyPath); - const isScrapyProject = projectType === PROJECT_TYPES.SCRAPY; - const { language, languageVersion } = detectLocalActorLanguage(cwd); + const projectRuntimeResult = await useCwdProject(); + + if (projectRuntimeResult.isErr()) { + error({ message: projectRuntimeResult.unwrapErr().message }); + process.exitCode = CommandExitCodes.InvalidActorJson; + return; + } + + const project = projectRuntimeResult.unwrap(); + const { type, entrypoint: cwdEntrypoint, runtime } = project; - if (!packageJsonExists && !mainPyExists && !isScrapyProject) { + if (type === ProjectLanguage.Unknown) { throw new Error( 'Actor is of an unknown format.' + - ` Make sure either the 'package.json' file or 'src/__main__.py' file exists or you are in a migrated Scrapy project.`, + ` Make sure your project is supported by Apify CLI (either a package.json file is present, or a Python entrypoint could be found) or you are in a migrated Scrapy project.`, ); } - // Defaults (for node, the start script, for python, the src module) - let runType: RunType = language === LANGUAGE.NODEJS ? RunType.Script : RunType.Module; - let entrypoint: string = language === LANGUAGE.NODEJS ? 'start' : 'src'; + if (!runtime) { + switch (type) { + case ProjectLanguage.JavaScript: + error({ + message: `No Node.js detected! Please install Node.js ${SUPPORTED_NODEJS_VERSION} (or higher) to be able to run Node.js Actors locally.`, + }); + break; + case ProjectLanguage.Scrapy: + case ProjectLanguage.Python: + error({ + message: `No Python detected! Please install Python ${MINIMUM_SUPPORTED_PYTHON_VERSION} (or higher) to be able to run Python Actors locally.`, + }); + break; + default: + error({ + message: `No runtime detected! Make sure you have Python ${MINIMUM_SUPPORTED_PYTHON_VERSION} (or higher) or Node.js ${SUPPORTED_NODEJS_VERSION} (or higher) installed.`, + }); + } + + return; + } + + let runType: RunType; + let entrypoint: string; - if (this.flags.entrypoint) { + if (cwdEntrypoint?.script) { + runType = RunType.Script; + entrypoint = cwdEntrypoint.script; + } else if (cwdEntrypoint?.path) { + runType = RunType.DirectFile; + entrypoint = cwdEntrypoint.path; + } else if (this.flags.entrypoint) { entrypoint = this.flags.entrypoint; const entrypointPath = join(cwd, this.flags.entrypoint); const entrypointStat = await stat(entrypointPath).catch(() => null); - // Directory -> We just try to run it as a module (in python, it needs to have a main.py file, in node.js, an index.(m)js file) if (entrypointStat?.isDirectory()) { + // Directory -> We just try to run it as a module (in python, it needs to have a main.py file, in node.js, an index.(m)js file) runType = RunType.Module; - // entrypoint -> ./src/file for example (running custom scripts) - } else if (entrypointStat?.isFile()) { + } + // File -> ./src/file for example (running custom scripts) + else if (entrypointStat?.isFile()) { runType = RunType.DirectFile; - // If it's not a file, or a directory, we just let it be a script - } else { + } + // If it's not a file, or a directory, we just let it be a script + else { runType = RunType.Script; } + } else { + error({ + message: `No entrypoint detected! Please provide an entrypoint using the --entrypoint flag, or make sure your project has an entrypoint.`, + }); + + return; } if (existsSync(LEGACY_LOCAL_STORAGE_DIR) && !existsSync(actualStoragePath)) { @@ -163,27 +196,28 @@ export class RunCommand extends ApifyCommand { }); } + const crawleeVersion = await useModuleVersion({ + moduleName: 'crawlee', + project, + }); + let CRAWLEE_PURGE_ON_START = '0'; // Purge stores // TODO: this needs to be cleaned up heavily - ideally logic should be in the project analyzers if (this.flags.purge) { - switch (projectType) { - case PROJECT_TYPES.PRE_CRAWLEE_APIFY_SDK: { - await Promise.all([purgeDefaultQueue(), purgeDefaultKeyValueStore(), purgeDefaultDataset()]); - info({ message: 'All default local stores were purged.' }); - break; - } - case PROJECT_TYPES.CRAWLEE: - default: { - CRAWLEE_PURGE_ON_START = '1'; - } - } + CRAWLEE_PURGE_ON_START = '1'; - if (language === LANGUAGE.PYTHON) { + if (crawleeVersion.isNone()) { await Promise.all([purgeDefaultQueue(), purgeDefaultKeyValueStore(), purgeDefaultDataset()]); info({ message: 'All default local stores were purged.' }); } + + // This might not be needed for python and scrapy projects + // if (type === ProjectLanguage.Python || type === ProjectLanguage.Scrapy) { + // await Promise.all([purgeDefaultQueue(), purgeDefaultKeyValueStore(), purgeDefaultDataset()]); + // info({ message: 'All default local stores were purged.' }); + // } } // TODO: deprecate these flags @@ -249,100 +283,100 @@ export class RunCommand extends ApifyCommand { } try { - if (language === LANGUAGE.NODEJS) { - // Actor is written in Node.js - const currentNodeVersion = languageVersion; - const minimumSupportedNodeVersion = minVersion(SUPPORTED_NODEJS_VERSION); - if (currentNodeVersion) { - // --max-http-header-size=80000 - // Increases default size of headers. The original limit was 80kb, but from node 10+ they decided to lower it to 8kb. - // However they did not think about all the sites there with large headers, - // so we put back the old limit of 80kb, which seems to work just fine. - if (isNodeVersionSupported(currentNodeVersion)) { + switch (type) { + case ProjectLanguage.JavaScript: { + const minimumSupportedNodeVersion = minVersion(SUPPORTED_NODEJS_VERSION); + + if (isNodeVersionSupported(runtime.version)) { + // --max-http-header-size=80000 + // Increases default size of headers. The original limit was 80kb, but from node 10+ they decided to lower it to 8kb. + // However they did not think about all the sites there with large headers, + // so we put back the old limit of 80kb, which seems to work just fine. env.NODE_OPTIONS = env.NODE_OPTIONS ? `${env.NODE_OPTIONS} --max-http-header-size=80000` : '--max-http-header-size=80000'; } else { warning({ message: - `You are running Node.js version ${currentNodeVersion}, which is no longer supported. ` + + `You are running Node.js version ${runtime.version}, which is no longer supported. ` + `Please upgrade to Node.js version ${minimumSupportedNodeVersion} or later.`, }); } - this.telemetryData.actorNodejsVersion = currentNodeVersion; - this.telemetryData.actorLanguage = LANGUAGE.NODEJS; - - // We allow "module" type directly in node too (it will work for a folder that has an `index.js` file) if (runType === RunType.DirectFile || runType === RunType.Module) { - await execWithLog('node', [entrypoint], { env, cwd }); + await execWithLog({ + cmd: runtime.executablePath, + args: [entrypoint], + opts: { env, cwd }, + }); } else { - // TODO(vladfrangu): what is this for? Some old template maybe? - // && !existsSync(serverJsFile) - // const serverJsFile = join(cwd, 'server.js'); - const packageJson = await loadJsonFile<{ - scripts: Record; - }>(packageJsonPath); - - if (!packageJson.scripts) { + // Assert the package.json content for scripts + const packageJson = await readFile(join(cwd, 'package.json'), 'utf8').catch(() => '{}'); + const packageJsonObj = JSON.parse(packageJson); + + if (!packageJsonObj.scripts) { throw new Error( 'No scripts were found in package.json. Please set it up for your project. ' + 'For more information about that call "apify help run".', ); } - if (!packageJson.scripts[entrypoint]) { + if (!packageJsonObj.scripts[entrypoint]) { throw new Error( `The script "${entrypoint}" was not found in package.json. Please set it up for your project. ` + 'For more information about that call "apify help run".', ); } - await execWithLog(getNpmCmd(), ['run', entrypoint], { env, cwd }); - } - } else { - error({ - message: `No Node.js detected! Please install Node.js ${minimumSupportedNodeVersion} or higher to be able to run Node.js Actors locally.`, - }); - } - } else if (language === LANGUAGE.PYTHON) { - const pythonVersion = languageVersion; - this.telemetryData.actorPythonVersion = pythonVersion; - this.telemetryData.actorLanguage = LANGUAGE.PYTHON; - if (pythonVersion) { - if (isPythonVersionSupported(pythonVersion)) { - const pythonCommand = getPythonCommand(cwd); - - if (isScrapyProject && !this.flags.entrypoint) { - const project = new ScrapyProjectAnalyzer(cwd); - project.loadScrapyCfg(); - if (project.configuration.hasKey('apify', 'mainpy_location')) { - entrypoint = project.configuration.get('apify', 'mainpy_location')!; - } + if (!runtime.pmPath) { + throw new Error( + 'No npm executable found! Please make sure your Node.js runtime has npm installed if you want to run package.json scripts locally.', + ); } - if (runType === RunType.Module) { - await execWithLog(pythonCommand, ['-m', entrypoint], { - env, - cwd, - }); - } else { - await execWithLog(pythonCommand, [entrypoint], { env, cwd }); - } - } else { + await execWithLog({ + cmd: runtime.pmPath, + args: ['run', entrypoint], + opts: { env, cwd }, + overrideCommand: runtime.runtimeShorthand || 'npm', + }); + } + + break; + } + case ProjectLanguage.Python: + case ProjectLanguage.Scrapy: { + if (!isPythonVersionSupported(runtime.version)) { error({ - message: `Python Actors require Python 3.9 or higher, but you have Python ${pythonVersion}!`, + message: `Python Actors require Python 3.9 or higher, but you have Python ${runtime.version}!`, }); error({ message: 'Please install Python 3.9 or higher to be able to run Python Actors locally.', }); + + return; } - } else { + + if (runType === RunType.Module) { + await execWithLog({ + cmd: runtime.executablePath, + args: ['-m', entrypoint], + opts: { env, cwd }, + }); + } else { + await execWithLog({ + cmd: runtime.executablePath, + args: [entrypoint], + opts: { env, cwd }, + }); + } + + break; + } + default: error({ - message: - 'No Python detected! Please install Python 3.9 or higher to be able to run Python Actors locally.', + message: `Failed to detect the language of your project. Please report this issue to the Apify team with your project structure over at https://github.com/apify/apify-cli/issues`, }); - } } } finally { if (storedInputResults) { @@ -397,7 +431,7 @@ export class RunCommand extends ApifyCommand { existingInput?.fileName ?? 'INPUT.json', ); - await ensureDir(dirname(inputFilePath)); + await mkdir(dirname(inputFilePath), { recursive: true }); await writeFile(inputFilePath, JSON.stringify(inputOverride.input, null, 2)); return { @@ -457,7 +491,7 @@ export class RunCommand extends ApifyCommand { ); } - await ensureDir(dirname(inputFilePath)); + await mkdir(dirname(inputFilePath), { recursive: true }); await writeFile(inputFilePath, JSON.stringify(fullInputOverride, null, 2)); return { @@ -468,7 +502,7 @@ export class RunCommand extends ApifyCommand { } if (!existingInput) { - await ensureDir(dirname(inputFilePath)); + await mkdir(dirname(inputFilePath), { recursive: true }); // No input -> use defaults for this run await writeFile(inputFilePath, JSON.stringify(defaults, null, 2)); @@ -503,7 +537,7 @@ export class RunCommand extends ApifyCommand { } // Step 4: store the input - await ensureDir(dirname(inputFilePath)); + await mkdir(dirname(inputFilePath), { recursive: true }); await writeFile(inputFilePath, JSON.stringify(fullInput, null, 2)); return { diff --git a/src/lib/apify_command.ts b/src/lib/apify_command.ts index ef67cb4fa..7df71b9d6 100644 --- a/src/lib/apify_command.ts +++ b/src/lib/apify_command.ts @@ -3,6 +3,7 @@ import process from 'node:process'; import { Command, type Interfaces, loadHelpClass } from '@oclif/core'; import { COMMANDS_WITHIN_ACTOR, LANGUAGE } from './consts.js'; +import { ProjectLanguage, useCwdProject } from './hooks/useCwdProject.js'; import { maybeTrackTelemetry } from './telemetry.js'; import { type KeysToCamelCase, argsToCamelCase } from './utils.js'; import { detectInstallationType } from './version_check.js'; @@ -54,15 +55,21 @@ export abstract class ApifyCommand extends Command { try { eventData.installationType = detectInstallationType(); + if (!this.telemetryData.actorLanguage && command && COMMANDS_WITHIN_ACTOR.includes(command)) { - const { language, languageVersion } = detectLocalActorLanguage(process.cwd()); - eventData.actorLanguage = language; - if (language === LANGUAGE.NODEJS) { - eventData.actorNodejsVersion = languageVersion; - } else if (language === LANGUAGE.PYTHON) { - eventData.actorPythonVersion = languageVersion; - } + const cwdProject = await useCwdProject(); + + cwdProject.inspect((project) => { + if (project.type === ProjectLanguage.JavaScript) { + eventData.actorLanguage = LANGUAGE.NODEJS; + eventData.actorNodejsVersion = project.runtime!.version; + } else if (project.type === ProjectLanguage.Python || project.type === ProjectLanguage.Scrapy) { + eventData.actorLanguage = LANGUAGE.PYTHON; + eventData.actorPythonVersion = project.runtime!.version; + } + }); } + await maybeTrackTelemetry({ eventName: `cli_command_${command}`, eventData, diff --git a/src/lib/exec.ts b/src/lib/exec.ts index f09261089..fcbb88ce3 100644 --- a/src/lib/exec.ts +++ b/src/lib/exec.ts @@ -25,7 +25,7 @@ const spawnPromised = async (cmd: string, args: string[], opts: SpawnOptionsWith // NOTE: This fix kills also puppeteer child node process process.on('SIGINT', () => { try { - childProcess.kill(); + childProcess.kill('SIGINT'); } catch { // SIGINT can come after the child process is finished, ignore it } @@ -40,7 +40,14 @@ const spawnPromised = async (cmd: string, args: string[], opts: SpawnOptionsWith }); }; -export async function execWithLog(cmd: string, args: string[] = [], opts: SpawnOptionsWithoutStdio = {}) { - run({ message: `${cmd} ${args.join(' ')}` }); +export interface ExecWithLogOptions { + cmd: string; + args?: string[]; + opts?: SpawnOptionsWithoutStdio; + overrideCommand?: string; +} + +export async function execWithLog({ cmd, args = [], opts = {}, overrideCommand }: ExecWithLogOptions) { + run({ message: `${overrideCommand || cmd} ${args.join(' ')}` }); await spawnPromised(cmd, args, opts); } diff --git a/src/lib/files.ts b/src/lib/files.ts index 5c3d6728c..2c584b2a6 100644 --- a/src/lib/files.ts +++ b/src/lib/files.ts @@ -1,5 +1,5 @@ import { existsSync, mkdirSync } from 'node:fs'; -import { stat, unlink } from 'node:fs/promises'; +import { readFile, stat, unlink, writeFile } from 'node:fs/promises'; import { join, sep } from 'node:path'; import { rimraf } from 'rimraf'; @@ -9,8 +9,9 @@ export const updateLocalJson = async ( updateAttrs: Record = {}, nestedObjectAttr = null, ) => { + const raw = await readFile(jsonFilePath, 'utf-8'); // eslint-disable-next-line @typescript-eslint/no-explicit-any - const currentObject = (await loadJsonFile(jsonFilePath)) as Record; + const currentObject = JSON.parse(raw) as Record; let newObject: Record; if (nestedObjectAttr) { @@ -20,7 +21,7 @@ export const updateLocalJson = async ( newObject = { ...currentObject, ...updateAttrs }; } - await writeJsonFile(jsonFilePath, newObject); + await writeFile(jsonFilePath, JSON.stringify(newObject, null, '\t')); }; /** diff --git a/src/lib/hooks/runtimes/javascript.ts b/src/lib/hooks/runtimes/javascript.ts index 05d0d300d..8ad2b4c25 100644 --- a/src/lib/hooks/runtimes/javascript.ts +++ b/src/lib/hooks/runtimes/javascript.ts @@ -1,9 +1,13 @@ +import process from 'node:process'; + import { none, some, type Option } from '@sapphire/result'; import { execa } from 'execa'; import which from 'which'; import type { Runtime } from '../useCwdProject.js'; +const cwdCache = new Map>(); + // Runtimes, in order of preference const runtimesToCheck = { node: ['--version'], @@ -42,9 +46,13 @@ async function getNpmVersion(npmPath: string) { return result.stdout.trim().replace(/^v/, ''); } -export async function useJavaScriptRuntime(): Promise< - Option -> { +export async function useJavaScriptRuntime(cwd = process.cwd()): Promise> { + const cached = cwdCache.get(cwd); + + if (cached) { + return cached; + } + for (const [runtime, args] of Object.entries(runtimesToCheck) as [keyof typeof runtimesToCheck, string[]][]) { try { const runtimePath = await which(runtime); @@ -52,10 +60,9 @@ export async function useJavaScriptRuntime(): Promise< const version = await getRuntimeVersion(runtimePath, args); if (version) { - const res: Runtime & { runtimeShorthand: keyof typeof runtimesToCheck } = { + const res: Runtime = { executablePath: runtimePath, version, - runtimeShorthand: runtime, }; // For npm, we also fetch the npm version @@ -63,10 +70,18 @@ export async function useJavaScriptRuntime(): Promise< const npmPath = await which('npm').catch(() => null); if (npmPath) { - res.pmVersion = await getNpmVersion(npmPath); res.pmPath = npmPath; + res.pmVersion = await getNpmVersion(npmPath); } } + // deno and bun support package.json scripts out of the box + else { + res.runtimeShorthand = runtime; + res.pmPath = runtimePath; + res.pmVersion = version; + } + + cwdCache.set(cwd, some(res)); return some(res); } @@ -75,5 +90,7 @@ export async function useJavaScriptRuntime(): Promise< } } + cwdCache.set(cwd, none); + return none; } diff --git a/src/lib/hooks/runtimes/python.ts b/src/lib/hooks/runtimes/python.ts index 4a5e89d2d..df3f13b64 100644 --- a/src/lib/hooks/runtimes/python.ts +++ b/src/lib/hooks/runtimes/python.ts @@ -9,6 +9,8 @@ import which from 'which'; import type { Runtime } from '../useCwdProject.js'; +const cwdCache = new Map>(); + async function getPythonVersion(runtimePath: string) { try { const result = await execa(runtimePath, ['-c', '"import platform; print(platform.python_version())"'], { @@ -30,6 +32,12 @@ async function getPythonVersion(runtimePath: string) { } export async function usePythonRuntime(cwd = process.cwd()): Promise> { + const cached = cwdCache.get(cwd); + + if (cached) { + return cached; + } + const isWindows = platform() === 'win32'; const pathParts = isWindows ? ['Scripts', 'python.exe'] : ['bin', 'python3']; @@ -47,6 +55,14 @@ export async function usePythonRuntime(cwd = process.cwd()): Promise> { +const cwdCache = new Map(); + +export async function useActorConfig( + { cwd = process.cwd(), migrateConfig = true, warnAboutOldConfig = true }: ActorConfigInput = { + cwd: process.cwd(), + migrateConfig: true, + warnAboutOldConfig: true, + }, +): Promise> { + const cached = cwdCache.get(cwd); + + if (cached) { + return ok(cached); + } + const newConfigPath = getLocalConfigPath(cwd); const deprecatedConfigPath = getDeprecatedLocalConfigPath(cwd); @@ -63,11 +75,13 @@ export async function useActorConfig({ } // Handle cleanup of deprecated config, we just ignore the deprecated config if both exist - if (config && deprecatedConfig) { + if (config && deprecatedConfig && warnAboutOldConfig) { await handleBothConfigVersionsFound(deprecatedConfigPath); } if (!config && !deprecatedConfig) { + cwdCache.set(cwd, { exists: false, migrated: false, config: {} }); + return ok({ exists: false, migrated: false, config: {} }); } @@ -84,6 +98,8 @@ export async function useActorConfig({ migrated = true; } + cwdCache.set(cwd, { exists: true, migrated, config: config || deprecatedConfig || {} }); + return ok({ exists: true, migrated, config: config || deprecatedConfig || {} }); } diff --git a/src/lib/hooks/useCwdProject.ts b/src/lib/hooks/useCwdProject.ts index b1ce606c2..d891dd74e 100644 --- a/src/lib/hooks/useCwdProject.ts +++ b/src/lib/hooks/useCwdProject.ts @@ -2,7 +2,7 @@ import { access, readFile } from 'node:fs/promises'; import { basename, join } from 'node:path'; import process from 'node:process'; -import { err, ok, type Result } from '@sapphire/result'; +import { ok, type Result } from '@sapphire/result'; import { useJavaScriptRuntime } from './runtimes/javascript.js'; import { usePythonRuntime } from './runtimes/python.js'; @@ -24,6 +24,7 @@ export interface Runtime { version: string; pmPath?: string | null; pmVersion?: string | null; + runtimeShorthand?: string; } export interface Entrypoint { @@ -41,7 +42,15 @@ export interface CwdProjectError { message: string; } +const cwdCache = new Map(); + export async function useCwdProject(cwd = process.cwd()): Promise> { + const cached = cwdCache.get(cwd); + + if (cached) { + return ok(cached); + } + const project: CwdProject = { type: ProjectLanguage.Unknown, }; @@ -51,6 +60,20 @@ export async function useCwdProject(cwd = process.cwd()): Promise + ` +(async () => { + const [nodeModule, process, path, fs] = await Promise.all([ + import('node:module'), + import('node:process'), + import('node:path'), + import('node:fs/promises'), + ]); + + /* we fake a script file here because otherwise node AND deno will fail to resolve modules, but bun works -.- */ + const dirname = path.join(process.cwd(), '__apify_cli_fetch_module_version__.js'); + + const _require = nodeModule.createRequire(dirname); + + try { + const modulePath = _require.resolve('${mod}'); + const moduleDir = path.dirname(modulePath); + + const packageJson = await fs.readFile(path.join(moduleDir, 'package.json'), 'utf8').catch(() => null); + + if (!packageJson) { + console.log('n/a'); + return; + } + + const packageJsonObj = JSON.parse(packageJson); + console.log(packageJsonObj.version); + } catch { + console.log('n/a'); + } +})(); +` + .replaceAll('\n', ' ') + .replaceAll('\t', ' '); + +const pyScript = (mod: string) => ` +try: + import ${mod} + print(${mod}.__version__) +except: + print('n/a') +`; + +const moduleVersionScripts: Record string[]> = { + node(mod) { + const script = jsScript(mod); + + return ['-e', `"${script}"`]; + }, + deno(mod) { + const script = jsScript(mod); + + return ['eval', `"${script}"`]; + }, + bun(mod) { + const script = jsScript(mod); + + return ['--eval', `"${script}"`]; + }, + python(mod) { + const script = pyScript(mod); + + return ['-c', `"${script}"`]; + }, +}; + +export async function useModuleVersion({ moduleName, project }: UseModuleVersionInput): Promise> { + if (!project.runtime) { + return none; + } + + let moduleVersionScriptKey: string; + + if (project.type === ProjectLanguage.JavaScript) { + moduleVersionScriptKey = project.runtime.runtimeShorthand || 'node'; + } else if (project.type === ProjectLanguage.Python || project.type === ProjectLanguage.Scrapy) { + moduleVersionScriptKey = 'python'; + } else { + return none; + } + + const args = moduleVersionScripts[moduleVersionScriptKey]?.(moduleName); + + if (!args) { + return none; + } + + try { + const result = await execa(project.runtime.executablePath, args, { + shell: true, + windowsHide: true, + }); + + if (result.stdout.trim() === 'n/a') { + return none; + } + + return some(result.stdout.trim()); + } catch { + return none; + } +} diff --git a/src/lib/utils.ts b/src/lib/utils.ts index fbd27dd52..4a056baeb 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -1,5 +1,5 @@ import { createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; -import { mkdir } from 'node:fs/promises'; +import { mkdir, readFile } from 'node:fs/promises'; import type { IncomingMessage } from 'node:http'; import { get } from 'node:https'; import { dirname, join } from 'node:path'; @@ -37,15 +37,12 @@ import { DEFAULT_LOCAL_STORAGE_DIR, GLOBAL_CONFIGS_FOLDER, INPUT_FILE_REG_EXP, - LANGUAGE, LOCAL_CONFIG_PATH, type Language, MINIMUM_SUPPORTED_PYTHON_VERSION, - PROJECT_TYPES, SUPPORTED_NODEJS_VERSION, } from './consts.js'; import { deleteFile, ensureFolderExistsSync, rimrafPromised } from './files.js'; -import { ProjectAnalyzer } from './project_analyzer.js'; import type { AuthJSON } from './types.js'; // Export AJV properly: https://github.com/ajv-validator/ajv/issues/2132 @@ -100,7 +97,8 @@ export const getLocalRequestQueuePath = (storeId?: string) => { export const getLocalUserInfo = async (): Promise => { let result: AuthJSON = {}; try { - result = await loadJsonFile(AUTH_FILE_PATH()); + const raw = await readFile(AUTH_FILE_PATH(), 'utf-8'); + result = JSON.parse(raw) as AuthJSON; } catch { return {}; } @@ -127,7 +125,8 @@ export const getLoggedClientOrThrow = async () => { const getTokenWithAuthFileFallback = (existingToken?: string) => { if (!existingToken && existsSync(GLOBAL_CONFIGS_FOLDER()) && existsSync(AUTH_FILE_PATH())) { - return loadJsonFileSync(AUTH_FILE_PATH()).token; + const raw = readFileSync(AUTH_FILE_PATH(), 'utf-8'); + return JSON.parse(raw).token; } return existingToken; From df4b9d7e710fb2e9ead12a4cd08e9c4f3f277bee Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Sun, 13 Apr 2025 21:27:37 +0300 Subject: [PATCH 03/13] fix: create --- src/commands/actors/pull.ts | 18 ++- src/commands/actors/push.ts | 36 +++-- src/commands/create.ts | 190 +++++++++++++++++---------- src/commands/run.ts | 2 +- src/lib/hooks/runtimes/javascript.ts | 2 + src/lib/hooks/useCwdProject.ts | 1 + test/utils.test.ts | 5 +- 7 files changed, 154 insertions(+), 100 deletions(-) diff --git a/src/commands/actors/pull.ts b/src/commands/actors/pull.ts index 2edd8e9c2..bef037d6f 100644 --- a/src/commands/actors/pull.ts +++ b/src/commands/actors/pull.ts @@ -11,6 +11,7 @@ import tiged from 'tiged'; import { ApifyCommand } from '../../lib/apify_command.js'; import { CommandExitCodes, LOCAL_CONFIG_PATH } from '../../lib/consts.js'; +import { useActorConfig } from '../../lib/hooks/useActorConfig.js'; import { error, success } from '../../lib/outputs.js'; import { getLocalUserInfo, getLoggedClientOrThrow } from '../../lib/utils.js'; @@ -51,19 +52,16 @@ export class ActorsPullCommand extends ApifyCommand { async run() { const cwd = process.cwd(); - let localConfig: Record; + const actorConfigResult = await useActorConfig(); - try { - localConfig = (await getLocalConfigOrThrow(cwd))!; - } catch (_error) { - const casted = _error as Error; - const cause = casted.cause as Error; - - error({ message: `${casted.message}\n ${cause.message}` }); + if (actorConfigResult.isErr()) { + error({ message: actorConfigResult.unwrapErr().message }); process.exitCode = CommandExitCodes.InvalidActorJson; return; } + const { config: actorConfig } = actorConfigResult.unwrap(); + const userInfo = await getLocalUserInfo(); const apifyClient = await getLoggedClientOrThrow(); @@ -72,8 +70,8 @@ export class ActorsPullCommand extends ApifyCommand { const actorId = this.args?.actorId || - (localConfig?.id as string | undefined) || - (localConfig?.name ? `${usernameOrId}/${localConfig.name}` : undefined); + (actorConfig?.id as string | undefined) || + (actorConfig?.name ? `${usernameOrId}/${actorConfig.name}` : undefined); if (!actorId) throw new Error('Cannot find Actor in this directory.'); diff --git a/src/commands/actors/push.ts b/src/commands/actors/push.ts index da7ad3ba7..bb33f83bf 100644 --- a/src/commands/actors/push.ts +++ b/src/commands/actors/push.ts @@ -13,6 +13,7 @@ import open from 'open'; import { ApifyCommand } from '../../lib/apify_command.js'; import { CommandExitCodes, DEPRECATED_LOCAL_CONFIG_NAME, LOCAL_CONFIG_PATH } from '../../lib/consts.js'; import { sumFilesSizeInBytes } from '../../lib/files.js'; +import { useActorConfig } from '../../lib/hooks/useActorConfig.js'; import { error, info, link, run, success, warning } from '../../lib/outputs.js'; import { transformEnvToEnvVars } from '../../lib/secrets.js'; import { @@ -130,19 +131,16 @@ export class ActorsPushCommand extends ApifyCommand { const apifyClient = await getLoggedClientOrThrow(); - let localConfig: Record; + const actorConfigResult = await useActorConfig(); - try { - localConfig = (await getLocalConfigOrThrow(cwd))!; - } catch (_error) { - const casted = _error as Error; - const cause = casted.cause as Error; - - error({ message: `${casted.message}\n ${cause.message}` }); + if (actorConfigResult.isErr()) { + error({ message: actorConfigResult.unwrapErr().message }); process.exitCode = CommandExitCodes.InvalidActorJson; return; } + const { config: actorConfig } = actorConfigResult.unwrap(); + const userInfo = await getLocalUserInfo(); const isOrganizationLoggedIn = !!userInfo.organizationOwnerUserId; const redirectUrlPart = isOrganizationLoggedIn ? `/organization/${userInfo.id}` : ''; @@ -153,9 +151,9 @@ export class ActorsPushCommand extends ApifyCommand { // User can override Actor version and build tag, attributes in localConfig will remain same. const version = - this.flags.version || (localConfig?.version as string | undefined) || DEFAULT_ACTOR_VERSION_NUMBER; + this.flags.version || (actorConfig?.version as string | undefined) || DEFAULT_ACTOR_VERSION_NUMBER; - let buildTag = this.flags.buildTag || (localConfig!.buildTag as string | undefined); + let buildTag = this.flags.buildTag || (actorConfig?.buildTag as string | undefined); // We can't add the default build tag to everything. If a user creates a new // version, e.g. for testing, but forgets to add a tag, it would use the default @@ -179,16 +177,16 @@ export class ActorsPushCommand extends ApifyCommand { actorId = actor.id; } else { const usernameOrId = userInfo.username || userInfo.id; - actor = (await apifyClient.actor(`${usernameOrId}/${localConfig!.name}`).get())!; + actor = (await apifyClient.actor(`${usernameOrId}/${actorConfig!.name}`).get())!; if (actor) { actorId = actor.id; } else { const { templates } = await fetchManifest(); - const actorTemplate = templates.find((t) => t.name === localConfig!.template); + const actorTemplate = templates.find((t) => t.name === actorConfig!.template); const defaultRunOptions = (actorTemplate?.defaultRunOptions || DEFAULT_RUN_OPTIONS) as ActorDefaultRunOptions; const newActor: ActorCollectionCreateOptions = { - name: localConfig!.name as string, + name: actorConfig!.name as string, defaultRunOptions, versions: [ { @@ -203,11 +201,11 @@ export class ActorsPushCommand extends ApifyCommand { actor = await apifyClient.actors().create(newActor); actorId = actor.id; isActorCreatedNow = true; - info({ message: `Created Actor with name ${localConfig!.name} on Apify.` }); + info({ message: `Created Actor with name ${actorConfig!.name} on Apify.` }); } } - info({ message: `Deploying Actor '${localConfig!.name}' to Apify.` }); + info({ message: `Deploying Actor '${actorConfig!.name}' to Apify.` }); const filesSize = await sumFilesSizeInBytes(filePathsToPush, cwd); const actorClient = apifyClient.actor(actorId); @@ -234,10 +232,10 @@ export class ActorsPushCommand extends ApifyCommand { !this.flags.force && actorModifiedMs && mostRecentModifiedFileMs < actorModifiedMs && - (localConfig?.name || forceActorId) + (actorConfig?.name || forceActorId) ) { throw new Error( - `Actor with identifier "${localConfig?.name || forceActorId}" is already on the platform and was modified there since modified locally. + `Actor with identifier "${actorConfig?.name || forceActorId}" is already on the platform and was modified there since modified locally. Skipping push. Use --force to override.`, ); } @@ -267,8 +265,8 @@ Skipping push. Use --force to override.`, // Update Actor version const actorCurrentVersion = await actorClient.version(version).get(); - const envVars = localConfig!.environmentVariables - ? transformEnvToEnvVars(localConfig!.environmentVariables as Record) + const envVars = actorConfig!.environmentVariables + ? transformEnvToEnvVars(actorConfig!.environmentVariables as Record) : undefined; if (actorCurrentVersion) { diff --git a/src/commands/create.ts b/src/commands/create.ts index b17c51ff5..91b308a7e 100644 --- a/src/commands/create.ts +++ b/src/commands/create.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdirSync } from 'node:fs'; +import { mkdirSync } from 'node:fs'; import { readdir, stat } from 'node:fs/promises'; import { join } from 'node:path'; import process from 'node:process'; @@ -12,12 +12,13 @@ import { EMPTY_LOCAL_CONFIG, LOCAL_CONFIG_PATH, PYTHON_VENV_PATH, SUPPORTED_NODE import { enhanceReadmeWithLocalSuffix, ensureValidActorName, getTemplateDefinition } from '../lib/create-utils.js'; import { execWithLog } from '../lib/exec.js'; import { updateLocalJson } from '../lib/files.js'; +import { usePythonRuntime } from '../lib/hooks/runtimes/python.js'; +import { ProjectLanguage, useCwdProject } from '../lib/hooks/useCwdProject.js'; import { createPrefilledInputFileFromInputSchema } from '../lib/input_schema.js'; import { error, info, success, warning } from '../lib/outputs.js'; import { downloadAndUnzip, getJsonFileContent, - getNpmCmd, isNodeVersionSupported, isPythonVersionSupported, setLocalConfig, @@ -140,7 +141,6 @@ export class CreateCommand extends ApifyCommand { await createPrefilledInputFileFromInputSchema(actFolderDir); const packageJsonPath = join(actFolderDir, 'package.json'); - const requirementsTxtPath = join(actFolderDir, 'requirements.txt'); const readmePath = join(actFolderDir, 'README.md'); // Add localReadmeSuffix which is fetched from manifest to README.md @@ -149,60 +149,119 @@ export class CreateCommand extends ApifyCommand { let dependenciesInstalled = false; if (!skipDependencyInstall) { - if (existsSync(packageJsonPath)) { - const currentNodeVersion = detectNodeVersion(); + const cwdProjectResult = await useCwdProject(actFolderDir); + + await cwdProjectResult.inspectAsync(async (project) => { const minimumSupportedNodeVersion = minVersion(SUPPORTED_NODEJS_VERSION); - if (currentNodeVersion) { - if (!isNodeVersionSupported(currentNodeVersion)) { - warning({ - message: - `You are running Node.js version ${currentNodeVersion}, which is no longer supported. ` + - `Please upgrade to Node.js version ${minimumSupportedNodeVersion} or later.`, - }); - } - // If the Actor is a Node.js Actor (has package.json), run `npm install` - await updateLocalJson(packageJsonPath, { name: actorName }); - // Run npm install in Actor dir. - // For efficiency, don't install Puppeteer for templates that don't use it - const cmdArgs = ['install']; - if (skipOptionalDeps) { - const currentNpmVersion = detectNpmVersion(); - if (gte(currentNpmVersion!, '7.0.0')) { - cmdArgs.push('--omit=optional'); - } else { - cmdArgs.push('--no-optional'); + + if (!project.runtime) { + switch (project.type) { + case ProjectLanguage.JavaScript: { + error({ + message: + `No Node.js detected! Please install Node.js ${minimumSupportedNodeVersion} or higher` + + ' to be able to run Node.js Actors locally.', + }); + break; } + case ProjectLanguage.Scrapy: + case ProjectLanguage.Python: { + break; + } + default: + // Do nothing } - await execWithLog(getNpmCmd(), cmdArgs, { cwd: actFolderDir }); - dependenciesInstalled = true; - } else { - error({ - message: - `No Node.js detected! Please install Node.js ${minimumSupportedNodeVersion} or higher` + - ' to be able to run Node.js Actors locally.', - }); + return; } - } else if (existsSync(requirementsTxtPath)) { - const pythonVersion = detectPythonVersion(actFolderDir); - if (pythonVersion) { - if (isPythonVersionSupported(pythonVersion)) { + + // eslint-disable-next-line prefer-destructuring + let runtime = project.runtime; + + switch (project.type) { + case ProjectLanguage.JavaScript: { + if (!isNodeVersionSupported(runtime.version)) { + warning({ + message: + `You are running Node.js version ${runtime.version}, which is no longer supported. ` + + `Please upgrade to Node.js version ${minimumSupportedNodeVersion} or later.`, + }); + } + + // If the Actor is a Node.js Actor (has package.json), run `npm install` + await updateLocalJson(packageJsonPath, { name: actorName }); + + // Run npm install in Actor dir. + // For efficiency, don't install Puppeteer for templates that don't use it + const cmdArgs = ['install']; + + if (skipOptionalDeps) { + switch (runtime.pmName) { + case 'npm': { + if (gte(runtime.pmVersion!, '7.0.0')) { + cmdArgs.push('--omit=optional'); + } else { + cmdArgs.push('--no-optional'); + } + break; + } + case 'bun': { + cmdArgs.push('--omit=optional'); + break; + } + case 'deno': { + // We want to make deno use the node_modules dir + cmdArgs.push('--node-modules-dir'); + break; + } + default: + // Do nothing + } + } + + await execWithLog({ + cmd: runtime.pmPath!, + args: cmdArgs, + opts: { cwd: actFolderDir }, + overrideCommand: runtime.pmName, + }); + + dependenciesInstalled = true; + + break; + } + case ProjectLanguage.Python: + case ProjectLanguage.Scrapy: { + if (!isPythonVersionSupported(runtime.version)) { + warning({ + message: `Python Actors require Python 3.9 or higher, but you have Python ${runtime.version}!`, + }); + warning({ + message: 'Please install Python 3.9 or higher to be able to run Python Actors locally.', + }); + return; + } + const venvPath = join(actFolderDir, '.venv'); - info({ message: `Python version ${pythonVersion} detected.` }); + info({ message: `Python version ${runtime.version} detected.` }); info({ message: `Creating a virtual environment in "${venvPath}" and installing dependencies from "requirements.txt"...`, }); - let pythonCommand = getPythonCommand(actFolderDir); + if (!process.env.VIRTUAL_ENV) { // If Python is not running in a virtual environment, create a new one - await execWithLog(pythonCommand, ['-m', 'venv', '--prompt', '.', PYTHON_VENV_PATH], { - cwd: actFolderDir, + await execWithLog({ + cmd: runtime.executablePath, + args: ['-m', 'venv', '--prompt', '.', PYTHON_VENV_PATH], + opts: { cwd: actFolderDir }, }); + // regenerate the `pythonCommand` after we create the virtual environment - pythonCommand = getPythonCommand(actFolderDir); + runtime = (await usePythonRuntime(actFolderDir)).unwrap(); } - await execWithLog( - pythonCommand, - [ + + await execWithLog({ + cmd: runtime.executablePath, + args: [ '-m', 'pip', 'install', @@ -213,11 +272,12 @@ export class CreateCommand extends ApifyCommand { 'setuptools', 'wheel', ], - { cwd: actFolderDir }, - ); - await execWithLog( - pythonCommand, - [ + opts: { cwd: actFolderDir }, + }); + + await execWithLog({ + cmd: runtime.executablePath, + args: [ '-m', 'pip', 'install', @@ -226,24 +286,17 @@ export class CreateCommand extends ApifyCommand { '-r', 'requirements.txt', ], - { cwd: actFolderDir }, - ); - dependenciesInstalled = true; - } else { - warning({ - message: `Python Actors require Python 3.9 or higher, but you have Python ${pythonVersion}!`, - }); - warning({ - message: 'Please install Python 3.9 or higher to be able to run Python Actors locally.', + opts: { cwd: actFolderDir }, }); + + dependenciesInstalled = true; + + break; } - } else { - warning({ - message: - 'No Python detected! Please install Python 3.9 or higher to be able to run Python Actors locally.', - }); + default: + // Do nothing } - } + }); } if (dependenciesInstalled) { @@ -252,10 +305,9 @@ export class CreateCommand extends ApifyCommand { if (messages?.postCreate) { info({ message: messages?.postCreate }); } - } else { - success({ - message: `Actor '${actorName}' was created. Please install its dependencies to be able to run it using "apify run".`, - }); - } + } else; + success({ + message: `Actor '${actorName}' was created. Please install its dependencies to be able to run it using "apify run".`, + }); } } diff --git a/src/commands/run.ts b/src/commands/run.ts index bf3b6d6ea..a2de5bd37 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -338,7 +338,7 @@ export class RunCommand extends ApifyCommand { cmd: runtime.pmPath, args: ['run', entrypoint], opts: { env, cwd }, - overrideCommand: runtime.runtimeShorthand || 'npm', + overrideCommand: runtime.pmName, }); } diff --git a/src/lib/hooks/runtimes/javascript.ts b/src/lib/hooks/runtimes/javascript.ts index 8ad2b4c25..9322a76ab 100644 --- a/src/lib/hooks/runtimes/javascript.ts +++ b/src/lib/hooks/runtimes/javascript.ts @@ -72,6 +72,7 @@ export async function useJavaScriptRuntime(cwd = process.cwd()): Promise { // Add in some retries for when the FS is slow to update await withRetries( async () => { - await execWithLog('unzip', ['-oq', zipName, '-d', tempFolder]); + await execWithLog({ + cmd: 'unzip', + args: ['-oq', zipName, '-d', tempFolder], + }); }, 3, 20, From d1be4b78bf3564c9dc75f9a827d3c2741b0efd03 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Sun, 13 Apr 2025 21:52:32 +0300 Subject: [PATCH 04/13] chore: tests --- test/commands/create.test.ts | 23 ++++++++------ test/commands/info.test.ts | 5 +-- test/commands/init.test.ts | 21 ++++++++----- test/commands/log_in_out.test.ts | 54 +++++++++++++++++++++++++------- test/commands/pull.test.ts | 12 +++---- test/commands/push.test.ts | 35 +++++++-------------- test/commands/run.test.ts | 42 ++++++++++--------------- test/python_support.test.ts | 14 ++++----- 8 files changed, 111 insertions(+), 95 deletions(-) diff --git a/test/commands/create.test.ts b/test/commands/create.test.ts index 36c882e61..ff9fc9afe 100644 --- a/test/commands/create.test.ts +++ b/test/commands/create.test.ts @@ -1,8 +1,7 @@ -import { existsSync } from 'node:fs'; +import { existsSync, readFileSync } from 'node:fs'; import { KEY_VALUE_STORE_KEYS } from '@apify/consts'; import { runCommand } from '@oclif/test'; -import { loadJsonFileSync } from 'load-json-file'; import { LOCAL_CONFIG_PATH } from '../../src/lib/consts.js'; import { getLocalKeyValueStorePath } from '../../src/lib/utils.js'; @@ -53,10 +52,12 @@ describe('apify create', () => { expect(existsSync(joinPath('package.json'))).toBeTruthy(); expect(existsSync(apifyJsonPath)).toBeFalsy(); expect(existsSync(actorJsonPath)).toBeTruthy(); - expect(loadJsonFileSync<{ name: string }>(actorJsonPath).name).to.be.eql(actName); - expect(loadJsonFileSync(joinPath(getLocalKeyValueStorePath(), `${KEY_VALUE_STORE_KEYS.INPUT}.json`))).to.be.eql( - expectedInput, - ); + expect(JSON.parse(readFileSync(actorJsonPath, 'utf8')).name).to.be.eql(actName); + expect( + JSON.parse( + readFileSync(joinPath(getLocalKeyValueStorePath(), `${KEY_VALUE_STORE_KEYS.INPUT}.json`), 'utf8'), + ), + ).to.be.eql(expectedInput); }); it('basic template structure with prefilled INPUT.json', async () => { @@ -78,10 +79,12 @@ describe('apify create', () => { expect(existsSync(joinPath('package.json'))).toBeTruthy(); expect(existsSync(apifyJsonPath)).toBeFalsy(); expect(existsSync(actorJsonPath)).toBeTruthy(); - expect(loadJsonFileSync<{ name: string }>(actorJsonPath)!.name).to.be.eql(actName); - expect(loadJsonFileSync(joinPath(getLocalKeyValueStorePath(), `${KEY_VALUE_STORE_KEYS.INPUT}.json`))).to.be.eql( - expectedInput, - ); + expect(JSON.parse(readFileSync(actorJsonPath, 'utf8')).name).to.be.eql(actName); + expect( + JSON.parse( + readFileSync(joinPath(getLocalKeyValueStorePath(), `${KEY_VALUE_STORE_KEYS.INPUT}.json`), 'utf8'), + ), + ).to.be.eql(expectedInput); }); it('should skip installing optional dependencies', async () => { diff --git a/test/commands/info.test.ts b/test/commands/info.test.ts index e068aa697..b358f1936 100644 --- a/test/commands/info.test.ts +++ b/test/commands/info.test.ts @@ -1,5 +1,6 @@ +import { readFileSync } from 'node:fs'; + import { runCommand } from '@oclif/test'; -import { loadJsonFileSync } from 'load-json-file'; import { InfoCommand } from '../../src/commands/info.js'; import { LoginCommand } from '../../src/commands/login.js'; @@ -22,7 +23,7 @@ describe('apify info', () => { await LoginCommand.run(['--token', TEST_USER_TOKEN], import.meta.url); await InfoCommand.run([], import.meta.url); - const userInfoFromConfig = loadJsonFileSync<{ id: string }>(AUTH_FILE_PATH()); + const userInfoFromConfig = JSON.parse(readFileSync(AUTH_FILE_PATH(), 'utf8')); expect(spy).toHaveBeenCalledTimes(2); expect(spy.mock.calls[1][0]).to.include(userInfoFromConfig.id); diff --git a/test/commands/init.test.ts b/test/commands/init.test.ts index c20d69dfe..68ff71af0 100644 --- a/test/commands/init.test.ts +++ b/test/commands/init.test.ts @@ -1,8 +1,7 @@ -import { existsSync } from 'node:fs'; +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { mkdir } from 'node:fs/promises'; import { KEY_VALUE_STORE_KEYS } from '@apify/consts'; -import { loadJsonFileSync } from 'load-json-file'; -import { writeJsonFile } from 'write-json-file'; import { EMPTY_LOCAL_CONFIG, LOCAL_CONFIG_PATH } from '../../src/lib/consts.js'; import { getLocalKeyValueStorePath } from '../../src/lib/utils.js'; @@ -35,11 +34,13 @@ describe('apify init', () => { const apifyJsonPath = 'apify.json'; expect(existsSync(joinPath(apifyJsonPath))).toBeFalsy(); - expect(loadJsonFileSync(joinPath(LOCAL_CONFIG_PATH))).toStrictEqual( + expect(JSON.parse(readFileSync(joinPath(LOCAL_CONFIG_PATH), 'utf8'))).toStrictEqual( Object.assign(EMPTY_LOCAL_CONFIG, { name: actName }), ); expect( - loadJsonFileSync(joinPath(getLocalKeyValueStorePath(), `${KEY_VALUE_STORE_KEYS.INPUT}.json`)), + JSON.parse( + readFileSync(joinPath(getLocalKeyValueStorePath(), `${KEY_VALUE_STORE_KEYS.INPUT}.json`), 'utf8'), + ), ).toStrictEqual({}); }); @@ -59,18 +60,22 @@ describe('apify init', () => { }, required: ['url'], }; + const defaultActorJson = Object.assign(EMPTY_LOCAL_CONFIG, { name: actName, input }); - await writeJsonFile(joinPath('.actor/actor.json'), defaultActorJson); + await mkdir(joinPath('.actor'), { recursive: true }); + writeFileSync(joinPath(LOCAL_CONFIG_PATH), JSON.stringify(defaultActorJson, null, '\t'), { flag: 'w' }); await InitCommand.run(['-y', actName], import.meta.url); // Check that it won't create deprecated config // TODO: We can remove this later const apifyJsonPath = 'apify.json'; expect(existsSync(joinPath(apifyJsonPath))).toBeFalsy(); - expect(loadJsonFileSync(joinPath(LOCAL_CONFIG_PATH))).toStrictEqual(defaultActorJson); + expect(JSON.parse(readFileSync(joinPath(LOCAL_CONFIG_PATH), 'utf8'))).toStrictEqual(defaultActorJson); expect( - loadJsonFileSync(joinPath(getLocalKeyValueStorePath(), `${KEY_VALUE_STORE_KEYS.INPUT}.json`)), + JSON.parse( + readFileSync(joinPath(getLocalKeyValueStorePath(), `${KEY_VALUE_STORE_KEYS.INPUT}.json`), 'utf8'), + ), ).toStrictEqual({ url: 'https://www.apify.com/' }); }); }); diff --git a/test/commands/log_in_out.test.ts b/test/commands/log_in_out.test.ts index b762f8dca..6a23aac6b 100644 --- a/test/commands/log_in_out.test.ts +++ b/test/commands/log_in_out.test.ts @@ -1,8 +1,7 @@ -import { existsSync } from 'node:fs'; +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { existsSync, readFileSync } from 'node:fs'; import axios from 'axios'; -import { loadJsonFileSync } from 'load-json-file'; -import _ from 'underscore'; import { AUTH_FILE_PATH } from '../../src/lib/consts.js'; import { TEST_USER_BAD_TOKEN, TEST_USER_TOKEN, testUserClient } from '../__setup__/config.js'; @@ -35,14 +34,31 @@ describe('apify login and logout', () => { it('should work with correct token', async () => { await LoginCommand.run(['--token', TEST_USER_TOKEN], import.meta.url); - const expectedUserInfo = Object.assign(await testUserClient.user('me').get(), { token: TEST_USER_TOKEN }); - const userInfoFromConfig = loadJsonFileSync(AUTH_FILE_PATH()); + const expectedUserInfo = Object.assign(await testUserClient.user('me').get(), { + token: TEST_USER_TOKEN, + }) as unknown as Record; + const userInfoFromConfig = JSON.parse(readFileSync(AUTH_FILE_PATH(), 'utf8')); expect(spy).toHaveBeenCalledTimes(1); expect(spy.mock.calls[0][0]).to.include('Success:'); + // Omit currentBillingPeriod, It can change during tests - const floatFields = ['currentBillingPeriod', 'plan', 'createdAt']; - expect(_.omit(expectedUserInfo, floatFields)).to.eql(_.omit(userInfoFromConfig, floatFields)); + + const { + currentBillingPeriod: _1, + plan: _2, + createdAt: _3, + ...expectedUserInfoWithoutFloatFields + } = expectedUserInfo; + + const { + currentBillingPeriod: _4, + plan: _5, + createdAt: _6, + ...userInfoFromConfigWithoutFloatFields + } = userInfoFromConfig; + + expect(expectedUserInfoWithoutFloatFields).to.eql(userInfoFromConfigWithoutFloatFields); await LogoutCommand.run([], import.meta.url); const isGlobalConfig = existsSync(AUTH_FILE_PATH()); @@ -69,12 +85,28 @@ describe('apify login and logout', () => { expect(response.status).to.be.eql(200); - const expectedUserInfo = Object.assign(await testUserClient.user('me').get(), { token: TEST_USER_TOKEN }); - const userInfoFromConfig = loadJsonFileSync(AUTH_FILE_PATH()); + const expectedUserInfo = Object.assign(await testUserClient.user('me').get(), { + token: TEST_USER_TOKEN, + }) as unknown as Record; + const userInfoFromConfig = JSON.parse(readFileSync(AUTH_FILE_PATH(), 'utf8')); expect(spy.mock.calls[2][0]).to.include('Success:'); + // Omit currentBillingPeriod, It can change during tests - const floatFields = ['currentBillingPeriod', 'plan', 'createdAt']; - expect(_.omit(expectedUserInfo, floatFields)).to.eql(_.omit(userInfoFromConfig, floatFields)); + + const { + currentBillingPeriod: _1, + plan: _2, + createdAt: _3, + ...expectedUserInfoWithoutFloatFields + } = expectedUserInfo; + const { + currentBillingPeriod: _4, + plan: _5, + createdAt: _6, + ...userInfoFromConfigWithoutFloatFields + } = userInfoFromConfig; + + expect(expectedUserInfoWithoutFloatFields).to.eql(userInfoFromConfigWithoutFloatFields); }); }); diff --git a/test/commands/pull.test.ts b/test/commands/pull.test.ts index f1be658b5..2e64ebd4a 100644 --- a/test/commands/pull.test.ts +++ b/test/commands/pull.test.ts @@ -1,12 +1,10 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { existsSync } from 'node:fs'; +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; import { rm } from 'node:fs/promises'; import { join } from 'node:path'; import { runCommand } from '@oclif/test'; import type { ActorCollectionCreateOptions } from 'apify-client'; -import { loadJsonFileSync } from 'load-json-file'; -import { writeJsonFile } from 'write-json-file'; import { LoginCommand } from '../../src/commands/login.js'; import { DEPRECATED_LOCAL_CONFIG_NAME, LOCAL_CONFIG_PATH } from '../../src/lib/consts.js'; @@ -146,7 +144,7 @@ describe('apify pull', () => { await ActorsPullCommand.run([testActor.id], import.meta.url); - const actorJson = loadJsonFileSync<{ name: string }>(join(testActor.name, LOCAL_CONFIG_PATH)); + const actorJson = JSON.parse(readFileSync(join(testActor.name, LOCAL_CONFIG_PATH), 'utf8')); expect(actorJson.name).to.be.eql(actorFromServer!.name); }); @@ -160,7 +158,7 @@ describe('apify pull', () => { await ActorsPullCommand.run([testActor.id], import.meta.url); - const actorPackageJson = loadJsonFileSync<{ name: string }>(join(testActor.name, 'package.json')); + const actorPackageJson = JSON.parse(readFileSync(join(testActor.name, 'package.json'), 'utf8')); expect(actorPackageJson.name).to.be.eql('act-in-gist'); }); @@ -174,7 +172,7 @@ describe('apify pull', () => { await ActorsPullCommand.run([testActor.id], import.meta.url); - const actorJson = loadJsonFileSync<{ name: string }>(join(testActor.name, DEPRECATED_LOCAL_CONFIG_NAME)); + const actorJson = JSON.parse(readFileSync(join(testActor.name, DEPRECATED_LOCAL_CONFIG_NAME), 'utf8')); expect(actorJson.name).to.be.eql('baidu-scraper'); }); @@ -190,7 +188,7 @@ describe('apify pull', () => { contentBeforeEdit.name = testActor.name; (TEST_ACTOR_SOURCE_FILES.versions![0] as any).sourceFiles[2].content = contentBeforeEdit; - await writeJsonFile( + writeFileSync( join('pull-test-no-name', LOCAL_CONFIG_PATH), (TEST_ACTOR_SOURCE_FILES.versions![0] as any).sourceFiles[2].content, ); diff --git a/test/commands/push.test.ts b/test/commands/push.test.ts index 6ac4d2a9f..c62888688 100644 --- a/test/commands/push.test.ts +++ b/test/commands/push.test.ts @@ -1,13 +1,11 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { existsSync, unlinkSync, writeFileSync } from 'node:fs'; +import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'; import { mkdir, writeFile } from 'node:fs/promises'; import { platform } from 'node:os'; import { ACTOR_SOURCE_TYPES, SOURCE_FILE_FORMATS } from '@apify/consts'; import { cryptoRandomObjectId } from '@apify/utilities'; import type { ActorCollectionCreateOptions } from 'apify-client'; -import { loadJsonFileSync } from 'load-json-file'; -import { writeJsonFileSync } from 'write-json-file'; import { LOCAL_CONFIG_PATH } from '../../src/lib/consts.js'; import { createSourceFiles, getActorLocalFilePaths, getLocalUserInfo } from '../../src/lib/utils.js'; @@ -72,15 +70,11 @@ describe('apify push', () => { }); it('should work without actorId', async () => { - const actorJson = loadJsonFileSync<{ - environmentVariables: Record; - name: string; - version: string; - }>(joinPath(LOCAL_CONFIG_PATH)); + const actorJson = JSON.parse(readFileSync(joinPath(LOCAL_CONFIG_PATH), 'utf8')); actorJson.environmentVariables = { MY_ENV_VAR: 'envVarValue', }; - writeJsonFileSync(joinPath(LOCAL_CONFIG_PATH), actorJson); + writeFileSync(joinPath(LOCAL_CONFIG_PATH), JSON.stringify(actorJson, null, '\t'), { flag: 'w' }); await ActorsPushCommand.run(['--no-prompt', '--force'], import.meta.url); @@ -113,7 +107,7 @@ describe('apify push', () => { it('should work with actorId', async () => { let testActor = await testUserClient.actors().create(TEST_ACTOR); const testActorClient = testUserClient.actor(testActor.id); - const actorJson = loadJsonFileSync<{ version: string }>(joinPath(LOCAL_CONFIG_PATH)); + const actorJson = JSON.parse(readFileSync(joinPath(LOCAL_CONFIG_PATH), 'utf8')); await ActorsPushCommand.run(['--no-prompt', '--force', testActor.id], import.meta.url); @@ -159,11 +153,9 @@ describe('apify push', () => { actorsForCleanup.add(testActor.id); const testActorClient = testUserClient.actor(testActor.id); - const actorJson = loadJsonFileSync<{ environmentVariables?: Record; version: string }>( - joinPath(LOCAL_CONFIG_PATH), - ); + const actorJson = JSON.parse(readFileSync(joinPath(LOCAL_CONFIG_PATH), 'utf8')); delete actorJson.environmentVariables; - writeJsonFileSync(joinPath(LOCAL_CONFIG_PATH), actorJson); + writeFileSync(joinPath(LOCAL_CONFIG_PATH), JSON.stringify(actorJson, null, '\t'), { flag: 'w' }); await ActorsPushCommand.run(['--no-prompt', testActor.id], import.meta.url); @@ -200,12 +192,10 @@ describe('apify push', () => { let testActor = await testUserClient.actors().create(testActorWithEnvVars); actorsForCleanup.add(testActor.id); const testActorClient = testUserClient.actor(testActor.id); - const actorJson = loadJsonFileSync<{ environmentVariables?: Record; version: string }>( - joinPath(LOCAL_CONFIG_PATH), - ); + const actorJson = JSON.parse(readFileSync(joinPath(LOCAL_CONFIG_PATH), 'utf8')); delete actorJson.environmentVariables; - writeJsonFileSync(joinPath(LOCAL_CONFIG_PATH), actorJson); + writeFileSync(joinPath(LOCAL_CONFIG_PATH), JSON.stringify(actorJson, null, '\t'), { flag: 'w' }); // Create large file to ensure Actor will be uploaded as zip writeFileSync(joinPath('3mb-file.txt'), Buffer.alloc(1024 * 1024 * 3)); @@ -234,11 +224,8 @@ describe('apify push', () => { }); it('typescript files should be treated as text', async () => { - const { name, version } = loadJsonFileSync<{ - environmentVariables?: Record; - version: string; - name: string; - }>(joinPath(LOCAL_CONFIG_PATH)); + const actorJson = JSON.parse(readFileSync(joinPath(LOCAL_CONFIG_PATH), 'utf8')); + const { name, version } = actorJson; writeFileSync(joinPath('some-typescript-file.ts'), `console.log('ok');`); @@ -265,7 +252,7 @@ describe('apify push', () => { const testActor = await testUserClient.actors().create(TEST_ACTOR); actorsForCleanup.add(testActor.id); const testActorClient = testUserClient.actor(testActor.id); - const actorJson = loadJsonFileSync<{ version: string }>(joinPath(LOCAL_CONFIG_PATH)); + const actorJson = JSON.parse(readFileSync(joinPath(LOCAL_CONFIG_PATH), 'utf8')); // @ts-expect-error Wrong typing of update method await testActorClient.version(actorJson.version).update({ buildTag: 'beta' }); diff --git a/test/commands/run.test.ts b/test/commands/run.test.ts index b9d214df3..3e60a96bd 100644 --- a/test/commands/run.test.ts +++ b/test/commands/run.test.ts @@ -3,8 +3,6 @@ import { fileURLToPath } from 'node:url'; import { APIFY_ENV_VARS } from '@apify/consts'; import { captureOutput } from '@oclif/test'; -import { loadJsonFileSync } from 'load-json-file'; -import { writeJsonFileSync } from 'write-json-file'; import { AUTH_FILE_PATH, EMPTY_LOCAL_CONFIG, LOCAL_CONFIG_PATH } from '../../src/lib/consts.js'; import { rimrafPromised } from '../../src/lib/files.js'; @@ -83,7 +81,7 @@ describe('apify run', () => { // check act output const actOutputPath = joinPath(getLocalKeyValueStorePath(), 'OUTPUT.json'); - const actOutput = loadJsonFileSync(actOutputPath); + const actOutput = JSON.parse(readFileSync(actOutputPath, 'utf8')); expect(actOutput).toStrictEqual(expectOutput); }); @@ -98,6 +96,7 @@ describe('apify run', () => { import { Actor } from 'apify'; Actor.main(async () => { + console.log(process.env); await Actor.setValue('OUTPUT', process.env); console.log('Done.'); }); @@ -106,17 +105,14 @@ describe('apify run', () => { const apifyJson = EMPTY_LOCAL_CONFIG; apifyJson.environmentVariables = testEnvVars; - writeJsonFileSync(joinPath(LOCAL_CONFIG_PATH), apifyJson); + writeFileSync(joinPath(LOCAL_CONFIG_PATH), JSON.stringify(apifyJson, null, '\t'), { flag: 'w' }); await RunCommand.run([], import.meta.url); const actOutputPath = joinPath(getLocalKeyValueStorePath(), 'OUTPUT.json'); - const localEnvVars = - loadJsonFileSync>( - actOutputPath, - ); - const auth = loadJsonFileSync<{ proxy: { password: string }; id: string; token: string }>(AUTH_FILE_PATH()); + const localEnvVars = JSON.parse(readFileSync(actOutputPath, 'utf8')); + const auth = JSON.parse(readFileSync(AUTH_FILE_PATH(), 'utf8')); expect(localEnvVars[APIFY_ENV_VARS.PROXY_PASSWORD]).toStrictEqual(auth.proxy.password); expect(localEnvVars[APIFY_ENV_VARS.USER_ID]).toStrictEqual(auth.id); @@ -145,7 +141,7 @@ describe('apify run', () => { const apifyJson = EMPTY_LOCAL_CONFIG; apifyJson.environmentVariables = testEnvVars; - writeJsonFileSync(joinPath(LOCAL_CONFIG_PATH), apifyJson); + writeFileSync(joinPath(LOCAL_CONFIG_PATH), JSON.stringify(apifyJson, null, '\t'), { flag: 'w' }); const pkgJson = readFileSync(joinPath('package.json'), 'utf8'); const parsedPkgJson = JSON.parse(pkgJson); @@ -156,11 +152,8 @@ describe('apify run', () => { const actOutputPath = joinPath(getLocalKeyValueStorePath(), 'OUTPUT.json'); - const localEnvVars = - loadJsonFileSync>( - actOutputPath, - ); - const auth = loadJsonFileSync<{ proxy: { password: string }; id: string; token: string }>(AUTH_FILE_PATH()); + const localEnvVars = JSON.parse(readFileSync(actOutputPath, 'utf8')); + const auth = JSON.parse(readFileSync(AUTH_FILE_PATH(), 'utf8')); expect(localEnvVars[APIFY_ENV_VARS.PROXY_PASSWORD]).toStrictEqual(auth.proxy.password); expect(localEnvVars[APIFY_ENV_VARS.USER_ID]).toStrictEqual(auth.id); @@ -168,7 +161,7 @@ describe('apify run', () => { expect(localEnvVars.TEST_LOCAL).toStrictEqual(testEnvVars.TEST_LOCAL); const actOutputPath2 = joinPath(getLocalKeyValueStorePath(), 'owo.json'); - const actOutput2 = loadJsonFileSync(actOutputPath2); + const actOutput2 = JSON.parse(readFileSync(actOutputPath2, 'utf8')); expect(actOutput2).toStrictEqual('uwu'); }); @@ -193,17 +186,14 @@ describe('apify run', () => { const apifyJson = EMPTY_LOCAL_CONFIG; apifyJson.environmentVariables = testEnvVars; - writeJsonFileSync(joinPath(LOCAL_CONFIG_PATH), apifyJson); + writeFileSync(joinPath(LOCAL_CONFIG_PATH), JSON.stringify(apifyJson, null, '\t'), { flag: 'w' }); await RunCommand.run(['--entrypoint', 'src/other.js'], import.meta.url); const actOutputPath = joinPath(getLocalKeyValueStorePath(), 'OUTPUT.json'); - const localEnvVars = - loadJsonFileSync>( - actOutputPath, - ); - const auth = loadJsonFileSync<{ proxy: { password: string }; id: string; token: string }>(AUTH_FILE_PATH()); + const localEnvVars = JSON.parse(readFileSync(actOutputPath, 'utf8')); + const auth = JSON.parse(readFileSync(AUTH_FILE_PATH(), 'utf8')); expect(localEnvVars[APIFY_ENV_VARS.PROXY_PASSWORD]).toStrictEqual(auth.proxy.password); expect(localEnvVars[APIFY_ENV_VARS.USER_ID]).toStrictEqual(auth.id); @@ -211,7 +201,7 @@ describe('apify run', () => { expect(localEnvVars.TEST_LOCAL).toStrictEqual(testEnvVars.TEST_LOCAL); const actOutputPath2 = joinPath(getLocalKeyValueStorePath(), 'two.json'); - const actOutput2 = loadJsonFileSync(actOutputPath2); + const actOutput2 = JSON.parse(readFileSync(actOutputPath2, 'utf8')); expect(actOutput2).toStrictEqual('can play'); }); @@ -222,7 +212,7 @@ describe('apify run', () => { const actInputPath = joinPath(getLocalKeyValueStorePath(), 'INPUT.json'); const testJsonPath = joinPath(getLocalKeyValueStorePath(), 'TEST.json'); - writeJsonFileSync(actInputPath, input); + writeFileSync(actInputPath, JSON.stringify(input, null, '\t'), { flag: 'w' }); let actCode = ` import { Actor } from 'apify'; @@ -336,7 +326,7 @@ describe('apify run', () => { await RunCommand.run([], import.meta.url); - const output = loadJsonFileSync(outputPath); + const output = JSON.parse(readFileSync(outputPath, 'utf8')); expect(output).toStrictEqual({ awesome: true, help: 'this_maze_is_not_meant_for_you' }); }); @@ -346,7 +336,7 @@ describe('apify run', () => { await RunCommand.run([], import.meta.url); - const output = loadJsonFileSync(outputPath); + const output = JSON.parse(readFileSync(outputPath, 'utf8')); expect(output).toStrictEqual({ awesome: true }); }); }); diff --git a/test/python_support.test.ts b/test/python_support.test.ts index dd42cc832..46d6ebfeb 100644 --- a/test/python_support.test.ts +++ b/test/python_support.test.ts @@ -1,10 +1,9 @@ -import { existsSync, writeFileSync } from 'node:fs'; +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; import { rm } from 'node:fs/promises'; -import { loadJsonFileSync } from 'load-json-file'; - import { useTempPath } from './__setup__/hooks/useTempPath.js'; -import { detectPythonVersion, getLocalKeyValueStorePath } from '../src/lib/utils.js'; +import { usePythonRuntime } from '../src/lib/hooks/runtimes/python.js'; +import { getLocalKeyValueStorePath } from '../src/lib/utils.js'; const actorName = 'my-python-actor'; const PYTHON_START_TEMPLATE_ID = 'python-start'; @@ -28,7 +27,8 @@ describe('Python support [python]', () => { }); it('Python templates work [python]', { timeout: 120_000 }, async () => { - const pythonVersion = detectPythonVersion('.'); + const pythonVersion = (await usePythonRuntime(tmpPath)).map((r) => r.pmVersion).unwrapOr(undefined); + // Don't fail this test when Python is not installed (it will be installed in the right CI workflow) if (!pythonVersion && !process.env.CI) { console.log('Skipping Python template test since Python is not installed'); @@ -65,7 +65,7 @@ async def main(): // Check Actor output const actorOutputPath = joinPath(getLocalKeyValueStorePath(), 'OUTPUT.json'); - const actorOutput = loadJsonFileSync(actorOutputPath); - expect(actorOutput).toStrictEqual(actorOutput); + const actorOutput = JSON.parse(readFileSync(actorOutputPath, 'utf8')); + expect(actorOutput).toStrictEqual(expectedOutput); }); }); From c81554613e1f930869f065264a60a0d3a1199111 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Sun, 13 Apr 2025 22:16:36 +0300 Subject: [PATCH 05/13] chore: test shenanigans --- src/lib/hooks/useActorConfig.ts | 2 +- src/lib/hooks/useCwdProject.ts | 2 +- test/__setup__/reset-cwd-caches.ts | 7 +++++++ test/commands/push.test.ts | 5 +++++ test/commands/run.test.ts | 5 +++++ test/python_support.test.ts | 9 ++++++++- 6 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 test/__setup__/reset-cwd-caches.ts diff --git a/src/lib/hooks/useActorConfig.ts b/src/lib/hooks/useActorConfig.ts index 4b146510d..cc8c23f6d 100644 --- a/src/lib/hooks/useActorConfig.ts +++ b/src/lib/hooks/useActorConfig.ts @@ -31,7 +31,7 @@ export interface ActorConfigInput { warnAboutOldConfig?: boolean; } -const cwdCache = new Map(); +export const cwdCache = new Map(); export async function useActorConfig( { cwd = process.cwd(), migrateConfig = true, warnAboutOldConfig = true }: ActorConfigInput = { diff --git a/src/lib/hooks/useCwdProject.ts b/src/lib/hooks/useCwdProject.ts index a1d38ab56..dc60cd828 100644 --- a/src/lib/hooks/useCwdProject.ts +++ b/src/lib/hooks/useCwdProject.ts @@ -43,7 +43,7 @@ export interface CwdProjectError { message: string; } -const cwdCache = new Map(); +export const cwdCache = new Map(); export async function useCwdProject(cwd = process.cwd()): Promise> { const cached = cwdCache.get(cwd); diff --git a/test/__setup__/reset-cwd-caches.ts b/test/__setup__/reset-cwd-caches.ts new file mode 100644 index 000000000..9695f9c96 --- /dev/null +++ b/test/__setup__/reset-cwd-caches.ts @@ -0,0 +1,7 @@ +import { cwdCache as actorConfigCache } from '../../src/lib/hooks/useActorConfig.js'; +import { cwdCache as cwdProjectCache } from '../../src/lib/hooks/useCwdProject.js'; + +export function resetCwdCaches() { + actorConfigCache.clear(); + cwdProjectCache.clear(); +} diff --git a/test/commands/push.test.ts b/test/commands/push.test.ts index c62888688..87b46dffa 100644 --- a/test/commands/push.test.ts +++ b/test/commands/push.test.ts @@ -13,6 +13,7 @@ import { TEST_USER_TOKEN, testUserClient } from '../__setup__/config.js'; import { useAuthSetup } from '../__setup__/hooks/useAuthSetup.js'; import { useConsoleSpy } from '../__setup__/hooks/useConsoleSpy.js'; import { useTempPath } from '../__setup__/hooks/useTempPath.js'; +import { resetCwdCaches } from '../__setup__/reset-cwd-caches.js'; const ACTOR_NAME = `push-cli-test-${cryptoRandomObjectId(6)}`; const TEST_ACTOR: ActorCollectionCreateOptions = { @@ -69,6 +70,10 @@ describe('apify push', () => { } }); + beforeEach(() => { + resetCwdCaches(); + }); + it('should work without actorId', async () => { const actorJson = JSON.parse(readFileSync(joinPath(LOCAL_CONFIG_PATH), 'utf8')); actorJson.environmentVariables = { diff --git a/test/commands/run.test.ts b/test/commands/run.test.ts index 3e60a96bd..f6a42f555 100644 --- a/test/commands/run.test.ts +++ b/test/commands/run.test.ts @@ -15,6 +15,7 @@ import { import { TEST_USER_TOKEN } from '../__setup__/config.js'; import { useAuthSetup } from '../__setup__/hooks/useAuthSetup.js'; import { useTempPath } from '../__setup__/hooks/useTempPath.js'; +import { resetCwdCaches } from '../__setup__/reset-cwd-caches.js'; const actName = 'run-my-actor'; const pathToDefaultsInputSchema = fileURLToPath(new URL('../__setup__/input-schemas/defaults.json', import.meta.url)); @@ -60,6 +61,10 @@ describe('apify run', () => { await afterAllCalls(); }); + beforeEach(() => { + resetCwdCaches(); + }); + it('run act with output', async () => { const expectOutput = { my: 'output', diff --git a/test/python_support.test.ts b/test/python_support.test.ts index 46d6ebfeb..7533178b7 100644 --- a/test/python_support.test.ts +++ b/test/python_support.test.ts @@ -2,6 +2,7 @@ import { existsSync, readFileSync, writeFileSync } from 'node:fs'; import { rm } from 'node:fs/promises'; import { useTempPath } from './__setup__/hooks/useTempPath.js'; +import { resetCwdCaches } from './__setup__/reset-cwd-caches.js'; import { usePythonRuntime } from '../src/lib/hooks/runtimes/python.js'; import { getLocalKeyValueStorePath } from '../src/lib/utils.js'; @@ -26,8 +27,14 @@ describe('Python support [python]', () => { await afterAllCalls(); }); + beforeEach(() => { + resetCwdCaches(); + }); + it('Python templates work [python]', { timeout: 120_000 }, async () => { - const pythonVersion = (await usePythonRuntime(tmpPath)).map((r) => r.pmVersion).unwrapOr(undefined); + const runtime = await usePythonRuntime(tmpPath); + + const pythonVersion = runtime.map((r) => r.version).unwrapOr(undefined); // Don't fail this test when Python is not installed (it will be installed in the right CI workflow) if (!pythonVersion && !process.env.CI) { From 09ed682fae4618316f6437597ec2079f98aa48fd Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Sun, 13 Apr 2025 22:23:58 +0300 Subject: [PATCH 06/13] chore: venv issue --- src/commands/actors/pull.ts | 2 +- src/commands/actors/push.ts | 2 +- src/commands/create.ts | 9 ++++----- src/commands/run.ts | 4 ++-- src/lib/hooks/runtimes/python.ts | 12 ++++++++++-- src/lib/hooks/useCwdProject.ts | 8 +++++--- 6 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/commands/actors/pull.ts b/src/commands/actors/pull.ts index bef037d6f..2121df19b 100644 --- a/src/commands/actors/pull.ts +++ b/src/commands/actors/pull.ts @@ -52,7 +52,7 @@ export class ActorsPullCommand extends ApifyCommand { async run() { const cwd = process.cwd(); - const actorConfigResult = await useActorConfig(); + const actorConfigResult = await useActorConfig({ cwd }); if (actorConfigResult.isErr()) { error({ message: actorConfigResult.unwrapErr().message }); diff --git a/src/commands/actors/push.ts b/src/commands/actors/push.ts index bb33f83bf..abf89cfac 100644 --- a/src/commands/actors/push.ts +++ b/src/commands/actors/push.ts @@ -131,7 +131,7 @@ export class ActorsPushCommand extends ApifyCommand { const apifyClient = await getLoggedClientOrThrow(); - const actorConfigResult = await useActorConfig(); + const actorConfigResult = await useActorConfig({ cwd }); if (actorConfigResult.isErr()) { error({ message: actorConfigResult.unwrapErr().message }); diff --git a/src/commands/create.ts b/src/commands/create.ts index 91b308a7e..18d972a7e 100644 --- a/src/commands/create.ts +++ b/src/commands/create.ts @@ -1,5 +1,4 @@ -import { mkdirSync } from 'node:fs'; -import { readdir, stat } from 'node:fs/promises'; +import { mkdir, readdir, stat } from 'node:fs/promises'; import { join } from 'node:path'; import process from 'node:process'; @@ -103,7 +102,7 @@ export class CreateCommand extends ApifyCommand { // Create Actor directory structure if (!folderExists) { - mkdirSync(actFolderDir); + await mkdir(actFolderDir, { recursive: true }); } break; } @@ -149,7 +148,7 @@ export class CreateCommand extends ApifyCommand { let dependenciesInstalled = false; if (!skipDependencyInstall) { - const cwdProjectResult = await useCwdProject(actFolderDir); + const cwdProjectResult = await useCwdProject({ cwd: actFolderDir }); await cwdProjectResult.inspectAsync(async (project) => { const minimumSupportedNodeVersion = minVersion(SUPPORTED_NODEJS_VERSION); @@ -256,7 +255,7 @@ export class CreateCommand extends ApifyCommand { }); // regenerate the `pythonCommand` after we create the virtual environment - runtime = (await usePythonRuntime(actFolderDir)).unwrap(); + runtime = (await usePythonRuntime({ cwd: actFolderDir, force: true })).unwrap(); } await execWithLog({ diff --git a/src/commands/run.ts b/src/commands/run.ts index a2de5bd37..e086c624d 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -99,7 +99,7 @@ export class RunCommand extends ApifyCommand { const { proxy, id: userId, token } = await getLocalUserInfo(); - const localConfigResult = await useActorConfig(); + const localConfigResult = await useActorConfig({ cwd }); if (localConfigResult.isErr()) { error({ message: localConfigResult.unwrapErr().message }); @@ -111,7 +111,7 @@ export class RunCommand extends ApifyCommand { const actualStoragePath = getLocalStorageDir(); - const projectRuntimeResult = await useCwdProject(); + const projectRuntimeResult = await useCwdProject({ cwd }); if (projectRuntimeResult.isErr()) { error({ message: projectRuntimeResult.unwrapErr().message }); diff --git a/src/lib/hooks/runtimes/python.ts b/src/lib/hooks/runtimes/python.ts index df3f13b64..b3aab76b9 100644 --- a/src/lib/hooks/runtimes/python.ts +++ b/src/lib/hooks/runtimes/python.ts @@ -31,10 +31,18 @@ async function getPythonVersion(runtimePath: string) { } } -export async function usePythonRuntime(cwd = process.cwd()): Promise> { +export interface UsePythonRuntimeInput { + cwd?: string; + force?: boolean; +} + +export async function usePythonRuntime({ + cwd = process.cwd(), + force = false, +}: UsePythonRuntimeInput = {}): Promise> { const cached = cwdCache.get(cwd); - if (cached) { + if (cached && !force) { return cached; } diff --git a/src/lib/hooks/useCwdProject.ts b/src/lib/hooks/useCwdProject.ts index dc60cd828..19c5b92b9 100644 --- a/src/lib/hooks/useCwdProject.ts +++ b/src/lib/hooks/useCwdProject.ts @@ -45,7 +45,9 @@ export interface CwdProjectError { export const cwdCache = new Map(); -export async function useCwdProject(cwd = process.cwd()): Promise> { +export async function useCwdProject({ + cwd = process.cwd(), +}: { cwd?: string } = {}): Promise> { const cached = cwdCache.get(cwd); if (cached) { @@ -62,7 +64,7 @@ export async function useCwdProject(cwd = process.cwd()): Promise Date: Sun, 13 Apr 2025 22:35:50 +0300 Subject: [PATCH 07/13] chore: python can bite me --- src/commands/create.ts | 9 +++++---- src/commands/run.ts | 16 +++++++++------- src/lib/hooks/useCwdProject.ts | 8 ++++++-- test/python_support.test.ts | 4 ++-- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/commands/create.ts b/src/commands/create.ts index 18d972a7e..2fc357eb1 100644 --- a/src/commands/create.ts +++ b/src/commands/create.ts @@ -304,9 +304,10 @@ export class CreateCommand extends ApifyCommand { if (messages?.postCreate) { info({ message: messages?.postCreate }); } - } else; - success({ - message: `Actor '${actorName}' was created. Please install its dependencies to be able to run it using "apify run".`, - }); + } else { + success({ + message: `Actor '${actorName}' was created. Please install its dependencies to be able to run it using "apify run".`, + }); + } } } diff --git a/src/commands/run.ts b/src/commands/run.ts index e086c624d..9017cfc08 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -122,6 +122,8 @@ export class RunCommand extends ApifyCommand { const project = projectRuntimeResult.unwrap(); const { type, entrypoint: cwdEntrypoint, runtime } = project; + console.log(runtime); + if (type === ProjectLanguage.Unknown) { throw new Error( 'Actor is of an unknown format.' + @@ -154,13 +156,7 @@ export class RunCommand extends ApifyCommand { let runType: RunType; let entrypoint: string; - if (cwdEntrypoint?.script) { - runType = RunType.Script; - entrypoint = cwdEntrypoint.script; - } else if (cwdEntrypoint?.path) { - runType = RunType.DirectFile; - entrypoint = cwdEntrypoint.path; - } else if (this.flags.entrypoint) { + if (this.flags.entrypoint) { entrypoint = this.flags.entrypoint; const entrypointPath = join(cwd, this.flags.entrypoint); @@ -179,6 +175,12 @@ export class RunCommand extends ApifyCommand { else { runType = RunType.Script; } + } else if (cwdEntrypoint?.script) { + runType = RunType.Script; + entrypoint = cwdEntrypoint.script; + } else if (cwdEntrypoint?.path) { + runType = type === ProjectLanguage.Python ? RunType.Module : RunType.DirectFile; + entrypoint = cwdEntrypoint.path; } else { error({ message: `No entrypoint detected! Please provide an entrypoint using the --entrypoint flag, or make sure your project has an entrypoint.`, diff --git a/src/lib/hooks/useCwdProject.ts b/src/lib/hooks/useCwdProject.ts index 19c5b92b9..9ee08fe35 100644 --- a/src/lib/hooks/useCwdProject.ts +++ b/src/lib/hooks/useCwdProject.ts @@ -1,5 +1,5 @@ import { access, readFile } from 'node:fs/promises'; -import { basename, join } from 'node:path'; +import { basename, dirname, join } from 'node:path'; import process from 'node:process'; import { ok, type Result } from '@sapphire/result'; @@ -188,7 +188,11 @@ async function checkPythonProject(cwd: string) { for (const path of filesToCheck) { try { await access(path); - return path; + + // By default in python, we run python3 -m + // For some unholy reason, python does NOT support absolute paths for this -.- + // Effectively, this returns `src` from `/cwd/src/__main__.py`, et al. + return basename(dirname(path)); } catch { // Ignore errors } diff --git a/test/python_support.test.ts b/test/python_support.test.ts index 7533178b7..d1d03647b 100644 --- a/test/python_support.test.ts +++ b/test/python_support.test.ts @@ -10,7 +10,7 @@ const actorName = 'my-python-actor'; const PYTHON_START_TEMPLATE_ID = 'python-start'; const { beforeAllCalls, afterAllCalls, joinPath, tmpPath, toggleCwdBetweenFullAndParentPath } = useTempPath(actorName, { create: true, - remove: true, + remove: false, cwd: true, cwdParent: true, }); @@ -32,7 +32,7 @@ describe('Python support [python]', () => { }); it('Python templates work [python]', { timeout: 120_000 }, async () => { - const runtime = await usePythonRuntime(tmpPath); + const runtime = await usePythonRuntime({ cwd: tmpPath, force: true }); const pythonVersion = runtime.map((r) => r.version).unwrapOr(undefined); From aef66e5e6c8b79aab86ded144bd687eb5346897c Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Sun, 13 Apr 2025 22:36:33 +0300 Subject: [PATCH 08/13] chore: console.log mishap --- src/commands/run.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/commands/run.ts b/src/commands/run.ts index 9017cfc08..d5873911b 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -122,8 +122,6 @@ export class RunCommand extends ApifyCommand { const project = projectRuntimeResult.unwrap(); const { type, entrypoint: cwdEntrypoint, runtime } = project; - console.log(runtime); - if (type === ProjectLanguage.Unknown) { throw new Error( 'Actor is of an unknown format.' + From 0f120d074672a064e3d86cdac15cc22afefe2e0a Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Sun, 13 Apr 2025 22:40:32 +0300 Subject: [PATCH 09/13] chore: debug python failing in CI --- .github/workflows/check.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 8a9ca3280..0abefaa69 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -90,6 +90,7 @@ jobs: env: TEST_USER_TOKEN: ${{ secrets.APIFY_TEST_USER_API_TOKEN }} APIFY_CLI_DISABLE_TELEMETRY: 1 + NO_SILENT_TESTS: 1 run: yarn test-python docs: From 3d18b5c9dff1a8d4b138b22ff92fba2686f91803 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Sun, 13 Apr 2025 22:45:58 +0300 Subject: [PATCH 10/13] chore: fix python for real --- src/commands/create.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/commands/create.ts b/src/commands/create.ts index 2fc357eb1..43ab1f8e3 100644 --- a/src/commands/create.ts +++ b/src/commands/create.ts @@ -256,6 +256,7 @@ export class CreateCommand extends ApifyCommand { // regenerate the `pythonCommand` after we create the virtual environment runtime = (await usePythonRuntime({ cwd: actFolderDir, force: true })).unwrap(); + project.runtime = runtime; } await execWithLog({ From 5020408c203719407221261f9428f4fea63a7467 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Sun, 13 Apr 2025 22:47:33 +0300 Subject: [PATCH 11/13] chore: format should lint smh --- src/commands/actors/rm.ts | 2 +- src/commands/builds/rm.ts | 2 +- src/commands/datasets/rm.ts | 2 +- src/commands/key-value-stores/delete-value.ts | 2 +- src/commands/key-value-stores/rm.ts | 2 +- src/commands/runs/rm.ts | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/commands/actors/rm.ts b/src/commands/actors/rm.ts index bcb5c1b42..f700b2dca 100644 --- a/src/commands/actors/rm.ts +++ b/src/commands/actors/rm.ts @@ -2,8 +2,8 @@ import { Args } from '@oclif/core'; import type { ApifyApiError } from 'apify-client'; import { ApifyCommand } from '../../lib/apify_command.js'; -import { confirmAction } from '../../lib/utils/confirm.js'; import { error, info, success } from '../../lib/outputs.js'; +import { confirmAction } from '../../lib/utils/confirm.js'; import { getLoggedClientOrThrow } from '../../lib/utils.js'; export class ActorRmCommand extends ApifyCommand { diff --git a/src/commands/builds/rm.ts b/src/commands/builds/rm.ts index fc9ddbe40..4838332e5 100644 --- a/src/commands/builds/rm.ts +++ b/src/commands/builds/rm.ts @@ -2,8 +2,8 @@ import { Args } from '@oclif/core'; import type { ActorTaggedBuild, ApifyApiError } from 'apify-client'; import { ApifyCommand } from '../../lib/apify_command.js'; -import { confirmAction } from '../../lib/utils/confirm.js'; import { error, info, success } from '../../lib/outputs.js'; +import { confirmAction } from '../../lib/utils/confirm.js'; import { getLoggedClientOrThrow } from '../../lib/utils.js'; export class BuildsRmCommand extends ApifyCommand { diff --git a/src/commands/datasets/rm.ts b/src/commands/datasets/rm.ts index 4c2297dd2..ceb185b88 100644 --- a/src/commands/datasets/rm.ts +++ b/src/commands/datasets/rm.ts @@ -3,9 +3,9 @@ import type { ApifyApiError } from 'apify-client'; import chalk from 'chalk'; import { ApifyCommand } from '../../lib/apify_command.js'; -import { confirmAction } from '../../lib/utils/confirm.js'; import { tryToGetDataset } from '../../lib/commands/storages.js'; import { error, info, success } from '../../lib/outputs.js'; +import { confirmAction } from '../../lib/utils/confirm.js'; import { getLoggedClientOrThrow } from '../../lib/utils.js'; export class DatasetsRmCommand extends ApifyCommand { diff --git a/src/commands/key-value-stores/delete-value.ts b/src/commands/key-value-stores/delete-value.ts index c756301ff..4040d7028 100644 --- a/src/commands/key-value-stores/delete-value.ts +++ b/src/commands/key-value-stores/delete-value.ts @@ -3,9 +3,9 @@ import type { ApifyApiError } from 'apify-client'; import chalk from 'chalk'; import { ApifyCommand } from '../../lib/apify_command.js'; -import { confirmAction } from '../../lib/utils/confirm.js'; import { tryToGetKeyValueStore } from '../../lib/commands/storages.js'; import { error, info } from '../../lib/outputs.js'; +import { confirmAction } from '../../lib/utils/confirm.js'; import { getLoggedClientOrThrow } from '../../lib/utils.js'; export class KeyValueStoresDeleteValueCommand extends ApifyCommand { diff --git a/src/commands/key-value-stores/rm.ts b/src/commands/key-value-stores/rm.ts index 53536dc01..e383c3937 100644 --- a/src/commands/key-value-stores/rm.ts +++ b/src/commands/key-value-stores/rm.ts @@ -3,9 +3,9 @@ import type { ApifyApiError } from 'apify-client'; import chalk from 'chalk'; import { ApifyCommand } from '../../lib/apify_command.js'; -import { confirmAction } from '../../lib/utils/confirm.js'; import { tryToGetKeyValueStore } from '../../lib/commands/storages.js'; import { error, info, success } from '../../lib/outputs.js'; +import { confirmAction } from '../../lib/utils/confirm.js'; import { getLoggedClientOrThrow } from '../../lib/utils.js'; export class KeyValueStoresRmCommand extends ApifyCommand { diff --git a/src/commands/runs/rm.ts b/src/commands/runs/rm.ts index 5c6afc995..f45c4b33d 100644 --- a/src/commands/runs/rm.ts +++ b/src/commands/runs/rm.ts @@ -3,8 +3,8 @@ import { Args } from '@oclif/core'; import type { ApifyApiError } from 'apify-client'; import { ApifyCommand } from '../../lib/apify_command.js'; -import { confirmAction } from '../../lib/utils/confirm.js'; import { error, info, success } from '../../lib/outputs.js'; +import { confirmAction } from '../../lib/utils/confirm.js'; import { getLoggedClientOrThrow } from '../../lib/utils.js'; const deletableStatuses = [ From 0fcc8c550051c7753b8d390c72ed43b1b2c015c8 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Mon, 14 Apr 2025 10:21:37 +0300 Subject: [PATCH 12/13] ci was not in fact unstable --- test/commands/pull.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/commands/pull.test.ts b/test/commands/pull.test.ts index 2e64ebd4a..48041513d 100644 --- a/test/commands/pull.test.ts +++ b/test/commands/pull.test.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import { existsSync, readFileSync, writeFileSync } from 'node:fs'; -import { rm } from 'node:fs/promises'; +import { mkdir, rm } from 'node:fs/promises'; import { join } from 'node:path'; import { runCommand } from '@oclif/test'; @@ -167,6 +167,7 @@ describe('apify pull', () => { const testActor = await testUserClient .actors() .create({ name: `pull-test-${Date.now()}`, ...TEST_ACTOR_GIT_REPO }); + actorsForCleanup.add(testActor.id); actorNamesForCleanup.add(testActor.name); @@ -181,12 +182,19 @@ describe('apify pull', () => { const testActor = await testUserClient .actors() .create({ name: `pull-test-${Date.now()}`, ...TEST_ACTOR_SOURCE_FILES }); + actorsForCleanup.add(testActor.id); actorNamesForCleanup.add('pull-test-no-name'); const contentBeforeEdit = JSON.parse((TEST_ACTOR_SOURCE_FILES.versions![0] as any).sourceFiles[2].content); contentBeforeEdit.name = testActor.name; - (TEST_ACTOR_SOURCE_FILES.versions![0] as any).sourceFiles[2].content = contentBeforeEdit; + (TEST_ACTOR_SOURCE_FILES.versions![0] as any).sourceFiles[2].content = JSON.stringify( + contentBeforeEdit, + null, + '\t', + ); + + await mkdir(join('pull-test-no-name', '.actor'), { recursive: true }); writeFileSync( join('pull-test-no-name', LOCAL_CONFIG_PATH), From dfaeac248fa72be3dec1131921c49b036b8b7412 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Mon, 14 Apr 2025 16:08:21 +0300 Subject: [PATCH 13/13] fix: missing error message --- src/commands/create.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/commands/create.ts b/src/commands/create.ts index 43ab1f8e3..b5d75fb61 100644 --- a/src/commands/create.ts +++ b/src/commands/create.ts @@ -7,7 +7,13 @@ import { Args, Flags } from '@oclif/core'; import { gte, minVersion } from 'semver'; import { ApifyCommand } from '../lib/apify_command.js'; -import { EMPTY_LOCAL_CONFIG, LOCAL_CONFIG_PATH, PYTHON_VENV_PATH, SUPPORTED_NODEJS_VERSION } from '../lib/consts.js'; +import { + EMPTY_LOCAL_CONFIG, + LOCAL_CONFIG_PATH, + MINIMUM_SUPPORTED_PYTHON_VERSION, + PYTHON_VENV_PATH, + SUPPORTED_NODEJS_VERSION, +} from '../lib/consts.js'; import { enhanceReadmeWithLocalSuffix, ensureValidActorName, getTemplateDefinition } from '../lib/create-utils.js'; import { execWithLog } from '../lib/exec.js'; import { updateLocalJson } from '../lib/files.js'; @@ -156,7 +162,7 @@ export class CreateCommand extends ApifyCommand { if (!project.runtime) { switch (project.type) { case ProjectLanguage.JavaScript: { - error({ + warning({ message: `No Node.js detected! Please install Node.js ${minimumSupportedNodeVersion} or higher` + ' to be able to run Node.js Actors locally.', @@ -165,6 +171,9 @@ export class CreateCommand extends ApifyCommand { } case ProjectLanguage.Scrapy: case ProjectLanguage.Python: { + warning({ + message: `No Python detected! Please install Python ${MINIMUM_SUPPORTED_PYTHON_VERSION} or higher to be able to run Python Actors locally.`, + }); break; } default: