diff --git a/scripts/backend/build-backend.mjs b/scripts/backend/build-backend.mjs index c42fe5a3..d7776c29 100644 --- a/scripts/backend/build-backend.mjs +++ b/scripts/backend/build-backend.mjs @@ -5,6 +5,8 @@ import { fileURLToPath } from 'node:url'; import { copyTree, + createPythonInstallEnv, + prunePythonBytecodeArtifacts, resolveAndValidateRuntimeSource, resolveRuntimePython, } from './runtime-layout-utils.mjs'; @@ -466,11 +468,13 @@ const installRuntimeDependencies = (runtimePython) => { 'pip', '--disable-pip-version-check', 'install', + '--no-compile', ...pipArgs, ]; return spawnSync(runtimePython.absolute, installArgs, { cwd: outputDir, stdio: 'inherit', + env: createPythonInstallEnv(), windowsHide: true, }); }; @@ -562,6 +566,20 @@ const installRuntimeDependencies = (runtimePython) => { ); } } + + const bytecodeCleanupStats = prunePythonBytecodeArtifacts(runtimeDir); + if ( + bytecodeCleanupStats.removedCacheDirs > 0 || + bytecodeCleanupStats.removedBytecodeFiles > 0 || + bytecodeCleanupStats.removedOrphanBytecodeFiles > 0 + ) { + console.log( + '[build-backend] removed Python bytecode artifacts ' + + `(${bytecodeCleanupStats.removedCacheDirs} cache dirs, ` + + `${bytecodeCleanupStats.removedBytecodeFiles} cached files, ` + + `${bytecodeCleanupStats.removedOrphanBytecodeFiles} orphan files).`, + ); + } }; const main = () => { diff --git a/scripts/backend/runtime-layout-utils.mjs b/scripts/backend/runtime-layout-utils.mjs index e598003d..0c4c7be0 100644 --- a/scripts/backend/runtime-layout-utils.mjs +++ b/scripts/backend/runtime-layout-utils.mjs @@ -1,6 +1,8 @@ import fs from 'node:fs'; import path from 'node:path'; +const isBytecodeFile = (entryName) => entryName.endsWith('.pyc') || entryName.endsWith('.pyo'); + const shouldCopy = (sourcePath) => { const base = path.basename(sourcePath); if (base === '__pycache__' || base === '.pytest_cache' || base === '.ruff_cache') { @@ -9,7 +11,7 @@ const shouldCopy = (sourcePath) => { if (base === '.git' || base === '.mypy_cache' || base === '.DS_Store') { return false; } - if (base.endsWith('.pyc') || base.endsWith('.pyo')) { + if (isBytecodeFile(base)) { return false; } return true; @@ -24,6 +26,59 @@ export const copyTree = (fromPath, toPath, { dereference = false } = {}) => { }); }; +export const createPythonInstallEnv = (env = process.env) => ({ + ...env, + PYTHONDONTWRITEBYTECODE: '1', +}); + +export const prunePythonBytecodeArtifacts = (rootDir) => { + const stats = { + removedCacheDirs: 0, + removedBytecodeFiles: 0, + removedOrphanBytecodeFiles: 0, + }; + + const visit = (directoryPath, { inPycache = false } = {}) => { + for (const entry of fs.readdirSync(directoryPath, { withFileTypes: true })) { + const entryPath = path.join(directoryPath, entry.name); + + if (entry.isDirectory()) { + const childInPycache = inPycache || entry.name === '__pycache__'; + + if (entry.name === '__pycache__' && !inPycache) { + stats.removedCacheDirs += 1; + } + + visit(entryPath, { inPycache: childInPycache }); + + if (childInPycache) { + fs.rmdirSync(entryPath); + } + + continue; + } + + if (inPycache) { + stats.removedBytecodeFiles += 1; + fs.rmSync(entryPath, { force: true }); + continue; + } + + if (isBytecodeFile(entry.name)) { + stats.removedOrphanBytecodeFiles += 1; + fs.rmSync(entryPath, { force: true }); + } + } + }; + + if (!fs.existsSync(rootDir)) { + return stats; + } + + visit(rootDir); + return stats; +}; + export const resolveAndValidateRuntimeSource = ({ projectRoot, outputDir, runtimeSource }) => { if (!runtimeSource) { throw new Error( diff --git a/scripts/backend/runtime-layout-utils.test.mjs b/scripts/backend/runtime-layout-utils.test.mjs new file mode 100644 index 00000000..c82d4be9 --- /dev/null +++ b/scripts/backend/runtime-layout-utils.test.mjs @@ -0,0 +1,54 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import * as runtimeLayoutUtils from './runtime-layout-utils.mjs'; + +test('createPythonInstallEnv forces PYTHONDONTWRITEBYTECODE while preserving other env vars', () => { + assert.equal(typeof runtimeLayoutUtils.createPythonInstallEnv, 'function'); + + const env = runtimeLayoutUtils.createPythonInstallEnv({ + PATH: '/tmp/bin', + PYTHONDONTWRITEBYTECODE: '0', + }); + + assert.equal(env.PATH, '/tmp/bin'); + assert.equal(env.PYTHONDONTWRITEBYTECODE, '1'); +}); + +test('prunePythonBytecodeArtifacts removes bytecode files and cache directories recursively', () => { + assert.equal(typeof runtimeLayoutUtils.prunePythonBytecodeArtifacts, 'function'); + + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'astrbot-bytecode-fixture-')); + const nestedPackageDir = path.join(fixtureRoot, 'python', 'lib', 'python3.12', 'site-packages', 'demo'); + const cacheDir = path.join(nestedPackageDir, '__pycache__'); + const nestedCacheDir = path.join(cacheDir, 'nested'); + const sourceFile = path.join(nestedPackageDir, 'module.py'); + const bytecodeFile = path.join(cacheDir, 'module.cpython-312.pyc'); + const nestedCacheFile = path.join(nestedCacheDir, 'metadata.txt'); + const orphanBytecodeFile = path.join(fixtureRoot, 'python', 'bin', 'tool.pyc'); + + fs.mkdirSync(nestedCacheDir, { recursive: true }); + fs.mkdirSync(path.dirname(orphanBytecodeFile), { recursive: true }); + fs.writeFileSync(sourceFile, 'value = 1\n', 'utf8'); + fs.writeFileSync(bytecodeFile, 'bytecode', 'utf8'); + fs.writeFileSync(nestedCacheFile, 'metadata', 'utf8'); + fs.writeFileSync(orphanBytecodeFile, 'bytecode', 'utf8'); + + const stats = runtimeLayoutUtils.prunePythonBytecodeArtifacts(fixtureRoot); + + assert.deepEqual(stats, { + removedCacheDirs: 1, + removedBytecodeFiles: 2, + removedOrphanBytecodeFiles: 1, + }); + assert.equal(fs.existsSync(cacheDir), false); + assert.equal(fs.existsSync(bytecodeFile), false); + assert.equal(fs.existsSync(nestedCacheFile), false); + assert.equal(fs.existsSync(orphanBytecodeFile), false); + assert.equal(fs.existsSync(sourceFile), true); + + fs.rmSync(fixtureRoot, { recursive: true, force: true }); +});