From 4fa98a2e9bd4705a94ea42f6ddfd6069ce9adba2 Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Wed, 4 Feb 2026 17:39:19 +0100 Subject: [PATCH 1/8] allow apt tests to fail --- .../tests/template/methods/aptInstall.test.ts | 30 ++++++++++++------- .../methods/test_apt_install.py | 2 ++ .../template_sync/methods/test_apt_install.py | 2 ++ 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/packages/js-sdk/tests/template/methods/aptInstall.test.ts b/packages/js-sdk/tests/template/methods/aptInstall.test.ts index 3e4ad777a4..ae319edada 100644 --- a/packages/js-sdk/tests/template/methods/aptInstall.test.ts +++ b/packages/js-sdk/tests/template/methods/aptInstall.test.ts @@ -2,22 +2,32 @@ import { Template } from '../../../src' import { buildTemplateTest } from '../../setup' buildTemplateTest('apt install', async ({ buildTemplate }) => { - const template = Template() - .fromImage('ubuntu:24.04') - .skipCache() - .aptInstall('rolldice') + try { + const template = Template() + .fromImage('ubuntu:24.04') + .skipCache() + .aptInstall('rolldice') - await buildTemplate(template) + await buildTemplate(template) + } catch (error) { + // aptInstall tests are optional and allowed to fail + console.warn('aptInstall test failed (optional):', error) + } }) buildTemplateTest( 'apt install with no install recommends', async ({ buildTemplate }) => { - const template = Template() - .fromImage('ubuntu:24.04') - .skipCache() - .aptInstall('rolldice', { noInstallRecommends: true }) + try { + const template = Template() + .fromImage('ubuntu:24.04') + .skipCache() + .aptInstall('rolldice', { noInstallRecommends: true }) - await buildTemplate(template) + await buildTemplate(template) + } catch (error) { + // aptInstall tests are optional and allowed to fail + console.warn('aptInstall test failed (optional):', error) + } } ) diff --git a/packages/python-sdk/tests/async/template_async/methods/test_apt_install.py b/packages/python-sdk/tests/async/template_async/methods/test_apt_install.py index 5ed6392072..00c93d7d08 100644 --- a/packages/python-sdk/tests/async/template_async/methods/test_apt_install.py +++ b/packages/python-sdk/tests/async/template_async/methods/test_apt_install.py @@ -4,6 +4,7 @@ @pytest.mark.skip_debug() +@pytest.mark.xfail(reason="aptInstall tests are optional and allowed to fail") async def test_apt_install(async_build): template = ( AsyncTemplate().from_image("ubuntu:24.04").skip_cache().apt_install("rolldice") @@ -13,6 +14,7 @@ async def test_apt_install(async_build): @pytest.mark.skip_debug() +@pytest.mark.xfail(reason="aptInstall tests are optional and allowed to fail") async def test_apt_install_no_install_recommends(async_build): template = ( AsyncTemplate() diff --git a/packages/python-sdk/tests/sync/template_sync/methods/test_apt_install.py b/packages/python-sdk/tests/sync/template_sync/methods/test_apt_install.py index a82b93c837..1f639d2eeb 100644 --- a/packages/python-sdk/tests/sync/template_sync/methods/test_apt_install.py +++ b/packages/python-sdk/tests/sync/template_sync/methods/test_apt_install.py @@ -4,6 +4,7 @@ @pytest.mark.skip_debug() +@pytest.mark.xfail(reason="aptInstall tests are optional and allowed to fail") def test_apt_install(build): template = ( Template().from_image("ubuntu:24.04").skip_cache().apt_install("rolldice") @@ -13,6 +14,7 @@ def test_apt_install(build): @pytest.mark.skip_debug() +@pytest.mark.xfail(reason="aptInstall tests are optional and allowed to fail") def test_apt_install_no_install_recommends(build): template = ( Template() From 82d12a4e9cd2195a470e3df8d86cf3f9ae20db66 Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Wed, 4 Feb 2026 18:07:54 +0100 Subject: [PATCH 2/8] fix: Use Vitest .fails() instead of try-catch for apt tests Replace try-catch blocks that silently swallowed errors with Vitest's .fails() modifier. This properly reports test results while still allowing expected failures to pass CI, similar to pytest's xfail. Co-Authored-By: Claude Opus 4.5 --- .../tests/template/methods/aptInstall.test.ts | 39 ++++++++----------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/packages/js-sdk/tests/template/methods/aptInstall.test.ts b/packages/js-sdk/tests/template/methods/aptInstall.test.ts index ae319edada..64ce7daade 100644 --- a/packages/js-sdk/tests/template/methods/aptInstall.test.ts +++ b/packages/js-sdk/tests/template/methods/aptInstall.test.ts @@ -1,33 +1,28 @@ import { Template } from '../../../src' import { buildTemplateTest } from '../../setup' -buildTemplateTest('apt install', async ({ buildTemplate }) => { - try { - const template = Template() - .fromImage('ubuntu:24.04') - .skipCache() - .aptInstall('rolldice') +// Mark tests as expected to fail (similar to pytest's @pytest.mark.xfail(strict=True)) +// Unlike try-catch which silently swallows errors, .fails() properly reports test results: +// - If test fails as expected → test passes (XFAIL behavior) +// - If test unexpectedly passes → test fails, signaling the issue may be fixed - await buildTemplate(template) - } catch (error) { - // aptInstall tests are optional and allowed to fail - console.warn('aptInstall test failed (optional):', error) - } +buildTemplateTest.fails('apt install', async ({ buildTemplate }) => { + const template = Template() + .fromImage('ubuntu:24.04') + .skipCache() + .aptInstall('rolldice') + + await buildTemplate(template) }) -buildTemplateTest( +buildTemplateTest.fails( 'apt install with no install recommends', async ({ buildTemplate }) => { - try { - const template = Template() - .fromImage('ubuntu:24.04') - .skipCache() - .aptInstall('rolldice', { noInstallRecommends: true }) + const template = Template() + .fromImage('ubuntu:24.04') + .skipCache() + .aptInstall('rolldice', { noInstallRecommends: true }) - await buildTemplate(template) - } catch (error) { - // aptInstall tests are optional and allowed to fail - console.warn('aptInstall test failed (optional):', error) - } + await buildTemplate(template) } ) From c0f21a0b9881ad89b6cf0ca6d5e4bb05d2e0d825 Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Wed, 4 Feb 2026 18:15:53 +0100 Subject: [PATCH 3/8] fix: Add buildTemplateTestXFail helper for optional tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add xfail-style test helper similar to pytest's @pytest.mark.xfail: - If test passes → test passes (ideal outcome) - If test fails → logs XFAIL with reason, test still passes This allows apt install tests to fail without breaking CI while still clearly reporting failures in test output. Co-Authored-By: Claude Opus 4.5 --- packages/js-sdk/tests/setup.ts | 23 ++++++++++++++ .../tests/template/methods/aptInstall.test.ts | 30 +++++++++---------- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/packages/js-sdk/tests/setup.ts b/packages/js-sdk/tests/setup.ts index 27b8f5dc3f..7700b8e889 100644 --- a/packages/js-sdk/tests/setup.ts +++ b/packages/js-sdk/tests/setup.ts @@ -108,6 +108,29 @@ export const buildTemplateTest = base.extend({ ], }) +/** + * Creates a test that is allowed to fail (similar to pytest's @pytest.mark.xfail). + * - If the test passes → test passes (ideal outcome) + * - If the test fails → logs XFAIL and test still passes (expected failure) + * + * Unlike try-catch that silently swallows errors, this clearly reports failures + * in test output while not breaking CI. + */ +export function buildTemplateTestXFail( + name: string, + fn: (fixtures: BuildTemplateFixture) => Promise, + reason?: string +) { + return buildTemplateTest(name, async (fixtures) => { + try { + await fn(fixtures) + } catch (error) { + const reasonStr = reason ? ` (${reason})` : '' + console.error(`\n[XFAIL] "${name}"${reasonStr}: ${error}`) + } + }) +} + export const isDebug = process.env.E2B_DEBUG !== undefined export const isIntegrationTest = process.env.E2B_INTEGRATION_TEST !== undefined diff --git a/packages/js-sdk/tests/template/methods/aptInstall.test.ts b/packages/js-sdk/tests/template/methods/aptInstall.test.ts index 64ce7daade..c58510636b 100644 --- a/packages/js-sdk/tests/template/methods/aptInstall.test.ts +++ b/packages/js-sdk/tests/template/methods/aptInstall.test.ts @@ -1,21 +1,20 @@ import { Template } from '../../../src' -import { buildTemplateTest } from '../../setup' +import { buildTemplateTestXFail } from '../../setup' -// Mark tests as expected to fail (similar to pytest's @pytest.mark.xfail(strict=True)) -// Unlike try-catch which silently swallows errors, .fails() properly reports test results: -// - If test fails as expected → test passes (XFAIL behavior) -// - If test unexpectedly passes → test fails, signaling the issue may be fixed - -buildTemplateTest.fails('apt install', async ({ buildTemplate }) => { - const template = Template() - .fromImage('ubuntu:24.04') - .skipCache() - .aptInstall('rolldice') +buildTemplateTestXFail( + 'apt install', + async ({ buildTemplate }) => { + const template = Template() + .fromImage('ubuntu:24.04') + .skipCache() + .aptInstall('rolldice') - await buildTemplate(template) -}) + await buildTemplate(template) + }, + 'aptInstall tests are optional and allowed to fail' +) -buildTemplateTest.fails( +buildTemplateTestXFail( 'apt install with no install recommends', async ({ buildTemplate }) => { const template = Template() @@ -24,5 +23,6 @@ buildTemplateTest.fails( .aptInstall('rolldice', { noInstallRecommends: true }) await buildTemplate(template) - } + }, + 'aptInstall tests are optional and allowed to fail' ) From 4a8f662ad04281960dc59c4a1aacd1e87da4ad36 Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Wed, 4 Feb 2026 18:26:51 +0100 Subject: [PATCH 4/8] fix: Use object destructuring in buildTemplateTestXFail Vitest fixtures require object destructuring pattern in the callback. Co-Authored-By: Claude Opus 4.5 --- packages/js-sdk/tests/setup.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/js-sdk/tests/setup.ts b/packages/js-sdk/tests/setup.ts index 7700b8e889..1dafcd6448 100644 --- a/packages/js-sdk/tests/setup.ts +++ b/packages/js-sdk/tests/setup.ts @@ -121,9 +121,9 @@ export function buildTemplateTestXFail( fn: (fixtures: BuildTemplateFixture) => Promise, reason?: string ) { - return buildTemplateTest(name, async (fixtures) => { + return buildTemplateTest(name, async ({ buildTemplate }) => { try { - await fn(fixtures) + await fn({ buildTemplate }) } catch (error) { const reasonStr = reason ? ` (${reason})` : '' console.error(`\n[XFAIL] "${name}"${reasonStr}: ${error}`) From eca14d639a442f6627e278c7286f2f218a879a74 Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Wed, 4 Feb 2026 18:38:59 +0100 Subject: [PATCH 5/8] refactor: Rename buildTemplateTestXFail to buildTemplateTestAllowFail Co-Authored-By: Claude Opus 4.5 --- packages/js-sdk/tests/setup.ts | 2 +- packages/js-sdk/tests/template/methods/aptInstall.test.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/js-sdk/tests/setup.ts b/packages/js-sdk/tests/setup.ts index 1dafcd6448..8effe56901 100644 --- a/packages/js-sdk/tests/setup.ts +++ b/packages/js-sdk/tests/setup.ts @@ -116,7 +116,7 @@ export const buildTemplateTest = base.extend({ * Unlike try-catch that silently swallows errors, this clearly reports failures * in test output while not breaking CI. */ -export function buildTemplateTestXFail( +export function buildTemplateTestAllowFail( name: string, fn: (fixtures: BuildTemplateFixture) => Promise, reason?: string diff --git a/packages/js-sdk/tests/template/methods/aptInstall.test.ts b/packages/js-sdk/tests/template/methods/aptInstall.test.ts index c58510636b..4c1a300892 100644 --- a/packages/js-sdk/tests/template/methods/aptInstall.test.ts +++ b/packages/js-sdk/tests/template/methods/aptInstall.test.ts @@ -1,7 +1,7 @@ import { Template } from '../../../src' -import { buildTemplateTestXFail } from '../../setup' +import { buildTemplateTestAllowFail } from '../../setup' -buildTemplateTestXFail( +buildTemplateTestAllowFail( 'apt install', async ({ buildTemplate }) => { const template = Template() @@ -14,7 +14,7 @@ buildTemplateTestXFail( 'aptInstall tests are optional and allowed to fail' ) -buildTemplateTestXFail( +buildTemplateTestAllowFail( 'apt install with no install recommends', async ({ buildTemplate }) => { const template = Template() From 67a6e740cbc5f9edea0c5cb97c01a7bdff58552d Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Wed, 4 Feb 2026 18:44:22 +0100 Subject: [PATCH 6/8] refactor: Add generic allowFail wrapper for tests Add a reusable allowFail() wrapper that works with any test fixture. Tests wrapped with allowFail will log failures but not break CI. Co-Authored-By: Claude Opus 4.5 --- packages/js-sdk/tests/setup.ts | 24 ++++++++----------- .../tests/template/methods/aptInstall.test.ts | 16 ++++++------- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/packages/js-sdk/tests/setup.ts b/packages/js-sdk/tests/setup.ts index 8effe56901..3e3b221a45 100644 --- a/packages/js-sdk/tests/setup.ts +++ b/packages/js-sdk/tests/setup.ts @@ -109,26 +109,22 @@ export const buildTemplateTest = base.extend({ }) /** - * Creates a test that is allowed to fail (similar to pytest's @pytest.mark.xfail). + * Wraps a test callback to allow it to fail without breaking CI. * - If the test passes → test passes (ideal outcome) - * - If the test fails → logs XFAIL and test still passes (expected failure) + * - If the test fails → logs ALLOW_FAIL and test still passes * - * Unlike try-catch that silently swallows errors, this clearly reports failures - * in test output while not breaking CI. + * Works with any test fixture type (sandboxTest, buildTemplateTest, etc.) */ -export function buildTemplateTestAllowFail( - name: string, - fn: (fixtures: BuildTemplateFixture) => Promise, - reason?: string -) { - return buildTemplateTest(name, async ({ buildTemplate }) => { +export function allowFail( + fn: (fixtures: T) => Promise +): (fixtures: T) => Promise { + return async (fixtures: T) => { try { - await fn({ buildTemplate }) + await fn(fixtures) } catch (error) { - const reasonStr = reason ? ` (${reason})` : '' - console.error(`\n[XFAIL] "${name}"${reasonStr}: ${error}`) + console.error(`\n[ALLOW_FAIL]: ${error}`) } - }) + } } export const isDebug = process.env.E2B_DEBUG !== undefined diff --git a/packages/js-sdk/tests/template/methods/aptInstall.test.ts b/packages/js-sdk/tests/template/methods/aptInstall.test.ts index 4c1a300892..063b3d5f9e 100644 --- a/packages/js-sdk/tests/template/methods/aptInstall.test.ts +++ b/packages/js-sdk/tests/template/methods/aptInstall.test.ts @@ -1,28 +1,26 @@ import { Template } from '../../../src' -import { buildTemplateTestAllowFail } from '../../setup' +import { allowFail, buildTemplateTest } from '../../setup' -buildTemplateTestAllowFail( +buildTemplateTest( 'apt install', - async ({ buildTemplate }) => { + allowFail(async ({ buildTemplate }) => { const template = Template() .fromImage('ubuntu:24.04') .skipCache() .aptInstall('rolldice') await buildTemplate(template) - }, - 'aptInstall tests are optional and allowed to fail' + }) ) -buildTemplateTestAllowFail( +buildTemplateTest( 'apt install with no install recommends', - async ({ buildTemplate }) => { + allowFail(async ({ buildTemplate }) => { const template = Template() .fromImage('ubuntu:24.04') .skipCache() .aptInstall('rolldice', { noInstallRecommends: true }) await buildTemplate(template) - }, - 'aptInstall tests are optional and allowed to fail' + }) ) From 79e5aa1ab2938107ad081b31d6b3fa8e479e981e Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Wed, 4 Feb 2026 18:50:48 +0100 Subject: [PATCH 7/8] fix: Use object destructuring in allowFail wrapper Vitest requires object destructuring pattern in fixture callbacks. Co-Authored-By: Claude Opus 4.5 --- packages/js-sdk/tests/setup.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/js-sdk/tests/setup.ts b/packages/js-sdk/tests/setup.ts index 3e3b221a45..9e8db00d86 100644 --- a/packages/js-sdk/tests/setup.ts +++ b/packages/js-sdk/tests/setup.ts @@ -115,12 +115,12 @@ export const buildTemplateTest = base.extend({ * * Works with any test fixture type (sandboxTest, buildTemplateTest, etc.) */ -export function allowFail( +export function allowFail( fn: (fixtures: T) => Promise ): (fixtures: T) => Promise { - return async (fixtures: T) => { + return async ({ ...fixtures }: T) => { try { - await fn(fixtures) + await fn(fixtures as T) } catch (error) { console.error(`\n[ALLOW_FAIL]: ${error}`) } From d3f0b677ecaf1bd10a1030fea18e0e28af603824 Mon Sep 17 00:00:00 2001 From: Mish Ushakov <10400064+mishushakov@users.noreply.github.com> Date: Wed, 4 Feb 2026 19:06:16 +0100 Subject: [PATCH 8/8] fix: Use specific buildTemplateTestAllowFail wrapper Vitest fixtures don't support generic wrappers with rest parameters. Use a specific wrapper function instead. Co-Authored-By: Claude Opus 4.5 --- packages/js-sdk/tests/setup.ts | 19 +++++++------- .../tests/template/methods/aptInstall.test.ts | 25 ++++++++----------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/packages/js-sdk/tests/setup.ts b/packages/js-sdk/tests/setup.ts index 9e8db00d86..76b8d27bc2 100644 --- a/packages/js-sdk/tests/setup.ts +++ b/packages/js-sdk/tests/setup.ts @@ -109,22 +109,21 @@ export const buildTemplateTest = base.extend({ }) /** - * Wraps a test callback to allow it to fail without breaking CI. + * Creates a build template test that is allowed to fail without breaking CI. * - If the test passes → test passes (ideal outcome) * - If the test fails → logs ALLOW_FAIL and test still passes - * - * Works with any test fixture type (sandboxTest, buildTemplateTest, etc.) */ -export function allowFail( - fn: (fixtures: T) => Promise -): (fixtures: T) => Promise { - return async ({ ...fixtures }: T) => { +export function buildTemplateTestAllowFail( + name: string, + fn: (fixtures: BuildTemplateFixture) => Promise +) { + return buildTemplateTest(name, async ({ buildTemplate }) => { try { - await fn(fixtures as T) + await fn({ buildTemplate }) } catch (error) { - console.error(`\n[ALLOW_FAIL]: ${error}`) + console.error(`\n[ALLOW_FAIL] ${name}: ${error}`) } - } + }) } export const isDebug = process.env.E2B_DEBUG !== undefined diff --git a/packages/js-sdk/tests/template/methods/aptInstall.test.ts b/packages/js-sdk/tests/template/methods/aptInstall.test.ts index 063b3d5f9e..01519837cf 100644 --- a/packages/js-sdk/tests/template/methods/aptInstall.test.ts +++ b/packages/js-sdk/tests/template/methods/aptInstall.test.ts @@ -1,26 +1,23 @@ import { Template } from '../../../src' -import { allowFail, buildTemplateTest } from '../../setup' +import { buildTemplateTestAllowFail } from '../../setup' -buildTemplateTest( - 'apt install', - allowFail(async ({ buildTemplate }) => { - const template = Template() - .fromImage('ubuntu:24.04') - .skipCache() - .aptInstall('rolldice') +buildTemplateTestAllowFail('apt install', async ({ buildTemplate }) => { + const template = Template() + .fromImage('ubuntu:24.04') + .skipCache() + .aptInstall('rolldice') - await buildTemplate(template) - }) -) + await buildTemplate(template) +}) -buildTemplateTest( +buildTemplateTestAllowFail( 'apt install with no install recommends', - allowFail(async ({ buildTemplate }) => { + async ({ buildTemplate }) => { const template = Template() .fromImage('ubuntu:24.04') .skipCache() .aptInstall('rolldice', { noInstallRecommends: true }) await buildTemplate(template) - }) + } )