diff --git a/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt b/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt index a3577df23b..31a5e277cb 100644 --- a/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt +++ b/core-tests/e2e-tests/e2e-tests-utils/src/test/kotlin/org/evomaster/e2etests/utils/BlackBoxUtils.kt @@ -36,7 +36,6 @@ object BlackBoxUtils { private fun mvn() = if (isWindows()) "mvn.cmd" else "mvn" - private fun runNpmInstall() { val command = listOf(npm(), "ci") @@ -106,7 +105,7 @@ object BlackBoxUtils { } } - fun runNpmTests(folderRelativePath: String) { + fun runNpmTests(folderRelativePath: String, isPlaywright: Boolean = false) { runNpmInstall() val path = if(folderRelativePath.endsWith("/")){ @@ -116,8 +115,12 @@ object BlackBoxUtils { "$folderRelativePath/" } - val command = listOf(npm(), "test", "--", "--testPathPattern=\"$path\"") - runTestsCommand(command, JS_BASE_PATH, "NPM") + val command = if (isPlaywright) { + listOf(npm(), "run", "test:playwright", "--", path) + } else { + listOf(npm(), "test", "--", "--testPathPattern=\"$path\"") + } + runTestsCommand(command, JS_BASE_PATH, if (isPlaywright) "Playwright" else "NPM") } fun runPythonTests(folderRelativePath: String) { @@ -151,4 +154,4 @@ object BlackBoxUtils { fun getOutputFilePrefixKotlin(outputFolderName: String) = "com.kotlin.$outputFolderName.EM" -} +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt b/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt index 30291759dc..399d12cd72 100644 --- a/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt +++ b/core-tests/e2e-tests/spring/spring-graphql-bb/src/test/kotlin/org/evomaster/e2etests/spring/graphql/bb/SpringTestBase.kt @@ -100,6 +100,7 @@ abstract class SpringTestBase : GraphQLTestBase() { fun runGeneratedTests(outputFormat: OutputFormat, outputFolderName: String){ when{ + outputFormat.isPlaywright() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName), true) outputFormat.isJavaScript() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName)) outputFormat.isPython() -> BlackBoxUtils.runPythonTests(BlackBoxUtils.relativePath(outputFolderName)) else -> throw IllegalArgumentException("Not supported output type $outputFormat") diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json index c85a0aca99..4333fffe1d 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json +++ b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package-lock.json @@ -7,6 +7,7 @@ "name": "evomaster-client-js-e2e-tests", "license": "LGPL-3.0-only", "devDependencies": { + "@playwright/test": "^1.59.1", "jest": "29.7.0", "superagent": "9.0.2", "supertest": "7.0.0", @@ -933,6 +934,21 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@playwright/test": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz", + "integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==", + "dev": true, + "dependencies": { + "playwright": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -3248,6 +3264,50 @@ "node": ">=8" } }, + "node_modules/playwright": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", + "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "dev": true, + "dependencies": { + "playwright-core": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", + "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "dev": true, + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/pretty-format": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", @@ -4567,6 +4627,15 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "@playwright/test": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz", + "integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==", + "dev": true, + "requires": { + "playwright": "1.59.1" + } + }, "@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -6292,6 +6361,31 @@ "find-up": "^4.0.0" } }, + "playwright": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", + "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "dev": true, + "requires": { + "fsevents": "2.3.2", + "playwright-core": "1.59.1" + }, + "dependencies": { + "fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "optional": true + } + } + }, + "playwright-core": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", + "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "dev": true + }, "pretty-format": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json index 5c69b06726..21e7571566 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json +++ b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/package.json @@ -6,13 +6,15 @@ "author": "EvoMaster Team", "license": "LGPL-3.0-only", "devDependencies": { + "@playwright/test": "^1.59.1", "jest": "29.7.0", "superagent": "9.0.2", "supertest": "7.0.0", "urijs": "1.19.6" }, "scripts": { - "test": "jest" + "test": "jest", + "test:playwright": "playwright test" }, "jest": { "testEnvironment": "node", diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/javascript/playwright.config.js b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/playwright.config.js new file mode 100644 index 0000000000..ce35c56d60 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-bb/javascript/playwright.config.js @@ -0,0 +1,6 @@ +const { defineConfig } = require('@playwright/test'); + +module.exports = defineConfig({ + testDir: './generated', + testMatch: /.*[tT]est\.js/, +}); diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/maven/pom.xml b/core-tests/e2e-tests/spring/spring-rest-bb/maven/pom.xml index 664c3532c2..a0736e954e 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/maven/pom.xml +++ b/core-tests/e2e-tests/spring/spring-rest-bb/maven/pom.xml @@ -11,7 +11,7 @@ --> org.evomaster - evomaster-e2e-tests + evomaster-e2e-tests-spring 6.0.1-SNAPSHOT ../../pom.xml diff --git a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt index 2ed918c6bb..2434bbc8fb 100644 --- a/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt +++ b/core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/SpringTestBase.kt @@ -150,7 +150,6 @@ abstract class SpringTestBase : RestTestBase() { } } - fun runBlackBoxEM( outputFormat: OutputFormat, outputFolderName: String, @@ -172,6 +171,7 @@ abstract class SpringTestBase : RestTestBase() { fun runGeneratedTests(outputFormat: OutputFormat, outputFolderName: String){ when{ + outputFormat.isPlaywright() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName), true) outputFormat.isJavaScript() -> BlackBoxUtils.runNpmTests(BlackBoxUtils.relativePath(outputFolderName)) outputFormat.isPython() -> BlackBoxUtils.runPythonTests(BlackBoxUtils.relativePath(outputFolderName)) outputFormat.isJava() -> BlackBoxUtils.runJavaTests(outputFolderName) @@ -230,4 +230,4 @@ abstract class SpringTestBase : RestTestBase() { } } -} +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml b/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml index 2fe734f2a0..4cacc8b78b 100644 --- a/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml +++ b/core-tests/e2e-tests/spring/spring-rest-rsa/pom.xml @@ -133,6 +133,7 @@ org.apache.maven.plugins maven-compiler-plugin + diff --git a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt index c5aeb96248..d18bf4a177 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/OutputFormat.kt @@ -18,6 +18,7 @@ enum class OutputFormat { KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, + JS_JEST_PLAYWRIGHT, //CSHARP_XUNIT, //no longer supported, but there is still legacy code not removed PYTHON_UNITTEST ; @@ -28,6 +29,8 @@ enum class OutputFormat { fun isJavaScript() = this.name.startsWith("js_", true) + fun isPlaywright() = this.name.endsWith("_playwright", true) + fun isJavaOrKotlin() = isJava() || isKotlin() fun isJUnit5() = this.name.endsWith("junit_5", true) @@ -41,4 +44,6 @@ enum class OutputFormat { fun isPython() = this.name.startsWith("python_", true) + fun isJsBased() = isJavaScript() || isPlaywright() + // Helper method for JavaScript based formats. Playwright is currently only supported for JavaScript (or TypeScript) // } diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt index b3791fe751..a106052d94 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/CookieWriter.kt @@ -50,7 +50,12 @@ object CookieWriter { when { format.isJava() -> lines.add("final Map ${cookiesName(k)} = ") format.isKotlin() -> lines.add("val ${cookiesName(k)} : Map = ") - format.isJavaScript() -> lines.add("const ${cookiesName(k)} = ") + format.isJavaScript() -> when { + format.isPlaywright() -> + lines.add("let ${cookiesName(k)};") + else -> + lines.add("const ${cookiesName(k)} = ") + } } if (!format.isPython()) { @@ -76,12 +81,32 @@ object CookieWriter { format.isPython() -> lines.append(".cookies") } - if(format.isJavaScript()){ - lines.add(".then((res) => res.headers['set-cookie'][0].split(';')[0])") - lines.add(".catch((err) => (err.status >= 300 && err.status <= 399) ? err.response.headers['set-cookie'][0].split(';')[0] : null)") + if (format.isJavaScript()) { + + // SuperAgent/Playwright cookie extraction + if (format.isPlaywright()) { + lines.add(".then((res) => {") + lines.add("const setCookie = res.headers()['set-cookie'];") + lines.add("if (setCookie) {") + lines.add("return setCookie.split('\\n')[0].split(';')[0];") + lines.add("}") + lines.add("return null;") + lines.add("})") + } else { + lines.add(".then((res) => res.headers['set-cookie'][0].split(';')[0])") + lines.add(".catch((err) => (err.status >= 300 && err.status <= 399) ? err.response.headers['set-cookie'][0].split(';')[0] : null)") + } + + if (format.isPlaywright()) { + // Playwright cookie extraction must NOT assume res is a response object + lines.add(".then(async (cookie) => {") + lines.add("${cookiesName(k)} = cookie;") + lines.add("})") + } lines.appendSemicolon() } + if (format.isPython()) { lines.add("${cookiesName(k)} = requests.utils.dict_from_cookiejar($targetCookieVariable)") } @@ -110,8 +135,12 @@ object CookieWriter { targetVariable: String ) { - if(format.isJavaScript()) { + if(format.isJavaScript() ) { callEndpoint(lines, k, format, baseUrlOfSut) + if (format.isPlaywright()) { + return + // to avoid two request calls for the same endpoint + } } if(format.isPython()) { @@ -122,6 +151,10 @@ object CookieWriter { if(contentType != null) { when { format.isJavaOrKotlin() -> lines.add(".contentType(\"${contentType.defaultValue}\")") + format.isPlaywright() -> { + // handled in request options 'data' or similar if needed, + // but usually Playwright sets it automatically if passed as object + } format.isJavaScript() -> lines.add(".set(\"content-type\", \"${contentType.defaultValue}\")") format.isPython() -> { lines.add("headers[\"content-type\"] = \"${contentType.defaultValue}\"") @@ -150,7 +183,14 @@ object CookieWriter { for(header in k.headers) { when { format.isJavaOrKotlin() -> lines.add(".header(\"${header.name}\", \"${header.value}\")") - format.isJavaScript() -> lines.add(".set(\"${header.name}\", \"${header.value}\")") + format.isJavaScript() -> when { + format.isPlaywright() -> + { + // handled in callEndPoint for Playwright + } + else -> + lines.add(".set(\"${header.name}\", \"${header.value}\")") + } format.isPython() -> { lines.add("headers[\"${header.name}\"] = \"${header.value}\"") } @@ -159,7 +199,11 @@ object CookieWriter { if (format.isJavaScript()){ // disable redirections - lines.add(".redirects(0)") + if (format.isPlaywright()) { + // handled in callEndPoint + } else { + lines.add(".redirects(0)") + } } /* @@ -170,6 +214,7 @@ object CookieWriter { callEndpoint(lines, k, format, baseUrlOfSut) } + if (format.isPython()) { lines.add("$targetVariable = requests \\") lines.indent(2) @@ -189,7 +234,13 @@ object CookieWriter { baseUrlOfSut: String ) { val verb = k.verb.name.lowercase() - lines.add(".$verb(") + + if (format.isPlaywright()) { + lines.add("await request.$verb(") + } else { + lines.add(".$verb(") + } + if (k.externalEndpointURL != null) { lines.append("\"${k.externalEndpointURL}\"") } else { @@ -200,6 +251,34 @@ object CookieWriter { } lines.append("${k.endpoint}\"") } + + if (format.isPlaywright()) { + lines.append(", {") + lines.indented { + if (k.headers.isNotEmpty() || k.contentType != null) { + lines.add("headers: {") + lines.indented { + if (k.contentType != null) { + lines.add("'Content-Type': '${k.contentType.defaultValue}',") + } + for (header in k.headers) { + lines.add("'${header.name}': '${header.value}',") + } + } + lines.add("},") + } + if (k.payload != null) { + if (k.contentType == ContentType.JSON) { + lines.add("data: ${k.payload},") + } else { + lines.add("data: '${k.payload}',") + } + } + lines.add("maxRedirects: 0,") + } + lines.add("}") + } + if (!format.isPython()) { lines.append(")") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt index ec2df92b7e..fb911ed9cd 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/auth/TokenWriter.kt @@ -52,11 +52,13 @@ object TokenWriter { when { format.isJava() -> lines.add("final String ${tokenName(k)} = ") format.isKotlin() -> lines.add("val ${tokenName(k)} : String = ") + format.isPlaywright() -> lines.add("let ${tokenName(k)};") format.isJavaScript() -> lines.add("let ${tokenName(k)} = ") } when{ format.isJavaOrKotlin() -> lines.append("given()") + format.isPlaywright() -> testCaseWriter.startRequest(lines) //handled in HttpWsTestCaseWriter with startRequest(lines) and await request format.isJavaScript() -> { lines.append("\"\"") lines.appendSemicolon() @@ -85,7 +87,9 @@ object TokenWriter { when(token.extractFrom){ TokenHandling.ExtractFrom.BODY -> { - if (format.isJavaScript()) { + if (format.isPlaywright()) { + lines.add(".then(async res => {${tokenName(k)} = (await res.json()).$path;})") + } else if (format.isJavaScript()) { lines.add(".then(res => {${tokenName(k)} = res.body.$path;},") lines.indented { lines.add("error => {console.log(error.response.body); throw Error(\"Auth failed.\")})") } } else if (format.isPython()) { @@ -100,7 +104,9 @@ object TokenWriter { } TokenHandling.ExtractFrom.HEADER -> { val header = token.extractSelector - if (format.isJavaScript()) { + if (format.isPlaywright()) { + lines.add(".then(async res => {${tokenName(k)} = res.headers()[\"${header.lowercase()}\"];})") + } else if (format.isJavaScript()) { lines.add(".then(res => {${tokenName(k)} = res.get(\"$header\");},") lines.indented { lines.add("error => {console.log(error.response.headers); throw Error(\"Auth failed.\")})") } } else if (format.isPython()) { diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index f84839f380..61716e6400 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -277,6 +277,7 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { //TODO would not this fail on recursive/nested calls??? format.isJava() -> ".body(\"${k}isEmpty()\", is(true))" format.isKotlin() -> ".body(\"${k}isEmpty()\", `is`(true))" //'is' is a keyword in Kotlin + format.isPlaywright() -> "expect(Object.keys(await $responseVariableName.json()${k}).length).toBe(0);" format.isJavaScript() -> "expect(Object.keys($responseVariableName.body${k}).length).toBe(0);" format.isCsharp() -> "Assert.True($responseVariableName${k}.ToString() == \"{}\");" format.isPython() -> "assert len($responseVariableName.json()${k}) == 0" @@ -340,12 +341,38 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { return text.replace("\$", "\\\$") } + private fun formatFieldPath(fieldPath: String): String { + if (fieldPath.isEmpty()) { + return "" + } + if (format.isJavaOrKotlin()) { + return if (fieldPath.startsWith("'")) "$fieldPath." else "'$fieldPath'." + } + return if (fieldPath.startsWith("[") || fieldPath.startsWith(".")) { + fieldPath + } else { + ".$fieldPath" + } + } + private fun handleAssertionsOnField(value: Any?, flakyValue: Any?, lines: Lines, fieldPath: String, responseVariableName: String?) { if (value == null) { + val field = when { + format.isJavaScript() -> { + if (format.isPlaywright()) { + "(await $responseVariableName.json())" + } else { + "$responseVariableName.body" + } + } + else -> "" + } + val fieldWithDot = if (fieldPath.isEmpty() || fieldPath.startsWith("[")) fieldPath else if (fieldPath.startsWith(".")) fieldPath else ".$fieldPath" val instruction = when { format.isJavaOrKotlin() -> ".body(\"${fieldPath}\", nullValue())" - format.isJavaScript() -> "expect($responseVariableName.body$fieldPath).toBe(null);" + format.isPlaywright() -> "expect(($field)$fieldWithDot).toBe(null);" + format.isJavaScript() -> "expect($field$fieldPath).toBe(null);" // ($field$)fieldPath format.isCsharp() -> "Assert.True($responseVariableName$fieldPath == null);" format.isPython() -> "assert $responseVariableName.json()$fieldPath is None" else -> throw IllegalStateException("Format not supported yet: $format") @@ -395,10 +422,24 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { if (isSuitableToPrint(toPrint)) { if (format.isJavaScript() || format.isPython()) { + val field = when { + format.isJavaScript() -> { + if (format.isPlaywright()) { + "(await $responseVariableName.json())" + } else { + "$responseVariableName.body" + } + } + else -> "" + } + + val fieldWithDot = if (fieldPath.isEmpty() || fieldPath.startsWith("[")) fieldPath else if (fieldPath.startsWith(".")) fieldPath else ".$fieldPath" val assertionContent = if (format.isPython()) { "assert $responseVariableName.json()$fieldPath == $toPrint" + }else if (format.isPlaywright()){ // playwright + "expect($field$fieldWithDot).toBe($toPrint);" }else { // javascript - "expect($responseVariableName.body$fieldPath).toBe($toPrint);" + "expect($field$fieldPath).toBe($toPrint);" // ($field$)fieldPath } if (flakyValue == null || flakyValue == value){ @@ -543,6 +584,9 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { } if (format.isJavaScript()) { + if (format.isPlaywright()) { + return "expect(await $responseVariableName.text()).toBe(\"\");" + } /* This is super ugly... but there is no clean solution for this in Jest nor SuperAgent... :( @@ -578,8 +622,15 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { val path = if (fieldPath.isEmpty()) "" else "$fieldPath." ".body(\"${path}size()\", equalTo($expectedSize))" } - format.isJavaScript() -> - "expect($responseVariableName.body$fieldPath.length).toBe($expectedSize);" + format.isJavaScript() -> { + if (format.isPlaywright()) { + val field = formatFieldPath(fieldPath) + "expect((await $responseVariableName.json())$field).toHaveLength($expectedSize);" + } else { + val field = "$responseVariableName.body" + "expect($field$fieldPath.length).toBe($expectedSize);" // ($field$)fieldPath + } + } format.isCsharp() -> "Assert.True($responseVariableName$fieldPath.Count == $expectedSize);" format.isPython() -> @@ -599,7 +650,8 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { } if (format.isJavaScript()) { - return "expect($responseVariableName.text).toBe(\"$content\");" + val contentCall = if (format.isPlaywright()) "await $responseVariableName.text()" else "$responseVariableName.text" + return "expect($contentCall).toBe(\"$content\");" } if (format.isCsharp()) { diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt index 30962d25e7..090947a5b5 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/GraphQLTestCaseWriter.kt @@ -52,6 +52,9 @@ class GraphQLTestCaseWriter : HttpWsTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.add(".contentType(\"application/json\")") + format.isPlaywright() -> { + // Handled in callEndpoint for Playwright + } format.isJavaScript() -> lines.add(".set('Content-Type','application/json')") format.isPython() -> lines.add("headers[\"content-type\"] = \"application/json\"") // format.isCsharp() -> lines.add("Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(\"application/json\"));") @@ -95,10 +98,14 @@ class GraphQLTestCaseWriter : HttpWsTestCaseWriter() { } override fun handleVerbEndpoint(baseUrlOfSut: String, _call: HttpWsAction, lines: Lines) { - // TODO maybe in future might want to have GET for QUERY types val verb = "post" - lines.add(".$verb(") + + if (format.isPlaywright()) { + lines.add("await request.$verb(") + } else { + lines.add(".$verb(") + } if(config.blackBox){ /* @@ -131,6 +138,14 @@ class GraphQLTestCaseWriter : HttpWsTestCaseWriter() { } } + if (format.isPlaywright()) { + lines.append(", {") + lines.indented { + lines.add("maxRedirects: 0,") + } + lines.add("}") + } + lines.append(")") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index e742406143..8e8b11930e 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -60,7 +60,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { fun startRequest(lines: Lines){ when { format.isJavaOrKotlin() -> lines.append("given()") - format.isJavaScript() -> lines.append("await superagent") + format.isPlaywright() -> lines.append("await request") + format.isJavaScript() && !format.isPlaywright() -> lines.append("await superagent") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") } @@ -103,6 +104,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isKotlin() -> lines.append("val $resVarName: ValidatableResponse = ") format.isJava() -> lines.append("ValidatableResponse $resVarName = ") + format.isPlaywright() -> lines.append("const $resVarName = ") format.isJavaScript() -> lines.append("const $resVarName = ") format.isPython() -> lines.append("$resVarName = ") format.isCsharp() -> lines.append("var $resVarName = ") @@ -111,6 +113,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.append("given()") + format.isPlaywright() -> {} // already handled in handleVerbEndpoint in RestTestCaseWriter format.isJavaScript() -> lines.append("await superagent") format.isCsharp() -> lines.append("await Client") format.isPython() -> lines.append("requests \\") @@ -186,6 +189,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun openAcceptHeader(): String { return when { format.isJavaOrKotlin() -> ".accept(" + format.isPlaywright() -> "'Accept': " format.isJavaScript() -> ".set('Accept', " format.isCsharp() -> "Client.DefaultRequestHeaders.Add(\"Accept\", " format.isPython() -> "headers['Accept'] = " @@ -195,8 +199,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun closeAcceptHeader(openedHeader: String): String { var result = openedHeader - if (!config.outputFormat.isPython()) { - result += ")" + when { + format.isPlaywright() -> result += "," + !config.outputFormat.isPython() -> result += ")" } if (format.isCsharp()){ result = "$result;" @@ -211,7 +216,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { but that is not the case for the other libraries used for example in JS and C# */ return config.enableBasicAssertions && - (config.outputFormat == OutputFormat.JS_JEST || config.outputFormat == OutputFormat.PYTHON_UNITTEST) + (config.outputFormat == OutputFormat.JS_JEST || config.outputFormat == OutputFormat.JS_JEST_PLAYWRIGHT || config.outputFormat == OutputFormat.PYTHON_UNITTEST) } protected fun handleHeaders(call: HttpWsAction, lines: Lines) { @@ -227,7 +232,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val set = when { format.isJavaOrKotlin() -> "header" - format.isJavaScript() -> "set" + format.isJavaScript() && !format.isPlaywright()-> "set" + format.isPlaywright() -> "" // headers are handled in a map in the options object format.isPython() -> "headers = {}" else -> throw IllegalArgumentException("Not supported format: $format") } @@ -236,10 +242,21 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add(set) } + val bodyParam = call.parameters.find { p -> p is BodyParam } as BodyParam? + + if (format.isPlaywright() && bodyParam != null) { + val contentType = bodyParam.contentType() + if (contentType != null) { + lines.add("'Content-Type': '$contentType',") + } + } + //headers in specified auth info call.auth.headers.forEach { if (format.isPython()) { lines.add("headers[\"${it.name}\"] = \"${it.value}\"") + } else if (format.isPlaywright()) { + lines.add("'${it.name}': '${it.value}', // ${call.auth.name}") } else { lines.add(".$set(\"${it.name}\", \"${it.value}\") // ${call.auth.name}") } @@ -257,6 +274,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val escapedHeader = GeneUtils.applyEscapes(x, GeneUtils.EscapeMode.BODY, format) if (format.isPython()) { lines.add("headers[\"${it.name}\"] = \"${escapedHeader}\"") + } else if (format.isPlaywright()) { + lines.add("'${it.name}': '${escapedHeader}',") } else { lines.add(".$set(\"${it.name}\", \"${escapedHeader}\")") @@ -270,13 +289,22 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val tokenHeader = elc.token!!.sendName if (format.isPython()) { lines.add("headers[\"$tokenHeader\"] = ${TokenWriter.authPayloadName(elc)} # ${call.auth.name}") + } else if (format.isPlaywright()) { + lines.add("'$tokenHeader': ${TokenWriter.authPayloadName(elc)}, // ${call.auth.name}") } else { lines.add(".$set(\"$tokenHeader\", ${TokenWriter.authPayloadName(elc)}) // ${call.auth.name}") } } else { when { format.isJavaOrKotlin() -> lines.add(".cookies(${CookieWriter.cookiesName(elc)})") - format.isJavaScript() -> lines.add(".set('Cookie', ${CookieWriter.cookiesName(elc)})") + format.isJavaScript() -> when { + format.isPlaywright() -> { + val cookieVar = CookieWriter.cookiesName(elc) + lines.add("'Cookie': $cookieVar,") + } + else -> + lines.add(".set('Cookie', ${CookieWriter.cookiesName(elc)})") + } // Python cookies are set alongside the headers and body when performing the request } } @@ -303,7 +331,19 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val code = res.getStatusCode() when { - format.isJavaScript() -> { + + format.isJavaScript() && format.isPlaywright() -> { + val statusAssert = "expect($responseVariableName.status()).toBe($code);" + if (res.getFlakyStatusCode() == null){ + lines.add(statusAssert) + }else{ + lines.addSingleCommentLine(flakyInfo("Status Code", code.toString(), res.getFlakyStatusCode().toString())) + lines.addSingleCommentLine(statusAssert) + } + lines.addEmpty() + } + + format.isJavaScript() && !format.isPlaywright() -> { val statusAssert = "expect($responseVariableName.status).toBe($code);" if (res.getFlakyStatusCode() == null){ lines.add(statusAssert) @@ -436,6 +476,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addStatement("val $varName = System.currentTimeMillis()") } else if(format.isPython()) { lines.addStatement("$varName = time.perf_counter() * 1000") + } else if(format.isPlaywright()) { + lines.addStatement("const $varName = performance.now()") } else if(format.isJavaScript()) { lines.addStatement("$varName = performance.now()") } @@ -455,6 +497,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addStatement("val $finalVarName = System.currentTimeMillis() - $varName") } else if(format.isPython()) { lines.addStatement("$finalVarName = (time.perf_counter() * 1000) - $varName") + } else if(format.isPlaywright()) { + lines.addStatement("const $finalVarName = performance.now() - $varName") } else if(format.isJavaScript()) { lines.addStatement("$finalVarName = performance.now() - $varName") } @@ -466,6 +510,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addSingleCommentLine("Note: SQL Injection vulnerability detected in this call. Expected response time (sqliInjectedSleepDurationMs) should be greater than ${config.sqliInjectedSleepDurationMs} ms.") when{ format.isJavaOrKotlin() -> lines.addStatement("assertTrue($finalVarName > ${config.sqliInjectedSleepDurationMs})") + format.isPlaywright() -> lines.addStatement("expect($finalVarName).toBeGreaterThan(${config.sqliInjectedSleepDurationMs})") format.isJavaScript() -> lines.addStatement("expect($finalVarName).toBeGreaterThan(${config.sqliInjectedSleepDurationMs})") format.isPython() -> lines.addStatement("assert $finalVarName > ${config.sqliInjectedSleepDurationMs}") else -> {} @@ -474,7 +519,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.addSingleCommentLine("Note: No SQL Injection vulnerability detected in this call. Expected response time (sqliBaselineMaxResponseTimeMs) should be less than ${config.sqliBaselineMaxResponseTimeMs} ms.") when{ format.isJavaOrKotlin() -> lines.addStatement("assertTrue($finalVarName < ${config.sqliBaselineMaxResponseTimeMs})") - format.isJavaScript() -> lines.addStatement("expect($finalVarName).toBeLessThan(${config.sqliBaselineMaxResponseTimeMs})") + format.isPlaywright() -> lines.addStatement("expect($finalVarName).toBeLessThan(${config.sqliBaselineMaxResponseTimeMs})") + format.isJavaScript()-> lines.addStatement("expect($finalVarName).toBeLessThan(${config.sqliBaselineMaxResponseTimeMs})") format.isPython() -> lines.addStatement("assert $finalVarName < ${config.sqliBaselineMaxResponseTimeMs}") else -> {} } @@ -508,11 +554,32 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isJavaScript() -> { lines.indent(2) - //in SuperAgent, verb must be first - handleVerbEndpoint(baseUrlOfSut, call, lines) - lines.append(getAcceptHeader(call, res)) - handleHeaders(call, lines) - handleBody(call, lines) + if (format.isPlaywright()) { + handleVerbEndpoint(baseUrlOfSut, call, lines) + lines.replaceInCurrent(Regex("\\)$"), "") + lines.append(", {") + lines.addEmpty() + lines.indented { + if (call is org.evomaster.core.problem.rest.data.RestCallAction) { + lines.add("method: \"${call.verb.name.uppercase()}\",") + } + lines.add("headers: {") + lines.indented { + lines.add(getAcceptHeader(call, res)) + handleHeaders(call, lines) + } + lines.add("},") + handleBody(call, lines, dtoVar) + lines.add("maxRedirects: 0,") + } + lines.add("})") + } else { + //in SuperAgent, verb must be first + handleVerbEndpoint(baseUrlOfSut, call, lines) + lines.append(getAcceptHeader(call, res)) + handleHeaders(call, lines) + handleBody(call, lines) + } } format.isCsharp() -> { @@ -540,7 +607,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { fun sendBodyCommand(): String { return when { format.isJavaOrKotlin() -> "body" - format.isJavaScript() -> "send" + format.isJavaScript() && !format.isPlaywright() -> "send" + format.isPlaywright() -> "data" format.isCsharp() -> "" format.isPython() -> "" else -> throw IllegalArgumentException("Format not supported $format") @@ -562,7 +630,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> lines.add(".contentType(\"${bodyParam.contentType()}\")") - format.isJavaScript() -> lines.add(".set('Content-Type','${bodyParam.contentType()}')") + format.isJavaScript() && !format.isPlaywright() -> lines.add(".set('Content-Type','${bodyParam.contentType()}')") + format.isPlaywright() -> { + // handled in makeHttpCall through options object + } format.isPython() -> lines.add("headers[\"content-type\"] = \"${bodyParam.contentType()}\"") } @@ -580,12 +651,13 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val body = bodyParam.getValueAsPrintableString(mode = GeneUtils.EscapeMode.TEXT, targetFormat = format) // handle body only if it is not black - if (body.isNotBlank()){ + if (body.isNotBlank()) { if (body != "\"\"") { when { format.isCsharp() -> { lines.append("new StringContent(\"$body\", Encoding.UTF8, \"${bodyParam.contentType()}\")") } + format.isPython() -> { if (body.trim().isNullOrBlank()) { lines.add("body = \"\"") @@ -593,16 +665,29 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add("body = $body") } } - else -> lines.add(".$send($body)") + + format.isJavaScript() && format.isPlaywright() -> { + lines.add("data: $body,") + } + + else -> { + lines.add(".$send($body)") + } } } else { when { format.isCsharp() -> { lines.append("new StringContent(\"${"""\"\""""}\", Encoding.UTF8, \"${bodyParam.contentType()}\")") } + format.isPython() -> { lines.add("body = \"\"") } + + format.isJavaScript() && format.isPlaywright() -> { + lines.add("data: \"${"""\"\""""}\",") + } + else -> lines.add(".$send(\"${"""\"\""""}\")") } } @@ -624,6 +709,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> { lines.add("body = \"$body\"") } + format.isJavaScript() && format.isPlaywright() -> { + lines.add("data: \"$body\",") + } else -> lines.add(".$send(\"$body\")") } } else if (bodyParam.isXml()) { @@ -636,6 +724,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> { lines.add("body = \"$escapedXml\"") } + format.isPlaywright() -> { + lines.add("data: \"$escapedXml\",") + } else -> lines.add(".$send(\"$escapedXml\")") } } else { @@ -663,7 +754,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> { lines.add("body = ${bodyLines.first()}") } - format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, false) + format.isPlaywright() -> { + writePlaywrightPayload(lines, bodyLines, false) + } + format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, false) // Needs to be checked! fwr else -> writeJavaOrKotlinJsonBody(lines, send, bodyLines, dtoVar, false) } } else { @@ -688,7 +782,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.add("${bodyLines.last()}") } } - format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, true) + format.isPlaywright() -> { + writePlaywrightPayload(lines, bodyLines, true) + } + format.isJavaScript() -> writeStringifiedPayload(lines, send, bodyLines, true) // Needs to be checked fwr else -> writeJavaOrKotlinJsonBody(lines, send, bodyLines, dtoVar, true) } } @@ -722,6 +819,20 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.append(")") } + private fun writePlaywrightPayload(lines: Lines, bodyLines: List, isMultiLine: Boolean) { + lines.add("data: ${bodyLines.first()}") + if (isMultiLine) { + lines.append(" + ") + lines.indented { + (1 until bodyLines.lastIndex).forEach { i -> + lines.add("${bodyLines[i]} + ") + } + lines.add("${bodyLines.last()}") + } + } + lines.append(",") + } + /** * This is done mainly for RestAssured */ @@ -741,6 +852,11 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } } + format.isPlaywright() -> { + // assertions for Playwright are handled in handleResponseAfterTheCall, + // as they cannot be chained directly in the request call + } + else -> throw IllegalStateException("No assertion in calls for format: $format") } @@ -757,7 +873,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { protected fun handleResponseAssertions(lines: Lines, res: HttpWsCallResult, responseVariableName: String?) { - assert(responseVariableName != null || format.isJavaOrKotlin()) + assert(responseVariableName != null || format.isJavaOrKotlin() || format.isPlaywright()) /* there are 2 cases: @@ -777,8 +893,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { if(!allow.isNullOrBlank()){ val instruction = when { format.isJavaOrKotlin() -> ".header(\"Allow\", \"$allow\")" - format.isJavaScript() -> - "expect($responseVariableName.header[\"allow\"].startsWith(\"$allow\")).toBe(true);" + format.isPlaywright() -> "expect($responseVariableName.headers()[\"allow\"]).toContain(\"$allow\")" + format.isJavaScript() -> "expect($responseVariableName.header[\"allow\"].startsWith(\"$allow\")).toBe(true);" format.isPython() -> "assert \"$allow\" in $responseVariableName.headers[\"allow\"]" else -> throw IllegalStateException("Unsupported format $format") } @@ -809,6 +925,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { val instruction = when { format.isJavaOrKotlin() -> ".contentType(\"$bodyTypeSimplified\")" + format.isPlaywright() -> "expect($responseVariableName.headers()[\"content-type\"]).toContain(\"$bodyTypeSimplified\")" format.isJavaScript() -> "expect($responseVariableName.header[\"content-type\"].startsWith(\"$bodyTypeSimplified\")).toBe(true);" @@ -873,8 +990,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } protected fun handleLastLine(call: HttpWsAction, res: HttpWsCallResult, lines: Lines, resVarName: String) { - - if (format.isJavaScript()) { + if (format.isPlaywright()) { + // No extra processing for Playwright + } else if(format.isJavaScript()) { /* This is to deal with very weird behavior in SuperAgent that crashes the tests for status codes different from 2xx... @@ -928,9 +1046,11 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { } val jsonPath = JsonUtils.fromPointerToPath(jsonPointer) + val dictAccess = JsonUtils.fromPointerToDictionaryAccess(jsonPointer) return when { format.isPython() -> "str($resVarName.json()${JsonUtils.fromPointerToDictionaryAccess(jsonPointer)})" + format.isPlaywright() -> " ((await $resVarName.json())$dictAccess)?.toString()" format.isJavaScript() -> "$resVarName.body.$jsonPath.toString()" format.isJavaOrKotlin() -> "$resVarName.extract().body().path$extraTypeInfo(\"$jsonPath\").toString()" else -> throw IllegalStateException("Unsupported format $format") diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index 9ae514e51d..dc4b61e532 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -201,7 +201,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { when { format.isJava() -> lines.add("String $name = ") format.isKotlin() -> lines.add("val $name : String? = ") - format.isJavaScript() -> lines.add("const $name = ") + format.isJsBased() -> lines.add("const $name = ") format.isPython() -> {lines.add("$name = ")} // should never happen else -> throw IllegalStateException("Unsupported format $format") @@ -217,14 +217,23 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { val call = _call as RestCallAction val verb = call.verb.name.lowercase() - if (format.isCsharp()) { - lines.append(".${StringUtils.capitalization(verb)}Async(") - } else { - if (verb == "trace" && format.isJavaOrKotlin()) { - //currently, RestAssured does not have a trace() method - lines.add(".request(io.restassured.http.Method.TRACE, ") - } else { - lines.add(".$verb(") + when { + format.isPlaywright() -> { + val verbToUse = call.verb.name.lowercase() + if (verbToUse.uppercase() == "OPTIONS") { + lines.add("await request.fetch(") + } else { + lines.add("await request.$verbToUse(") + } + } + format.isCsharp() -> lines.append(".${StringUtils.capitalization(verb)}Async(") + else -> { + if (verb == "trace" && format.isJavaOrKotlin()) { + //currently, RestAssured does not have a trace() method + lines.add(".request(io.restassured.http.Method.TRACE, ") + } else { + lines.add(".$verb(") + } } } @@ -413,7 +422,13 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { lines.add("assertTrue(isValidURIorEmpty($location));") } format.isJavaScript() -> { - lines.add("const $location = $resVarName.header['location'];") + + if (format.isPlaywright()) { + lines.add("const $location = $resVarName") + lines.append(".headers()['location'];") + } else { + lines.add("const $location = $resVarName.header['location'];") + } val validCheck = "${TestSuiteWriter.jsImport}.isValidURIorEmpty($location)" lines.add("expect($validCheck).toBe(true);") } @@ -445,7 +460,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { val extract = extractValueFromJsonResponse(resVarName, idPointer) when{ - format.isJavaScript() -> lines.add("const ") + format.isJsBased() -> lines.add("const ") format.isJava() -> lines.add("String ") format.isKotlin() -> lines.add("val ") format.isPython() -> lines.add("")/* nothing to do in Python */ diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt index 7f286c394a..5be50cd510 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt @@ -117,7 +117,8 @@ abstract class TestCaseWriter { when { format.isJava() -> lines.add("public void ${test.name}() throws Exception {") format.isKotlin() -> lines.add("fun ${test.name}() {") - format.isJavaScript() -> lines.add("test(\"${test.name}\", async () => {") + format.isJavaScript() && !format.isPlaywright()-> lines.add("test(\"${test.name}\", async () => {") + format.isJavaScript() && format.isPlaywright() -> lines.add("test(\"${test.name}\", async ({ request }) => {") format.isCsharp() -> lines.add("public async Task ${test.name}() {") format.isPython() -> lines.add("def ${test.name}(self):") } @@ -310,6 +311,12 @@ abstract class TestCaseWriter { testSuitePath: Path?, baseUrlOfSut: String ) { + val playwrightExpectException = format.isPlaywright() && shouldFailIfExceptionNotThrown(res) + val hasThrownVar = if (playwrightExpectException) "hasThrown_${counter++}" else "" + + if (playwrightExpectException) { + lines.add("let $hasThrownVar = false;") + } when { /* TODO do we need to handle differently in JS due to Promises? @@ -372,6 +379,12 @@ abstract class TestCaseWriter { format.isPython() -> lines.add("except Exception as e:") } + if (playwrightExpectException) { + lines.indented { + lines.add("$hasThrownVar = true;") + } + } + res.getErrorMessage()?.let { lines.indented { lines.addSingleCommentLine("${it.replace('\n', ' ').replace('\r', ' ')}") @@ -385,6 +398,9 @@ abstract class TestCaseWriter { } else { lines.add("}") } + if (playwrightExpectException) { + lines.add("expect($hasThrownVar).toBe(true);") + } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index fe08c169ec..cfdfe05de0 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -539,7 +539,11 @@ class TestSuiteWriter { } if (format.isJavaScript()) { - lines.add("const superagent = require(\"superagent\");") + if (format.isPlaywright()) { + lines.add("const { test, expect, request } = require('@playwright/test');") + } else { + lines.add("const superagent = require(\"superagent\");") + } val jsUtils = JsLoader::class.java.getResource("/$javascriptUtilsFilename").readText() saveToDisk(jsUtils, Paths.get(config.outputFolder, javascriptUtilsFilename)) @@ -548,7 +552,8 @@ class TestSuiteWriter { if (controllerName != null) { lines.add("const $controllerName = require(\"${config.jsControllerPath}\");") } - if (config.testTimeout > 0) { + + if (config.testTimeout > 0 && !format.isPlaywright()) { lines.add("jest.setTimeout(${config.testTimeout * 1000});") } } @@ -741,6 +746,7 @@ class TestSuiteWriter { lines.add("let $baseUrlOfSut;") } else { lines.add("const $baseUrlOfSut = \"${BlackBoxUtils.targetUrl(config, sampler)}\";") + } } else if (config.outputFormat.isCsharp()) { lines.add("private static readonly HttpClient Client = new HttpClient ();") @@ -791,7 +797,8 @@ class TestSuiteWriter { lines.add("@JvmStatic") lines.add("fun initClass()") } - format.isJavaScript() -> lines.add("beforeAll( async () =>") + format.isJavaScript() && !format.isPlaywright()-> lines.add("beforeAll(async () =>") + format.isJavaScript() && format.isPlaywright() -> lines.add("(async ({ request }) =>") } lines.block { @@ -923,7 +930,7 @@ class TestSuiteWriter { testCaseWriter.addExtraInitStatement(lines) } - if (format.isJavaScript()) { + if (format.isJavaScript()) { // End statement of blocks. Valid for playwright too lines.append(");") } } @@ -946,7 +953,8 @@ class TestSuiteWriter { lines.add("@JvmStatic") lines.add("fun tearDown()") } - format.isJavaScript() -> lines.add("afterAll( async () =>") + format.isJavaScript() && !format.isPlaywright()-> lines.add("afterAll( async () =>") + format.isJavaScript() && format.isPlaywright() -> lines.add("test.afterAll(async () =>") } if (!format.isCsharp()) { @@ -999,7 +1007,9 @@ class TestSuiteWriter { format.isKotlin() -> { lines.add("fun initTest()") } - format.isJavaScript() -> lines.add("beforeEach(async () => ") + format.isJavaScript() && !format.isPlaywright()-> lines.add("beforeEach( async () =>") + format.isJavaScript() && format.isPlaywright() -> lines.add("test.beforeEach(async () =>") + //for C# we are actually setting up the constructor for the test class format.isCsharp() -> lines.add("public ${name.getClassName()} ($fixtureClass fixture)") } diff --git a/docs/options.md b/docs/options.md index 1eea634712..cfb45bfbb3 100644 --- a/docs/options.md +++ b/docs/options.md @@ -42,7 +42,7 @@ There are 3 types of options: |`configPath`| __String__. File path for file with configuration settings. Supported formats are YAML and TOML. When EvoMaster starts, it will read such file and import all configurations from it. *Constraints*: `regex .*\.(yml\|yaml\|toml)`. *Default value*: `em.yaml`.| |`outputFilePrefix`| __String__. The name prefix of generated file(s) with the test cases, without file type extension. In JVM languages, if the name contains '.', folders will be created to represent the given package structure. Also, in JVM languages, should not use '-' in the file name, as not valid symbol for class identifiers. This prefix be combined with the outputFileSuffix to combined the final name. As EvoMaster can split the generated tests among different files, each will get a label, and the names will be in the form prefix+label+suffix. *Constraints*: `regex [-a-zA-Z$_][-0-9a-zA-Z$_]*(.[-a-zA-Z$_][-0-9a-zA-Z$_]*)*`. *Default value*: `EvoMaster`.| |`outputFileSuffix`| __String__. The name suffix for the generated file(s), to be added before the file type extension. As EvoMaster can split the generated tests among different files, each will get a label, and the names will be in the form prefix+label+suffix. *Constraints*: `regex [-a-zA-Z$_][-0-9a-zA-Z$_]*(.[-a-zA-Z$_][-0-9a-zA-Z$_]*)*`. *Default value*: `Test`.| -|`outputFormat`| __Enum__. Specify in which format the tests should be outputted. If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used. On the other hand, for black-box testing it will default to a predefined type (e.g., Python). *Valid values*: `DEFAULT, JAVA_JUNIT_5, JAVA_JUNIT_4, KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, PYTHON_UNITTEST`. *Default value*: `DEFAULT`.| +|`outputFormat`| __Enum__. Specify in which format the tests should be outputted. If left on `DEFAULT`, for white-box testing then the value specified in the _EvoMaster Driver_ will be used. On the other hand, for black-box testing it will default to a predefined type (e.g., Python). *Valid values*: `DEFAULT, JAVA_JUNIT_5, JAVA_JUNIT_4, KOTLIN_JUNIT_4, KOTLIN_JUNIT_5, JS_JEST, JS_JEST_PLAYWRIGHT, PYTHON_UNITTEST`. *Default value*: `DEFAULT`.| |`testTimeout`| __Int__. Enforce timeout (in seconds) in the generated tests. This feature might not be supported in all frameworks. If 0 or negative, the timeout is not applied. *Default value*: `60`.| |`ratePerMinute`| __Int__. Rate limiter, of how many actions to do per minute. For example, when making HTTP calls towards an external service, might want to limit the number of calls to avoid bombarding such service (which could end up becoming equivalent to a DoS attack). A value of zero or negative means that no limiter is applied. This is needed only for black-box testing of remote services. Note that, evan without this parameter, EvoMaster will still respect the Retry-After given back in 429 responses. *Default value*: `0`.| |`header0`| __String__. In black-box testing, we still need to deal with authentication of the HTTP requests. With this parameter it is possible to specify a HTTP header that is going to be added to most requests. This should be provided in the form _name:value_. If more than 1 header is needed, use as well the other options _header1_ and _header2_. *Constraints*: `regex (.+:.+)\|(^$)`. *Default value*: `""`.|