Skip to content

Commit 1562c58

Browse files
saw-jansaw-jan
authored
Merge pull request #1052 from nextcloud/chore/port-pr1044
chore: implement expected failure
2 parents afdbcba + bbc0d91 commit 1562c58

7 files changed

Lines changed: 175 additions & 17 deletions

File tree

‎.github/workflows/shared_workflow.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -326,5 +326,6 @@ jobs:
326326
- name: API Tests
327327
env:
328328
NEXTCLOUD_BASE_URL: http://localhost
329-
BEHAT_FILTER_TAGS: '~@skip-${{ matrix.nextcloudVersion }}'
329+
NEXTCLOUD_VERSION: ${{ matrix.nextcloudVersion }}
330+
BEHAT_FILTER_TAGS: '~@skip'
330331
run: make api-test

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ log/
1717
vendor/
1818
.php-cs-fixer.cache
1919
tests/pact/
20+
tests/acceptance/reports
2021
docker-compose.override.yml
2122
compose.override.yaml
2223
.env

‎docs/running_API_test.md‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,42 @@
22
- SPDX-FileCopyrightText: 2024 Jankari Tech Pvt. Ltd.
33
- SPDX-License-Identifier: AGPL-3.0-or-later
44
-->
5-
### 🧪 Running API tests
65

7-
> **_NOTE:_**
6+
## 🧪 Running API tests
7+
8+
> **_NOTE:_**
89
> Before running the API tests, the nextcloud instance needs to be ready, and also integration app needs to be enabled
910
10-
To run the whole of the acceptance tests locally run the command below.
11+
To run the whole of the acceptance tests locally run the command below:
12+
1113
```shell
12-
NEXTCLOUD_BASE_URL=http://<nextcloud_host> make api-test
14+
NEXTCLOUD_BASE_URL=http://<nextcloud_host> \
15+
BEHAT_FILTER_TAGS="~@skip" \
16+
make api-test
1317
```
1418

15-
In order to run only a specific scenario
19+
In order to run only a specific scenario:
20+
1621
```shell
17-
NEXTCLOUD_BASE_URL=http://<nextcloud_host> \
18-
make api-test \
19-
FEATURE_PATH=tests/acceptance/features/api/directUpload.feature:15
22+
NEXTCLOUD_BASE_URL=http://<nextcloud_host> \
23+
BEHAT_FEATURE_PATH=tests/acceptance/features/api/directUpload.feature:15 \
24+
make api-test
2025
```
26+
27+
## Mark Scenario as Expected Failure
28+
29+
We can mark a scenario as expected to fail for all Nextcloud versions or for specific versions using tags.
30+
31+
- `@expect-fail`: Marks a scenario as expected failure for all Nextcloud versions
32+
- `@expect-fail-on-nc<major-version>`: Marks a scenario as expected failure for a specific Nextcloud version. (E.g.: `@expect-fail-on-nc33`)
33+
34+
> **_NOTE:_**
35+
> We MUST provide `NEXTCLOUD_VERSION` environment variable while running the tests that are tagged `@expect-fail-on-nc<major-version>`.
36+
>
37+
> Possible values for `NEXTCLOUD_VERSION`: `stable33` (any major version number), `33` (any major version number) and `master`
38+
>
39+
> ```shell
40+
> NEXTCLOUD_BASE_URL=http://<nextcloud_host> \
41+
> NEXTCLOUD_VERSION=33 \
42+
> make api-test
43+
> ```

‎makefile‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,6 @@ build_tools_directory=$(CURDIR)/build/tools
1414
npm=$(shell which npm 2> /dev/null)
1515
composer=$(shell which composer 2> /dev/null)
1616

17-
#check for BEHAT_FILTER_TAGS env is set any
18-
FILTER_TAGS := ~@skip
19-
ifdef BEHAT_FILTER_TAGS
20-
FILTER_TAGS:=$(FILTER_TAGS)&&${BEHAT_FILTER_TAGS}
21-
endif
22-
2317
all: build
2418

2519
.PHONY: build
@@ -108,7 +102,7 @@ jsunit:
108102

109103
.PHONY: api-test
110104
api-test:
111-
composer run test:api -- --tags '${FILTER_TAGS}' ${FEATURE_PATH}
105+
bash tests/acceptance/run.sh
112106

113107
.PHONY: test
114108
test: phpunit jsunit api-test

‎tests/acceptance/features/api/directUpload.feature‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -678,7 +678,7 @@ Feature: API endpoint for direct upload
678678
}
679679
"""
680680

681-
@skip-stable34
681+
@expect-fail-on-nc34
682682
Scenario: Upload a file that just fits into the users quota
683683
Given the quota of user "Carol" has been set to "10 B"
684684
And user "Carol" got a direct-upload token for "/"

‎tests/acceptance/features/bootstrap/FeatureContext.php‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,11 @@
88

99
use Behat\Behat\Context\Context;
1010
use Behat\Behat\Context\Environment\InitializedContextEnvironment;
11+
use Behat\Behat\Hook\Scope\AfterScenarioScope;
1112
use Behat\Behat\Hook\Scope\BeforeScenarioScope;
1213
use Behat\Gherkin\Node\PyStringNode;
1314
use Behat\Gherkin\Node\TableNode;
15+
use Behat\Testwork\Tester\Result\TestResult;
1416
use GuzzleHttp\Client;
1517
use GuzzleHttp\Cookie\CookieJar;
1618
use GuzzleHttp\Exception\GuzzleException;
@@ -100,6 +102,19 @@ public function setResponse(?ResponseInterface $response): void {
100102
$this->response = $response;
101103
}
102104

105+
/**
106+
* @return string
107+
*/
108+
public function getServerVersion(): string {
109+
$version = getenv('NEXTCLOUD_VERSION');
110+
if (!$version) {
111+
return "";
112+
}
113+
$version = preg_replace("/^stable/", "", $version);
114+
$version = explode(".", $version)[0];
115+
return "nc{$version}";
116+
}
117+
103118
public function __construct(
104119
string $baseUrl,
105120
string $adminUsername,
@@ -944,6 +959,7 @@ public function sendHttpRequest(
944959
if ($user !== null && $password !== null) {
945960
$options['auth'] = [$user, $password];
946961
}
962+
$options['verify'] = false;
947963
$client = new Client($options);
948964
if ($headers === null) {
949965
$headers = [];
@@ -1369,4 +1385,64 @@ public function after():void {
13691385
}
13701386
$this->createdAppPasswords = [];
13711387
}
1388+
1389+
/**
1390+
* @AfterScenario
1391+
*
1392+
* @param AfterScenarioScope $scope
1393+
*
1394+
* @return void
1395+
*/
1396+
public function checkExpectedFailure(AfterScenarioScope $scope): void {
1397+
$reportDir = dirname(dirname(__DIR__)) . "/reports";
1398+
if (!is_dir($reportDir) && !mkdir($reportDir, 0755, true)) {
1399+
throw new \RuntimeException("Failed to create report directory: {$reportDir}");
1400+
}
1401+
1402+
$serverVersion = $this->getServerVersion();
1403+
$tag = "expect-fail-on-{$serverVersion}";
1404+
$scenario = $scope->getScenario();
1405+
$feature = $scope->getFeature();
1406+
$result = $scope->getTestResult();
1407+
1408+
$featurePath = $feature->getFile();
1409+
$featurePath = "tests/" . explode("/tests/", $featurePath)[1];
1410+
$title = $scenario->getTitle();
1411+
$keyword = $scenario->getKeyword();
1412+
$lineNumber = $scenario->getLine();
1413+
$scenarioLine = " - $keyword: $title ($featurePath:$lineNumber)";
1414+
1415+
$hasExpectFailTag = "";
1416+
foreach ($scenario->getTags() as $t) {
1417+
if (str_starts_with($t, $tag)) {
1418+
$hasExpectFailTag = $t;
1419+
break;
1420+
}
1421+
}
1422+
1423+
$reportFile = null;
1424+
if ($hasExpectFailTag && !$serverVersion) {
1425+
echo "[ERROR] Scenario has tag '$hasExpectFailTag' but could not determine server version. Set 'NEXTCLOUD_VERSION' env";
1426+
if ($result->isPassed()) {
1427+
$reportFile = "$reportDir/unexpected-passes.txt";
1428+
}
1429+
}
1430+
1431+
if ($scenario->hasTag($tag) || $scenario->hasTag("expect-fail")) {
1432+
if ($result->getResultCode() === TestResult::FAILED) {
1433+
$reportFile = "$reportDir/expected-failures.txt";
1434+
} elseif ($result->isPassed()) {
1435+
$reportFile = "$reportDir/unexpected-passes.txt";
1436+
echo "[ERROR] Scenario was expected to fail but it did not.";
1437+
} else {
1438+
$reportFile = "$reportDir/failures.txt";
1439+
}
1440+
} elseif (!$result->isPassed()) {
1441+
$reportFile = "$reportDir/failures.txt";
1442+
}
1443+
1444+
if ($reportFile) {
1445+
file_put_contents($reportFile, $scenarioLine . PHP_EOL, FILE_APPEND);
1446+
}
1447+
}
13721448
}

‎tests/acceptance/run.sh‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/bin/bash
2+
3+
# SPDX-FileCopyrightText: 2026 Jankari Tech Pvt. Ltd.
4+
# SPDX-License-Identifier: AGPL-3.0-or-later
5+
6+
SCRIPT_DIR="$(dirname "$0")"
7+
REPORTS_DIR="${SCRIPT_DIR}/reports"
8+
EXPECTED_FAILURES_FILE="$REPORTS_DIR/expected-failures.txt"
9+
FAILURES_FILE="$REPORTS_DIR/failures.txt"
10+
UNEXPECTED_PASSES_FILE="$REPORTS_DIR/unexpected-passes.txt"
11+
12+
# options for behat command
13+
BEHAT_OPTIONS=(-f pretty)
14+
15+
if [ -n "$BEHAT_FILTER_TAGS" ]; then
16+
BEHAT_OPTIONS+=(--tags "$BEHAT_FILTER_TAGS")
17+
fi
18+
19+
function run_test() {
20+
local args=("${BEHAT_OPTIONS[@]}")
21+
if [ -n "$BEHAT_FEATURE_PATH" ]; then
22+
args+=("$BEHAT_FEATURE_PATH")
23+
fi
24+
composer run test:api -- "${args[@]}"
25+
}
26+
27+
# cleanup reports directory
28+
rm -rf "$REPORTS_DIR"
29+
30+
# run the tests
31+
run_test
32+
33+
exit_code=$?
34+
35+
if [ ! -f "$FAILURES_FILE" ] && [ ! -f "$UNEXPECTED_PASSES_FILE" ]; then
36+
# pass the test execution if the exit code is 1
37+
# but there are expected failures defined.
38+
if [ $exit_code -eq 1 ] && [ -s "$EXPECTED_FAILURES_FILE" ]; then
39+
exit_code=0
40+
fi
41+
else
42+
if [ -s "$FAILURES_FILE" ]; then
43+
echo ""
44+
echo "[ERROR] Failed scenarios:"
45+
cat "$FAILURES_FILE"
46+
exit_code=1
47+
fi
48+
if [ -s "$UNEXPECTED_PASSES_FILE" ]; then
49+
echo ""
50+
echo "[ERROR] Unexpected passed scenarios:"
51+
cat "$UNEXPECTED_PASSES_FILE"
52+
exit_code=1
53+
fi
54+
fi
55+
56+
if [ -s "$EXPECTED_FAILURES_FILE" ]; then
57+
echo ""
58+
echo "[INFO] Expected failed scenarios:"
59+
cat "$EXPECTED_FAILURES_FILE"
60+
fi
61+
62+
echo ""
63+
exit $exit_code

0 commit comments

Comments
 (0)