From 2fba0679894c55ae1cba872b5d408c88730c5bcc Mon Sep 17 00:00:00 2001 From: Henk van den Akker Date: Sun, 19 Apr 2020 22:37:32 +0200 Subject: [PATCH 01/63] Added console logging --- robotstatuschecker.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index d1f04c8..a36fedf 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -43,11 +43,16 @@ import re import sys -from robot.api import ExecutionResult, ResultVisitor +from robot.api import ExecutionResult, ResultVisitor, logger +from robot.output import LOGGER from robot.utils import Matcher __version__ = 'devel' +width = 78 +status_length = len('| PASS |') +suite_separator = '%s' % ('=' * width) +test_separator = '%s' % ('-' * width) def process_output(inpath, outpath=None, verbose=True): @@ -73,16 +78,39 @@ def process_output(inpath, outpath=None, verbose=True): class StatusChecker(ResultVisitor): + def __init__(self): + logger.console(suite_separator) + logger.console('Postprocessing of results by StatusChecker...') + logger.console(suite_separator) + def process_output(self, inpath, outpath=None): result = ExecutionResult(inpath) result.suite.visit(self) result.save(outpath) return result + def start_suite(self, suite): + LOGGER.start_suite(suite) + + def end_suite(self, suite): + LOGGER.end_suite(suite) + def visit_test(self, test): expected = Expected(test.doc) if TestStatusChecker(expected).check(test): LogMessageChecker(expected).check(test) + test.keywords.visit(self) + self.end_test(test) + + def end_test(self, test): + # LOGGER.end_test(test) + name_length = len(test.name) + space_length = width - status_length - name_length + space = '%s' % (" " * space_length) + logger.console('%s%s| %s |' % (test.name, space, test.status)) + if test.message: + logger.console(test.message) + logger.console(test_separator) def visit_keyword(self, kw): pass From a26215cd3b7a07ef79d15afbcd6a117db9ff506f Mon Sep 17 00:00:00 2001 From: Henk van den Akker Date: Mon, 20 Apr 2020 21:01:37 +0200 Subject: [PATCH 02/63] Update console logging --- robotstatuschecker.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index a36fedf..60ed59c 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -49,10 +49,6 @@ __version__ = 'devel' -width = 78 -status_length = len('| PASS |') -suite_separator = '%s' % ('=' * width) -test_separator = '%s' % ('-' * width) def process_output(inpath, outpath=None, verbose=True): @@ -79,8 +75,10 @@ def process_output(inpath, outpath=None, verbose=True): class StatusChecker(ResultVisitor): def __init__(self): + width = 78 + suite_separator = '%s' % ('=' * width) logger.console(suite_separator) - logger.console('Postprocessing of results by StatusChecker...') + logger.console('*************** Postprocessing of results by StatusChecker.... ***************') logger.console(suite_separator) def process_output(self, inpath, outpath=None): @@ -96,21 +94,17 @@ def end_suite(self, suite): LOGGER.end_suite(suite) def visit_test(self, test): - expected = Expected(test.doc) - if TestStatusChecker(expected).check(test): - LogMessageChecker(expected).check(test) - test.keywords.visit(self) - self.end_test(test) + if self.start_test(test) is not False: + expected = Expected(test.doc) + if TestStatusChecker(expected).check(test): + LogMessageChecker(expected).check(test) + self.end_test(test) + + def start_test(self, test): + LOGGER.start_test(test) def end_test(self, test): - # LOGGER.end_test(test) - name_length = len(test.name) - space_length = width - status_length - name_length - space = '%s' % (" " * space_length) - logger.console('%s%s| %s |' % (test.name, space, test.status)) - if test.message: - logger.console(test.message) - logger.console(test_separator) + LOGGER.end_test(test) def visit_keyword(self, kw): pass From e8addb97d9bd1f9497c26c2bdc12a6b77ea824f3 Mon Sep 17 00:00:00 2001 From: Henk van den Akker Date: Wed, 22 Apr 2020 00:13:49 +0200 Subject: [PATCH 03/63] Added options to suppress console logging --- robotstatuschecker.py | 54 +++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 60ed59c..4347bfc 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -27,17 +27,24 @@ Command-line usage: - python -m robotstatuschecker infile [outfile] + python -m robotstatuschecker infile [outfile] [--quiet] + robot --prerebotmodifier robotstatuschecker.StatusChecker data_sources + rebot --prerebotmodifier robotstatuschecker.StatusChecker robot_outputs Programmatic usage: from robotstatuschecker import process_output - process_output('infile.xml', 'outfile.xml') + process_output('infile.xml', 'outfile.xml', verbose=True) If an output file is not given, the input file is edited in place. + +By default status checker prints logging to the console. To suppress console +logging use --quiet on the command line, +--prerebotmodifier robotstatuschecker.StatusChecker:False in conjunction with +robot or rebot and verbose=False with the process_output function """ -from __future__ import print_function +# from __future__ import print_function from os.path import abspath import re @@ -65,21 +72,25 @@ def process_output(inpath, outpath=None, verbose=True): int: Number of failed critical tests after post-processing. """ if verbose: - print('Checking %s' % abspath(inpath)) - result = StatusChecker().process_output(inpath, outpath) + logger.console('Checking %s' % abspath(inpath)) + result = StatusChecker(verbose).process_output(inpath, outpath) if verbose and outpath: - print('Output: %s' % abspath(outpath)) + logger.console('Output: %s' % abspath(outpath)) return result.return_code class StatusChecker(ResultVisitor): - def __init__(self): - width = 78 - suite_separator = '%s' % ('=' * width) - logger.console(suite_separator) - logger.console('*************** Postprocessing of results by StatusChecker.... ***************') - logger.console(suite_separator) + def __init__(self, verbose=True): + self.verbose = verbose + if str(self.verbose).lower() == 'false': + self.verbose = False + if self.verbose: + width = 78 + suite_separator = '%s' % ('=' * width) + logger.console(suite_separator) + logger.console('************** Post-processing of results by status checker.... **************') + logger.console(suite_separator) def process_output(self, inpath, outpath=None): result = ExecutionResult(inpath) @@ -88,10 +99,12 @@ def process_output(self, inpath, outpath=None): return result def start_suite(self, suite): - LOGGER.start_suite(suite) + if self.verbose: + LOGGER.start_suite(suite) def end_suite(self, suite): - LOGGER.end_suite(suite) + if self.verbose: + LOGGER.end_suite(suite) def visit_test(self, test): if self.start_test(test) is not False: @@ -101,10 +114,12 @@ def visit_test(self, test): self.end_test(test) def start_test(self, test): - LOGGER.start_test(test) + if self.verbose: + LOGGER.start_test(test) def end_test(self, test): - LOGGER.end_test(test) + if self.verbose: + LOGGER.end_test(test) def visit_keyword(self, kw): pass @@ -285,8 +300,13 @@ def _check_msg_message(self, test, kw, msg, expected): if '-h' in sys.argv or '--help' in sys.argv: print(__doc__) sys.exit(251) + args = sys.argv[1:] try: - rc = process_output(*sys.argv[1:]) + if '--quiet' in args: + args.remove('--quiet') + rc = process_output(*args, verbose=False) + else: + rc = process_output(*args) except TypeError: print(__doc__) sys.exit(252) From 2b00dfed95e0c1cac38eaac7765e080ed9b5c6cc Mon Sep 17 00:00:00 2001 From: Tatu Aalto <2665023+aaltat@users.noreply.github.com> Date: Mon, 29 Mar 2021 13:38:31 +0300 Subject: [PATCH 04/63] Add GitHub actions (#12) --- .github/workflows/push.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/push.yaml diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml new file mode 100644 index 0000000..975aeab --- /dev/null +++ b/.github/workflows/push.yaml @@ -0,0 +1,28 @@ +name: Robot Framework statuschecker CI + +on: [push, pull_request] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.9] + rf-version: [3.2.2, 4.0.0] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} with Robot Framework ${{ matrix.rf-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + if: matrix.python-version != 'pypy3' + run: | + python -m pip install --upgrade pip + pip install robotframework-${{ matrix.rf-version }} + - name: Run tests + run: | + python --version + python test/run.py \ No newline at end of file From 0fbb1caf63dac3a8a0e4d3fee618341b5e2eec43 Mon Sep 17 00:00:00 2001 From: Tatu Aalto <2665023+aaltat@users.noreply.github.com> Date: Mon, 29 Mar 2021 13:39:39 +0300 Subject: [PATCH 05/63] Fix RF install --- .github/workflows/push.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 975aeab..b09cecb 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -21,8 +21,8 @@ jobs: if: matrix.python-version != 'pypy3' run: | python -m pip install --upgrade pip - pip install robotframework-${{ matrix.rf-version }} + pip install robotframework==${{ matrix.rf-version }} - name: Run tests run: | python --version - python test/run.py \ No newline at end of file + python test/run.py From 552ea524603c580a843da6287ce0f17d421e364c Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Sat, 27 Mar 2021 21:00:06 +0200 Subject: [PATCH 06/63] For RF4 use body instead of keyword attribute --- robotstatuschecker.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index d1f04c8..74b64ad 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -43,12 +43,15 @@ import re import sys +from robot import __version__ as rf_version from robot.api import ExecutionResult, ResultVisitor from robot.utils import Matcher __version__ = 'devel' +RF4 = not rf_version.startswith("3") + def process_output(inpath, outpath=None, verbose=True): """The main programmatic entry point to status checker. @@ -222,13 +225,18 @@ def _get_keyword(self, test, expected): kw = None try: for index in expected.kw_index: - kw = (kw or test).keywords[index] + kw = self._get_keyword_rf_version(kw, test, index) return kw except IndexError: message = "No keyword with index '%s'." % expected.kw_index_str self._fail(test, message) return None + def _get_keyword_rf_version(self, kw, test, index): + if not RF4: + return (kw or test).keywords[index] + return (kw or test).body[index] + def _check_message(self, test, kw, expected): try: msg = kw.messages[expected.msg_index] From 2c5e101b9b9c2177002d6c2e4ff99a0ce3d62074 Mon Sep 17 00:00:00 2001 From: Tatu Aalto <2665023+aaltat@users.noreply.github.com> Date: Mon, 29 Mar 2021 14:10:19 +0300 Subject: [PATCH 07/63] Add pypy3 in CI --- .github/workflows/push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index b09cecb..2cedbda 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.9] + python-version: [3.6, 3.9, pypy3] rf-version: [3.2.2, 4.0.0] steps: From 3aba9868f9c9f2c161e5208b7e58b5f68371c523 Mon Sep 17 00:00:00 2001 From: Tatu Aalto <2665023+aaltat@users.noreply.github.com> Date: Mon, 29 Mar 2021 14:40:27 +0300 Subject: [PATCH 08/63] Fix CI for pypy --- .github/workflows/push.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 2cedbda..02261b2 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -18,7 +18,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies - if: matrix.python-version != 'pypy3' run: | python -m pip install --upgrade pip pip install robotframework==${{ matrix.rf-version }} From 1fd62358ece2c73a4ee5fd77136780bab065f274 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Mon, 29 Mar 2021 23:42:31 +0300 Subject: [PATCH 09/63] Updated __version__ to 1.5.0 --- robotstatuschecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 74b64ad..aa0dcd5 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -48,7 +48,7 @@ from robot.utils import Matcher -__version__ = 'devel' +__version__ = "1.5.0" RF4 = not rf_version.startswith("3") From 0552d10f77200511db807c26bc8b9f6bb04d1b97 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Mon, 29 Mar 2021 23:43:52 +0300 Subject: [PATCH 10/63] Rellu and inv to set version --- BUILD.rst | 10 ++++++---- requirements-dev.txt | 5 +++++ tasks.py | 47 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 requirements-dev.txt create mode 100644 tasks.py diff --git a/BUILD.rst b/BUILD.rst index c0e7aee..81c463b 100644 --- a/BUILD.rst +++ b/BUILD.rst @@ -10,9 +10,10 @@ Releasing StatusChecker 3. Update ``__version__`` in ``_:: - sed -i "s/__version__ = .*/__version__ = '$VERSION'/" robotstatuschecker.py + inv set-version 1.5.0 git diff # verify changes - git commit -m "Updated __version__ to $VERSION" robotstatuschecker.py && git push + git commit -m "Updated __version__ to $VERSION" robotstatuschecker.py + git push 4. Tag:: @@ -31,9 +32,10 @@ Releasing StatusChecker 8. ``__version__`` back to ``devel``:: - sed -i "s/__version__ = .*/__version__ = 'devel'/" robotstatuschecker.py + inv set-version devel git diff # verify changes - git commit -m "__version__ back to devel" robotstatuschecker.py && git push + git commit -m "__version__ back to devel" robotstatuschecker.py + git push 9. Advertise on mailing lists, `Twitter `_, `LinkedIn `_, and elsewhere as diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..707f594 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +invoke >= 1.4.1 +rellu >= 0.6 +twine >= 3.4.1 +wheel >= 0.36.2 +black >= 20.8b1 diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..50f5839 --- /dev/null +++ b/tasks.py @@ -0,0 +1,47 @@ +from pathlib import Path + +from invoke import task +from rellu import initialize_labels, ReleaseNotesGenerator, Version +from rellu.tasks import clean # noqa + +VERSION_PATTERN = '__version__ = "(.*)"' +REPOSITORY = "robotframework/statuschecker" +VERSION_PATH = Path("robotstatuschecker.py") +RELEASE_NOTES_PATH = Path("docs/releasenotes/robotstatuschecker-{version}.rst") +RELEASE_NOTES_TITLE = "robotstatuschecker {version}" +RELEASE_NOTES_INTRO = """ +StatusChecker is a tool for validating that executed `Robot Framework`_ test cases +have expected statuses and log messages. It is mainly useful for Robot Framework +test library developers who want to use Robot Framework to also test their libraries. +StatusChecker 1.4 and newer are compatible both with Python 2 and Python 3. + +StatusChecker project is hosted at GitHub and downloads are at PyPI_ +.. _Robot Framework: http://robotframework.org +.. _PyPI: https://github.com/robotframework/statuschecker +.. _issue tracker: https://github.com/robotframework/SeleniumLibrary/issues?q=milestone%3A{version.milestone} +""" + +@task +def set_version(ctx, version): + """Set project version in `robotstatuschecker.py`` file. + Args: + version: Project version to set or ``dev`` to set development version. + Following PEP-440 compatible version numbers are supported: + - Final version like 3.0 or 3.1.2. + - Alpha, beta or release candidate with ``a``, ``b`` or ``rc`` postfix, + respectively, and an incremented number like 3.0a1 or 3.0.1rc1. + - Development version with ``.dev`` postfix and an incremented number like + 3.0.dev1 or 3.1a1.dev2. + When the given version is ``dev``, the existing version number is updated + to the next suitable development version. For example, 3.0 -> 3.0.1.dev1, + 3.1.1 -> 3.1.2.dev1, 3.2a1 -> 3.2a2.dev1, 3.2.dev1 -> 3.2.dev2. + """ + version = Version(version, VERSION_PATH, VERSION_PATTERN) + version.write() + print(version) + + +@task +def print_version(ctx): + """Print the current project version.""" + print(Version(path=VERSION_PATH, pattern=VERSION_PATTERN)) From a3bccf72f0917bfa5726c83db10fb677a8301f32 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Tue, 30 Mar 2021 00:07:57 +0300 Subject: [PATCH 11/63] Release notes for 1.5.0 --- .../releasenotes/robotstatuschecker-1.5.0.rst | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 docs/releasenotes/robotstatuschecker-1.5.0.rst diff --git a/docs/releasenotes/robotstatuschecker-1.5.0.rst b/docs/releasenotes/robotstatuschecker-1.5.0.rst new file mode 100644 index 0000000..703ba1a --- /dev/null +++ b/docs/releasenotes/robotstatuschecker-1.5.0.rst @@ -0,0 +1,48 @@ +======================== +robotstatuschecker 1.5.0 +======================== + + +.. default-role:: code + + +StatusChecker is a tool for validating that executed `Robot Framework`_ test cases +have expected statuses and log messages. It is mainly useful for Robot Framework +test library developers who want to use Robot Framework to also test their libraries. +StatusChecker 1.4 and newer are compatible both with Python 2 and Python 3. + +StatusChecker project is hosted at GitHub and downloads are at PyPI_ +.. _Robot Framework: http://robotframework.org +.. _PyPI: https://github.com/robotframework/statuschecker +.. _issue tracker: https://github.com/robotframework/SeleniumLibrary/issues?q=milestone%3Av1.5.0 + + +.. contents:: + :depth: 2 + :local: + +Most important enhancements +=========================== + +Broken with RF4 (`#11`_) +------------------------ +Fixes deprecation warning with RF 4.0 + +Full list of fixes and enhancements +=================================== + +.. list-table:: + :header-rows: 1 + + * - ID + - Type + - Priority + - Summary + * - `#11`_ + - bug + - critical + - Broken with RF4 + +Altogether 1 issue. View on the `issue tracker `__. + +.. _#11: https://github.com/robotframework/statuschecker/issues/11 From a7c73f686964c187f9b12528bac6e7ac8ccf9c55 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Tue, 30 Mar 2021 00:09:32 +0300 Subject: [PATCH 12/63] Updated BUILD instructions --- BUILD.rst | 171 +++++++++++++++++++++++++++++++++++++++++++++++------- tasks.py | 41 +++++++++++++ 2 files changed, 192 insertions(+), 20 deletions(-) diff --git a/BUILD.rst b/BUILD.rst index 81c463b..5c8e90d 100644 --- a/BUILD.rst +++ b/BUILD.rst @@ -1,42 +1,173 @@ Releasing StatusChecker ======================= -1. Execute tests using different Python implementations and versions. + +Using Invoke +~~~~~~~~~~~~ + +Invoke tasks are defined in the ``_ file and they are executed from +the command line like:: + + inv[oke] task [options] + +Run ``invoke`` without arguments for help. All tasks can be listed using +``invoke --list`` and each task's usage with ``invoke --help task``. + +Preparation +----------- + +1. Check that you are on the master branch and have nothing left to commit, + pull, or push:: + + git branch + git status + git pull --rebase + git push + +2. Clean up:: + + invoke clean + +3. Execute tests using different Python implementations and versions. See ``_ for instructions. -2. Set ``$VERSION`` shell variable to ease copy-pasting further commands:: +4. Set version information to a shell variable to ease copy-pasting further + commands. Add ``aN``, ``bN`` or ``rcN`` postfix if creating a pre-release:: + + VERSION= + + For example, ``VERSION=3.0.1`` or ``VERSION=3.1a2``. + +Release notes +------------- + +1. Set GitHub user information into shell variables to ease copy-pasting the + following command:: + + GITHUB_USERNAME= + GITHUB_PASSWORD= + + Alternatively, supply the credentials when running that command. + +2. Generate a template for the release notes:: + + invoke release-notes -w -v $VERSION -u $GITHUB_USERNAME -p $GITHUB_PASSWORD + + The ``-v $VERSION`` option can be omitted if `version is already set + `__. Omit the ``-w`` option if you just want to get release + notes printed to the console, not written to a file. + + When generating release notes for a preview release like ``3.0.2rc1``, + the list of issues is only going to contain issues with that label + (e.g. ``rc1``) or with a label of an earlier preview release (e.g. + ``alpha1``, ``beta2``). + +2. Fill the missing details in the generated release notes template. + +3. Make sure that issues have correct information: + + - All issues should have type (bug, enhancement or task) and priority set. + Notice that issues with the task type are automatically excluded from + the release notes. + - Issue priorities should be consistent. + - Issue titles should be informative. Consistency is good here too, but + no need to overdo it. + + If information needs to be added or edited, its better to edit it in the + issue tracker than in the generated release notes. This allows re-generating + the list of issues later if more issues are added. - VERSION=x.y +4. Add, commit and push:: -3. Update ``__version__`` in ``_:: + git add docs/releasenotes/robotstatuschecker-$VERSION.rst + git commit -m "Release notes for $VERSION" docs/releasenotes/robotstatuschecker-$VERSION.rst + git push + +5. Update later if necessary. Writing release notes is typically the biggest + task when generating releases, and getting everything done in one go is + often impossible. + + +Set version +----------- + +1. Set version information in ``_:: + + invoke set-version $VERSION + +2. Commit and push changes:: - inv set-version 1.5.0 - git diff # verify changes - git commit -m "Updated __version__ to $VERSION" robotstatuschecker.py + git commit -m "Updated version to $VERSION" robotstatuschecker.py git push -4. Tag:: - git tag -a $VERSION -m "Release $VERSION" && git push --tags -5. Create distribution:: +Tagging +------- + +1. Create an annotated tag and push it:: + + git tag -a v$VERSION -m "Release $VERSION" + git push --tags + +2. Add short release notes to GitHub's `releases page + `_ + with a link to the full release notes. + +Creating distributions +---------------------- + +1. Checkout the earlier created tag if necessary:: + + git checkout v$VERSION - python setup.py sdist register upload + This isn't necessary if continuing right after tagging_. -6. Verify that `PyPI pages `_ - look good. +2. Cleanup (again). This removes temporary files as well as ``build`` and + ``dist`` directories:: -7. Test that installation works:: + invoke clean - pip install robotstatuschecker --upgrade +3. Create source distribution and universal (i.e. Python 2 and 3 compatible) + `wheel `_:: -8. ``__version__`` back to ``devel``:: + python setup.py sdist bdist_wheel --universal + ls -l dist - inv set-version devel - git diff # verify changes - git commit -m "__version__ back to devel" robotstatuschecker.py + Distributions can be tested locally if needed. + +4. Upload distributions to PyPI:: + + twine upload dist/* + +5. Verify that project the page at `PyPI + `_ + looks good. + +6. Test installation (add ``--pre`` with pre-releases):: + + pip install --upgrade robotframework-seleniumlibrary + +Post actions +------------ + +1. Back to master if needed:: + + git checkout master + +2. Set dev version based on the previous version:: + + invoke set-version dev + git commit -m "Back to dev version" src/SeleniumLibrary/__init__.py git push -9. Advertise on mailing lists, `Twitter `_, + For example, ``1.2.3`` is changed to ``1.2.4.dev1`` and ``2.0.1a1`` + to ``2.0.1a2.dev1``. + +3. Close the `issue tracker milestone + `_. + Create also new milestone for the next release unless one exists already. + +4. Advertise on mailing lists, `Twitter `_, `LinkedIn `_, and elsewhere as needed. diff --git a/tasks.py b/tasks.py index 50f5839..0d27bf0 100644 --- a/tasks.py +++ b/tasks.py @@ -1,3 +1,4 @@ +import sys from pathlib import Path from invoke import task @@ -45,3 +46,43 @@ def set_version(ctx, version): def print_version(ctx): """Print the current project version.""" print(Version(path=VERSION_PATH, pattern=VERSION_PATTERN)) + + +@task +def release_notes(ctx, version=None, username=None, password=None, write=False): + """Generates release notes based on issues in the issue tracker. + Args: + version: Generate release notes for this version. If not given, + generated them for the current version. + username: GitHub username. + password: GitHub password. + write: When set to True, write release notes to a file overwriting + possible existing file. Otherwise just print them to the + terminal. + Username and password can also be specified using ``GITHUB_USERNAME`` and + ``GITHUB_PASSWORD`` environment variable, respectively. If they aren't + specified at all, communication with GitHub is anonymous and typically + pretty slow. + """ + version = Version(version, VERSION_PATH, VERSION_PATTERN) + folder = RELEASE_NOTES_PATH.parent.resolve() + folder.mkdir(parents=True, exist_ok=True) + file = RELEASE_NOTES_PATH if write else sys.stdout + generator = ReleaseNotesGenerator( + REPOSITORY, RELEASE_NOTES_TITLE, RELEASE_NOTES_INTRO + ) + generator.generate(version, username, password, file) + + +@task +def init_labels(ctx, username=None, password=None): + """Initialize project by setting labels in the issue tracker. + Args: + username: GitHub username. + password: GitHub password. + Username and password can also be specified using ``GITHUB_USERNAME`` and + ``GITHUB_PASSWORD`` environment variable, respectively. + Should only be executed once when taking ``rellu`` tooling to use or + when labels it uses have changed. + """ + initialize_labels(REPOSITORY, username, password) \ No newline at end of file From 4b38490e7b55230525af5afe030ac62805ccc664 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Tue, 30 Mar 2021 00:18:21 +0300 Subject: [PATCH 13/63] Fixes release instructions --- BUILD.rst | 6 +++--- setup.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/BUILD.rst b/BUILD.rst index 5c8e90d..8cc2e2f 100644 --- a/BUILD.rst +++ b/BUILD.rst @@ -111,7 +111,7 @@ Tagging git push --tags 2. Add short release notes to GitHub's `releases page - `_ + `_ with a link to the full release notes. Creating distributions @@ -141,7 +141,7 @@ Creating distributions twine upload dist/* 5. Verify that project the page at `PyPI - `_ + `_ looks good. 6. Test installation (add ``--pre`` with pre-releases):: @@ -165,7 +165,7 @@ Post actions to ``2.0.1a2.dev1``. 3. Close the `issue tracker milestone - `_. + `_. Create also new milestone for the next release unless one exists already. 4. Advertise on mailing lists, `Twitter `_, diff --git a/setup.py b/setup.py index 59a1eb3..01a12a6 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ """.strip().splitlines() CURDIR = dirname(abspath(__file__)) with open(join(CURDIR, NAME+'.py')) as f: - VERSION = re.search("\n__version__ = '(.*)'\n", f.read()).group(1) + VERSION = re.search('\n__version__ = "(.*)"\n', f.read()).group(1) with open(join(CURDIR, 'README.rst')) as f: README = f.read() From e9ec28a9c0d2dfb11332221247a681d51efb53ad Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Tue, 30 Mar 2021 00:34:18 +0300 Subject: [PATCH 14/63] Revert "For RF4 use body instead of keyword attribute" This reverts commit 552ea524603c580a843da6287ce0f17d421e364c. --- robotstatuschecker.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index aa0dcd5..aacd95a 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -43,15 +43,12 @@ import re import sys -from robot import __version__ as rf_version from robot.api import ExecutionResult, ResultVisitor from robot.utils import Matcher __version__ = "1.5.0" -RF4 = not rf_version.startswith("3") - def process_output(inpath, outpath=None, verbose=True): """The main programmatic entry point to status checker. @@ -225,18 +222,13 @@ def _get_keyword(self, test, expected): kw = None try: for index in expected.kw_index: - kw = self._get_keyword_rf_version(kw, test, index) + kw = (kw or test).keywords[index] return kw except IndexError: message = "No keyword with index '%s'." % expected.kw_index_str self._fail(test, message) return None - def _get_keyword_rf_version(self, kw, test, index): - if not RF4: - return (kw or test).keywords[index] - return (kw or test).body[index] - def _check_message(self, test, kw, expected): try: msg = kw.messages[expected.msg_index] From 2dc9245665f33e9cc083a0e3100465e8ce4c95bc Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Tue, 30 Mar 2021 00:37:54 +0300 Subject: [PATCH 15/63] Release notes for 1.5.1 --- .../releasenotes/robotstatuschecker-1.5.1.rst | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 docs/releasenotes/robotstatuschecker-1.5.1.rst diff --git a/docs/releasenotes/robotstatuschecker-1.5.1.rst b/docs/releasenotes/robotstatuschecker-1.5.1.rst new file mode 100644 index 0000000..a2fe74b --- /dev/null +++ b/docs/releasenotes/robotstatuschecker-1.5.1.rst @@ -0,0 +1,43 @@ +======================== +robotstatuschecker 1.5.1 +======================== + + +.. default-role:: code + + +StatusChecker is a tool for validating that executed `Robot Framework`_ test cases +have expected statuses and log messages. It is mainly useful for Robot Framework +test library developers who want to use Robot Framework to also test their libraries. +StatusChecker 1.4 and newer are compatible both with Python 2 and Python 3. + +This reverts previous release because it does not work. + +StatusChecker project is hosted at GitHub and downloads are at PyPI_ +.. _Robot Framework: http://robotframework.org +.. _PyPI: https://github.com/robotframework/statuschecker +.. _issue tracker: https://github.com/robotframework/SeleniumLibrary/issues?q=milestone%3Av1.5.1 + + +.. contents:: + :depth: 2 + :local: + +Full list of fixes and enhancements +=================================== + +.. list-table:: + :header-rows: 1 + + * - ID + - Type + - Priority + - Summary + * - `#17`_ + - --- + - --- + - Revert RF 4.0 support because it does not work + +Altogether 1 issue. View on the `issue tracker `__. + +.. _#17: https://github.com/robotframework/statuschecker/issues/17 From 6f5e7849f15da025c8f33353f21dd5f96b24b845 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Tue, 30 Mar 2021 00:38:40 +0300 Subject: [PATCH 16/63] Updated version to 1.5.1 --- robotstatuschecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index aacd95a..746b400 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -47,7 +47,7 @@ from robot.utils import Matcher -__version__ = "1.5.0" +__version__ = "1.5.1" def process_output(inpath, outpath=None, verbose=True): From d7c66b37dd1f7bd190b160dd91b7bfbe351cdf4d Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Tue, 30 Mar 2021 00:40:10 +0300 Subject: [PATCH 17/63] Back to dev version --- robotstatuschecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 746b400..78f1a67 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -47,7 +47,7 @@ from robot.utils import Matcher -__version__ = "1.5.1" +__version__ = "1.5.2.dev1" def process_output(inpath, outpath=None, verbose=True): From 26c10734cad0fa4b3ceaf43a8c834d47e1ada58c Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Tue, 30 Mar 2021 00:42:38 +0300 Subject: [PATCH 18/63] More fixes to release --- BUILD.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/BUILD.rst b/BUILD.rst index 8cc2e2f..6df9453 100644 --- a/BUILD.rst +++ b/BUILD.rst @@ -29,7 +29,7 @@ Preparation invoke clean 3. Execute tests using different Python implementations and versions. - See ``_ for instructions. + See ``_ for instructions. 4. Set version information to a shell variable to ease copy-pasting further commands. Add ``aN``, ``bN`` or ``rcN`` postfix if creating a pre-release:: @@ -146,7 +146,7 @@ Creating distributions 6. Test installation (add ``--pre`` with pre-releases):: - pip install --upgrade robotframework-seleniumlibrary + pip install --upgrade robotstatuschecker Post actions ------------ @@ -158,7 +158,7 @@ Post actions 2. Set dev version based on the previous version:: invoke set-version dev - git commit -m "Back to dev version" src/SeleniumLibrary/__init__.py + git commit -m "Back to dev version" robotstatuschecker.py git push For example, ``1.2.3`` is changed to ``1.2.4.dev1`` and ``2.0.1a1`` From 807881687d1463404bc591fee269dc5eba04508d Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 2 Apr 2021 22:05:23 +0300 Subject: [PATCH 19/63] Add .venv to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5e3dfbb..13245df 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ dist/ build/ *egg-info *~ +.venv From 649444747570cf0c68e92c1f0c3db5bbde7a529e Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Sat, 3 Apr 2021 01:45:22 +0300 Subject: [PATCH 20/63] Drop Python 2 support --- robotstatuschecker.py | 8 +++----- setup.py | 9 +++------ test/run.py | 2 -- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 78f1a67..ee63a1d 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -37,8 +37,6 @@ If an output file is not given, the input file is edited in place. """ -from __future__ import print_function - from os.path import abspath import re import sys @@ -88,7 +86,7 @@ def visit_keyword(self, kw): pass -class Expected(object): +class Expected: def __init__(self, doc): self.status = self._get_status(doc) @@ -108,7 +106,7 @@ def _get_logs(self, doc): return [ExpectedLog(item) for item in doc.split('LOG')[1:]] -class ExpectedLog(object): +class ExpectedLog: def __init__(self, doc): index, message = doc.strip().split(' ', 1) @@ -139,7 +137,7 @@ def _split_level(self, message): return 'INFO', message -class BaseChecker(object): +class BaseChecker: def _message_matches(self, actual, expected): if actual == expected: diff --git a/setup.py b/setup.py index 01a12a6..7034d08 100755 --- a/setup.py +++ b/setup.py @@ -1,9 +1,6 @@ #!/usr/bin/env python -try: - from setuptools import setup -except ImportError: - from distutils.core import setup +from setuptools import setup from os.path import abspath, dirname, join import re @@ -13,7 +10,6 @@ Development Status :: 5 - Production/Stable License :: OSI Approved :: Apache Software License Operating System :: OS Independent -Programming Language :: Python :: 2 Programming Language :: Python :: 3 Topic :: Software Development :: Testing Framework :: Robot Framework @@ -39,5 +35,6 @@ platforms = 'any', classifiers = CLASSIFIERS, py_modules = ['robotstatuschecker'], - install_requires = ['robotframework'] + install_requires = ['robotframework'], + python_requires = '>=3.6,<4.0' ) diff --git a/test/run.py b/test/run.py index 1cb3341..dd82f17 100755 --- a/test/run.py +++ b/test/run.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function - from os.path import abspath, dirname, exists, join from platform import python_implementation, python_version from shutil import rmtree From a2fb0704957dee1ccd9b1630a398b155eafa4989 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Sat, 3 Apr 2021 01:48:05 +0300 Subject: [PATCH 21/63] Run Black --- robotstatuschecker.py | 97 +++++++++++++++++++++---------------------- test/run.py | 37 +++++++---------- 2 files changed, 63 insertions(+), 71 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index ee63a1d..1fcd150 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -62,15 +62,14 @@ def process_output(inpath, outpath=None, verbose=True): int: Number of failed critical tests after post-processing. """ if verbose: - print('Checking %s' % abspath(inpath)) + print("Checking %s" % abspath(inpath)) result = StatusChecker().process_output(inpath, outpath) if verbose and outpath: - print('Output: %s' % abspath(outpath)) + print("Output: %s" % abspath(outpath)) return result.return_code class StatusChecker(ResultVisitor): - def process_output(self, inpath, outpath=None): result = ExecutionResult(inpath) result.suite.visit(self) @@ -87,72 +86,69 @@ def visit_keyword(self, kw): class Expected: - def __init__(self, doc): self.status = self._get_status(doc) self.message = self._get_message(doc) self.logs = self._get_logs(doc) def _get_status(self, doc): - return 'FAIL' if 'FAIL' in doc else 'PASS' + return "FAIL" if "FAIL" in doc else "PASS" def _get_message(self, doc): - if 'FAIL' not in doc and 'PASS' not in doc: - return '' + if "FAIL" not in doc and "PASS" not in doc: + return "" status = self._get_status(doc) - return doc.split(status, 1)[1].split('LOG', 1)[0].strip() + return doc.split(status, 1)[1].split("LOG", 1)[0].strip() def _get_logs(self, doc): - return [ExpectedLog(item) for item in doc.split('LOG')[1:]] + return [ExpectedLog(item) for item in doc.split("LOG")[1:]] class ExpectedLog: - def __init__(self, doc): - index, message = doc.strip().split(' ', 1) + index, message = doc.strip().split(" ", 1) self.kw_index, self.msg_index = self._split_index(index) self.level, self.message = self._split_level(message) @property def kw_index_str(self): - return '.'.join(str(index + 1) for index in self.kw_index) + return ".".join(str(index + 1) for index in self.kw_index) @property def msg_index_str(self): return str(self.msg_index + 1) def _split_index(self, index): - if ':' in index: - kw_index, msg_index = index.split(':') + if ":" in index: + kw_index, msg_index = index.split(":") else: kw_index, msg_index = index, 1 - kw_index = [int(index) - 1 for index in kw_index.split('.')] + kw_index = [int(index) - 1 for index in kw_index.split(".")] msg_index = int(msg_index) - 1 return kw_index, msg_index def _split_level(self, message): - for level in ['TRACE', 'DEBUG', 'INFO', 'WARN', 'FAIL', 'ERROR']: + for level in ["TRACE", "DEBUG", "INFO", "WARN", "FAIL", "ERROR"]: if message.startswith(level): - return level, message[len(level):].strip() - return 'INFO', message + return level, message[len(level) :].strip() + return "INFO", message class BaseChecker: - def _message_matches(self, actual, expected): if actual == expected: return True - if expected.startswith('REGEXP:'): - pattern = '^%s$' % expected.replace('REGEXP:', '', 1).strip() + if expected.startswith("REGEXP:"): + pattern = "^%s$" % expected.replace("REGEXP:", "", 1).strip() if re.match(pattern, actual, re.DOTALL): return True - if expected.startswith('GLOB:'): - pattern = expected.replace('GLOB:', '', 1).strip() + if expected.startswith("GLOB:"): + pattern = expected.replace("GLOB:", "", 1).strip() matcher = Matcher(pattern, caseless=False, spaceless=False) if matcher.match(actual): return True - if expected.startswith('STARTS:'): - start = expected.replace('STARTS:', '', 1).strip() + if expected.startswith("STARTS:"): + start = expected.replace("STARTS:", "", 1).strip() if actual.startswith(start): return True return False @@ -163,25 +159,24 @@ def _assert(self, condition, test, message): return True def _fail(self, test, message): - test.status = 'FAIL' + test.status = "FAIL" self._set_message(test, message) return False def _pass(self, test, message): - test.status = 'PASS' + test.status = "PASS" self._set_message(test, message) return True def _set_message(self, test, message): if test.message: - original = '\n\nOriginal message:\n%s' % test.message + original = "\n\nOriginal message:\n%s" % test.message else: - original = '' + original = "" test.message = message + original class TestStatusChecker(BaseChecker): - def __init__(self, expected): self.status = expected.status self.message = expected.message @@ -192,21 +187,19 @@ def check(self, test): def _check_status(self, test): condition = test.status == self.status - message = ('Test was expected to %s but it %sED.' - % (self.status, test.status)) + message = "Test was expected to %s but it %sED." % (self.status, test.status) return self._assert(condition, test, message) def _check_message(self, test): if not self._message_matches(test.message, self.message): - message = 'Wrong message.\n\nExpected:\n%s' % self.message + message = "Wrong message.\n\nExpected:\n%s" % self.message return self._fail(test, message) - if test.status == 'FAIL': - return self._pass(test, 'Test failed as expected.') + if test.status == "FAIL": + return self._pass(test, "Test failed as expected.") return True class LogMessageChecker(BaseChecker): - def __init__(self, expected): self.logs = expected.logs @@ -231,10 +224,8 @@ def _check_message(self, test, kw, expected): try: msg = kw.messages[expected.msg_index] except IndexError: - condition = expected.message == 'NONE' - message = ( - "Keyword '%s' (index %s) does not have message %s." - % (kw.name, expected.kw_index_str, expected.msg_index_str)) + condition = expected.message == "NONE" + message = "Keyword '%s' (index %s) does not have message %s." % (kw.name, expected.kw_index_str, expected.msg_index_str) self._assert(condition, test, message) else: if self._check_msg_level(test, kw, msg, expected): @@ -242,23 +233,29 @@ def _check_message(self, test, kw, expected): def _check_msg_level(self, test, kw, msg, expected): condition = msg.level == expected.level - message = ("Keyword '%s' (index %s) message %s has wrong level.\n\n" - "Expected: %s\nActual: %s" - % (kw.name, expected.kw_index_str, expected.msg_index_str, - expected.level, msg.level)) + message = "Keyword '%s' (index %s) message %s has wrong level.\n\n" "Expected: %s\nActual: %s" % ( + kw.name, + expected.kw_index_str, + expected.msg_index_str, + expected.level, + msg.level, + ) return self._assert(condition, test, message) def _check_msg_message(self, test, kw, msg, expected): condition = self._message_matches(msg.message.strip(), expected.message) - message = ("Keyword '%s' (index %s) message %s has wrong content.\n\n" - "Expected:\n%s\n\nActual:\n%s" - % (kw.name, expected.kw_index_str, expected.msg_index_str, - expected.message, msg.message)) + message = "Keyword '%s' (index %s) message %s has wrong content.\n\n" "Expected:\n%s\n\nActual:\n%s" % ( + kw.name, + expected.kw_index_str, + expected.msg_index_str, + expected.message, + msg.message, + ) return self._assert(condition, test, message) -if __name__ == '__main__': - if '-h' in sys.argv or '--help' in sys.argv: +if __name__ == "__main__": + if "-h" in sys.argv or "--help" in sys.argv: print(__doc__) sys.exit(251) try: diff --git a/test/run.py b/test/run.py index dd82f17..f250a48 100755 --- a/test/run.py +++ b/test/run.py @@ -24,19 +24,17 @@ def check_tests(robot_file): def _run_tests_and_process_output(robot_file): - results = join(CURDIR, 'results') - output = join(results, 'output.xml') + results = join(CURDIR, "results") + output = join(results, "output.xml") if exists(results): rmtree(results) - run(join(CURDIR, robot_file), output=output, log=None, report=None, - loglevel='DEBUG') + run(join(CURDIR, robot_file), output=output, log=None, report=None, loglevel="DEBUG") process_output(output) rebot(output, outputdir=results) return output class StatusCheckerChecker(ResultVisitor): - def __init__(self): self.errors = [] self.tests = 0 @@ -44,33 +42,30 @@ def __init__(self): def visit_test(self, test): self.tests += 1 status, message = self._get_expected(test) - errors = [self._verify(test.status, status, 'status'), - self._verify(test.message, message, 'message')] - errors = ['- %s' % e for e in errors if e] + errors = [self._verify(test.status, status, "status"), self._verify(test.message, message, "message")] + errors = ["- %s" % e for e in errors if e] if errors: - self.errors.append('%s:\n%s' % (test.name, '\n'.join(errors))) + self.errors.append("%s:\n%s" % (test.name, "\n".join(errors))) def _get_expected(self, test): kw = test.keywords[0] - assert kw.name == 'Status', "No 'Status' keyword found." - return (kw.keywords[1].messages[0].message, - kw.keywords[2].messages[0].message) + assert kw.name == "Status", "No 'Status' keyword found." + return (kw.keywords[1].messages[0].message, kw.keywords[2].messages[0].message) def _verify(self, actual, expected, explanation): if actual == expected: - return '' - return ('Expected %s to be "%s" but it was "%s".' - % (explanation, expected, actual)) + return "" + return 'Expected %s to be "%s" but it was "%s".' % (explanation, expected, actual) def print_status(self): print() if self.errors: - print('%d/%d test failed:' % (len(self.errors), self.tests)) - print('\n-------------------------------------\n'.join(self.errors)) + print("%d/%d test failed:" % (len(self.errors), self.tests)) + print("\n-------------------------------------\n".join(self.errors)) else: - print('All %d tests passed/failed/logged as expected.' % self.tests) - print('Run on %s %s.' % (python_implementation(), python_version())) + print("All %d tests passed/failed/logged as expected." % self.tests) + print("Run on %s %s." % (python_implementation(), python_version())) -if __name__ == '__main__': - check_tests('tests.robot') +if __name__ == "__main__": + check_tests("tests.robot") From 72d6a9e05171643a16a7a2b6f8f9c013e93233c1 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Sat, 3 Apr 2021 02:01:38 +0300 Subject: [PATCH 22/63] F-strings --- robotstatuschecker.py | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 1fcd150..4edbe44 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -62,10 +62,10 @@ def process_output(inpath, outpath=None, verbose=True): int: Number of failed critical tests after post-processing. """ if verbose: - print("Checking %s" % abspath(inpath)) + print(f"Checking {abspath(inpath)}") result = StatusChecker().process_output(inpath, outpath) if verbose and outpath: - print("Output: %s" % abspath(outpath)) + print(f"Output: {abspath(outpath)}") return result.return_code @@ -139,7 +139,7 @@ def _message_matches(self, actual, expected): if actual == expected: return True if expected.startswith("REGEXP:"): - pattern = "^%s$" % expected.replace("REGEXP:", "", 1).strip() + pattern = f"^{expected.replace('REGEXP:', '', 1).strip()}$" if re.match(pattern, actual, re.DOTALL): return True if expected.startswith("GLOB:"): @@ -170,7 +170,7 @@ def _pass(self, test, message): def _set_message(self, test, message): if test.message: - original = "\n\nOriginal message:\n%s" % test.message + original = f"\n\nOriginal message:\n{test.message}" else: original = "" test.message = message + original @@ -187,12 +187,12 @@ def check(self, test): def _check_status(self, test): condition = test.status == self.status - message = "Test was expected to %s but it %sED." % (self.status, test.status) + message = f"Test was expected to {self.status} but it {test.status}ED." return self._assert(condition, test, message) def _check_message(self, test): if not self._message_matches(test.message, self.message): - message = "Wrong message.\n\nExpected:\n%s" % self.message + message = f"Wrong message.\n\nExpected:\n{self.message}" return self._fail(test, message) if test.status == "FAIL": return self._pass(test, "Test failed as expected.") @@ -216,7 +216,7 @@ def _get_keyword(self, test, expected): kw = (kw or test).keywords[index] return kw except IndexError: - message = "No keyword with index '%s'." % expected.kw_index_str + message = f"No keyword with index '{expected.kw_index_str}'." self._fail(test, message) return None @@ -225,7 +225,9 @@ def _check_message(self, test, kw, expected): msg = kw.messages[expected.msg_index] except IndexError: condition = expected.message == "NONE" - message = "Keyword '%s' (index %s) does not have message %s." % (kw.name, expected.kw_index_str, expected.msg_index_str) + message = ( + f"Keyword '{kw.name}' (index {expected.kw_index_str}) does not have message {expected.msg_index_str}." + ) self._assert(condition, test, message) else: if self._check_msg_level(test, kw, msg, expected): @@ -233,23 +235,17 @@ def _check_message(self, test, kw, expected): def _check_msg_level(self, test, kw, msg, expected): condition = msg.level == expected.level - message = "Keyword '%s' (index %s) message %s has wrong level.\n\n" "Expected: %s\nActual: %s" % ( - kw.name, - expected.kw_index_str, - expected.msg_index_str, - expected.level, - msg.level, + message = ( + f"Keyword '{kw.name}' (index {expected.kw_index_str}) message {expected.msg_index_str} has wrong level." + f"\n\nExpected: {expected.level}\nActual: {msg.level}" ) return self._assert(condition, test, message) def _check_msg_message(self, test, kw, msg, expected): condition = self._message_matches(msg.message.strip(), expected.message) - message = "Keyword '%s' (index %s) message %s has wrong content.\n\n" "Expected:\n%s\n\nActual:\n%s" % ( - kw.name, - expected.kw_index_str, - expected.msg_index_str, - expected.message, - msg.message, + message = ( + f"Keyword '{kw.name}' (index {expected.kw_index_str}) message {expected.msg_index_str} has wrong content." + f"\n\nExpected:\n{expected.message}\n\nActual:\n{msg.message}" ) return self._assert(condition, test, message) From 35a6c948cb7fce1b77046f757b860af28a6cefc3 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Mon, 5 Apr 2021 21:40:06 +0300 Subject: [PATCH 23/63] Mark test setup logs with SUITE marker --- robotstatuschecker.py | 29 +++++++++++++++++++++++++---- test/tests.robot | 14 ++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 4edbe44..afcc38e 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -106,8 +106,12 @@ def _get_logs(self, doc): class ExpectedLog: def __init__(self, doc): - index, message = doc.strip().split(" ", 1) - self.kw_index, self.msg_index = self._split_index(index) + index, message = doc.strip().split(' ', 1) + test_setup, kw_index, msg_index, test_teardown = self._split_index(index) + self.test_setup = test_setup + self.kw_index = kw_index + self.msg_index = msg_index + self.test_teardown = test_teardown self.level, self.message = self._split_level(message) @property @@ -123,9 +127,18 @@ def _split_index(self, index): kw_index, msg_index = index.split(":") else: kw_index, msg_index = index, 1 - kw_index = [int(index) - 1 for index in kw_index.split(".")] + new_kw_index = [] + test_setup = False + test_teardown = False + for index in kw_index.split("."): + if index.upper() == "SUITE": + test_setup = True + elif index.upper() == "TEARDOWN": + test_teardown = True + else: + new_kw_index.append(int(index) - 1) msg_index = int(msg_index) - 1 - return kw_index, msg_index + return test_setup, new_kw_index, msg_index, test_teardown def _split_level(self, message): for level in ["TRACE", "DEBUG", "INFO", "WARN", "FAIL", "ERROR"]: @@ -213,6 +226,14 @@ def _get_keyword(self, test, expected): kw = None try: for index in expected.kw_index: + if expected.test_setup and test.keywords.setup: + kw = test.keywords.setup + elif expected.test_setup and not test.keywords.setup: + message = "Expected test setup but setup is not present." + self._fail(test, message) + return None + elif test.keywords.setup: + index += 1 kw = (kw or test).keywords[index] return kw except IndexError: diff --git a/test/tests.robot b/test/tests.robot index c9672df..b690790 100644 --- a/test/tests.robot +++ b/test/tests.robot @@ -106,6 +106,20 @@ NONE log message No Operation Log Message +Test Setup Check Is Done By SUITE Marker + [Documentation] ... + ... LOG SUITE.2:1 PASS + ... LOG 1:1 KALA + [Setup] Status PASS + Log KALA + +Error When No Setup + [Documentation] ... + ... LOG SUITE.1:1 PASS + ... LOG 2:1 KALA + Status FAIL Expected test setup but setup is not present. + Log KALA + Expected FAIL and log messages [Documentation] This text is ignored. FAIL Told ya!! ... LOG 2 Failing soon! From aa0b4e75e905be515ef341c2451e6ca349206a74 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Mon, 5 Apr 2021 23:05:17 +0300 Subject: [PATCH 24/63] Simplify logic --- robotstatuschecker.py | 8 ++++---- test/tests.robot | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index afcc38e..5e3076d 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -133,8 +133,10 @@ def _split_index(self, index): for index in kw_index.split("."): if index.upper() == "SUITE": test_setup = True + new_kw_index.append(0) elif index.upper() == "TEARDOWN": test_teardown = True + new_kw_index.append(-1) else: new_kw_index.append(int(index) - 1) msg_index = int(msg_index) - 1 @@ -226,13 +228,11 @@ def _get_keyword(self, test, expected): kw = None try: for index in expected.kw_index: - if expected.test_setup and test.keywords.setup: - kw = test.keywords.setup - elif expected.test_setup and not test.keywords.setup: + if expected.test_setup and not test.keywords.setup: message = "Expected test setup but setup is not present." self._fail(test, message) return None - elif test.keywords.setup: + if test.keywords.setup and not expected.test_setup: index += 1 kw = (kw or test).keywords[index] return kw diff --git a/test/tests.robot b/test/tests.robot index b690790..8cc7efb 100644 --- a/test/tests.robot +++ b/test/tests.robot @@ -108,6 +108,7 @@ NONE log message Test Setup Check Is Done By SUITE Marker [Documentation] ... + ... LOG SUITE:1 NONE ... LOG SUITE.2:1 PASS ... LOG 1:1 KALA [Setup] Status PASS From 4044386c70f6c9ffe2fcef587eb761d32d19f229 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Mon, 5 Apr 2021 23:12:49 +0300 Subject: [PATCH 25/63] Teardown tests --- robotstatuschecker.py | 4 ++++ test/tests.robot | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 5e3076d..fe465da 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -232,6 +232,10 @@ def _get_keyword(self, test, expected): message = "Expected test setup but setup is not present." self._fail(test, message) return None + if expected.test_teardown and not test.keywords.teardown: + message = "Expected test setup but setup is not present." + self._fail(test, message) + return None if test.keywords.setup and not expected.test_setup: index += 1 kw = (kw or test).keywords[index] diff --git a/test/tests.robot b/test/tests.robot index 8cc7efb..88fc5bf 100644 --- a/test/tests.robot +++ b/test/tests.robot @@ -121,6 +121,16 @@ Error When No Setup Status FAIL Expected test setup but setup is not present. Log KALA +Test Teardown Check Is Done By TEARDOWN Marker + [Documentation] LOG TEARDOWN:1 foobar + Status PASS + [Teardown] Log foobar + +Error When No Teardown + [Documentation] LOG TEARDOWN:1 foobar + Status FAIL Expected test setup but setup is not present. + Log KALA + Expected FAIL and log messages [Documentation] This text is ignored. FAIL Told ya!! ... LOG 2 Failing soon! From 2e78b190d73853612a4d31239be9deacca641dca Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Mon, 5 Apr 2021 23:23:59 +0300 Subject: [PATCH 26/63] Added docs --- README.rst | 8 ++++++++ robotstatuschecker.py | 2 +- test/tests.robot | 13 ++++++++----- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index bf82173..0c3d2d4 100644 --- a/README.rst +++ b/README.rst @@ -64,6 +64,9 @@ If a test is expected to *PASS* with a certain message, the word ``PASS`` must be added to its documentation explicitly and the expected message given after that. +If a message check should happen in test setup or teardown, that check +must be prefixed with ``SETUP`` or ``TEARDOWN`` word. + The expected message can also be specified as a regular expression by prefixing it with ``REGEXP:``. The specified regular expression must match the error message fully. Having spaces between the status, @@ -87,6 +90,11 @@ statuses and messages: [Documentation] FAIL Expected error message Steps + Check in test setup is done by SETUP marker + [Documentation] LOG SETUP This first log message in test setup + [Setup] Test specific setup + Steps + Exclude documentation before marker [Documentation] This text is ignored FAIL Expected error message Steps diff --git a/robotstatuschecker.py b/robotstatuschecker.py index fe465da..a2387ea 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -131,7 +131,7 @@ def _split_index(self, index): test_setup = False test_teardown = False for index in kw_index.split("."): - if index.upper() == "SUITE": + if index.upper() == "SETUP": test_setup = True new_kw_index.append(0) elif index.upper() == "TEARDOWN": diff --git a/test/tests.robot b/test/tests.robot index 88fc5bf..c38ed10 100644 --- a/test/tests.robot +++ b/test/tests.robot @@ -106,23 +106,26 @@ NONE log message No Operation Log Message -Test Setup Check Is Done By SUITE Marker +Test Setup Check Is Done By SETUP Marker [Documentation] ... - ... LOG SUITE:1 NONE - ... LOG SUITE.2:1 PASS + ... LOG SETUP:1 NONE + ... LOG SETUP.2:1 PASS + ... LOG SETUP.2 PASS ... LOG 1:1 KALA [Setup] Status PASS Log KALA Error When No Setup [Documentation] ... - ... LOG SUITE.1:1 PASS + ... LOG SETUP.1:1 PASS ... LOG 2:1 KALA Status FAIL Expected test setup but setup is not present. Log KALA Test Teardown Check Is Done By TEARDOWN Marker - [Documentation] LOG TEARDOWN:1 foobar + [Documentation] ... + ... LOG TEARDOWN:1 foobar + ... LOG TEARDOWN foobar Status PASS [Teardown] Log foobar From 4deb4f23430fc5c8b4f73d581286ef325af77e2c Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Tue, 6 Apr 2021 00:19:14 +0300 Subject: [PATCH 27/63] Use 4.0.1b1 in CI --- .github/workflows/push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 02261b2..651a206 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -9,7 +9,7 @@ jobs: strategy: matrix: python-version: [3.6, 3.9, pypy3] - rf-version: [3.2.2, 4.0.0] + rf-version: [3.2.2, 4.0.1b1] steps: - uses: actions/checkout@v2 From 9375c2ab9098171af374b458055c4ff56ff3b934 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Tue, 6 Apr 2021 00:03:31 +0300 Subject: [PATCH 28/63] RF 4 support for checker --- robotstatuschecker.py | 52 ++++++++++++++++++++++++++++++++++--------- test/tests.robot | 2 +- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index a2387ea..6ce740b 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -41,11 +41,13 @@ import re import sys +from robot import __version__ as rf_version from robot.api import ExecutionResult, ResultVisitor from robot.utils import Matcher __version__ = "1.5.2.dev1" +RF3 = rf_version.startswith("3") def process_output(inpath, outpath=None, verbose=True): @@ -215,6 +217,10 @@ def _check_message(self, test): class LogMessageChecker(BaseChecker): + + _no_setup_message = "Expected test setup but setup is not present." + _no_teardown_message = "Expected test teardown but teardown is not present." + def __init__(self, expected): self.logs = expected.logs @@ -228,23 +234,47 @@ def _get_keyword(self, test, expected): kw = None try: for index in expected.kw_index: - if expected.test_setup and not test.keywords.setup: - message = "Expected test setup but setup is not present." - self._fail(test, message) - return None - if expected.test_teardown and not test.keywords.teardown: - message = "Expected test setup but setup is not present." - self._fail(test, message) - return None - if test.keywords.setup and not expected.test_setup: - index += 1 - kw = (kw or test).keywords[index] + kw = self._get_keyword_rf3_rf4(test, expected, kw, index) + if kw is None: + return kw return kw except IndexError: message = f"No keyword with index '{expected.kw_index_str}'." self._fail(test, message) return None + def _get_keyword_rf3_rf4(self, test, expected, kw, index): + if RF3: + return self._get_keyword_rf3(test, expected, kw, index) + return self._get_keyword_rf4(test, expected, kw, index) + + def _get_keyword_rf3(self, test, expected, kw, index): + if expected.test_setup and not test.keywords.setup: + self._fail(test, self._no_setup_message) + return None + if expected.test_teardown and not test.keywords.teardown: + self._fail(test, self._no_teardown_message) + return None + if test.keywords.setup and not expected.test_setup: + index += 1 + kw = (kw or test).keywords[index] + return kw + + def _get_keyword_rf4(self, test, expected, kw, index): + if expected.test_setup and not test.setup: + self._fail(test, self._no_setup_message) + return None + if expected.test_teardown and not test.teardown: + self._fail(test, self._no_teardown_message) + return None + if expected.test_setup and not kw: + kw = test.setup + elif expected.test_teardown and not kw: + kw = test.teardown + else: + kw = (kw or test).body[index] + return kw + def _check_message(self, test, kw, expected): try: msg = kw.messages[expected.msg_index] diff --git a/test/tests.robot b/test/tests.robot index c38ed10..78edd2f 100644 --- a/test/tests.robot +++ b/test/tests.robot @@ -131,7 +131,7 @@ Test Teardown Check Is Done By TEARDOWN Marker Error When No Teardown [Documentation] LOG TEARDOWN:1 foobar - Status FAIL Expected test setup but setup is not present. + Status FAIL Expected test teardown but teardown is not present. Log KALA Expected FAIL and log messages From 35b644258103d9784c9be3dc568177c2f92c6d2f Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Tue, 6 Apr 2021 00:10:29 +0300 Subject: [PATCH 29/63] RF 4 support also run run.py --- test/run.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/run.py b/test/run.py index f250a48..43fd62a 100755 --- a/test/run.py +++ b/test/run.py @@ -12,6 +12,7 @@ sys.path.insert(0, dirname(CURDIR)) from robotstatuschecker import process_output +from robotstatuschecker import RF3 def check_tests(robot_file): @@ -48,6 +49,19 @@ def visit_test(self, test): self.errors.append("%s:\n%s" % (test.name, "\n".join(errors))) def _get_expected(self, test): + if RF3: + return self._get_expected_rf3(test) + return self._get_expected_rf4(test) + + def _get_expected_rf4(self, test): + if len(test.setup.body) == 0: + kw = test.body[0] + else: + kw = test.setup + return (kw.body[1].messages[0].message, + kw.body[2].messages[0].message) + + def _get_expected_rf3(self, test): kw = test.keywords[0] assert kw.name == "Status", "No 'Status' keyword found." return (kw.keywords[1].messages[0].message, kw.keywords[2].messages[0].message) From 710a437f112880434a1c1ca47d540c4663a684e2 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Tue, 6 Apr 2021 00:12:11 +0300 Subject: [PATCH 30/63] Clean code --- robotstatuschecker.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 6ce740b..5eccb0d 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -257,8 +257,7 @@ def _get_keyword_rf3(self, test, expected, kw, index): return None if test.keywords.setup and not expected.test_setup: index += 1 - kw = (kw or test).keywords[index] - return kw + return (kw or test).keywords[index] def _get_keyword_rf4(self, test, expected, kw, index): if expected.test_setup and not test.setup: From ccc2081011b5c32453efdbd59f3b50fbca72802f Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 9 Apr 2021 00:00:27 +0300 Subject: [PATCH 31/63] Release notes for 2.0.0 --- .../releasenotes/robotstatuschecker-2.0.0.rst | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 docs/releasenotes/robotstatuschecker-2.0.0.rst diff --git a/docs/releasenotes/robotstatuschecker-2.0.0.rst b/docs/releasenotes/robotstatuschecker-2.0.0.rst new file mode 100644 index 0000000..34f2f32 --- /dev/null +++ b/docs/releasenotes/robotstatuschecker-2.0.0.rst @@ -0,0 +1,77 @@ +======================== +robotstatuschecker 2.0.0 +======================== + + +.. default-role:: code + + +StatusChecker is a tool for validating that executed `Robot Framework`_ test cases +have expected statuses and log messages. It is mainly useful for Robot Framework +test library developers who want to use Robot Framework to also test their libraries. +StatusChecker 2 and newer are compatible both with Python 3.6+ and Robot Framework 3.2 +4.0. + +StatusChecker project is hosted at GitHub and downloads are at PyPI_ +.. _Robot Framework: http://robotframework.org +.. _PyPI: https://github.com/robotframework/statuschecker +.. _issue tracker: https://github.com/robotframework/SeleniumLibrary/issues?q=milestone%3Av2.0.0 + + +.. contents:: + :depth: 2 + :local: + +Most important enhancements +=========================== + +Broken with RF4 (`#11`_) +------------------------ +Status checker is not compatible with RF 4.0 and does not anymore display +warning when it uses RF result parser. + +Add SETUP and TEARDOWN markers for checking logs from test setup and teardown (`#21`_) +-------------------------------------------------------------------------------------- +In previous releases keywords in setup and teardown could have been referenced +index starting from 1. If setup was present, then index one would point the +setup keyword. But if setup was not present, index 1 would point to the first +keyword in test body. This is somewhat confusing and now keywords is setup and +teardown must be targeted by using SETUP and TEARDOWN markers. Also index one +will always point to the first keyword in the test body. + +Backwards incompatible changes +============================== + +Drop Python 2 support and use Python 3.6 is minimum version. (`#14`_) +---------------------------------------------------------------------- +This release drops support for Python 2 and raises minimum version Python to +3.6. Also Robot Framework 3.2 and 4.0 are supported by this release. + +Full list of fixes and enhancements +=================================== + +.. list-table:: + :header-rows: 1 + + * - ID + - Type + - Priority + - Summary + * - `#11`_ + - bug + - critical + - Broken with RF4 + * - `#14`_ + - enhancement + - critical + - Drop Python 2 support and use Python 3.6 is minimum version. + * - `#21`_ + - enhancement + - high + - Add SETUP and TEARDOWN markers for checking logs from test setup and teardown + +Altogether 3 issues. View on the `issue tracker `__. + +.. _#11: https://github.com/robotframework/statuschecker/issues/11 +.. _#14: https://github.com/robotframework/statuschecker/issues/14 +.. _#21: https://github.com/robotframework/statuschecker/issues/21 From d3cb6fd85336396916882335e4819c9cc2c37efb Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 9 Apr 2021 00:00:59 +0300 Subject: [PATCH 32/63] Updated version to 2.0.0 --- robotstatuschecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 5eccb0d..624007a 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -46,7 +46,7 @@ from robot.utils import Matcher -__version__ = "1.5.2.dev1" +__version__ = "2.0.0" RF3 = rf_version.startswith("3") From 8ec68b69063031dba04741683293fed4b8074de9 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 9 Apr 2021 00:03:19 +0300 Subject: [PATCH 33/63] Back to dev version --- robotstatuschecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 624007a..9302995 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -46,7 +46,7 @@ from robot.utils import Matcher -__version__ = "2.0.0" +__version__ = "2.0.1.dev1" RF3 = rf_version.startswith("3") From dfb6e0ffb5105ec5b05b42bc6be52f2928744b60 Mon Sep 17 00:00:00 2001 From: Tatu Aalto <2665023+aaltat@users.noreply.github.com> Date: Tue, 13 Apr 2021 09:35:47 +0300 Subject: [PATCH 34/63] Use RF4.0.1 in CI --- .github/workflows/push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 651a206..fe54701 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -9,7 +9,7 @@ jobs: strategy: matrix: python-version: [3.6, 3.9, pypy3] - rf-version: [3.2.2, 4.0.1b1] + rf-version: [3.2.2, 4.0.1] steps: - uses: actions/checkout@v2 From 7572bca540ab1b7c44ed75461ece2faa072e813e Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Wed, 14 Apr 2021 22:53:44 +0300 Subject: [PATCH 35/63] Add Py3.8 --- .github/workflows/push.yaml | 2 +- test/run.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index fe54701..da9ff3b 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.9, pypy3] + python-version: [3.6, 3.8, 3.9, pypy3] rf-version: [3.2.2, 4.0.1] steps: diff --git a/test/run.py b/test/run.py index 43fd62a..2eeab4f 100755 --- a/test/run.py +++ b/test/run.py @@ -11,6 +11,8 @@ CURDIR = dirname(abspath(__file__)) sys.path.insert(0, dirname(CURDIR)) +from robot.version import VERSION + from robotstatuschecker import process_output from robotstatuschecker import RF3 @@ -21,6 +23,7 @@ def check_tests(robot_file): checker = StatusCheckerChecker() result.suite.visit(checker) checker.print_status() + print(f"Robot Framework version: {VERSION}") sys.exit(len(checker.errors)) From 7cf296575e47fd62cfb9ab1ab74b254e249df0db Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 16 Apr 2021 20:05:23 +0300 Subject: [PATCH 36/63] FIx bug with RF 3.2.2 --- robotstatuschecker.py | 4 +++- test/tests.robot | 25 +++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 9302995..bee53ee 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -115,6 +115,7 @@ def __init__(self, doc): self.msg_index = msg_index self.test_teardown = test_teardown self.level, self.message = self._split_level(message) + self.visited_setup = False @property def kw_index_str(self): @@ -255,8 +256,9 @@ def _get_keyword_rf3(self, test, expected, kw, index): if expected.test_teardown and not test.keywords.teardown: self._fail(test, self._no_teardown_message) return None - if test.keywords.setup and not expected.test_setup: + if test.keywords.setup and not expected.test_setup and not expected.visited_setup: index += 1 + expected.visited_setup = True return (kw or test).keywords[index] def _get_keyword_rf4(self, test, expected, kw, index): diff --git a/test/tests.robot b/test/tests.robot index 78edd2f..674f47f 100644 --- a/test/tests.robot +++ b/test/tests.robot @@ -72,13 +72,34 @@ Trailing and leading whitespace is ignored in log messages Log ${SPACE*10}xxx${SPACE*10} Log messages deeper - [Documentation] LOG 2:1 Hello LOG 2:2 World + [Documentation] + ... LOG 2:1 Hello + ... LOG 2:2 World ... LOG 3.1 DEBUG User Keyword - ... LOG 4.1:1 User LOG 4.1:2 Keyword + ... LOG 4.1:1 User + ... LOG 4.1:2 Keyword + ... LOG 5.1:2 DEBUG STARTS: Traceback (most recent call last): Status PASS Log Many Hello World Logging User Keyword Logging User Keyword 2 + Run Keyword And Ignore Error + ... Fail My Error Here + +Log messages deeper with setup + [Documentation] + ... LOG 1:1 Hello + ... LOG 1:2 World + ... LOG 2.1 DEBUG User Keyword + ... LOG 3.1:1 User + ... LOG 3.1:2 Keyword + ... LOG 4.1:2 DEBUG STARTS: Traceback (most recent call last): + [Setup] Status PASS + Log Many Hello World + Logging User Keyword + Logging User Keyword 2 + Run Keyword And Ignore Error + ... Fail My Error Here Log message with REGEXP [Documentation] LOG 2 REGEXP: H[ei]l{2}o w\\w+! LOG 2 REGEXP: Hell.* From 486a0082e9ec937a71f22d30d951d4d89ac5bc4c Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 16 Apr 2021 20:17:13 +0300 Subject: [PATCH 37/63] No Py 3.8 in CI --- .github/workflows/push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index da9ff3b..fe54701 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.8, 3.9, pypy3] + python-version: [3.6, 3.9, pypy3] rf-version: [3.2.2, 4.0.1] steps: From fa6bb87d704e0f7d812ad23b0ebbdc473600bb73 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 16 Apr 2021 20:26:02 +0300 Subject: [PATCH 38/63] Release notes for 2.0.1 --- .../releasenotes/robotstatuschecker-2.0.1.rst | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 docs/releasenotes/robotstatuschecker-2.0.1.rst diff --git a/docs/releasenotes/robotstatuschecker-2.0.1.rst b/docs/releasenotes/robotstatuschecker-2.0.1.rst new file mode 100644 index 0000000..e784cd0 --- /dev/null +++ b/docs/releasenotes/robotstatuschecker-2.0.1.rst @@ -0,0 +1,51 @@ +======================== +robotstatuschecker 2.0.1 +======================== + + +.. default-role:: code + + +StatusChecker is a tool for validating that executed `Robot Framework`_ test cases +have expected statuses and log messages. It is mainly useful for Robot Framework +test library developers who want to use Robot Framework to also test their libraries. +StatusChecker 2.0.1 is compatible with Python 3.6+ and RF 3.2.2 and RF 4.0.1. StatusChecker +2.0.1 fixes regression with Rf 3.2.2 when test has test setup. + +StatusChecker project is hosted at GitHub and downloads are at PyPI_ +.. _Robot Framework: http://robotframework.org +.. _PyPI: https://github.com/robotframework/statuschecker +.. _issue tracker: https://github.com/robotframework/SeleniumLibrary/issues?q=milestone%3Av2.0.1 + + +.. contents:: + :depth: 2 + :local: + +Most important enhancements +=========================== + +statuschecker works different with different RF versions (`#26`_) +----------------------------------------------------------------- +Release 2.0.0 added support for SETUP and TEARDOWN markers, but also +introduced regression when test had test setup and log messages +where checked. + +Full list of fixes and enhancements +=================================== + +.. list-table:: + :header-rows: 1 + + * - ID + - Type + - Priority + - Summary + * - `#26`_ + - bug + - critical + - statuschecker works different with different RF versions + +Altogether 1 issue. View on the `issue tracker `__. + +.. _#26: https://github.com/robotframework/statuschecker/issues/26 From f93364a0f46cca8df5fa6d34cbcec3d984135163 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 16 Apr 2021 20:26:21 +0300 Subject: [PATCH 39/63] Updated version to 2.0.1 --- robotstatuschecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index bee53ee..05f09b4 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -46,7 +46,7 @@ from robot.utils import Matcher -__version__ = "2.0.1.dev1" +__version__ = "2.0.1" RF3 = rf_version.startswith("3") From cf5271a0dc9f9f530b47b04fe6253f093d08d53c Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 16 Apr 2021 20:32:49 +0300 Subject: [PATCH 40/63] Back to dev version --- robotstatuschecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 05f09b4..cc9a0ae 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -46,7 +46,7 @@ from robot.utils import Matcher -__version__ = "2.0.1" +__version__ = "2.0.2.dev1" RF3 = rf_version.startswith("3") From 086276fe99541649adb843aaf996acf2262328fd Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 16 Apr 2021 20:36:32 +0300 Subject: [PATCH 41/63] Run Black --- robotstatuschecker.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index cc9a0ae..bc0a0ac 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -108,7 +108,7 @@ def _get_logs(self, doc): class ExpectedLog: def __init__(self, doc): - index, message = doc.strip().split(' ', 1) + index, message = doc.strip().split(" ", 1) test_setup, kw_index, msg_index, test_teardown = self._split_index(index) self.test_setup = test_setup self.kw_index = kw_index @@ -256,7 +256,11 @@ def _get_keyword_rf3(self, test, expected, kw, index): if expected.test_teardown and not test.keywords.teardown: self._fail(test, self._no_teardown_message) return None - if test.keywords.setup and not expected.test_setup and not expected.visited_setup: + if ( + test.keywords.setup + and not expected.test_setup + and not expected.visited_setup + ): index += 1 expected.visited_setup = True return (kw or test).keywords[index] @@ -281,9 +285,7 @@ def _check_message(self, test, kw, expected): msg = kw.messages[expected.msg_index] except IndexError: condition = expected.message == "NONE" - message = ( - f"Keyword '{kw.name}' (index {expected.kw_index_str}) does not have message {expected.msg_index_str}." - ) + message = f"Keyword '{kw.name}' (index {expected.kw_index_str}) does not have message {expected.msg_index_str}." self._assert(condition, test, message) else: if self._check_msg_level(test, kw, msg, expected): From 8ac863670992fb8e454b0a54afc0d320589c708d Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 16 Apr 2021 21:56:34 +0300 Subject: [PATCH 42/63] Fixed bug when accessing keywords in teardown when not marked as teardown --- robotstatuschecker.py | 30 ++++++++++++++++++++++++------ test/tests.robot | 10 ++++++++-- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index bc0a0ac..889fe87 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -219,8 +219,14 @@ def _check_message(self, test): class LogMessageChecker(BaseChecker): - _no_setup_message = "Expected test setup but setup is not present." - _no_teardown_message = "Expected test teardown but teardown is not present." + _no_setup_message = "Expected test {} to have setup but setup is not present." + _no_teardown_message = ( + "Expected test {} to have teardown but teardown is not present." + ) + _teardown_access_message = ( + "In test '{}' keyword is in teardown but " + "was expected to ne in test body index {}" + ) def __init__(self, expected): self.logs = expected.logs @@ -251,10 +257,16 @@ def _get_keyword_rf3_rf4(self, test, expected, kw, index): def _get_keyword_rf3(self, test, expected, kw, index): if expected.test_setup and not test.keywords.setup: - self._fail(test, self._no_setup_message) + self._fail(test, self._no_setup_message.format(test.name)) return None if expected.test_teardown and not test.keywords.teardown: - self._fail(test, self._no_teardown_message) + self._fail(test, self._no_teardown_message.format(test.name)) + return None + if not expected.test_teardown and test.keywords.teardown: + self._fail( + test, + self._teardown_access_message.format(test.name, expected.kw_index_str), + ) return None if ( test.keywords.setup @@ -267,10 +279,16 @@ def _get_keyword_rf3(self, test, expected, kw, index): def _get_keyword_rf4(self, test, expected, kw, index): if expected.test_setup and not test.setup: - self._fail(test, self._no_setup_message) + self._fail(test, self._no_setup_message.format(test.name)) return None if expected.test_teardown and not test.teardown: - self._fail(test, self._no_teardown_message) + self._fail(test, self._no_teardown_message.format(test.name)) + return None + if not expected.test_teardown and test.teardown: + self._fail( + test, + self._teardown_access_message.format(test.name, expected.kw_index_str), + ) return None if expected.test_setup and not kw: kw = test.setup diff --git a/test/tests.robot b/test/tests.robot index 674f47f..0d41df2 100644 --- a/test/tests.robot +++ b/test/tests.robot @@ -140,7 +140,7 @@ Error When No Setup [Documentation] ... ... LOG SETUP.1:1 PASS ... LOG 2:1 KALA - Status FAIL Expected test setup but setup is not present. + Status FAIL Expected test Error When No Setup to have setup but setup is not present. Log KALA Test Teardown Check Is Done By TEARDOWN Marker @@ -152,7 +152,7 @@ Test Teardown Check Is Done By TEARDOWN Marker Error When No Teardown [Documentation] LOG TEARDOWN:1 foobar - Status FAIL Expected test teardown but teardown is not present. + Status FAIL Expected test Error When No Teardown to have teardown but teardown is not present. Log KALA Expected FAIL and log messages @@ -209,6 +209,12 @@ FAILURE: Wrong log message ... Actual:\nHi world! Log Hi world! +FAILURE: Access teardown without TEADOWN + [Documentation] LOG 3:1 Hi from teardown + Status FAIL In test 'FAILURE: Access teardown without TEADOWN' keyword is in teardown but was expected to ne in test body index 3 + Log Hi world! + [Teardown] Log Hi from teardown + FAILURE: Wrong log level [Documentation] LOG 2.1 Hello world! Status FAIL Keyword 'BuiltIn.Log' (index 2.1) message 1 has wrong level.\n\n From 814a8342f83cd8b77347969a5eb8f96afccee8f4 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 16 Apr 2021 22:05:07 +0300 Subject: [PATCH 43/63] Release notes for 2.0.2 --- .../releasenotes/robotstatuschecker-2.0.2.rst | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/releasenotes/robotstatuschecker-2.0.2.rst diff --git a/docs/releasenotes/robotstatuschecker-2.0.2.rst b/docs/releasenotes/robotstatuschecker-2.0.2.rst new file mode 100644 index 0000000..bf85676 --- /dev/null +++ b/docs/releasenotes/robotstatuschecker-2.0.2.rst @@ -0,0 +1,50 @@ +======================== +robotstatuschecker 2.0.2 +======================== + + +.. default-role:: code + + +StatusChecker is a tool for validating that executed `Robot Framework`_ test cases +have expected statuses and log messages. It is mainly useful for Robot Framework +test library developers who want to use Robot Framework to also test their libraries. +StatusChecker 2.0.1 is compatible with Python 3.6+ and RF 3.2.2 and RF 4.0.1. StatusChecker +2.0.1 fixes regression with Rf 3.2.2 when test has test setup. + +StatusChecker project is hosted at GitHub and downloads are at PyPI_ +.. _Robot Framework: http://robotframework.org +.. _PyPI: https://github.com/robotframework/statuschecker +.. _issue tracker: https://github.com/robotframework/SeleniumLibrary/issues?q=milestone%3Av2.0.2 + + +.. contents:: + :depth: 2 + :local: + +Most important enhancements +=========================== + +Keywords teardown in RF 3 could be accessed without TEARDOWN marker (`#28`_) +---------------------------------------------------------------------------- +Keywords teardown in RF 3 could be accessed without TEARDOWN marker. It as +used as is, but it should raise an error. + +Full list of fixes and enhancements +=================================== + +.. list-table:: + :header-rows: 1 + + * - ID + - Type + - Priority + - Summary + * - `#28`_ + - bug + - critical + - Keywords teardown in RF 3 could be accessed without TEARDOWN marker + +Altogether 1 issue. View on the `issue tracker `__. + +.. _#28: https://github.com/robotframework/statuschecker/issues/28 From c23a6261ffce887db75cf7f7f840a625ae77a196 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 16 Apr 2021 22:05:23 +0300 Subject: [PATCH 44/63] Updated version to 2.0.2 --- robotstatuschecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 889fe87..75d07e3 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -46,7 +46,7 @@ from robot.utils import Matcher -__version__ = "2.0.2.dev1" +__version__ = "2.0.2" RF3 = rf_version.startswith("3") From fddc6fb5c4a9f66d9de2874985306998de53b85d Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 16 Apr 2021 22:08:39 +0300 Subject: [PATCH 45/63] Back to dev version --- robotstatuschecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 75d07e3..1d3b481 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -46,7 +46,7 @@ from robot.utils import Matcher -__version__ = "2.0.2" +__version__ = "2.0.3.dev1" RF3 = rf_version.startswith("3") From ee68b5d96de1a22f3085b30c0bc7654586c13043 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 16 Apr 2021 23:11:14 +0300 Subject: [PATCH 46/63] No fail in RF 3 if accessing keywords in teardown without TEARDOWN marker --- robotstatuschecker.py | 12 ------------ test/tests.robot | 13 +++++++------ 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 1d3b481..ac523bd 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -262,12 +262,6 @@ def _get_keyword_rf3(self, test, expected, kw, index): if expected.test_teardown and not test.keywords.teardown: self._fail(test, self._no_teardown_message.format(test.name)) return None - if not expected.test_teardown and test.keywords.teardown: - self._fail( - test, - self._teardown_access_message.format(test.name, expected.kw_index_str), - ) - return None if ( test.keywords.setup and not expected.test_setup @@ -284,12 +278,6 @@ def _get_keyword_rf4(self, test, expected, kw, index): if expected.test_teardown and not test.teardown: self._fail(test, self._no_teardown_message.format(test.name)) return None - if not expected.test_teardown and test.teardown: - self._fail( - test, - self._teardown_access_message.format(test.name, expected.kw_index_str), - ) - return None if expected.test_setup and not kw: kw = test.setup elif expected.test_teardown and not kw: diff --git a/test/tests.robot b/test/tests.robot index 0d41df2..cb9467b 100644 --- a/test/tests.robot +++ b/test/tests.robot @@ -178,6 +178,13 @@ Expected PASS and log messages Log Any time now... Pass Execution Told ya!! +Expected PASS and teadown does not affect + [Documentation] This text is ignored. + ... LOG 2 Passing soon! + Status PASS ${EMPTY} + Log Passing soon! + [Teardown] Log This is logged + FAILURE: Unexpected PASS [Documentation] FAIL Expected failure does not occur Status FAIL Test was expected to FAIL but it PASSED. @@ -209,12 +216,6 @@ FAILURE: Wrong log message ... Actual:\nHi world! Log Hi world! -FAILURE: Access teardown without TEADOWN - [Documentation] LOG 3:1 Hi from teardown - Status FAIL In test 'FAILURE: Access teardown without TEADOWN' keyword is in teardown but was expected to ne in test body index 3 - Log Hi world! - [Teardown] Log Hi from teardown - FAILURE: Wrong log level [Documentation] LOG 2.1 Hello world! Status FAIL Keyword 'BuiltIn.Log' (index 2.1) message 1 has wrong level.\n\n From 0a645d1af236f61e8529269c86da2ec36acf2464 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 16 Apr 2021 23:17:33 +0300 Subject: [PATCH 47/63] Release notes for 2.0.3 --- .../releasenotes/robotstatuschecker-2.0.3.rst | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/releasenotes/robotstatuschecker-2.0.3.rst diff --git a/docs/releasenotes/robotstatuschecker-2.0.3.rst b/docs/releasenotes/robotstatuschecker-2.0.3.rst new file mode 100644 index 0000000..e7a8fb8 --- /dev/null +++ b/docs/releasenotes/robotstatuschecker-2.0.3.rst @@ -0,0 +1,50 @@ +======================== +robotstatuschecker 2.0.3 +======================== + + +.. default-role:: code + + +StatusChecker is a tool for validating that executed `Robot Framework`_ test cases +have expected statuses and log messages. It is mainly useful for Robot Framework +test library developers who want to use Robot Framework to also test their libraries. +StatusChecker 2.0.1 is compatible with Python 3.6+ and RF 3.2.2 and RF 4.0.1. StatusChecker +2.0.2 fixes regression with 2.0.2 release + +StatusChecker project is hosted at GitHub and downloads are at PyPI_ + +.. _Robot Framework: http://robotframework.org +.. _PyPI: https://github.com/robotframework/statuschecker +.. _issue tracker: https://github.com/robotframework/SeleniumLibrary/issues?q=milestone%3Av2.0.3 + + +.. contents:: + :depth: 2 + :local: + +Most important enhancements +=========================== + +**EXPLAIN** or remove these. + +- Revert changes in release 2.0.2, it causes too much other failures (`#29`_) + +Full list of fixes and enhancements +=================================== + +.. list-table:: + :header-rows: 1 + + * - ID + - Type + - Priority + - Summary + * - `#29`_ + - bug + - critical + - Revert changes in release 2.0.2, it causes too much other failures + +Altogether 1 issue. View on the `issue tracker `__. + +.. _#29: https://github.com/robotframework/statuschecker/issues/29 From 81b6b8c9b831a53e759b3686146ea0fe3d39640e Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 16 Apr 2021 23:17:49 +0300 Subject: [PATCH 48/63] Updated version to 2.0.3 --- robotstatuschecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index ac523bd..dee2b25 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -46,7 +46,7 @@ from robot.utils import Matcher -__version__ = "2.0.3.dev1" +__version__ = "2.0.3" RF3 = rf_version.startswith("3") From 0217eb060973379de000efbe5e68743986d8765e Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Fri, 16 Apr 2021 23:20:42 +0300 Subject: [PATCH 49/63] Back to dev version --- robotstatuschecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index dee2b25..66b770b 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -46,7 +46,7 @@ from robot.utils import Matcher -__version__ = "2.0.3" +__version__ = "2.0.4.dev1" RF3 = rf_version.startswith("3") From b89a5f7736bcbc82b3edcf5b85f9c00f45632d12 Mon Sep 17 00:00:00 2001 From: Jari Date: Fri, 15 Oct 2021 19:27:39 +0200 Subject: [PATCH 50/63] Wild card support for log verification - message index ':*' - log level 'ANY' --- robotstatuschecker.py | 47 +++++++++++++++++++----------- test/run.py | 13 ++++++--- test/tests.robot | 67 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 20 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 66b770b..0c1eac0 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -123,7 +123,7 @@ def kw_index_str(self): @property def msg_index_str(self): - return str(self.msg_index + 1) + return str(self.msg_index + 1) if isinstance(self.msg_index, int) else self.msg_index def _split_index(self, index): if ":" in index: @@ -142,11 +142,11 @@ def _split_index(self, index): new_kw_index.append(-1) else: new_kw_index.append(int(index) - 1) - msg_index = int(msg_index) - 1 + msg_index = "*" if msg_index=="*" else int(msg_index) - 1 return test_setup, new_kw_index, msg_index, test_teardown def _split_level(self, message): - for level in ["TRACE", "DEBUG", "INFO", "WARN", "FAIL", "ERROR"]: + for level in ["TRACE", "DEBUG", "INFO", "WARN", "FAIL", "ERROR", "ANY"]: if message.startswith(level): return level, message[len(level) :].strip() return "INFO", message @@ -171,9 +171,9 @@ def _message_matches(self, actual, expected): return True return False - def _assert(self, condition, test, message): + def _assert(self, condition, test, message, fail=True): if not condition: - return self._fail(test, message) + return self._fail(test, message) if fail else False return True def _fail(self, test, message): @@ -287,31 +287,46 @@ def _get_keyword_rf4(self, test, expected, kw, index): return kw def _check_message(self, test, kw, expected): - try: - msg = kw.messages[expected.msg_index] - except IndexError: - condition = expected.message == "NONE" - message = f"Keyword '{kw.name}' (index {expected.kw_index_str}) does not have message {expected.msg_index_str}." - self._assert(condition, test, message) + if expected.msg_index != "*": + try: + msg = kw.messages[expected.msg_index] + except IndexError: + condition = expected.message == "NONE" + message = f"Keyword '{kw.name}' (index {expected.kw_index_str}) does not have message {expected.msg_index_str}." + self._assert(condition, test, message) + else: + if self._check_msg_level(test, kw, msg, expected): + self._check_msg_message(test, kw, msg, expected) else: - if self._check_msg_level(test, kw, msg, expected): - self._check_msg_message(test, kw, msg, expected) + if expected.message == "NONE": + message = f"Message index wildcard '*' is not supported with expected message 'NONE'" + self._fail(test, message) + return + + for msg in kw.messages: + if self._check_msg_message(test, kw, msg, expected, fail=False): + self._check_msg_level(test, kw, msg, expected) + break + else: + message = f"Keyword '{kw.name}' (index {expected.kw_index_str}) does not contain any logs with message '{expected.message}'" + self._fail(test, message) + def _check_msg_level(self, test, kw, msg, expected): - condition = msg.level == expected.level + condition = msg.level == expected.level if expected.level != "ANY" else True message = ( f"Keyword '{kw.name}' (index {expected.kw_index_str}) message {expected.msg_index_str} has wrong level." f"\n\nExpected: {expected.level}\nActual: {msg.level}" ) return self._assert(condition, test, message) - def _check_msg_message(self, test, kw, msg, expected): + def _check_msg_message(self, test, kw, msg, expected, fail=True): condition = self._message_matches(msg.message.strip(), expected.message) message = ( f"Keyword '{kw.name}' (index {expected.kw_index_str}) message {expected.msg_index_str} has wrong content." f"\n\nExpected:\n{expected.message}\n\nActual:\n{msg.message}" ) - return self._assert(condition, test, message) + return self._assert(condition, test, message, fail) if __name__ == "__main__": diff --git a/test/run.py b/test/run.py index 2eeab4f..51abd99 100755 --- a/test/run.py +++ b/test/run.py @@ -7,6 +7,7 @@ from robot import run, rebot from robot.api import ExecutionResult, ResultVisitor +from robot.output.console.highlighting import HighlightingStream CURDIR = dirname(abspath(__file__)) sys.path.insert(0, dirname(CURDIR)) @@ -75,13 +76,17 @@ def _verify(self, actual, expected, explanation): return 'Expected %s to be "%s" but it was "%s".' % (explanation, expected, actual) def print_status(self): + sout = HighlightingStream(sys.stdout) print() if self.errors: - print("%d/%d test failed:" % (len(self.errors), self.tests)) - print("\n-------------------------------------\n".join(self.errors)) + sout.error("%d/%d test failed:" % (len(self.errors), self.tests), "FAIL") + for err_line in "\n-------------------------------------\n".join(self.errors).splitlines(): + sout.error(err_line, "FAIL") else: - print("All %d tests passed/failed/logged as expected." % self.tests) - print("Run on %s %s." % (python_implementation(), python_version())) + sout.write('[ ', flush=False) + sout.highlight("PASS", flush=False) + sout.write(' ] All %d tests passed/failed/logged as expected.\n' % self.tests) + print("-------------------------------------\nRun on %s %s." % (python_implementation(), python_version())) if __name__ == "__main__": diff --git a/test/tests.robot b/test/tests.robot index cb9467b..7c17e50 100644 --- a/test/tests.robot +++ b/test/tests.robot @@ -101,6 +101,36 @@ Log messages deeper with setup Run Keyword And Ignore Error ... Fail My Error Here +Log messages deeper with wildcard + [Documentation] + ... LOG 2:1 Hello + ... LOG 2:2 ANY World + ... LOG 3.1 DEBUG User Keyword + ... LOG 4.1:* User + ... LOG 4.1:* ANY Keyword + ... LOG 5.1:* DEBUG STARTS: Traceback (most recent call last): + Status PASS + Log Many Hello World + Logging User Keyword + Logging User Keyword 2 + Run Keyword And Ignore Error + ... Fail My Error Here + +Log messages deeper with wildcard and setup + [Documentation] + ... LOG 1:1 Hello + ... LOG 1:2 ANY World + ... LOG 2.1 DEBUG User Keyword + ... LOG 3.1:* User + ... LOG 3.1:* ANY Keyword + ... LOG 4.1:* DEBUG STARTS: Traceback (most recent call last): + [Setup] Status PASS + Log Many Hello World + Logging User Keyword + Logging User Keyword 2 + Run Keyword And Ignore Error + ... Fail My Error Here + Log message with REGEXP [Documentation] LOG 2 REGEXP: H[ei]l{2}o w\\w+! LOG 2 REGEXP: Hell.* ... LOG 3 REGEXP: Multi.*message @@ -143,6 +173,22 @@ Error When No Setup Status FAIL Expected test Error When No Setup to have setup but setup is not present. Log KALA +Test Setup Check Is Done By SETUP Marker and wildcard is used + [Documentation] ... + ... LOG SETUP:1 NONE + ... LOG SETUP.2:* PASS + ... LOG SETUP.2 PASS + ... LOG 1:* HAUKI + [Setup] Status PASS + Log HAUKI + +Error When No Setup and wildcard is used + [Documentation] ... + ... LOG SETUP.1:* PASS + ... LOG 2:* KALA + Status FAIL Expected test Error When No Setup and wildcard is used to have setup but setup is not present. + Log KALA + Test Teardown Check Is Done By TEARDOWN Marker [Documentation] ... ... LOG TEARDOWN:1 foobar @@ -155,6 +201,23 @@ Error When No Teardown Status FAIL Expected test Error When No Teardown to have teardown but teardown is not present. Log KALA +Test Teardown Check Is Done By TEARDOWN Marker and wildcard is used + [Documentation] ... + ... LOG TEARDOWN:* foobar + ... LOG TEARDOWN foobar + Status PASS + [Teardown] Log foobar + +Error When No Teardown and wildcard is used + [Documentation] LOG TEARDOWN:* foobar + Status FAIL Expected test Error When No Teardown and wildcard is used to have teardown but teardown is not present. + Log KALA + +Error When NONE is used with wildcard + [Documentation] LOG 2.1:* INFO NONE + Status FAIL Message index wildcard '*' is not supported with expected message 'NONE' + Logging User Keyword 2 + Expected FAIL and log messages [Documentation] This text is ignored. FAIL Told ya!! ... LOG 2 Failing soon! @@ -239,6 +302,10 @@ FAILURE: Non-existing log message Status FAIL Keyword 'BuiltIn.Log' (index 2) does not have message 2. Log Message +FAILURE: Non-existing log message wildcard + [Documentation] LOG 1:* Bogus message + Status FAIL Keyword 'Status' (index 1) does not contain any logs with message 'Bogus message' + *** Keywords *** Logging User Keyword Log User Keyword DEBUG From 40cadacc17ca85becddf91050d91ea4498ba8a7e Mon Sep 17 00:00:00 2001 From: Jari Date: Tue, 19 Oct 2021 07:45:41 +0200 Subject: [PATCH 51/63] fixes from review comments --- robotstatuschecker.py | 61 +++++++++++++++++++++++++------------------ test/tests.robot | 8 ++++-- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 0c1eac0..22dd620 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -142,7 +142,7 @@ def _split_index(self, index): new_kw_index.append(-1) else: new_kw_index.append(int(index) - 1) - msg_index = "*" if msg_index=="*" else int(msg_index) - 1 + msg_index = "*" if msg_index == "*" else int(msg_index) - 1 return test_setup, new_kw_index, msg_index, test_teardown def _split_level(self, message): @@ -286,39 +286,50 @@ def _get_keyword_rf4(self, test, expected, kw, index): kw = (kw or test).body[index] return kw - def _check_message(self, test, kw, expected): - if expected.msg_index != "*": - try: - msg = kw.messages[expected.msg_index] - except IndexError: - condition = expected.message == "NONE" - message = f"Keyword '{kw.name}' (index {expected.kw_index_str}) does not have message {expected.msg_index_str}." - self._assert(condition, test, message) - else: - if self._check_msg_level(test, kw, msg, expected): - self._check_msg_message(test, kw, msg, expected) + def _check_message_by_index(self, test, kw, expected): + try: + msg = kw.messages[expected.msg_index] + except IndexError: + condition = expected.message == "NONE" + message = ( + f"Keyword '{kw.name}' (index {expected.kw_index_str}) does not " + f"have message {expected.msg_index_str}." + ) + self._assert(condition, test, message) else: - if expected.message == "NONE": - message = f"Message index wildcard '*' is not supported with expected message 'NONE'" - self._fail(test, message) - return - - for msg in kw.messages: - if self._check_msg_message(test, kw, msg, expected, fail=False): - self._check_msg_level(test, kw, msg, expected) + if self._check_msg_level(test, kw, msg, expected): + self._check_msg_message(test, kw, msg, expected) + + def _check_message_by_wildcard(self, test, kw, expected): + if expected.message == "NONE": + message = f"Message index wildcard '*' is not supported with expected message 'NONE'." + self._fail(test, message) + return + + for msg in kw.messages: + if self._check_msg_message(test, kw, msg, expected, fail=False): + if self._check_msg_level(test, kw, msg, expected, fail=False): break - else: - message = f"Keyword '{kw.name}' (index {expected.kw_index_str}) does not contain any logs with message '{expected.message}'" - self._fail(test, message) + else: + message = ( + f"Keyword '{kw.name}' (index {expected.kw_index_str}) does not contain any logs " + f"with level {expected.level} and message '{expected.message}'." + ) + self._fail(test, message) + def _check_message(self, test, kw, expected): + if expected.msg_index != "*": + self._check_message_by_index(test, kw, expected) + else: + self._check_message_by_wildcard(test, kw, expected) - def _check_msg_level(self, test, kw, msg, expected): + def _check_msg_level(self, test, kw, msg, expected, fail=True): condition = msg.level == expected.level if expected.level != "ANY" else True message = ( f"Keyword '{kw.name}' (index {expected.kw_index_str}) message {expected.msg_index_str} has wrong level." f"\n\nExpected: {expected.level}\nActual: {msg.level}" ) - return self._assert(condition, test, message) + return self._assert(condition, test, message, fail) def _check_msg_message(self, test, kw, msg, expected, fail=True): condition = self._message_matches(msg.message.strip(), expected.message) diff --git a/test/tests.robot b/test/tests.robot index 7c17e50..9e72c53 100644 --- a/test/tests.robot +++ b/test/tests.robot @@ -109,12 +109,16 @@ Log messages deeper with wildcard ... LOG 4.1:* User ... LOG 4.1:* ANY Keyword ... LOG 5.1:* DEBUG STARTS: Traceback (most recent call last): + ... LOG 6.1:* ANY REGEXP: .*recent.* + ... LOG 6.1:* DEBUG REGEXP: .*recent.* Status PASS Log Many Hello World Logging User Keyword Logging User Keyword 2 Run Keyword And Ignore Error ... Fail My Error Here + Run Keyword And Ignore Error + ... Fail 'recent call' in two different log levels Log messages deeper with wildcard and setup [Documentation] @@ -215,7 +219,7 @@ Error When No Teardown and wildcard is used Error When NONE is used with wildcard [Documentation] LOG 2.1:* INFO NONE - Status FAIL Message index wildcard '*' is not supported with expected message 'NONE' + Status FAIL Message index wildcard '*' is not supported with expected message 'NONE'. Logging User Keyword 2 Expected FAIL and log messages @@ -304,7 +308,7 @@ FAILURE: Non-existing log message FAILURE: Non-existing log message wildcard [Documentation] LOG 1:* Bogus message - Status FAIL Keyword 'Status' (index 1) does not contain any logs with message 'Bogus message' + Status FAIL Keyword 'Status' (index 1) does not contain any logs with level INFO and message 'Bogus message'. *** Keywords *** Logging User Keyword From 4fb6d3b7e43299a5876e84dbaf4322fe5157c571 Mon Sep 17 00:00:00 2001 From: Jari Date: Tue, 19 Oct 2021 07:50:19 +0200 Subject: [PATCH 52/63] reverted colouring from test/run.py --- test/run.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/test/run.py b/test/run.py index 51abd99..2eeab4f 100755 --- a/test/run.py +++ b/test/run.py @@ -7,7 +7,6 @@ from robot import run, rebot from robot.api import ExecutionResult, ResultVisitor -from robot.output.console.highlighting import HighlightingStream CURDIR = dirname(abspath(__file__)) sys.path.insert(0, dirname(CURDIR)) @@ -76,17 +75,13 @@ def _verify(self, actual, expected, explanation): return 'Expected %s to be "%s" but it was "%s".' % (explanation, expected, actual) def print_status(self): - sout = HighlightingStream(sys.stdout) print() if self.errors: - sout.error("%d/%d test failed:" % (len(self.errors), self.tests), "FAIL") - for err_line in "\n-------------------------------------\n".join(self.errors).splitlines(): - sout.error(err_line, "FAIL") + print("%d/%d test failed:" % (len(self.errors), self.tests)) + print("\n-------------------------------------\n".join(self.errors)) else: - sout.write('[ ', flush=False) - sout.highlight("PASS", flush=False) - sout.write(' ] All %d tests passed/failed/logged as expected.\n' % self.tests) - print("-------------------------------------\nRun on %s %s." % (python_implementation(), python_version())) + print("All %d tests passed/failed/logged as expected." % self.tests) + print("Run on %s %s." % (python_implementation(), python_version())) if __name__ == "__main__": From b0013f3873dce5a013bd69c36fa6a495e10e8f67 Mon Sep 17 00:00:00 2001 From: Jari Date: Wed, 20 Oct 2021 06:23:42 +0200 Subject: [PATCH 53/63] doc update for wildcard support --- README.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 0c3d2d4..f56daa1 100644 --- a/README.rst +++ b/README.rst @@ -134,11 +134,14 @@ The part after the colon species the message. For example, ``1:2`` means the second message of the first keyword and ``1.2:3`` is the third message of the second child keyword of the first keyword. The message index is optional and defaults to ``1``. +The message index also supports wildcard ``*``. For example ``1:*`` +matches any message of the first keyword. Message level is specified before the actual message, and it can be any of the valid log levels in capital letters. If the level is not given it defaults to ``INFO``. Starting from 1.4 release also -``ERROR`` level is supported. +``ERROR`` level is supported. The message level also supports wildcard +``ANY`` which will match all log levels. Possible leading and trailing whitespace is ignored both in the expected and in the actual log message. From e36d95b8cdbac6354abe72e6a56528fb0d8e1d80 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Mon, 18 Oct 2021 22:44:27 +0300 Subject: [PATCH 54/63] Use linting --- .flake8 | 9 +++++++++ .github/workflows/push.yaml | 17 ++++++++++++++--- pyproject.toml | 3 +++ requirements-dev.txt | 5 ++++- robotstatuschecker.py | 27 ++++++++++++--------------- tasks.py | 33 +++++++++++++++++++++++++++------ test/run.py | 16 +++++++++------- test/tests.robot | 11 +++++------ 8 files changed, 83 insertions(+), 38 deletions(-) create mode 100644 .flake8 create mode 100644 pyproject.toml diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..36e209e --- /dev/null +++ b/.flake8 @@ -0,0 +1,9 @@ +[flake8] +exclude = + __pycache__, + .venv, + .git, + dist, + build +max-line-length = 100 +ignore = E203 diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index fe54701..ec299df 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -8,8 +8,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.9, pypy3] - rf-version: [3.2.2, 4.0.1] + python-version: [3.7, 3.9, pypy3] + rf-version: [4.0.3, 4.1.2] steps: - uses: actions/checkout@v2 @@ -17,10 +17,21 @@ jobs: uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install dependencies on Python + if: matrix.python-version != 'pypy3' run: | python -m pip install --upgrade pip + pip install -r requirements-dev.txt pip install robotframework==${{ matrix.rf-version }} + - name: Install dependencies on pypy3 + if: matrix.python-version == 'pypy3' + run: | + python -m pip install --upgrade pip + pip install robotframework==${{ matrix.rf-version }} + - name: Run lint + if: matrix.python-version != 'pypy3' + run: | + inv lint - name: Run tests run: | python --version diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..021cb23 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.black] +line-length = 100 +target-version = ['py36'] diff --git a/requirements-dev.txt b/requirements-dev.txt index 707f594..ed0b8ed 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,4 +2,7 @@ invoke >= 1.4.1 rellu >= 0.6 twine >= 3.4.1 wheel >= 0.36.2 -black >= 20.8b1 +black >= 21.9b0 +flake8 >= 3.9.2 +robotframework-tidy >= 1.6.1 +isort >= 5.9.3 \ No newline at end of file diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 66b770b..b599060 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -37,15 +37,14 @@ If an output file is not given, the input file is edited in place. """ -from os.path import abspath import re import sys +from os.path import abspath from robot import __version__ as rf_version from robot.api import ExecutionResult, ResultVisitor from robot.utils import Matcher - __version__ = "2.0.4.dev1" RF3 = rf_version.startswith("3") @@ -220,12 +219,9 @@ def _check_message(self, test): class LogMessageChecker(BaseChecker): _no_setup_message = "Expected test {} to have setup but setup is not present." - _no_teardown_message = ( - "Expected test {} to have teardown but teardown is not present." - ) + _no_teardown_message = "Expected test {} to have teardown but teardown is not present." _teardown_access_message = ( - "In test '{}' keyword is in teardown but " - "was expected to ne in test body index {}" + "In test '{}' keyword is in teardown but " "was expected to ne in test body index {}" ) def __init__(self, expected): @@ -262,11 +258,7 @@ def _get_keyword_rf3(self, test, expected, kw, index): if expected.test_teardown and not test.keywords.teardown: self._fail(test, self._no_teardown_message.format(test.name)) return None - if ( - test.keywords.setup - and not expected.test_setup - and not expected.visited_setup - ): + if test.keywords.setup and not expected.test_setup and not expected.visited_setup: index += 1 expected.visited_setup = True return (kw or test).keywords[index] @@ -291,7 +283,10 @@ def _check_message(self, test, kw, expected): msg = kw.messages[expected.msg_index] except IndexError: condition = expected.message == "NONE" - message = f"Keyword '{kw.name}' (index {expected.kw_index_str}) does not have message {expected.msg_index_str}." + message = ( + f"Keyword '{kw.name}' (index {expected.kw_index_str}) does " + f"not have message {expected.msg_index_str}." + ) self._assert(condition, test, message) else: if self._check_msg_level(test, kw, msg, expected): @@ -300,7 +295,8 @@ def _check_message(self, test, kw, expected): def _check_msg_level(self, test, kw, msg, expected): condition = msg.level == expected.level message = ( - f"Keyword '{kw.name}' (index {expected.kw_index_str}) message {expected.msg_index_str} has wrong level." + f"Keyword '{kw.name}' (index {expected.kw_index_str}) " + f"message {expected.msg_index_str} has wrong level." f"\n\nExpected: {expected.level}\nActual: {msg.level}" ) return self._assert(condition, test, message) @@ -308,7 +304,8 @@ def _check_msg_level(self, test, kw, msg, expected): def _check_msg_message(self, test, kw, msg, expected): condition = self._message_matches(msg.message.strip(), expected.message) message = ( - f"Keyword '{kw.name}' (index {expected.kw_index_str}) message {expected.msg_index_str} has wrong content." + f"Keyword '{kw.name}' (index {expected.kw_index_str}) " + f"message {expected.msg_index_str} has wrong content." f"\n\nExpected:\n{expected.message}\n\nActual:\n{msg.message}" ) return self._assert(condition, test, message) diff --git a/tasks.py b/tasks.py index 0d27bf0..2940d24 100644 --- a/tasks.py +++ b/tasks.py @@ -2,7 +2,7 @@ from pathlib import Path from invoke import task -from rellu import initialize_labels, ReleaseNotesGenerator, Version +from rellu import ReleaseNotesGenerator, Version, initialize_labels from rellu.tasks import clean # noqa VERSION_PATTERN = '__version__ = "(.*)"' @@ -20,7 +20,8 @@ .. _Robot Framework: http://robotframework.org .. _PyPI: https://github.com/robotframework/statuschecker .. _issue tracker: https://github.com/robotframework/SeleniumLibrary/issues?q=milestone%3A{version.milestone} -""" +""" # noqa + @task def set_version(ctx, version): @@ -68,9 +69,7 @@ def release_notes(ctx, version=None, username=None, password=None, write=False): folder = RELEASE_NOTES_PATH.parent.resolve() folder.mkdir(parents=True, exist_ok=True) file = RELEASE_NOTES_PATH if write else sys.stdout - generator = ReleaseNotesGenerator( - REPOSITORY, RELEASE_NOTES_TITLE, RELEASE_NOTES_INTRO - ) + generator = ReleaseNotesGenerator(REPOSITORY, RELEASE_NOTES_TITLE, RELEASE_NOTES_INTRO) generator.generate(version, username, password, file) @@ -85,4 +84,26 @@ def init_labels(ctx, username=None, password=None): Should only be executed once when taking ``rellu`` tooling to use or when labels it uses have changed. """ - initialize_labels(REPOSITORY, username, password) \ No newline at end of file + initialize_labels(REPOSITORY, username, password) + + +@task +def lint(ctx): + """Run linters + + Flake8, Black and robotframework-tidy + """ + ctx.run("black --config pyproject.toml tasks.py robotstatuschecker.py") + ctx.run("flake8 --config .flake8 tasks.py robotstatuschecker.py") + ctx.run("isort tasks.py robotstatuschecker.py") + tidy_command = [ + "robotidy", + "--lineseparator", + "unix", + "--configure", + "NormalizeAssignments:equal_sign_type=space_and_equal_sign", + "--configure", + "NormalizeAssignments:equal_sign_type_variables=space_and_equal_sign", + "test", + ] + ctx.run(" ".join(tidy_command)) diff --git a/test/run.py b/test/run.py index 2eeab4f..1b81e42 100755 --- a/test/run.py +++ b/test/run.py @@ -1,9 +1,9 @@ #!/usr/bin/env python +import sys from os.path import abspath, dirname, exists, join from platform import python_implementation, python_version from shutil import rmtree -import sys from robot import run, rebot from robot.api import ExecutionResult, ResultVisitor @@ -11,10 +11,10 @@ CURDIR = dirname(abspath(__file__)) sys.path.insert(0, dirname(CURDIR)) -from robot.version import VERSION +from robot.version import VERSION # noqa -from robotstatuschecker import process_output -from robotstatuschecker import RF3 +from robotstatuschecker import process_output # noqa +from robotstatuschecker import RF3 # noqa def check_tests(robot_file): @@ -46,7 +46,10 @@ def __init__(self): def visit_test(self, test): self.tests += 1 status, message = self._get_expected(test) - errors = [self._verify(test.status, status, "status"), self._verify(test.message, message, "message")] + errors = [ + self._verify(test.status, status, "status"), + self._verify(test.message, message, "message"), + ] errors = ["- %s" % e for e in errors if e] if errors: self.errors.append("%s:\n%s" % (test.name, "\n".join(errors))) @@ -61,8 +64,7 @@ def _get_expected_rf4(self, test): kw = test.body[0] else: kw = test.setup - return (kw.body[1].messages[0].message, - kw.body[2].messages[0].message) + return (kw.body[1].messages[0].message, kw.body[2].messages[0].message) def _get_expected_rf3(self, test): kw = test.keywords[0] diff --git a/test/tests.robot b/test/tests.robot index cb9467b..ff695f8 100644 --- a/test/tests.robot +++ b/test/tests.robot @@ -1,7 +1,6 @@ *** Settings *** -Suite Setup Log Suite setup -Suite Teardown Log Suite teardown - +Suite Setup Log Suite setup +Suite Teardown Log Suite teardown *** Test Cases *** Implicit PASS @@ -145,13 +144,13 @@ Error When No Setup Test Teardown Check Is Done By TEARDOWN Marker [Documentation] ... - ... LOG TEARDOWN:1 foobar - ... LOG TEARDOWN foobar + ... LOG TEARDOWN:1 foobar + ... LOG TEARDOWN foobar Status PASS [Teardown] Log foobar Error When No Teardown - [Documentation] LOG TEARDOWN:1 foobar + [Documentation] LOG TEARDOWN:1 foobar Status FAIL Expected test Error When No Teardown to have teardown but teardown is not present. Log KALA From 76bfa95f9bafb3d080048184b668e5884f73c823 Mon Sep 17 00:00:00 2001 From: Jari Date: Thu, 21 Oct 2021 06:33:52 +0200 Subject: [PATCH 55/63] conflict resolution + linting --- robotstatuschecker.py | 2 +- test/tests.robot | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 5b4866f..ea9cd39 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -294,7 +294,7 @@ def _check_message_by_index(self, test, kw, expected): def _check_message_by_wildcard(self, test, kw, expected): if expected.message == "NONE": - message = f"Message index wildcard '*' is not supported with expected message 'NONE'." + message = "Message index wildcard '*' is not supported with expected message 'NONE'." self._fail(test, message) return diff --git a/test/tests.robot b/test/tests.robot index 196d4eb..4c10b53 100644 --- a/test/tests.robot +++ b/test/tests.robot @@ -206,14 +206,15 @@ Error When No Teardown Test Teardown Check Is Done By TEARDOWN Marker and wildcard is used [Documentation] ... - ... LOG TEARDOWN:* foobar - ... LOG TEARDOWN foobar + ... LOG TEARDOWN:* foobar + ... LOG TEARDOWN foobar Status PASS [Teardown] Log foobar Error When No Teardown and wildcard is used - [Documentation] LOG TEARDOWN:* foobar - Status FAIL Expected test Error When No Teardown and wildcard is used to have teardown but teardown is not present. + [Documentation] LOG TEARDOWN:* foobar + Status FAIL + ... Expected test Error When No Teardown and wildcard is used to have teardown but teardown is not present. Log KALA Error When NONE is used with wildcard From 7c6b040286514503acb5038b0e1eb6eedc51d4ac Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Mon, 25 Oct 2021 20:52:11 +0300 Subject: [PATCH 56/63] Release notes for 2.1.0 --- .../releasenotes/robotstatuschecker-2.1.0.rst | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/releasenotes/robotstatuschecker-2.1.0.rst diff --git a/docs/releasenotes/robotstatuschecker-2.1.0.rst b/docs/releasenotes/robotstatuschecker-2.1.0.rst new file mode 100644 index 0000000..a9e98b1 --- /dev/null +++ b/docs/releasenotes/robotstatuschecker-2.1.0.rst @@ -0,0 +1,50 @@ +======================== +robotstatuschecker 2.1.0 +======================== + + +.. default-role:: code + + +StatusChecker is a tool for validating that executed `Robot Framework`_ test cases +have expected statuses and log messages. It is mainly useful for Robot Framework +test library developers who want to use Robot Framework to also test their libraries. +StatusChecker 1.4 and newer are compatible with Python 3.6+. + +StatusChecker project is hosted at GitHub and downloads are at PyPI_ +.. _Robot Framework: http://robotframework.org +.. _PyPI: https://github.com/robotframework/statuschecker +.. _issue tracker: https://github.com/robotframework/SeleniumLibrary/issues?q=milestone%3Av2.1.0 + + +.. contents:: + :depth: 2 + :local: + +Most important enhancements +=========================== + +Wildcard support for the log verification (`#32`_) +-------------------------------------------------- +Now it possible to verify keyword log messages with a wildcard (*), example with 1:*. +This allows to verify that log massage is somewhere in the keyword, but specific index +of the log message is not mandatory anymore. + +Full list of fixes and enhancements +=================================== + +.. list-table:: + :header-rows: 1 + + * - ID + - Type + - Priority + - Summary + * - `#32`_ + - enhancement + - high + - Wildcard support for the log verification + +Altogether 1 issue. View on the `issue tracker `__. + +.. _#32: https://github.com/robotframework/statuschecker/issues/32 From 1f007cf4303204fc57bf2249073d23041b4976f8 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Mon, 25 Oct 2021 20:52:28 +0300 Subject: [PATCH 57/63] Updated version to 2.1.0 --- robotstatuschecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index ea9cd39..228de1a 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -45,7 +45,7 @@ from robot.api import ExecutionResult, ResultVisitor from robot.utils import Matcher -__version__ = "2.0.4.dev1" +__version__ = "2.1.0" RF3 = rf_version.startswith("3") From 3647e973c7d0159fb298ce903646163c790146cc Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Mon, 25 Oct 2021 20:56:00 +0300 Subject: [PATCH 58/63] Back to dev version --- robotstatuschecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 228de1a..3c49b8a 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -45,7 +45,7 @@ from robot.api import ExecutionResult, ResultVisitor from robot.utils import Matcher -__version__ = "2.1.0" +__version__ = "2.1.1.dev1" RF3 = rf_version.startswith("3") From 3f0ee3d39aebb2b6825b3b1cf95f1d0c69f51eb6 Mon Sep 17 00:00:00 2001 From: Eric Jones Date: Thu, 11 Nov 2021 10:51:20 -0700 Subject: [PATCH 59/63] support rf4 skip status --- README.rst | 6 ++++++ robotstatuschecker.py | 11 +++++++++-- test/README.rst | 9 +++++++++ test/run.py | 10 ++++++++-- test/tests.robot | 20 ++++++++++++++++++++ 5 files changed, 52 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index f56daa1..aa5fcca 100644 --- a/README.rst +++ b/README.rst @@ -60,6 +60,12 @@ the word ``FAIL`` (in uppercase) somewhere in the test case documentation. The expected error message must then follow the ``FAIL`` marker. +For robotframework version 4 you can also change the expected status +to *SKIP* by adding the word ``SKIP`` in the test case documentation. +Like Fail, the expected skip message must follow the word ``SKIP``. +If a test documentation contains the words ``FAIL`` and ``SKIP``, ``SKIP`` +will be ignored and the expected status will be *FAIL*. + If a test is expected to *PASS* with a certain message, the word ``PASS`` must be added to its documentation explicitly and the expected message given after that. diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 3c49b8a..7fc150d 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -93,10 +93,17 @@ def __init__(self, doc): self.logs = self._get_logs(doc) def _get_status(self, doc): - return "FAIL" if "FAIL" in doc else "PASS" + if RF3: + return "FAIL" if "FAIL" in doc else "PASS" + if "FAIL" not in doc: + return "SKIP" if "SKIP" in doc else "PASS" + return "FAIL" def _get_message(self, doc): - if "FAIL" not in doc and "PASS" not in doc: + if RF3: + if "FAIL" not in doc and "PASS" not in doc: + return "" + if all(status not in doc for status in ["FAIL", "SKIP", "PASS"]): return "" status = self._get_status(doc) return doc.split(status, 1)[1].split("LOG", 1)[0].strip() diff --git a/test/README.rst b/test/README.rst index 510f917..0aed144 100644 --- a/test/README.rst +++ b/test/README.rst @@ -17,3 +17,12 @@ other tests should pass. Test statuses and messages set by StatusChecker are verified by ``run.py``. Expected statuses and messages are logged by ``Status`` keyword that all tests must use as their first keyword. + +Test cases that use features only available in RF4+ (e.g. SKIP status) should +be tagged with + +.. sourcecode:: robotframework + + [Tags] rf3unsupported + +and will be excluded in run.py if the installed robotframework is RF3. \ No newline at end of file diff --git a/test/run.py b/test/run.py index 1b81e42..0ca2664 100755 --- a/test/run.py +++ b/test/run.py @@ -32,7 +32,10 @@ def _run_tests_and_process_output(robot_file): output = join(results, "output.xml") if exists(results): rmtree(results) - run(join(CURDIR, robot_file), output=output, log=None, report=None, loglevel="DEBUG") + if RF3: + run(join(CURDIR, robot_file), output=output, log=None, report=None, loglevel="DEBUG", exclude="rf3unsupported") + else: + run(join(CURDIR, robot_file), output=output, log=None, report=None, loglevel="DEBUG") process_output(output) rebot(output, outputdir=results) return output @@ -82,7 +85,10 @@ def print_status(self): print("%d/%d test failed:" % (len(self.errors), self.tests)) print("\n-------------------------------------\n".join(self.errors)) else: - print("All %d tests passed/failed/logged as expected." % self.tests) + if RF3: + print("All %d tests passed/failed/logged as expected." % self.tests) + else: + print("All %d tests passed/failed/logged/skipped as expected." % self.tests) print("Run on %s %s." % (python_implementation(), python_version())) diff --git a/test/tests.robot b/test/tests.robot index 4c10b53..95926ae 100644 --- a/test/tests.robot +++ b/test/tests.robot @@ -12,12 +12,32 @@ Explicit PASS with message Status PASS The message Pass Execution The message +Explicit SKIP with message + [Documentation] SKIP The message + [Tags] rf3unsupported + Status SKIP The message + Skip The message + Expected FAIL [Documentation] FAIL Expected failure Status PASS Test failed as expected.\n\n ... Original message:\nExpected failure Fail Expected failure +SKIP Plus FAIL Expected FAIL + [Documentation] SKIP FAIL Expected failure + [Tags] rf3unsupported + Status PASS Test failed as expected.\n\n + ... Original message:\nExpected failure + Fail Expected failure + +FAIL Plus SKIP Expected FAIL + [Documentation] FAIL Expected failure SKIP + [Tags] rf3unsupported + Status PASS Test failed as expected.\n\n + ... Original message:\nExpected failure SKIP + Fail Expected failure SKIP + Ignore documentation before marker [Documentation] This text is ignored. FAIL Expected failure Status PASS Test failed as expected.\n\n From b68297006ecd5ef0b392e863a71d797e6a6f24db Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Thu, 11 Nov 2021 23:39:07 +0200 Subject: [PATCH 60/63] Release notes for 2.2.0 --- .../releasenotes/robotstatuschecker-2.2.0.rst | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 docs/releasenotes/robotstatuschecker-2.2.0.rst diff --git a/docs/releasenotes/robotstatuschecker-2.2.0.rst b/docs/releasenotes/robotstatuschecker-2.2.0.rst new file mode 100644 index 0000000..c344d62 --- /dev/null +++ b/docs/releasenotes/robotstatuschecker-2.2.0.rst @@ -0,0 +1,42 @@ +======================== +robotstatuschecker 2.2.0 +======================== + + +.. default-role:: code + + +StatusChecker is a tool for validating that executed `Robot Framework`_ test cases +have expected statuses and log messages. It is mainly useful for Robot Framework +test library developers who want to use Robot Framework to also test their libraries. +StatusChecker is compatible both with Python 3.6+. This release add support for RF 4 +SKIP status. + +StatusChecker project is hosted at GitHub and downloads are at PyPI_ +.. _Robot Framework: http://robotframework.org +.. _PyPI: https://github.com/robotframework/statuschecker +.. _issue tracker: https://github.com/robotframework/SeleniumLibrary/issues?q=milestone%3Av2.2.0 + + +.. contents:: + :depth: 2 + :local: + +Full list of fixes and enhancements +=================================== + +.. list-table:: + :header-rows: 1 + + * - ID + - Type + - Priority + - Summary + * - `#36`_ + - --- + - --- + - Support RF 4 style SKIP + +Altogether 1 issue. View on the `issue tracker `__. + +.. _#36: https://github.com/robotframework/statuschecker/issues/36 From 3ea7001eb7d86b744c79deb7ab607e14000caa80 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Thu, 11 Nov 2021 23:39:32 +0200 Subject: [PATCH 61/63] Updated version to 2.2.0 --- robotstatuschecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 7fc150d..4ab94b6 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -45,7 +45,7 @@ from robot.api import ExecutionResult, ResultVisitor from robot.utils import Matcher -__version__ = "2.1.1.dev1" +__version__ = "2.2.0" RF3 = rf_version.startswith("3") From 0450a61036b5ab12d83cec21fdcdd7e323bb7109 Mon Sep 17 00:00:00 2001 From: Tatu Aalto Date: Thu, 11 Nov 2021 23:41:42 +0200 Subject: [PATCH 62/63] Back to dev version --- robotstatuschecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index 4ab94b6..b8873f4 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -45,7 +45,7 @@ from robot.api import ExecutionResult, ResultVisitor from robot.utils import Matcher -__version__ = "2.2.0" +__version__ = "2.2.1.dev1" RF3 = rf_version.startswith("3") From 88414717b978932ef7a1853b454cdc26c076c116 Mon Sep 17 00:00:00 2001 From: Henk van den Akker Date: Sun, 19 Apr 2020 22:37:32 +0200 Subject: [PATCH 63/63] Added console logging --- robotstatuschecker.py | 63 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/robotstatuschecker.py b/robotstatuschecker.py index b8873f4..9f90a8b 100755 --- a/robotstatuschecker.py +++ b/robotstatuschecker.py @@ -27,14 +27,21 @@ Command-line usage: - python -m robotstatuschecker infile [outfile] + python -m robotstatuschecker infile [outfile] [--quiet] + robot --prerebotmodifier robotstatuschecker.StatusChecker data_sources + rebot --prerebotmodifier robotstatuschecker.StatusChecker robot_outputs Programmatic usage: from robotstatuschecker import process_output - process_output('infile.xml', 'outfile.xml') + process_output('infile.xml', 'outfile.xml', verbose=True) If an output file is not given, the input file is edited in place. + +By default status checker prints logging to the console. To suppress console +logging use --quiet on the command line, +--prerebotmodifier robotstatuschecker.StatusChecker:False in conjunction with +robot or rebot and verbose=False with the process_output function """ import re @@ -42,7 +49,8 @@ from os.path import abspath from robot import __version__ as rf_version -from robot.api import ExecutionResult, ResultVisitor +from robot.api import ExecutionResult, ResultVisitor, logger +from robot.output import LOGGER from robot.utils import Matcher __version__ = "2.2.1.dev1" @@ -63,24 +71,54 @@ def process_output(inpath, outpath=None, verbose=True): int: Number of failed critical tests after post-processing. """ if verbose: - print(f"Checking {abspath(inpath)}") - result = StatusChecker().process_output(inpath, outpath) + logger.console(f"Checking {abspath(inpath)}") + result = StatusChecker(verbose).process_output(inpath, outpath) if verbose and outpath: - print(f"Output: {abspath(outpath)}") + logger.console(f"Output: {abspath(outpath)}") return result.return_code class StatusChecker(ResultVisitor): + + def __init__(self, verbose=True): + self.verbose = verbose + if str(self.verbose).lower() == 'false': + self.verbose = False + if self.verbose: + width = 78 + suite_separator = '%s' % ('=' * width) + logger.console(suite_separator) + logger.console('************** Post-processing of results by status checker.... **************') + logger.console(suite_separator) + def process_output(self, inpath, outpath=None): result = ExecutionResult(inpath) result.suite.visit(self) result.save(outpath) return result + def start_suite(self, suite): + if self.verbose: + LOGGER.start_suite(suite) + + def end_suite(self, suite): + if self.verbose: + LOGGER.end_suite(suite) + def visit_test(self, test): - expected = Expected(test.doc) - if TestStatusChecker(expected).check(test): - LogMessageChecker(expected).check(test) + if self.start_test(test) is not False: + expected = Expected(test.doc) + if TestStatusChecker(expected).check(test): + LogMessageChecker(expected).check(test) + self.end_test(test) + + def start_test(self, test): + if self.verbose: + LOGGER.start_test(test) + + def end_test(self, test): + if self.verbose: + LOGGER.end_test(test) def visit_keyword(self, kw): pass @@ -345,8 +383,13 @@ def _check_msg_message(self, test, kw, msg, expected, fail=True): if "-h" in sys.argv or "--help" in sys.argv: print(__doc__) sys.exit(251) + args = sys.argv[1:] try: - rc = process_output(*sys.argv[1:]) + if '--quiet' in args: + args.remove('--quiet') + rc = process_output(*args, verbose=False) + else: + rc = process_output(*args) except TypeError: print(__doc__) sys.exit(252)