|
| 1 | +@article{lauwaerts24, |
| 2 | + title = {Latch: {{Enabling}} Large-Scale Automated Testing on Constrained |
| 3 | + Systems}, |
| 4 | + author = {Lauwaerts, Tom and Marr, Stefan and Scholliers, Christophe}, |
| 5 | + year = {2024}, |
| 6 | + journal = {Science of Computer Programming}, |
| 7 | + volume = {238}, |
| 8 | + pages = {103157}, |
| 9 | + issn = {0167-6423}, |
| 10 | + doi = {10.1016/j.scico.2024.103157}, |
| 11 | + abstract = {Testing is an essential part of the software development cycle. |
| 12 | + Unfortunately, testing on constrained devices is currently very |
| 13 | + challenging. First, the limited memory of constrained devices |
| 14 | + severely restricts the size of test suites. Second, the limited |
| 15 | + processing power causes test suites to execute slowly, preventing |
| 16 | + a fast feedback loop. Third, when the constrained device becomes |
| 17 | + unresponsive, it is impossible to distinguish between the test |
| 18 | + failing or taking very long, forcing the developer to work with |
| 19 | + timeouts. Unfortunately, timeouts can cause tests to be flaky, |
| 20 | + i.e., have unpredictable outcomes independent of code changes. |
| 21 | + Given these problems, most IoT developers rely on laborious |
| 22 | + manual testing. In this paper, we propose the novel testing |
| 23 | + framework Latch (Large-scale Automated Testing on Constrained |
| 24 | + Hardware) to overcome the three main challenges of running large |
| 25 | + test suites on constrained hardware, as well as automate manual |
| 26 | + testing scenarios through a novel testing methodology based on |
| 27 | + debugger-like operations---we call this new testing approach |
| 28 | + managed testing. The core idea of Latch is to enable testing on |
| 29 | + constrained devices without those devices maintaining the whole |
| 30 | + test suite in memory. Therefore, programmers script and run tests |
| 31 | + on a workstation which then step-wise instructs the constrained |
| 32 | + device to execute each test, thereby overcoming the memory |
| 33 | + constraints. Our testing framework further allows developers to |
| 34 | + mark tests as depending on other tests. This way, Latch can skip |
| 35 | + tests that depend on previously failing tests resulting in a |
| 36 | + faster feedback loop. Finally, Latch addresses the issue of |
| 37 | + timeouts and flaky tests by including an analysis mode that |
| 38 | + provides feedback on timeouts and the flakiness of tests. To |
| 39 | + illustrate the expressiveness of Latch, we present testing |
| 40 | + scenarios representing unit testing, integration testing, and |
| 41 | + end-to-end testing. We evaluate the performance of Latch by |
| 42 | + testing a virtual machine against the WebAssembly specification, |
| 43 | + with a large test suite consisting of 10,213 tests running on an |
| 44 | + ESP32 microcontroller. Our experience shows that the testing |
| 45 | + framework is expressive, reliable and reasonably fast, making it |
| 46 | + suitable to run large test suites on constrained devices. |
| 47 | + Furthermore, the debugger-like operations enable to closely mimic |
| 48 | + manual testing.}, |
| 49 | + keywords = {Automated testing,Embedded devices,Flaky tests}, |
| 50 | +} |
0 commit comments