Skip to content

Commit aaa3c11

Browse files
angelrtiClarkTucker
angelrti
and
ClarkTucker
authored
Add new tests - python (#79)
* Adding new tests in python for the new options added to the executable * The basic check is that the sample has not 0 size * Replaced test test_ownership_receivers that relies on samples_sent by test_size_receivers that relies on the size of the samples, less error prone * Removed debug message * Added basic check for all tests * Replaced function of test_reliability_4 * Fixed lifespan tests in a corner case when the first sample is the only one from its group of samples * Fixed a potential infinite loop in the FinalInstanceState tests * Fixed disposing test * Adding EOF to all expect functions * Fixed large data and coherent sets test that were returning false-positives * Added a way of stopping coherent_sets and ordered_access tests * Added more lifespan tests and improved test check functions * Removed debug message * Fixed some issues * Fixing coherent sets check function * Added a way of removing bytes from publisher PTY that may block itself * Fixing interoperability_report after merging master * Increasing timeout to 120mins * Move the CFT test that uses color to -c param as it is different in each product * Adding a vendor coverage of tests now that we have a way of detecting unsupported tests * Documentation updates * Added tests for compatibility between ordered and coherent access. Also changed ordered access check function * Fixed ordered access test and coherent set tests. Also modified the test_order to accept instances * Updated the generated xlsx to show the summary in two columns * Use current repo executables for testing and and added an error if a executable does not exist * Added continue-on-error in case a executable is not found * Modified legend of the xlsx report * Fixing warnings with python strings * Added basic check to test_size_less_than_20 * Changed how to calculate the color of the single-product test, now it uses the passed tests instead of supported tests * Fixing color total tests and vendor coverage * Update generate_xlsx_report.py Add HDDS to generate_xlsx_report.py * Added hdds to the automation * Removed preRelease flag * Homogenize color code in the spreadsheet --------- Co-authored-by: Clark Tucker <ctucker@twinoakscomputing.com>
1 parent daede00 commit aaa3c11

5 files changed

Lines changed: 1751 additions & 374 deletions

File tree

‎.github/workflows/1_run_interoperability_tests.yml‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@ on:
66
publishers:
77
description: Publishers to use
88
type: string
9-
default: '["connext_dds","dust_dds","eprosima_fastdds","intercom_dds","opendds","toc_coredx_dds","eclipse_cyclone"]'
9+
default: '["connext_dds","dust_dds","eprosima_fastdds","intercom_dds","opendds","toc_coredx_dds", "hdds", "eclipse_cyclone"]'
1010
subscribers:
1111
description: Subscribers to use
1212
type: string
13-
default: '["connext_dds","dust_dds","eprosima_fastdds","intercom_dds","opendds","toc_coredx_dds","eclipse_cyclone"]'
13+
default: '["connext_dds","dust_dds","eprosima_fastdds","intercom_dds","opendds","toc_coredx_dds", "hdds", "eclipse_cyclone"]'
1414
jobs:
1515
generate_timestamp:
1616
runs-on: ubuntu-latest
@@ -41,22 +41,32 @@ jobs:
4141
- name: Download shape_main executables
4242
uses: robinraju/release-downloader@v1.10
4343
with:
44-
repository: omg-dds/dds-rtps
4544
latest: true
4645
fileName: "*"
4746
out-file-path: zipped_executables
48-
preRelease: true
47+
preRelease: false
4948
- name: Unzip executables
5049
run: unzip 'zipped_executables/*.zip' -d executables
5150
- name: Install Python requirements
5251
run: pip install --requirement requirements.txt
5352
- name: Run Interoperability script
54-
timeout-minutes: 60
53+
timeout-minutes: 120
54+
continue-on-error: true
5555
run: |
5656
publisher_exe=executables/${{ matrix.publisher }}*shape_main_linux
5757
subscriber_exe=executables/${{ matrix.subscriber }}*shape_main_linux
5858
output_file=junit_report-${{ matrix.publisher }}-${{ matrix.subscriber }}.xml
5959
extra_args=""
60+
# Check if publisher executable exists
61+
if ! ls $publisher_exe 1> /dev/null 2>&1; then
62+
echo "ERROR: Publisher executable not found: $publisher_exe"
63+
exit 1
64+
fi
65+
# Check if subscriber executable exists
66+
if ! ls $subscriber_exe 1> /dev/null 2>&1; then
67+
echo "ERROR: Subscriber executable not found: $subscriber_exe"
68+
exit 1
69+
fi
6070
if [[ "${subscriber_exe,,}" == *opendds* && "${publisher_exe,,}" == *connext_dds* ]]; then
6171
extra_args="--periodic-announcement 5000"
6272
fi

‎generate_xlsx_report.py‎

Lines changed: 76 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ def get_company_name(product:str) -> str:
7979
return 'eProsima'
8080
elif 'dust' in product.lower():
8181
return 'S2E Software Systems'
82+
elif 'hdds' in product.lower():
83+
return 'Naskel'
8284
elif 'cyclone' in product.lower():
8385
return 'Eclipse Foundation'
8486
else:
@@ -102,8 +104,10 @@ def get_product_name(product:str) -> str:
102104
return 'FastDDS ' + re.search(r'([\d.]+)', product).group(1)
103105
elif 'dust_dds' in product.lower():
104106
return 'Dust DDS ' + re.search(r'([\d.]+)', product).group(1)
107+
elif 'hdds' in product.lower():
108+
return 'HDDS ' + re.search(r'([\d.]+)', product).group(1)
105109
elif 'cyclone' in product.lower():
106-
return 'Cyclone DDS'
110+
return 'Cyclone DDS ' + re.search(r'([\d.]+)', product).group(1)
107111
else:
108112
raise RuntimeError('Impossible to get product name: ' + product)
109113

@@ -408,7 +412,7 @@ def create_summary_worksheet(self, name: str = 'Summary'):
408412
# rows.
409413
# The tables leave the first column (value 0) as gap
410414
self.add_data_summary_worksheet(
411-
starting_row=9,
415+
starting_row=13,
412416
starting_column=1,
413417
worksheet=summary_worksheet)
414418
# After having all data that may have an unknown length, we call
@@ -756,13 +760,10 @@ def add_data_summary_worksheet(self,
756760
'Product', self.__formats['bold_w_border'])
757761
worksheet.write(
758762
current_row, current_column + 2,
759-
'Tests Passed', self.__formats['bold_w_border'])
763+
'Single-Product Tests', self.__formats['bold_w_border'])
760764
worksheet.write(
761765
current_row, current_column + 3,
762-
'Supported Tests', self.__formats['bold_w_border'])
763-
worksheet.write(
764-
current_row, current_column + 4,
765-
'Supported Tests Passed', self.__formats['bold_w_border'])
766+
'Cross Product Tests', self.__formats['bold_w_border'])
766767

767768
current_row += 1
768769

@@ -778,35 +779,42 @@ def add_data_summary_worksheet(self,
778779
current_row, current_column + 1,
779780
product_name,
780781
self.__formats['bold_w_border'])
781-
# test passed
782-
worksheet.write(
783-
current_row, current_column + 2,
784-
str(value.get_passed_tests()) + ' / ' +
785-
str(value.get_total_tests()),
786-
self.get_format_color(value.get_passed_tests(),
787-
value.get_total_tests()))
788-
# supported tests
782+
783+
# vendor coverage
784+
785+
# for the vendor coverage, we need to find the right data in the
786+
# product_summary_dict, which is the one with the same product as
787+
# publisher and subscriber
788+
for product_key, product_value in self.__data.product_summary_dict.items():
789+
if product_name == product_key[0] and product_name == product_key[1]:
790+
product_coverage_supported_tests = product_value.get_supported_tests()
791+
product_coverage_total_tests = product_value.get_total_tests()
792+
product_coverage_passed_tests = product_value.get_passed_tests()
793+
794+
# vendor coverage
795+
worksheet.write(
796+
current_row, current_column + 2,
797+
str(product_coverage_passed_tests) + ' / ' +
798+
str(product_coverage_supported_tests) + ' / ' +
799+
str(product_coverage_total_tests),
800+
self.get_format_color(
801+
product_coverage_passed_tests,
802+
product_coverage_total_tests))
803+
804+
# total tests
789805
worksheet.write(
790806
current_row, current_column + 3,
791-
str(value.get_supported_tests()) + ' / ' +
792-
str(value.get_total_tests()),
793-
self.__formats['result_yellow'] if value.get_unsupported_tests() > 0
794-
else self.__formats['result_green'])
795-
# supported tests passed
796-
worksheet.write(
797-
current_row, current_column + 4,
798807
str(value.get_passed_tests()) + ' / ' +
799-
str(value.get_supported_tests()),
800-
self.get_format_color(value.get_passed_tests(),
801-
value.get_supported_tests()))
808+
str(value.get_supported_tests()) + ' / ' +
809+
str(value.get_total_tests()),
810+
self.get_format_color(
811+
value.get_passed_tests(), value.get_total_tests()))
812+
802813
current_row += 1
803814

815+
804816
# Add 2 rows of gap for the next table
805817
current_row += 2
806-
worksheet.write(
807-
current_row, current_column,
808-
'Test Result: passed / supported / total', self.__formats['bold_w_border'])
809-
current_row += 1
810818
worksheet.write(
811819
current_row, current_column,
812820
'Publisher (row)/Subscriber (column)', self.__formats['bold_w_border'])
@@ -823,7 +831,7 @@ def add_data_summary_worksheet(self,
823831

824832
# Add the table passed_tests/total_tests with all combinations of product
825833
# as publishers and as subscribers
826-
for (publisher_name, subscriber_name), value in self.__data.product_summary_dict.items():
834+
for (publisher_name, subscriber_name), product_value in self.__data.product_summary_dict.items():
827835
# if the publisher hasn't been already processed yet, determine
828836
# what is the process_row by selecting the next free row
829837
# (current_row+1)
@@ -853,10 +861,10 @@ def add_data_summary_worksheet(self,
853861
process_column = column_dict[subscriber_name]
854862

855863
worksheet.write(process_row, process_column,
856-
str(value.get_passed_tests()) + ' / ' +
857-
str(value.get_supported_tests()) + ' / ' +
858-
str(value.get_total_tests()),
859-
self.get_format_color(value.get_passed_tests(), value.get_supported_tests()))
864+
str(product_value.get_passed_tests()) + ' / ' +
865+
str(product_value.get_supported_tests()) + ' / ' +
866+
str(product_value.get_total_tests()),
867+
self.get_format_color(product_value.get_passed_tests(), product_value.get_total_tests()))
860868

861869
def add_static_data_summary_worksheet(self,
862870
worksheet: xlsxwriter.Workbook.worksheet_class,
@@ -889,10 +897,9 @@ def add_static_data_summary_worksheet(self,
889897
worksheet.insert_image(
890898
row=current_row, col=starting_column,
891899
filename=dds_logo_path,
892-
options={'x_scale': 0.4, 'y_scale': 0.4, 'decorative': True, 'object_position': 2})
900+
options={'x_scale': 0.6, 'y_scale': 0.6, 'decorative': True, 'object_position': 2})
893901

894902
# Add date
895-
current_row += 1
896903
worksheet.write(current_row, starting_column + 1, 'Date')
897904
date_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
898905
worksheet.write(current_row, starting_column + 2, date_time)
@@ -907,6 +914,39 @@ def add_static_data_summary_worksheet(self,
907914
worksheet.write(current_row, starting_column + 1,'Documentation')
908915
worksheet.write(current_row, starting_column + 2, self.REPO_DOC)
909916

917+
# Add number of tests
918+
919+
# Find the total number of unique tests by looking for a
920+
# (publisher, subscriber) pair where both are the same
921+
test_count = 0
922+
for key, value in self.__data.product_summary_dict.items():
923+
if key[0] == key[1]:
924+
test_count = value.get_total_tests()
925+
break
926+
current_row += 1
927+
worksheet.write(current_row, starting_column + 1,'Unique tests count')
928+
worksheet.write(current_row, starting_column + 2, test_count)
929+
930+
# add legend
931+
current_row += 2
932+
worksheet.write(current_row, starting_column + 1,
933+
'Single-Product Tests',
934+
self.__formats['bold'])
935+
worksheet.write(current_row, starting_column + 2,
936+
'Results where each product is tested only with itself')
937+
current_row += 1
938+
worksheet.write(current_row, starting_column + 1,
939+
'Cross-Product Tests',
940+
self.__formats['bold'])
941+
worksheet.write(current_row, starting_column + 2,
942+
'Results where each product is tested against all others (including itself)')
943+
current_row += 1
944+
worksheet.write(current_row, starting_column + 1,
945+
'Format X / Y / Z',
946+
self.__formats['bold'])
947+
worksheet.write(current_row, starting_column + 2,
948+
'Represents test passed / supported / total')
949+
910950
def add_static_data_description_worksheet(self,
911951
worksheet: xlsxwriter.Workbook.worksheet_class,
912952
name: str = 'Test Descriptions',

‎interoperability_report.py‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -383,7 +383,8 @@ def run_publisher_shape_main(
383383
r'\[[0-9]+\]', # index = 0
384384
'on_offered_deadline_missed()', # index = 1
385385
re.compile('not supported', re.IGNORECASE), # index = 2
386-
pexpect.TIMEOUT # index = 3
386+
pexpect.TIMEOUT, # index = 3
387+
pexpect.EOF # index = 4
387388
],
388389
timeout)
389390
if index == 1:
@@ -392,7 +393,7 @@ def run_publisher_shape_main(
392393
elif index == 2:
393394
produced_code[produced_code_index] = ReturnCode.PUB_UNSUPPORTED_FEATURE
394395
break
395-
elif index == 3:
396+
elif index == 3 or index == 4:
396397
produced_code[produced_code_index] = ReturnCode.DATA_NOT_SENT
397398
break
398399
last_sample_saved.put(last_sample)
@@ -401,8 +402,18 @@ def run_publisher_shape_main(
401402

402403
log_message(f'Publisher {publisher_index}: Waiting for Subscribers to finish',
403404
verbosity)
404-
for element in subscribers_finished:
405-
element.wait() # wait for all subscribers to finish
405+
# Check if all subscribers finished
406+
while True:
407+
if all(e.is_set() for e in subscribers_finished):
408+
break
409+
410+
# Drain publisher output
411+
try:
412+
child_pub.read_nonblocking(1024, timeout=0.1)
413+
except pexpect.TIMEOUT:
414+
pass
415+
except pexpect.EOF:
416+
break
406417
publisher_finished.set() # set publisher as finished
407418
# Stop process
408419
if not stop_process(child_pub):

0 commit comments

Comments
 (0)