From 7549d6e53d60b4e50d2dc8524e2e5574a56e69a4 Mon Sep 17 00:00:00 2001 From: "fabian.wuellhorst" Date: Mon, 17 Jun 2024 14:37:41 +0200 Subject: [PATCH 1/3] add coverage script from Mans with changes based on review in #315 #243 --- buildingspy/development/regressiontest.py | 108 +++++++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/buildingspy/development/regressiontest.py b/buildingspy/development/regressiontest.py index 1deb9451..36bd084d 100644 --- a/buildingspy/development/regressiontest.py +++ b/buildingspy/development/regressiontest.py @@ -351,6 +351,9 @@ def __init__( self._data = [] self._reporter = rep.Reporter(os.path.join(os.getcwd(), "unitTests-{}.log".format(tool))) + # List to store tested packages, used for coverage report + self._packages = [] + # By default, include export of FMUs. self._include_fmu_test = True @@ -589,6 +592,7 @@ def getModelicaCommand(self): elif self._modelica_tool != 'dymola': return 'jm_ipython.sh' else: + return "C://Program Files//Dymola 2023x//bin64//Dymola" return self._modelica_tool def isExecutable(self, program): @@ -771,11 +775,12 @@ def setSinglePackage(self, packageName): # Set data dictionary as it may have been generated earlier for the whole library. self._data = [] - + self._packages = [] for pac in packages: pacSep = pac.find('.') pacPat = pac[pacSep + 1:] pacPat = pacPat.replace('.', os.sep) + self._packages.append(pacPat) rooPat = os.path.join(self._libHome, 'Resources', 'Scripts', 'Dymola', pacPat) # Verify that the directory indeed exists if not os.path.isdir(rooPat): @@ -4288,3 +4293,104 @@ def _model_from_mo(self, mo_file): model = '.'.join(splt[root:]) # remove the '.mo' at the end return model[:-3] + + def getCoverage(self): + """ + Analyse how many examples are tested. + If ``setSinglePackage`` is called before this function, + only packages set will be included. Else, the whole library + will be checked. + + Returns: + - The coverage rate in percent as float + - The number of examples tested as int + - The total number of examples as int + - The list of models not tested as List[str] + - The list of packages included in the analysis as List[str] + """ + # first lines copy and paste from run function + if self.get_number_of_tests() == 0: + self.setDataDictionary(self._rootPackage) + + # Remove all data that do not require a simulation or an FMU export. + # Otherwise, some processes may have no simulation to run and then + # the json output file would have an invalid syntax + + # to not interact with other code here, we use the temp_data list + + temp_data = [ + element for element in self._data[:] + if element['mustSimulate'] or element['mustExportFMU'] + ] + + # now we got clean _data to compare + # next step get all examples in the package (whether whole library or + # single package) + if self._packages: + packages = self._packages + else: + packages = list(dict.fromkeys( + [pac['ScriptFile'].split(os.sep)[0] for pac in self._data]) + ) + + all_examples = [] + for package in packages: + package_path = os.path.join(self._libHome, package) + for dirpath, dirnames, filenames in os.walk(package_path): + for filename in filenames: + if any( + xs in filename for xs in ['Examples', 'Validation'] + ) and not filename.endswith(('package.mo', '.order')): + all_examples.append(os.path.abspath( + os.path.join(dirpath, filename)) + ) + + coverage = round(len(temp_data) / len(all_examples), 2) * 100 + + tested_model_names = [ + nam['ScriptFile'].split(os.sep)[-1][:-1] for nam in temp_data + ] + + missing_examples = [ + i for i in all_examples if not any( + xs in i for xs in tested_model_names) + ] + + n_tested_examples = len(temp_data) + n_examples = len(all_examples) + return coverage, n_tested_examples, n_examples, missing_examples, packages + + def printCoverage( + self, + coverage: float, + n_tested_examples: int, + n_examples: int, + missing_examples: list, + packages: list, + printer: callable = None + ) -> None: + """ + Print the output of getCoverage to inform about + coverage rate and missing models. + The default printer is the ``reporter.writeOutput``. + If another printing method is required, e.g. ``print`` or + ``logging.info``, it may be passed via the ``printer`` argument. + """ + if printer is None: + printer = self._reporter.writeOutput + printer('***\n\nModel Coverage: ', str(int(coverage)) + '%') + printer( + '***\n\nYou are testing : ', + n_tested_examples, + ' out of ', + n_examples, + 'total examples in ' + ) + for package in packages: + printer(package) + printer('\n') + + if missing_examples: + print('***\n\nThe following examples are not tested\n') + for i in missing_examples: + print(i.split(self._libHome)[1]) From 13346024da7a3b8555acff610940d474ad190ca7 Mon Sep 17 00:00:00 2001 From: "fabian.wuellhorst" Date: Mon, 17 Jun 2024 15:12:36 +0200 Subject: [PATCH 2/3] add unit-test and fix minor bug --- buildingspy/development/regressiontest.py | 53 +++++++++++-------- .../tests/test_development_regressiontest.py | 35 ++++++++++++ 2 files changed, 65 insertions(+), 23 deletions(-) diff --git a/buildingspy/development/regressiontest.py b/buildingspy/development/regressiontest.py index 36bd084d..f8602272 100644 --- a/buildingspy/development/regressiontest.py +++ b/buildingspy/development/regressiontest.py @@ -4307,6 +4307,15 @@ def getCoverage(self): - The total number of examples as int - The list of models not tested as List[str] - The list of packages included in the analysis as List[str] + + Example: + >>> from buildingspy.development.regressiontest import Tester + >>> import os + >>> ut = Tester(tool='dymola') + >>> myMoLib = os.path.join("buildingspy", "tests", "MyModelicaLibrary") + >>> ut.setLibraryRoot(myMoLib) + >>> ut.setSinglePackage('Examples') + >>> coverage_result = ut.getCoverage() """ # first lines copy and paste from run function if self.get_number_of_tests() == 0: @@ -4316,13 +4325,6 @@ def getCoverage(self): # Otherwise, some processes may have no simulation to run and then # the json output file would have an invalid syntax - # to not interact with other code here, we use the temp_data list - - temp_data = [ - element for element in self._data[:] - if element['mustSimulate'] or element['mustExportFMU'] - ] - # now we got clean _data to compare # next step get all examples in the package (whether whole library or # single package) @@ -4338,17 +4340,16 @@ def getCoverage(self): package_path = os.path.join(self._libHome, package) for dirpath, dirnames, filenames in os.walk(package_path): for filename in filenames: + filepath = os.path.abspath(os.path.join(dirpath, filename)) if any( - xs in filename for xs in ['Examples', 'Validation'] - ) and not filename.endswith(('package.mo', '.order')): - all_examples.append(os.path.abspath( - os.path.join(dirpath, filename)) - ) + xs in filepath for xs in ['Examples', 'Validation'] + ) and not filepath.endswith(('package.mo', '.order')): + all_examples.append(filepath) - coverage = round(len(temp_data) / len(all_examples), 2) * 100 + coverage = round(len(self._data) / len(all_examples), 2) * 100 tested_model_names = [ - nam['ScriptFile'].split(os.sep)[-1][:-1] for nam in temp_data + nam['ScriptFile'].split(os.sep)[-1][:-1] for nam in self._data ] missing_examples = [ @@ -4356,7 +4357,7 @@ def getCoverage(self): xs in i for xs in tested_model_names) ] - n_tested_examples = len(temp_data) + n_tested_examples = len(self._data) n_examples = len(all_examples) return coverage, n_tested_examples, n_examples, missing_examples, packages @@ -4375,22 +4376,28 @@ def printCoverage( The default printer is the ``reporter.writeOutput``. If another printing method is required, e.g. ``print`` or ``logging.info``, it may be passed via the ``printer`` argument. + + Example: + >>> from buildingspy.development.regressiontest import Tester + >>> import os + >>> ut = Tester(tool='dymola') + >>> myMoLib = os.path.join("buildingspy", "tests", "MyModelicaLibrary") + >>> ut.setLibraryRoot(myMoLib) + >>> ut.setSinglePackage('Examples') + >>> coverage_result = ut.getCoverage() + >>> ut.printCoverage(*coverage_result, printer=print) """ if printer is None: printer = self._reporter.writeOutput - printer('***\n\nModel Coverage: ', str(int(coverage)) + '%') + printer(f'***\nModel Coverage: {int(coverage)} %') printer( - '***\n\nYou are testing : ', - n_tested_examples, - ' out of ', - n_examples, - 'total examples in ' + f'***\nYou are testing: {n_tested_examples} ' + f'out of {n_examples} examples in package{"s" if len(packages) > 1 else ""}:', ) for package in packages: printer(package) - printer('\n') if missing_examples: - print('***\n\nThe following examples are not tested\n') + print('***\nThe following examples are not tested\n') for i in missing_examples: print(i.split(self._libHome)[1]) diff --git a/buildingspy/tests/test_development_regressiontest.py b/buildingspy/tests/test_development_regressiontest.py index 6d48fbe3..bf34f718 100644 --- a/buildingspy/tests/test_development_regressiontest.py +++ b/buildingspy/tests/test_development_regressiontest.py @@ -326,6 +326,41 @@ def test_expand_packages(self): self.assertRaises(ValueError, r.Tester.expand_packages, "AB}a{") + def test_get_coverage_single_package(self): + coverage_result = self._test_get_and_print_coverage(package="Examples") + self.assertEqual(coverage_result[0], 88) + self.assertEqual(coverage_result[1], 7) + self.assertEqual(coverage_result[2], 8) + self.assertTrue(coverage_result[3][0].endswith("ParameterEvaluation.mo")) + self.assertEqual(coverage_result[4], ["Examples"]) + + def test_get_coverage_all_packages(self): + coverage_result = self._test_get_and_print_coverage(package=None) + self.assertEqual(coverage_result[0], 89) + self.assertEqual(coverage_result[1], 8) + self.assertEqual(coverage_result[2], 9) + self.assertEqual(len(coverage_result[3]), 1) + self.assertEqual(len(coverage_result[4]), 2) + + def _test_get_and_print_coverage(self, package: str = None): + import buildingspy.development.regressiontest as r + ut = r.Tester(tool='dymola') + myMoLib = os.path.join("buildingspy", "tests", "MyModelicaLibrary") + ut.setLibraryRoot(myMoLib) + if package is not None: + ut.setSinglePackage(package) + coverage_result = ut.getCoverage() + self.assertIsInstance(coverage_result, tuple) + self.assertIsInstance(coverage_result[0], float) + self.assertIsInstance(coverage_result[1], int) + self.assertIsInstance(coverage_result[2], int) + self.assertIsInstance(coverage_result[3], list) + self.assertIsInstance(coverage_result[4], list) + # Check print with both custom and standard printer + ut.printCoverage(*coverage_result, printer=print) + ut.printCoverage(*coverage_result) + return coverage_result + if __name__ == '__main__': unittest.main() From cdbd098be8d7810c74226ad0ce7268f22f6dfa5e Mon Sep 17 00:00:00 2001 From: "fabian.wuellhorst" Date: Wed, 19 Jun 2024 21:01:02 +0200 Subject: [PATCH 3/3] Catch case for no examples --- buildingspy/development/regressiontest.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/buildingspy/development/regressiontest.py b/buildingspy/development/regressiontest.py index f8602272..0f6af5a1 100644 --- a/buildingspy/development/regressiontest.py +++ b/buildingspy/development/regressiontest.py @@ -4346,7 +4346,12 @@ def getCoverage(self): ) and not filepath.endswith(('package.mo', '.order')): all_examples.append(filepath) - coverage = round(len(self._data) / len(all_examples), 2) * 100 + n_tested_examples = len(temp_data) + n_examples = len(all_examples) + if n_examples > 0: + coverage = round(n_tested_examples / n_examples, 2) * 100 + else: + coverage = 100 tested_model_names = [ nam['ScriptFile'].split(os.sep)[-1][:-1] for nam in self._data @@ -4357,8 +4362,6 @@ def getCoverage(self): xs in i for xs in tested_model_names) ] - n_tested_examples = len(self._data) - n_examples = len(all_examples) return coverage, n_tested_examples, n_examples, missing_examples, packages def printCoverage(