diff --git a/bundles/org.eclipse.test.performance/META-INF/MANIFEST.MF b/bundles/org.eclipse.test.performance/META-INF/MANIFEST.MF index 04d3a3946..62eb38abc 100644 --- a/bundles/org.eclipse.test.performance/META-INF/MANIFEST.MF +++ b/bundles/org.eclipse.test.performance/META-INF/MANIFEST.MF @@ -1,7 +1,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: %Plugin.name -Bundle-SymbolicName: org.eclipse.test.performance +Bundle-SymbolicName: org.eclipse.test.performance; singleton:=true Bundle-Version: 3.18.100.qualifier Bundle-Activator: org.eclipse.test.internal.performance.PerformanceTestPlugin Bundle-Vendor: %Plugin.providerName diff --git a/bundles/org.eclipse.test.performance/build.properties b/bundles/org.eclipse.test.performance/build.properties index 2b8e8a77c..4068d1267 100644 --- a/bundles/org.eclipse.test.performance/build.properties +++ b/bundles/org.eclipse.test.performance/build.properties @@ -15,6 +15,7 @@ bin.includes = .,\ plugin.properties,\ about.html,\ + plugin.xml,\ META-INF/ source.. = src/ diff --git a/bundles/org.eclipse.test.performance/plugin.xml b/bundles/org.eclipse.test.performance/plugin.xml new file mode 100644 index 000000000..a54f96818 --- /dev/null +++ b/bundles/org.eclipse.test.performance/plugin.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + diff --git a/bundles/org.eclipse.test.performance/src/org/eclipse/test/internal/performance/data/ResultsData.java b/bundles/org.eclipse.test.performance/src/org/eclipse/test/internal/performance/data/ResultsData.java new file mode 100644 index 000000000..806532fa1 --- /dev/null +++ b/bundles/org.eclipse.test.performance/src/org/eclipse/test/internal/performance/data/ResultsData.java @@ -0,0 +1,169 @@ +/******************************************************************************* + * Copyright (c) 2022 Samantha Dawley and others. + * + * This program and the accompanying materials are made available under the terms of the Eclipse Public License 2.0 which + * accompanies this distribution, and is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: Samantha Dawley - initial API and implementation + *******************************************************************************/ + +package org.eclipse.test.internal.performance.data; + +import java.io.EOFException; +//import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.nio.file.Files; +//import java.nio.file.Paths; +import java.nio.file.Path; +import java.util.Set; +import java.util.Map; +import java.util.HashMap; + +import org.eclipse.test.internal.performance.data.Sample; +import org.eclipse.test.internal.performance.data.DataPoint; +import org.eclipse.test.internal.performance.data.Scalar; +import org.eclipse.test.internal.performance.data.Dim; +import org.eclipse.test.internal.performance.db.Variations; + + +public class ResultsData{ + //Static because there shouldn't be different groups of results + private static Map CURRENT_SCENARIO_DATA; //Map of scenarioID to Sample for the current build + private static Map BASELINE_SCENARIO_DATA; //Map of scenarioID to Sample for the baseline build + + private static String CURRENT_BUILD, BASELINE_BUILD = null; + + private final static String[] ECLIPSE_COMPONENTS = { + "org.eclipse.ant", + "org.eclipse.compare", + "org.eclipse.core", + //"org.eclipse.help", + "org.eclipse.jdt.core", + "org.eclipse.jdt.debug", + "org.eclipse.jdt.text", + "org.eclipse.jdt.ui", + //"org.eclipse.jface", + //"org.eclipse.osgi", + "org.eclipse.pde.api.tools", + "org.eclipse.pde.ui", + "org.eclipse.swt", + //"org.eclipse.team", + "org.eclipse.ua", + "org.eclipse.ui" + }; + + public ResultsData(String current, String baseline) { + CURRENT_BUILD = current; + BASELINE_BUILD = baseline; + CURRENT_SCENARIO_DATA = new HashMap(); + BASELINE_SCENARIO_DATA = new HashMap(); + } + + public void importData(Path inputFile) { + System.out.println("INFO: Reading data from " + inputFile); + Variations variations = null; + try (InputStream is = Files.newInputStream(inputFile)) { + while (true) { // loop will end on input stream EOF + ObjectInputStream ois = new ObjectInputStream(is); + String scenarioID = null; + Sample sample = null; + while (scenarioID == null || variations == null || sample == null) { + Object o = ois.readObject(); + if (String.class.equals(o.getClass())) { + scenarioID = (String) o; + } else if (Variations.class.equals(o.getClass())) { + if (variations == null) { //variations only needs to be captured once + variations = (Variations) o; + } + } else if (Sample.class.equals(o.getClass())) { + sample = (Sample) o; + } else { + System.err.println("WARN: Input contains unexpected object of type " + o.getClass().getCanonicalName()); + } + } + + //System.out.println("DEBUG: Store data for scenario " + scenarioID); + String build = variations.getProperty("build"); + if ((build.equals(CURRENT_BUILD)) && (!CURRENT_SCENARIO_DATA.containsKey(scenarioID))) { + CURRENT_SCENARIO_DATA.put(scenarioID, sample); + } else if ((build.equals(BASELINE_BUILD)) && (!BASELINE_SCENARIO_DATA.containsKey(scenarioID))) { + BASELINE_SCENARIO_DATA.put(scenarioID, sample); + } else { + System.err.println("WARN: Input contains Data from the wrong build or baseline"); + } + } + } catch (EOFException ex) { + // EOFException is the intended way to end the loop + System.out.println("Finished reading data from " + inputFile); + } + catch (Exception ex) { + System.err.println("ERROR: IOException reading: " + inputFile); + System.exit(1); + } + } + + public String[] getComponents() { + return ECLIPSE_COMPONENTS; + } + + public Set getCurrentScenarios() { + return CURRENT_SCENARIO_DATA.keySet(); + } + + public Set getBaselineScenarios() { + return BASELINE_SCENARIO_DATA.keySet(); + } + + public double[] getData(String build, String scenarioID) { + Sample sample = null; + + if (build == "current") { + sample = CURRENT_SCENARIO_DATA.get(scenarioID); + } else { + sample = BASELINE_SCENARIO_DATA.get(scenarioID); + } + + DataPoint[] data = sample.getDataPoints(); + + double elapsedProcess = 0; + double cpuTime = 0; + + for (DataPoint datum : data) { + Dim[] dimensions = datum.getDimensions(); + for (Dim dim : dimensions) { + if (dim.getName().contains("Elapsed Process")) { + Scalar scalar = datum.getScalar(dim); + String value = dim.getDisplayValue(scalar); + //check for big values (1.21k etc) + if (value.substring(value.length() - 1).equals("K")){ + value = value.substring(0, value.length()-1); + Double decimalValue = Double.parseDouble(value); + decimalValue = decimalValue * 1000; + elapsedProcess += decimalValue; + } else { + elapsedProcess += Double.parseDouble(value); + } + } + if (dim.getName().contains("CPU Time")) { + Scalar scalar = datum.getScalar(dim); + String value = dim.getDisplayValue(scalar); + //check for big values (1.21k etc) + if (value.substring(value.length() - 1).equals("K")){ + value = value.substring(0, value.length()-1); + Double decimalValue = Double.parseDouble(value); + decimalValue = decimalValue * 1000; + cpuTime += decimalValue; + } else { + cpuTime += Double.parseDouble(value); + } + } + } + } + + double[] currentData = {elapsedProcess, cpuTime}; + return currentData; + } +} diff --git a/bundles/org.eclipse.test.performance/src/org/eclipse/test/performance/BasicResultsTable.java b/bundles/org.eclipse.test.performance/src/org/eclipse/test/performance/BasicResultsTable.java new file mode 100644 index 000000000..72aa20181 --- /dev/null +++ b/bundles/org.eclipse.test.performance/src/org/eclipse/test/performance/BasicResultsTable.java @@ -0,0 +1,365 @@ +/******************************************************************************* + * Copyright (c) 2022 Samantha Dawley and others. + * + * This program and the accompanying materials are made available under the terms of the Eclipse Public License 2.0 which + * accompanies this distribution, and is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: Samantha Dawley - initial API and implementation + *******************************************************************************/ +package org.eclipse.test.performance; + +import java.io.IOException; +import java.io.FileOutputStream; +import java.io.OutputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.Path; +import java.util.Set; +import java.util.HashMap; +import java.util.ArrayList; + +import org.eclipse.equinox.app.IApplication; +import org.eclipse.equinox.app.IApplicationContext; +import org.eclipse.test.internal.performance.data.ResultsData; + +public class BasicResultsTable implements IApplication{ + private static String CURRENT_BUILD, BASELINE_BUILD=null; + private static ArrayList inputFiles = new ArrayList(); + private static Path phpTemplateFile = null; + private static String EOL = System.lineSeparator(); + private static String buildDirectory = ""; + + @Override + public Object start(IApplicationContext context) { + String[] args = (String[]) context.getArguments().get("application.args"); + //DEBUG + if (args.length > 0) { + System.out.println("\n\t= = Raw arguments ('application.args') passed to performance import application: = ="); + for (String arg : args) { + System.out.println("\t\t>" + arg + "<"); + } + } + //Stuff + run(args); + return this.EXIT_OK; + } + + @Override + public void stop() { + // Do nothing + } + + public static void run(String[] args) { + + parse(args); + + //Initialize results data + ResultsData results = new ResultsData(CURRENT_BUILD, BASELINE_BUILD); + try { + // import data + System.out.println("INFO: Start importing " + inputFiles.size() + " performance data files."); + for (Path inputFile : inputFiles) { + results.importData(inputFile); + } + } catch (Exception ex) { + System.out.println("Performance data import failed with exception!" + ex); + System.exit(1); + } + + //Sort all scenarios into components, then make html file per component + Set scenarioIDs = results.getCurrentScenarios(); + ArrayList usedComponents = new ArrayList(); + //group scenarios by component for making tables + HashMap> componentMap = new HashMap>(); + + //get components from scenario name, for simplicity I'm just grabing everthing before .test/.tests but eventually should be mapped to actual components + for (String scenarioID : scenarioIDs) { + String[] scenarioParts = scenarioID.split("\\."); + String scenarioComponent = ""; + for (String part : scenarioParts) { + if (part.equals("tests")) { + break; + } + //stw is different + if (part.equals("test")) { + break; + } + scenarioComponent = scenarioComponent + part + "."; + } + //trim final . + scenarioComponent = scenarioComponent.substring(0, scenarioComponent.length()-1); + + //check if component in used components list + if (usedComponents.contains(scenarioComponent)) { + //Update HashMap entry, add scenario to components list + ArrayList componentScenarios = componentMap.get(scenarioComponent); + componentScenarios.add(scenarioID); + componentMap.replace(scenarioComponent, componentScenarios); + } else { + //Add component to used components and make new entry into HashMap + ArrayList componentScenarios = new ArrayList(); + componentScenarios.add(scenarioID); + componentMap.put(scenarioComponent, componentScenarios); + usedComponents.add(scenarioComponent); + } + } + + //for checking if a test has a baseline to reference + Set baselineScenarios = results.getBaselineScenarios(); + + //Make component html files + for (String component : usedComponents) { + ArrayList scenarioList = componentMap.get(component); + scenarioList.sort(String::compareToIgnoreCase); + + //set up html string + String htmlString = ""; + //htmlString = htmlString + EOL + "

Performance of " + component + ": " + CURRENT_BUILD + " relative to " + BASELINE_BUILD + "

" + EOL; + //htmlString = htmlString + EOL + "Back to global results" + EOL; + htmlString = htmlString + EOL + "

All " + scenarioList.size() + " scenarios:

" + EOL; + htmlString = htmlString + EOL + "

Times are given in milliseconds.

" + EOL; + htmlString = htmlString + "" + EOL + "" + EOL; + htmlString = htmlString + "" + EOL; + htmlString = htmlString + "" + EOL; + htmlString = htmlString + "" + EOL; + htmlString = htmlString + "" + EOL; + htmlString = htmlString + "" + EOL; + htmlString = htmlString + "" + EOL; + htmlString = htmlString + "" + EOL; + htmlString = htmlString + "

Class

Name

Elapsed Process (Current)

Elapsed Process (Baseline)

Difference

CPU Time (Current)

CPU Time (Baseline)

" + EOL; + htmlString = htmlString + "

Difference

" + EOL; + + for (String scenario : scenarioList) { + String[] scenarioParts = null; + String componentClass = null; + String componentName = null; + //swt is different + if (scenario.contains("swt")) { + scenarioParts = scenario.split("\\."); + componentName = scenarioParts[scenarioParts.length - 1]; + for (int i=0; i < (scenarioParts.length - 1); i++ ) { + componentClass = componentClass + scenarioParts[i] + "."; + } + //trim final . + componentClass = componentClass.substring(0, componentClass.length()-1); + } else { + scenarioParts = scenario.split("#"); + componentClass = scenarioParts[0]; + componentName = scenarioParts[1]; + } + + double[] currentData = results.getData("current", scenario); + String elapsedCurrent = String.valueOf(currentData[0]); + String cpuCurrent = String.valueOf(currentData[1]); + + String elapsedBaseline = "N/A"; + String cpuBaseline = "N/A"; + + String elapsedPercent = "N/A"; + String cpuPercent = "N/A"; + + String elapsedColor = "#4CE600"; + String cpuColor = "#4CE600"; + + if (baselineScenarios.contains(scenario)) { + double[] baselineData = results.getData("baseline", scenario); + + elapsedBaseline = String.valueOf(baselineData[0]); + cpuBaseline = String.valueOf(baselineData[1]); + + double elapsedDifference = baselineData[0] - currentData[0]; + double cpuDifference = baselineData[1] - currentData[1]; + + double elapsedPercentValue = Math.abs(elapsedDifference / baselineData[0]) * 100; + double cpuPercentValue = Math.abs(cpuDifference / baselineData[1]) * 100; + + elapsedPercent = String.format("%.2f", elapsedPercentValue) + "%"; + cpuPercent = String.format("%.2f", cpuPercentValue) + "%"; + + if (elapsedDifference < 0) { + elapsedColor = "D7191C"; + } + if (cpuDifference < 0) { + cpuColor = "D7191C"; + } + + } + + htmlString = htmlString + "
" + componentClass + EOL; + htmlString = htmlString + "" + componentName + EOL; + htmlString = htmlString + "" + elapsedCurrent + EOL; + htmlString = htmlString + "" + elapsedBaseline + EOL; + htmlString = htmlString + "" + elapsedPercent + EOL; + htmlString = htmlString + "" + cpuCurrent + EOL; + htmlString = htmlString + "" + cpuBaseline + EOL; + htmlString = htmlString + "" + cpuPercent + EOL; + } + + htmlString = htmlString + "
" + EOL; + + //create file + String outputFileName = buildDirectory + "/" + component + "_BasicTable.html"; + File outputFile = new File(outputFileName); + + try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(outputFile))){ + outputStream.write(htmlString.getBytes()); + } + catch (final FileNotFoundException ex) { + System.err.println("ERROR: File not found exception while writing: " + outputFile.getPath()); + System.exit(1); + } + catch (final IOException ex) { + System.err.println("ERROR: IOException writing: " + outputFile.getPath()); + System.exit(1); + } + + } + + //make basicResultsIndex.html file + String htmlString = ""; + htmlString = htmlString + EOL; + + for (String component : usedComponents) { + htmlString = htmlString + "" + component + "*
" + EOL; + } + + //create file + String outputFileName = buildDirectory + "/BasicResultsIndex.html"; + File outputFile = new File(outputFileName); + + try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(outputFile))){ + outputStream.write(htmlString.getBytes()); + } + catch (final FileNotFoundException ex) { + System.err.println("ERROR: File not found exception while writing: " + outputFile.getPath()); + System.exit(1); + } + catch (final IOException ex) { + System.err.println("ERROR: IOException writing: " + outputFile.getPath()); + System.exit(1); + } + + //copy basicPerformance.php from templatefiles + String phpFileName = buildDirectory + "/basicPerformance.php"; + File phpFile = new File(phpFileName); + + try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(phpFile))){ + outputStream.write(Files.readAllBytes(phpTemplateFile)); + } + catch (final FileNotFoundException ex) { + System.err.println(EOL + "ERROR: File not found exception while writing: " + phpFile.getPath()); + System.exit(1); + } + catch (final IOException ex) { + System.err.println(EOL + "ERROR: IOException writing: " + phpFile.getPath()); + System.exit(1); + } + + } + + //args = baseline, current build, input file array + private static void parse(String[] args) { + if (args.length == 0) { + printUsage(); + } + + int i = 0; + while (i < args.length) { + String arg = args[i]; + if (arg.equals("-current")){ + CURRENT_BUILD = args[i+1]; + if (CURRENT_BUILD.startsWith("-")) { + System.out.println("Missing value for "+arg+" parameter"); + printUsage(); + } + i++; + i++; + continue; + } + if (arg.equals("-baseline")){ + BASELINE_BUILD = args[i+1]; + if (BASELINE_BUILD.startsWith("-")) { + System.out.println("Missing value for "+arg+" parameter"); + printUsage(); + } + i++; + i++; + continue; + } + if (arg.equals("-buildDirectory")){ + buildDirectory = args[i+1]; + if (buildDirectory.startsWith("-")) { + System.out.println("Missing value for "+arg+" parameter"); + printUsage(); + } + i++; + i++; + continue; + } + if (arg.equals("-phpFile")){ + String inputFile = args[i+1]; + + if (inputFile.startsWith("-")) { + System.out.println("Missing value for "+arg+" parameter"); + printUsage(); + } + //check real file + Path inputFilePath = Paths.get(inputFile); + if (Files.isReadable(inputFilePath)) { + phpTemplateFile = inputFilePath; + } else { + System.err.println("ERROR: invalid input argument. Cannot read file: " + inputFile); + } + + i++; + i++; + continue; + } + if (arg.equals("-inputFiles")){ + for (int j=1; j < 5; j++) { + String inputFile = args[i+j]; + + if (inputFile.startsWith("-")) { + System.out.println("Missing value for "+arg+" parameter"); + printUsage(); + } + //check real file + //Path inputFilePath = Paths.get(buildDirectory + "/" + inputFile); + Path inputFilePath = Paths.get(inputFile); + if (Files.isReadable(inputFilePath)) { + inputFiles.add(inputFilePath); + } else { + System.err.println("ERROR: invalid input argument. Cannot read file: " + inputFile); + } + } + i = i+5; + continue; + } + System.err.println("ERROR: Unrecognized argument (arg) found, with value of >" + arg + "<"); + i++; + continue; + } + + } + + private static void printUsage() { + System.out.println( + "Usage:\n" + + "-baseline: Build id for the baseline build.\n" + + "-current: Build id for the current build.\n" + + "-buildDirectory: Directory of performance.php file, usually /home/data/httpd/download.eclipse.org/eclipse/downloads/drops4/${BUILD_ID}/performance.\n" + + "-phpFile: Location of the basicPerformance.php file, also known as template file.\n" + + "-inputFiles: List of the dat files from which to extract performance data (will grab the next 4 args as filenames).\n" + ); + System.exit(1); + } +}