diff --git a/build-scripts/build_xccdf.py b/build-scripts/build_xccdf.py index 1c99dde7fd6..aab2137effe 100644 --- a/build-scripts/build_xccdf.py +++ b/build-scripts/build_xccdf.py @@ -136,7 +136,7 @@ def main(): ssg.xml.ElementTree.ElementTree(xccdftree).write( args.xccdf, xml_declaration=True, encoding="utf-8") - if args.thin_ds_components_dir != "off": + if args.thin_ds_components_dir is not None and args.thin_ds_components_dir != "off": if not os.path.exists(args.thin_ds_components_dir): os.makedirs(args.thin_ds_components_dir) store_xccdf_per_profile(loader, oval_linker, args.thin_ds_components_dir) diff --git a/build-scripts/compose_ds.py b/build-scripts/compose_ds.py index 0b36a8a3668..c8f13cd1aee 100755 --- a/build-scripts/compose_ds.py +++ b/build-scripts/compose_ds.py @@ -311,8 +311,11 @@ def _compose_multiple_ds(args): for xccdf in glob.glob("{}/xccdf*.xml".format(args.multiple_ds)): oval = xccdf.replace("xccdf", "oval") ocil = xccdf.replace("xccdf", "ocil") + cpe_dict = xccdf.replace("xccdf", "cpe_dict") + cpe_oval = xccdf.replace("xccdf", "cpe_oval") + ds = compose_ds( - xccdf, oval, ocil, args.cpe_dict, args.cpe_oval, args.enable_sce + xccdf, oval, ocil, cpe_dict, cpe_oval, args.enable_sce ) output_13 = _get_thin_ds_output_path(args.output_13, xccdf.replace("xccdf_", "")) output_12 = None diff --git a/build-scripts/cpe_generate.py b/build-scripts/cpe_generate.py index 78e7c82ffea..abfe546ab7f 100755 --- a/build-scripts/cpe_generate.py +++ b/build-scripts/cpe_generate.py @@ -6,6 +6,7 @@ import os import ssg import argparse +import glob import ssg.build_cpe import ssg.id_translate @@ -13,16 +14,13 @@ import ssg.xml import ssg.yaml import ssg.oval_object_model -from ssg.constants import XCCDF12_NS +from ssg.constants import XCCDF12_NS, cpe_language_namespace # This script requires two arguments: an OVAL file and a CPE dictionary file. # It is designed to extract any inventory definitions and the tests, states, # objects and variables it references and then write them into a standalone # OVAL CPE file, along with a synchronized CPE dictionary file. -oval_ns = "http://oval.mitre.org/XMLSchema/oval-definitions-5" -cpe_ns = "http://cpe.mitre.org/dictionary/2.0" - def parse_args(): p = argparse.ArgumentParser( @@ -53,30 +51,20 @@ def parse_args(): "--cpe-items-dir", help="the directory where compiled CPE items are stored" ) + p.add_argument( + "--thin-ds-components-dir", + help="Directory to store CPE OVAL for thin data stream. (off: to disable)" + "e.g.: ~/scap-security-guide/build/rhel7/thin_ds_component/" + "Fake profiles are used to create thin DS. Components are generated for each profile." + "The minimal cpe will be generated from the minimal XCCDF, " + "which is in the same directory.", + ) return p.parse_args() -def process_cpe_oval(oval_file_path): - oval_document = ssg.oval_object_model.load_oval_document(ssg.xml.parse_file(oval_file_path)) - oval_document.product_name = os.path.basename(__file__) - - references_to_keep = ssg.oval_object_model.OVALDefinitionReference() - for oval_def in oval_document.definitions.values(): - if oval_def.class_ != "inventory": - continue - references_to_keep += oval_document.get_all_references_of_definition(oval_def.id_) - - oval_document.keep_referenced_components(references_to_keep) - - translator = ssg.id_translate.IDTranslator("ssg") - oval_document = translator.translate_oval_document(oval_document, store_defname=True) - - return oval_document - - -def get_benchmark_cpe_names(xccdf_file): +def get_benchmark_cpe_names(xccdf_el_root_xml): benchmark_cpe_names = set() - xccdf_el_root_xml = ssg.xml.parse_file(xccdf_file) + for platform in xccdf_el_root_xml.findall(".//{%s}platform" % XCCDF12_NS): cpe_name = platform.get("idref") # skip CPE AL platforms (they are handled later) @@ -85,9 +73,7 @@ def get_benchmark_cpe_names(xccdf_file): continue benchmark_cpe_names.add(cpe_name) - for fact_ref in xccdf_el_root_xml.findall( - ".//{%s}fact-ref" % ssg.constants.PREFIX_TO_NS["cpe-lang"] - ): + for fact_ref in xccdf_el_root_xml.findall(".//{%s}fact-ref" % cpe_language_namespace): cpe_fact_ref_name = fact_ref.get("name") benchmark_cpe_names.add(cpe_fact_ref_name) return benchmark_cpe_names @@ -103,6 +89,51 @@ def load_cpe_dictionary(benchmark_cpe_names, product_yaml, cpe_items_dir): return cpe_list +def _get_all_check_fact_ref(xccdf_el_root_xml): + return xccdf_el_root_xml.findall(".//{%s}check-fact-ref" % cpe_language_namespace) + + +def get_all_cpe_oval_def_ids(xccdf_el_root_xml, cpe_dict, benchmark_cpe_names): + out = set() + + for cpe_item in cpe_dict.cpe_items: + if cpe_item.name in benchmark_cpe_names: + out.add(cpe_item.check_id) + + for check_fact_ref in _get_all_check_fact_ref(xccdf_el_root_xml): + out.add(check_fact_ref.get("id-ref")) + return out + + +def _save_minimal_cpe_oval(oval_document, path, oval_def_ids): + references_to_keep = ssg.oval_object_model.OVALDefinitionReference() + for oval_def_id in oval_def_ids: + references_to_keep += oval_document.get_all_references_of_definition(oval_def_id) + + oval_document.save_as_xml(path, references_to_keep) + + +def _update_oval_href_in_xccdf(xccdf_el_root_xml, file_name): + for check_fact_ref in _get_all_check_fact_ref(xccdf_el_root_xml): + check_fact_ref.set("href", file_name) + + +def _generate_cpe_for_thin_xccdf(thin_ds_components_dir, oval_document, cpe_dict): + for xccdf_path in glob.glob("{}/xccdf*.xml".format(thin_ds_components_dir)): + cpe_oval_path = xccdf_path.replace("xccdf_", "cpe_oval_") + cpe_dict_path = xccdf_path.replace("xccdf_", "cpe_dict_") + + xccdf_el_root_xml = ssg.xml.parse_file(xccdf_path) + benchmark_cpe_names = get_benchmark_cpe_names(xccdf_el_root_xml) + used_cpe_oval_def_ids = get_all_cpe_oval_def_ids( + xccdf_el_root_xml, cpe_dict, benchmark_cpe_names + ) + cpe_dict.to_file(cpe_dict_path, os.path.basename(cpe_oval_path), benchmark_cpe_names) + _save_minimal_cpe_oval(oval_document, cpe_oval_path, used_cpe_oval_def_ids) + _update_oval_href_in_xccdf(xccdf_el_root_xml, os.path.basename(cpe_oval_path)) + ssg.xml.ElementTree.ElementTree(xccdf_el_root_xml).write(xccdf_path, encoding="utf-8") + + def main(): args = parse_args() @@ -116,14 +147,24 @@ def main(): cpe_dict_filename = "ssg-{}-cpe-dictionary.xml".format(product) cpe_dict_path = os.path.join(args.cpeoutdir, cpe_dict_filename) - oval_document = process_cpe_oval(args.ovalfile) - oval_document.save_as_xml(oval_file_path) + xccdf_el_root_xml = ssg.xml.parse_file(args.xccdfFile) - benchmark_cpe_names = get_benchmark_cpe_names(args.xccdfFile) + benchmark_cpe_names = get_benchmark_cpe_names(xccdf_el_root_xml) cpe_dict = load_cpe_dictionary(benchmark_cpe_names, product_yaml, args.cpe_items_dir) cpe_dict.translate_cpe_oval_def_ids() cpe_dict.to_file(cpe_dict_path, oval_filename) + used_cpe_oval_def_ids = get_all_cpe_oval_def_ids( + xccdf_el_root_xml, cpe_dict, benchmark_cpe_names + ) + oval_document = ssg.build_cpe.get_linked_cpe_oval_document(args.ovalfile) + _save_minimal_cpe_oval(oval_document, oval_file_path, used_cpe_oval_def_ids) + + if args.thin_ds_components_dir is not None and args.thin_ds_components_dir != "off": + if not os.path.exists(args.thin_ds_components_dir): + os.makedirs(args.thin_ds_components_dir) + _generate_cpe_for_thin_xccdf(args.thin_ds_components_dir, oval_document, cpe_dict) + sys.exit(0) diff --git a/build-scripts/enable_derivatives.py b/build-scripts/enable_derivatives.py index 53e5eae1d0f..4a81a85a645 100755 --- a/build-scripts/enable_derivatives.py +++ b/build-scripts/enable_derivatives.py @@ -16,8 +16,8 @@ import sys from optparse import OptionParser -import ssg.constants import ssg.build_derivatives +import ssg.constants import ssg.xccdf import ssg.xml @@ -48,6 +48,12 @@ def parse_args(): parser.add_option( "--cpe-items-dir", dest="cpe_items_dir", help="path to the directory where compiled cpe items are stored") + parser.add_option( + "--unlinked-cpe-oval-path", + dest="unlinked_oval_file_path", + help="path to the unlinked cpe oval" + ) + (options, args) = parser.parse_args() if options.centos and options.sl: @@ -63,6 +69,12 @@ def parse_args(): return options, args +def store_xml(tree, path): + if hasattr(ssg.xml.ElementTree, "indent"): + ssg.xml.ElementTree.indent(tree, space=" ", level=0) + tree.write(path, encoding="utf-8", xml_declaration=True) + + def main(): options, args = parse_args() @@ -112,10 +124,15 @@ def main(): ) ssg.build_derivatives.replace_platform(root, oval_ns, derivative) - ssg.build_derivatives.add_cpe_item_to_dictionary( - root, args[0], args[1], options.id_name, options.cpe_items_dir) + oval_def_id = ssg.build_derivatives.add_cpe_item_to_dictionary( + root, args[0], args[1], options.id_name, options.cpe_items_dir + ) + if oval_def_id is not None: + ssg.build_derivatives.add_oval_definition_to_cpe_oval( + root, options.unlinked_oval_file_path, oval_def_id + ) - tree.write(options.output) + store_xml(tree, options.output) if __name__ == "__main__": diff --git a/cmake/SSGCommon.cmake b/cmake/SSGCommon.cmake index 6c734c0b3c7..50f655c9530 100644 --- a/cmake/SSGCommon.cmake +++ b/cmake/SSGCommon.cmake @@ -383,7 +383,7 @@ macro(ssg_build_cpe_dictionary PRODUCT) add_custom_command( OUTPUT "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-dictionary.xml" OUTPUT "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-oval.xml" - COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/cpe_generate.py" --product-yaml "${CMAKE_CURRENT_BINARY_DIR}/product.yml" --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" "${CMAKE_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml" "${CMAKE_CURRENT_BINARY_DIR}/cpe-oval-unlinked.xml" + COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/cpe_generate.py" --product-yaml "${CMAKE_CURRENT_BINARY_DIR}/product.yml" --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" "${CMAKE_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml" "${CMAKE_CURRENT_BINARY_DIR}/cpe-oval-unlinked.xml" --thin-ds-components-dir "${SSG_THIN_DS_COMPONENTS_DIR}" COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-dictionary.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-dictionary.xml" COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-oval.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-oval.xml" DEPENDS generate-${PRODUCT}-xccdf-oval-ocil "${CMAKE_CURRENT_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml" "${CMAKE_CURRENT_BINARY_DIR}/ssg-${PRODUCT}-oval.xml" "${CMAKE_CURRENT_BINARY_DIR}/ssg-${PRODUCT}-ocil.xml" @@ -932,7 +932,7 @@ macro(ssg_build_derivative_product ORIGINAL SHORTNAME DERIVATIVE) add_custom_command( OUTPUT "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-xccdf.xml" - COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-xccdf.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-xccdf.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" + COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-xccdf.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-xccdf.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" --unlinked-cpe-oval-path "${CMAKE_CURRENT_BINARY_DIR}/cpe-oval-unlinked.xml" DEPENDS generate-ssg-${ORIGINAL}-xccdf.xml "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-xccdf.xml" DEPENDS ${PRODUCT}-compile-all "${CMAKE_CURRENT_BINARY_DIR}/ssg_build_compile_all-${PRODUCT}" COMMENT "[${DERIVATIVE}-content] generating ssg-${DERIVATIVE}-xccdf.xml" @@ -944,7 +944,7 @@ macro(ssg_build_derivative_product ORIGINAL SHORTNAME DERIVATIVE) add_custom_command( OUTPUT "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds.xml" - COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" + COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" --unlinked-cpe-oval-path "${CMAKE_CURRENT_BINARY_DIR}/cpe-oval-unlinked.xml" COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds.xml" "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds.xml" DEPENDS generate-ssg-${ORIGINAL}-ds.xml "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds.xml" DEPENDS ${PRODUCT}-compile-all "${CMAKE_CURRENT_BINARY_DIR}/ssg_build_compile_all-${PRODUCT}" @@ -958,7 +958,7 @@ macro(ssg_build_derivative_product ORIGINAL SHORTNAME DERIVATIVE) if(SSG_BUILD_SCAP_12_DS) add_custom_command( OUTPUT "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds-1.2.xml" - COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds-1.2.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds-1.2.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" + COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds-1.2.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds-1.2.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" --unlinked-cpe-oval-path "${CMAKE_CURRENT_BINARY_DIR}/cpe-oval-unlinked.xml" COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds-1.2.xml" "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds-1.2.xml" DEPENDS generate-ssg-${ORIGINAL}-ds.xml "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds.xml" "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds-1.2.xml" DEPENDS ${PRODUCT}-compile-all "${CMAKE_CURRENT_BINARY_DIR}/ssg_build_compile_all-${PRODUCT}" diff --git a/ssg/build_cpe.py b/ssg/build_cpe.py index dca04276852..3c3b981e545 100644 --- a/ssg/build_cpe.py +++ b/ssg/build_cpe.py @@ -15,6 +15,9 @@ from .boolean_expression import Algebra, Symbol, Function from .entities.common import XCCDFEntity, Templatable from .yaml import convert_string_to_bool +from .oval_object_model import load_oval_document, OVALDefinitionReference +from .id_translate import IDTranslator +from .xml import parse_file class CPEDoesNotExist(Exception): @@ -147,21 +150,35 @@ def __init__(self): def add(self, cpe_item): self.cpe_items.append(cpe_item) - def to_xml_element(self, cpe_oval_file): + @staticmethod + def _create_cpe_list_xml_skeleton(): cpe_list = ET.Element("{%s}cpe-list" % CPEList.ns) cpe_list.set("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance") cpe_list.set("xsi:schemaLocation", "http://cpe.mitre.org/dictionary/2.0 " "http://cpe.mitre.org/files/cpe-dictionary_2.1.xsd") + return cpe_list + def _add_cpe_items_xml(self, cpe_list, cpe_oval_file, selection_of_cpe_names): self.cpe_items.sort(key=lambda cpe: cpe.name) for cpe_item in self.cpe_items: - cpe_list.append(cpe_item.to_xml_element(cpe_oval_file)) + if cpe_item.name in selection_of_cpe_names: + cpe_list.append(cpe_item.to_xml_element(cpe_oval_file)) + + def to_xml_element(self, cpe_oval_file, selection_of_cpe_names=None): + cpe_list = self._create_cpe_list_xml_skeleton() + if selection_of_cpe_names is None: + selection_of_cpe_names = [cpe_item.name for cpe_item in self.cpe_items] + + self._add_cpe_items_xml(cpe_list, cpe_oval_file, selection_of_cpe_names) + + if hasattr(ET, "indent"): + ET.indent(cpe_list, space=" ", level=0) return cpe_list - def to_file(self, file_name, cpe_oval_file): - root = self.to_xml_element(cpe_oval_file) + def to_file(self, file_name, cpe_oval_file, selection_of_cpe_names=None): + root = self.to_xml_element(cpe_oval_file, selection_of_cpe_names) tree = ET.ElementTree(root) tree.write(file_name, encoding="utf-8") @@ -439,3 +456,23 @@ def extract_referred_nodes(tree_with_refs, tree_with_ids, attrname): elementlist.append(element) return elementlist + + +def get_linked_cpe_oval_document(unlinked_oval_file_path): + oval_document = load_oval_document(parse_file(unlinked_oval_file_path)) + oval_document.product_name = os.path.basename(__file__) + + references_to_keep = OVALDefinitionReference() + for oval_def in oval_document.definitions.values(): + if oval_def.class_ != "inventory": + continue + references_to_keep += oval_document.get_all_references_of_definition( + oval_def.id_ + ) + + oval_document.keep_referenced_components(references_to_keep) + + translator = IDTranslator("ssg") + oval_document = translator.translate_oval_document(oval_document) + + return oval_document diff --git a/ssg/build_derivatives.py b/ssg/build_derivatives.py index 11c8adee30e..79fc3eb3089 100644 --- a/ssg/build_derivatives.py +++ b/ssg/build_derivatives.py @@ -7,9 +7,14 @@ import re from .xml import ElementTree -from .constants import standard_profiles, OSCAP_VENDOR, PREFIX_TO_NS, oval_namespace -from .build_cpe import ProductCPEs -from .id_translate import IDTranslator +from .constants import ( + standard_profiles, + OSCAP_VENDOR, + cpe_dictionary_namespace, + oval_namespace, + datastream_namespace, +) +from .build_cpe import ProductCPEs, get_linked_cpe_oval_document from .products import load_product_yaml @@ -44,16 +49,65 @@ def add_cpes(elem, namespace, mapping): return affected -def add_cpe_item_to_dictionary(tree_root, product_yaml_path, cpe_ref, id_name, cpe_items_dir): - cpe_list = tree_root.find(".//{%s}cpe-list" % (PREFIX_TO_NS["cpe-dict"])) +def get_cpe_item(product_yaml, cpe_ref, cpe_items_dir): + product_cpes = ProductCPEs() + product_cpes.load_cpes_from_directory_tree(cpe_items_dir, product_yaml) + return product_cpes.get_cpe(cpe_ref) + + +def add_cpe_item_to_dictionary( + tree_root, product_yaml_path, cpe_ref, id_name, cpe_items_dir +): + cpe_list = tree_root.find(".//{%s}cpe-list" % cpe_dictionary_namespace) if cpe_list is not None: product_yaml = load_product_yaml(product_yaml_path) - product_cpes = ProductCPEs() - product_cpes.load_cpes_from_directory_tree(cpe_items_dir, product_yaml) - cpe_item = product_cpes.get_cpe(cpe_ref) - translator = IDTranslator(id_name) - cpe_item.check_id = translator.generate_id("{" + oval_namespace + "}definition", cpe_item.check_id) - cpe_list.append(cpe_item.to_xml_element("ssg-%s-cpe-oval.xml" % product_yaml.get("product"))) + cpe_item = get_cpe_item(product_yaml, cpe_ref, cpe_items_dir) + cpe_item.content_id = id_name + cpe_item.set_cpe_oval_def_id() + cpe_list.append( + cpe_item.to_xml_element("ssg-%s-cpe-oval.xml" % product_yaml.get("product")) + ) + return cpe_item.cpe_oval_short_def_id + return None + + +def add_element_to(oval_root, tag_name, component_element): + xml_el = oval_root.find(".//{%s}%s" % (oval_namespace, tag_name)) + if xml_el is None: + xml_el = ElementTree.Element("{%s}%s" % (oval_namespace, tag_name)) + oval_root.append(xml_el) + xml_el.append(component_element) + + +def add_oval_components_to_oval_xml(oval_root, tag_name, component_dict): + for component in component_dict.values(): + add_element_to(oval_root, tag_name, component.get_xml_element()) + + +def get_cpe_oval_root(root): + for component_el in root.findall("./{%s}component" % datastream_namespace): + if "cpe-oval" in component_el.get("id", ""): + return component_el + return None + + +def add_oval_definition_to_cpe_oval(root, unlinked_oval_file_path, oval_def_id): + oval_cpe_root = get_cpe_oval_root(root) + if oval_cpe_root is None: + raise Exception("CPE OVAL is missing in base DS!") + + oval_document = get_linked_cpe_oval_document(unlinked_oval_file_path) + + references_to_keep = oval_document.get_all_references_of_definition(oval_def_id) + oval_document.keep_referenced_components(references_to_keep) + + add_oval_components_to_oval_xml( + oval_cpe_root, "definitions", oval_document.definitions + ) + add_oval_components_to_oval_xml(oval_cpe_root, "tests", oval_document.tests) + add_oval_components_to_oval_xml(oval_cpe_root, "objects", oval_document.objects) + add_oval_components_to_oval_xml(oval_cpe_root, "states", oval_document.states) + add_oval_components_to_oval_xml(oval_cpe_root, "variables", oval_document.variables) def add_notice(benchmark, namespace, notice, warning): diff --git a/ssg/build_yaml.py b/ssg/build_yaml.py index 0d4cc9e9d56..cdee5506067 100644 --- a/ssg/build_yaml.py +++ b/ssg/build_yaml.py @@ -49,6 +49,14 @@ from .entities.profile import Profile, ProfileWithInlinePolicies +def _get_cpe_platforms_of_sub_groups(group, rule_ids_list): + cpe_platforms = set() + for sub_group in group.groups.values(): + cpe_platforms_of_sub_group = sub_group.get_used_cpe_platforms(rule_ids_list) + cpe_platforms.update(cpe_platforms_of_sub_group) + return cpe_platforms + + def reorder_according_to_ordering(unordered, ordering, regex=None): ordered = [] if regex is None: @@ -366,6 +374,19 @@ def get_components_not_included_in_a_profiles(self, profiles): groups.update(groups_of_sub_group) return rules, groups + def get_used_cpe_platforms(self, profiles): + selected_rules = self.get_rules_selected_in_all_profiles(profiles) + cpe_platforms = _get_cpe_platforms_of_sub_groups(self, selected_rules) + return cpe_platforms + + def get_not_used_cpe_platforms(self, profiles): + used_cpe_platforms = self.get_used_cpe_platforms(profiles) + out = set() + for cpe_platform in self.product_cpes.platforms.keys(): + if cpe_platform not in used_cpe_platforms: + out.add(cpe_platform) + return out + def get_rules_selected_in_all_profiles(self, profiles=None): selected_rules = set() if profiles is None: @@ -402,14 +423,17 @@ def _create_benchmark_xml_skeleton(self, env_yaml): return root - def _add_cpe_xml(self, root, product_cpes=None): + def _add_cpe_xml(self, root, cpe_platforms_to_not_include, product_cpes=None): # if there are no platforms, do not output platform-specification at all - if len(self.product_cpes.platforms) > 0: - cpe_platform_spec = ET.Element( - "{%s}platform-specification" % PREFIX_TO_NS["cpe-lang"]) - for platform in self.product_cpes.platforms.values(): - cpe_platform_spec.append(platform.to_xml_element()) + cpe_platform_spec = ET.Element( + "{%s}platform-specification" % PREFIX_TO_NS["cpe-lang"]) + for platform_id, platform in self.product_cpes.platforms.items(): + if platform_id in cpe_platforms_to_not_include: + continue + cpe_platform_spec.append(platform.to_xml_element()) + + if len(cpe_platform_spec) > 0: root.append(cpe_platform_spec) # The Benchmark applicability is determined by the CPEs @@ -456,13 +480,16 @@ def _add_version_xml(self, root): def to_xml_element(self, env_yaml=None, product_cpes=None, components_to_not_include=None): if components_to_not_include is None: - components_to_not_include = {} + cpe_platforms = self.get_not_used_cpe_platforms(self.profiles) + components_to_not_include = {"cpe_platforms": cpe_platforms} root = self._create_benchmark_xml_skeleton(env_yaml) add_reference_title_elements(root, env_yaml) - self._add_cpe_xml(root, product_cpes) + self._add_cpe_xml( + root, components_to_not_include.get("cpe_platforms", set()), product_cpes + ) self._add_version_xml(root) @@ -511,6 +538,7 @@ def __str__(self): def get_benchmark_xml_for_profile(self, env_yaml, profile): rules, groups = self.get_components_not_included_in_a_profiles([profile]) + cpe_platforms = self.get_not_used_cpe_platforms([profile]) profiles = set(filter( lambda id_, profile_id=profile.id_: id_ != profile_id, [profile.id_ for profile in self.profiles] @@ -518,7 +546,8 @@ def get_benchmark_xml_for_profile(self, env_yaml, profile): components_to_not_include = { "rules": rules, "groups": groups, - "profiles": profiles + "profiles": profiles, + "cpe_platforms": cpe_platforms } return profile.id_, self.to_xml_element( env_yaml, @@ -773,6 +802,18 @@ def get_not_included_components(self, rule_ids_list): groups.update(groups_of_sub_group) return rules, groups + def get_used_cpe_platforms(self, rule_ids_list): + cpe_platforms = set() + for rule_id in rule_ids_list: + if rule_id not in self.rules: + continue + rule = self.rules[rule_id] + cpe_platforms.update(rule.cpe_platform_names) + cpe_platforms.update(rule.inherited_cpe_platform_names) + + cpe_platforms.update(_get_cpe_platforms_of_sub_groups(self, rule_ids_list)) + return cpe_platforms + def __str__(self): return self.id_ diff --git a/ssg/constants.py b/ssg/constants.py index e555f8211ab..bf639c7f62b 100644 --- a/ssg/constants.py +++ b/ssg/constants.py @@ -72,6 +72,7 @@ dc_namespace = "http://purl.org/dc/elements/1.1/" ocil_namespace = "http://scap.nist.gov/schema/ocil/2.0" cpe_language_namespace = "http://cpe.mitre.org/language/2.0" +cpe_dictionary_namespace = "http://cpe.mitre.org/dictionary/2.0" oval_footer = "" oval_namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5" xlink_namespace = "http://www.w3.org/1999/xlink" @@ -139,9 +140,9 @@ "xccdf-1.2": XCCDF12_NS, "html": xhtml_namespace, "xlink": xlink_namespace, - "cpe-dict": "http://cpe.mitre.org/dictionary/2.0", + "cpe-dict": cpe_dictionary_namespace, "cat": cat_namespace, - "cpe-lang": "http://cpe.mitre.org/language/2.0", + "cpe-lang": cpe_language_namespace, "sce": sce_namespace, }