Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduction of CPE content in DS #11648

Merged
merged 16 commits into from
Mar 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build-scripts/build_xccdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ def main():
ssg.xml.ElementTree.ElementTree(xccdftree).write(
args.xccdf, xml_declaration=True, encoding="utf-8")

if args.thin_ds_components_dir != "off":
if args.thin_ds_components_dir is not None and args.thin_ds_components_dir != "off":
if not os.path.exists(args.thin_ds_components_dir):
os.makedirs(args.thin_ds_components_dir)
store_xccdf_per_profile(loader, oval_linker, args.thin_ds_components_dir)
Expand Down
5 changes: 4 additions & 1 deletion build-scripts/compose_ds.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,8 +311,11 @@ def _compose_multiple_ds(args):
for xccdf in glob.glob("{}/xccdf*.xml".format(args.multiple_ds)):
oval = xccdf.replace("xccdf", "oval")
ocil = xccdf.replace("xccdf", "ocil")
cpe_dict = xccdf.replace("xccdf", "cpe_dict")
cpe_oval = xccdf.replace("xccdf", "cpe_oval")

ds = compose_ds(
xccdf, oval, ocil, args.cpe_dict, args.cpe_oval, args.enable_sce
xccdf, oval, ocil, cpe_dict, cpe_oval, args.enable_sce
)
output_13 = _get_thin_ds_output_path(args.output_13, xccdf.replace("xccdf_", ""))
output_12 = None
Expand Down
101 changes: 71 additions & 30 deletions build-scripts/cpe_generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,21 @@
import os
import ssg
import argparse
import glob

import ssg.build_cpe
import ssg.id_translate
import ssg.products
import ssg.xml
import ssg.yaml
import ssg.oval_object_model
from ssg.constants import XCCDF12_NS
from ssg.constants import XCCDF12_NS, cpe_language_namespace

# This script requires two arguments: an OVAL file and a CPE dictionary file.
# It is designed to extract any inventory definitions and the tests, states,
# objects and variables it references and then write them into a standalone
# OVAL CPE file, along with a synchronized CPE dictionary file.

oval_ns = "http://oval.mitre.org/XMLSchema/oval-definitions-5"
cpe_ns = "http://cpe.mitre.org/dictionary/2.0"


def parse_args():
p = argparse.ArgumentParser(
Expand Down Expand Up @@ -53,30 +51,20 @@ def parse_args():
"--cpe-items-dir",
help="the directory where compiled CPE items are stored"
)
p.add_argument(
"--thin-ds-components-dir",
help="Directory to store CPE OVAL for thin data stream. (off: to disable)"
"e.g.: ~/scap-security-guide/build/rhel7/thin_ds_component/"
"Fake profiles are used to create thin DS. Components are generated for each profile."
"The minimal cpe will be generated from the minimal XCCDF, "
"which is in the same directory.",
)
return p.parse_args()


def process_cpe_oval(oval_file_path):
oval_document = ssg.oval_object_model.load_oval_document(ssg.xml.parse_file(oval_file_path))
oval_document.product_name = os.path.basename(__file__)

references_to_keep = ssg.oval_object_model.OVALDefinitionReference()
for oval_def in oval_document.definitions.values():
if oval_def.class_ != "inventory":
continue
references_to_keep += oval_document.get_all_references_of_definition(oval_def.id_)

oval_document.keep_referenced_components(references_to_keep)

translator = ssg.id_translate.IDTranslator("ssg")
oval_document = translator.translate_oval_document(oval_document, store_defname=True)

return oval_document


def get_benchmark_cpe_names(xccdf_file):
def get_benchmark_cpe_names(xccdf_el_root_xml):
benchmark_cpe_names = set()
xccdf_el_root_xml = ssg.xml.parse_file(xccdf_file)

for platform in xccdf_el_root_xml.findall(".//{%s}platform" % XCCDF12_NS):
cpe_name = platform.get("idref")
# skip CPE AL platforms (they are handled later)
Expand All @@ -85,9 +73,7 @@ def get_benchmark_cpe_names(xccdf_file):
continue
benchmark_cpe_names.add(cpe_name)

for fact_ref in xccdf_el_root_xml.findall(
".//{%s}fact-ref" % ssg.constants.PREFIX_TO_NS["cpe-lang"]
):
for fact_ref in xccdf_el_root_xml.findall(".//{%s}fact-ref" % cpe_language_namespace):
cpe_fact_ref_name = fact_ref.get("name")
benchmark_cpe_names.add(cpe_fact_ref_name)
return benchmark_cpe_names
Expand All @@ -103,6 +89,51 @@ def load_cpe_dictionary(benchmark_cpe_names, product_yaml, cpe_items_dir):
return cpe_list


def _get_all_check_fact_ref(xccdf_el_root_xml):
return xccdf_el_root_xml.findall(".//{%s}check-fact-ref" % cpe_language_namespace)


def get_all_cpe_oval_def_ids(xccdf_el_root_xml, cpe_dict, benchmark_cpe_names):
out = set()

for cpe_item in cpe_dict.cpe_items:
if cpe_item.name in benchmark_cpe_names:
out.add(cpe_item.check_id)

for check_fact_ref in _get_all_check_fact_ref(xccdf_el_root_xml):
out.add(check_fact_ref.get("id-ref"))
return out


def _save_minimal_cpe_oval(oval_document, path, oval_def_ids):
references_to_keep = ssg.oval_object_model.OVALDefinitionReference()
for oval_def_id in oval_def_ids:
references_to_keep += oval_document.get_all_references_of_definition(oval_def_id)

oval_document.save_as_xml(path, references_to_keep)


def _update_oval_href_in_xccdf(xccdf_el_root_xml, file_name):
for check_fact_ref in _get_all_check_fact_ref(xccdf_el_root_xml):
check_fact_ref.set("href", file_name)


def _generate_cpe_for_thin_xccdf(thin_ds_components_dir, oval_document, cpe_dict):
for xccdf_path in glob.glob("{}/xccdf*.xml".format(thin_ds_components_dir)):
cpe_oval_path = xccdf_path.replace("xccdf_", "cpe_oval_")
cpe_dict_path = xccdf_path.replace("xccdf_", "cpe_dict_")

xccdf_el_root_xml = ssg.xml.parse_file(xccdf_path)
benchmark_cpe_names = get_benchmark_cpe_names(xccdf_el_root_xml)
used_cpe_oval_def_ids = get_all_cpe_oval_def_ids(
xccdf_el_root_xml, cpe_dict, benchmark_cpe_names
)
cpe_dict.to_file(cpe_dict_path, os.path.basename(cpe_oval_path), benchmark_cpe_names)
_save_minimal_cpe_oval(oval_document, cpe_oval_path, used_cpe_oval_def_ids)
_update_oval_href_in_xccdf(xccdf_el_root_xml, os.path.basename(cpe_oval_path))
ssg.xml.ElementTree.ElementTree(xccdf_el_root_xml).write(xccdf_path, encoding="utf-8")


def main():
args = parse_args()

Expand All @@ -116,14 +147,24 @@ def main():
cpe_dict_filename = "ssg-{}-cpe-dictionary.xml".format(product)
cpe_dict_path = os.path.join(args.cpeoutdir, cpe_dict_filename)

oval_document = process_cpe_oval(args.ovalfile)
oval_document.save_as_xml(oval_file_path)
xccdf_el_root_xml = ssg.xml.parse_file(args.xccdfFile)

benchmark_cpe_names = get_benchmark_cpe_names(args.xccdfFile)
benchmark_cpe_names = get_benchmark_cpe_names(xccdf_el_root_xml)
cpe_dict = load_cpe_dictionary(benchmark_cpe_names, product_yaml, args.cpe_items_dir)
cpe_dict.translate_cpe_oval_def_ids()
cpe_dict.to_file(cpe_dict_path, oval_filename)

used_cpe_oval_def_ids = get_all_cpe_oval_def_ids(
xccdf_el_root_xml, cpe_dict, benchmark_cpe_names
)
oval_document = ssg.build_cpe.get_linked_cpe_oval_document(args.ovalfile)
_save_minimal_cpe_oval(oval_document, oval_file_path, used_cpe_oval_def_ids)

if args.thin_ds_components_dir is not None and args.thin_ds_components_dir != "off":
if not os.path.exists(args.thin_ds_components_dir):
os.makedirs(args.thin_ds_components_dir)
_generate_cpe_for_thin_xccdf(args.thin_ds_components_dir, oval_document, cpe_dict)

sys.exit(0)


Expand Down
25 changes: 21 additions & 4 deletions build-scripts/enable_derivatives.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
import sys
from optparse import OptionParser

import ssg.constants
import ssg.build_derivatives
import ssg.constants
import ssg.xccdf
import ssg.xml

Expand Down Expand Up @@ -48,6 +48,12 @@ def parse_args():
parser.add_option(
"--cpe-items-dir",
dest="cpe_items_dir", help="path to the directory where compiled cpe items are stored")
parser.add_option(
"--unlinked-cpe-oval-path",
dest="unlinked_oval_file_path",
help="path to the unlinked cpe oval"
)

(options, args) = parser.parse_args()

if options.centos and options.sl:
Expand All @@ -63,6 +69,12 @@ def parse_args():
return options, args


def store_xml(tree, path):
if hasattr(ssg.xml.ElementTree, "indent"):
ssg.xml.ElementTree.indent(tree, space=" ", level=0)
tree.write(path, encoding="utf-8", xml_declaration=True)


def main():
options, args = parse_args()

Expand Down Expand Up @@ -112,10 +124,15 @@ def main():
)

ssg.build_derivatives.replace_platform(root, oval_ns, derivative)
ssg.build_derivatives.add_cpe_item_to_dictionary(
root, args[0], args[1], options.id_name, options.cpe_items_dir)
oval_def_id = ssg.build_derivatives.add_cpe_item_to_dictionary(
root, args[0], args[1], options.id_name, options.cpe_items_dir
)
if oval_def_id is not None:
ssg.build_derivatives.add_oval_definition_to_cpe_oval(
root, options.unlinked_oval_file_path, oval_def_id
)

tree.write(options.output)
store_xml(tree, options.output)


if __name__ == "__main__":
Expand Down
8 changes: 4 additions & 4 deletions cmake/SSGCommon.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -383,7 +383,7 @@ macro(ssg_build_cpe_dictionary PRODUCT)
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-dictionary.xml"
OUTPUT "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-oval.xml"
COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/cpe_generate.py" --product-yaml "${CMAKE_CURRENT_BINARY_DIR}/product.yml" --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" "${CMAKE_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml" "${CMAKE_CURRENT_BINARY_DIR}/cpe-oval-unlinked.xml"
COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/cpe_generate.py" --product-yaml "${CMAKE_CURRENT_BINARY_DIR}/product.yml" --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" "${CMAKE_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml" "${CMAKE_CURRENT_BINARY_DIR}/cpe-oval-unlinked.xml" --thin-ds-components-dir "${SSG_THIN_DS_COMPONENTS_DIR}"
COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-dictionary.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-dictionary.xml"
COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-oval.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-oval.xml"
DEPENDS generate-${PRODUCT}-xccdf-oval-ocil "${CMAKE_CURRENT_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml" "${CMAKE_CURRENT_BINARY_DIR}/ssg-${PRODUCT}-oval.xml" "${CMAKE_CURRENT_BINARY_DIR}/ssg-${PRODUCT}-ocil.xml"
Expand Down Expand Up @@ -932,7 +932,7 @@ macro(ssg_build_derivative_product ORIGINAL SHORTNAME DERIVATIVE)

add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-xccdf.xml"
COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-xccdf.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-xccdf.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items"
COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-xccdf.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-xccdf.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" --unlinked-cpe-oval-path "${CMAKE_CURRENT_BINARY_DIR}/cpe-oval-unlinked.xml"
DEPENDS generate-ssg-${ORIGINAL}-xccdf.xml "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-xccdf.xml"
DEPENDS ${PRODUCT}-compile-all "${CMAKE_CURRENT_BINARY_DIR}/ssg_build_compile_all-${PRODUCT}"
COMMENT "[${DERIVATIVE}-content] generating ssg-${DERIVATIVE}-xccdf.xml"
Expand All @@ -944,7 +944,7 @@ macro(ssg_build_derivative_product ORIGINAL SHORTNAME DERIVATIVE)

add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds.xml"
COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items"
COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" --unlinked-cpe-oval-path "${CMAKE_CURRENT_BINARY_DIR}/cpe-oval-unlinked.xml"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that the xmllint reformat needs to stay there because apparently OpenSCAP has problems with not pretty formatted XMLs and on RHEL 7 the Python doesn't produce pretty outputs.

The segfault is reproducible, it's not a random glitch.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, fixed, after RHEL 7 is not supported we can remove the formatting tool.

COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds.xml" "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds.xml"
DEPENDS generate-ssg-${ORIGINAL}-ds.xml "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds.xml"
DEPENDS ${PRODUCT}-compile-all "${CMAKE_CURRENT_BINARY_DIR}/ssg_build_compile_all-${PRODUCT}"
Expand All @@ -958,7 +958,7 @@ macro(ssg_build_derivative_product ORIGINAL SHORTNAME DERIVATIVE)
if(SSG_BUILD_SCAP_12_DS)
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds-1.2.xml"
COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds-1.2.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds-1.2.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items"
COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/enable_derivatives.py" --enable-${SHORTNAME} -i "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds-1.2.xml" -o "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds-1.2.xml" "${CMAKE_CURRENT_BINARY_DIR}/product.yml" ${DERIVATIVE} --id-name ssg --cpe-items-dir "${CMAKE_CURRENT_BINARY_DIR}/cpe_items" --unlinked-cpe-oval-path "${CMAKE_CURRENT_BINARY_DIR}/cpe-oval-unlinked.xml"
COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds-1.2.xml" "${CMAKE_BINARY_DIR}/ssg-${DERIVATIVE}-ds-1.2.xml"
DEPENDS generate-ssg-${ORIGINAL}-ds.xml "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds.xml" "${CMAKE_BINARY_DIR}/ssg-${ORIGINAL}-ds-1.2.xml"
DEPENDS ${PRODUCT}-compile-all "${CMAKE_CURRENT_BINARY_DIR}/ssg_build_compile_all-${PRODUCT}"
Expand Down
45 changes: 41 additions & 4 deletions ssg/build_cpe.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@
from .boolean_expression import Algebra, Symbol, Function
from .entities.common import XCCDFEntity, Templatable
from .yaml import convert_string_to_bool
from .oval_object_model import load_oval_document, OVALDefinitionReference
from .id_translate import IDTranslator
from .xml import parse_file


class CPEDoesNotExist(Exception):
Expand Down Expand Up @@ -147,21 +150,35 @@ def __init__(self):
def add(self, cpe_item):
self.cpe_items.append(cpe_item)

def to_xml_element(self, cpe_oval_file):
@staticmethod
def _create_cpe_list_xml_skeleton():
cpe_list = ET.Element("{%s}cpe-list" % CPEList.ns)
cpe_list.set("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance")
cpe_list.set("xsi:schemaLocation",
"http://cpe.mitre.org/dictionary/2.0 "
"http://cpe.mitre.org/files/cpe-dictionary_2.1.xsd")
return cpe_list

def _add_cpe_items_xml(self, cpe_list, cpe_oval_file, selection_of_cpe_names):
self.cpe_items.sort(key=lambda cpe: cpe.name)
for cpe_item in self.cpe_items:
cpe_list.append(cpe_item.to_xml_element(cpe_oval_file))
if cpe_item.name in selection_of_cpe_names:
cpe_list.append(cpe_item.to_xml_element(cpe_oval_file))

def to_xml_element(self, cpe_oval_file, selection_of_cpe_names=None):
cpe_list = self._create_cpe_list_xml_skeleton()

if selection_of_cpe_names is None:
selection_of_cpe_names = [cpe_item.name for cpe_item in self.cpe_items]

self._add_cpe_items_xml(cpe_list, cpe_oval_file, selection_of_cpe_names)

if hasattr(ET, "indent"):
ET.indent(cpe_list, space=" ", level=0)
return cpe_list

def to_file(self, file_name, cpe_oval_file):
root = self.to_xml_element(cpe_oval_file)
def to_file(self, file_name, cpe_oval_file, selection_of_cpe_names=None):
root = self.to_xml_element(cpe_oval_file, selection_of_cpe_names)
tree = ET.ElementTree(root)
tree.write(file_name, encoding="utf-8")

Expand Down Expand Up @@ -439,3 +456,23 @@ def extract_referred_nodes(tree_with_refs, tree_with_ids, attrname):
elementlist.append(element)

return elementlist


def get_linked_cpe_oval_document(unlinked_oval_file_path):
oval_document = load_oval_document(parse_file(unlinked_oval_file_path))
oval_document.product_name = os.path.basename(__file__)

references_to_keep = OVALDefinitionReference()
for oval_def in oval_document.definitions.values():
if oval_def.class_ != "inventory":
continue
references_to_keep += oval_document.get_all_references_of_definition(
oval_def.id_
)

oval_document.keep_referenced_components(references_to_keep)

translator = IDTranslator("ssg")
oval_document = translator.translate_oval_document(oval_document)

return oval_document
Loading
Loading