Merge pull request #10606 from matejak/product_stability

Add a product stability test

tests/CMakeLists.txt

@@ -62,6 +62,12 @@ add_test(
 )
 set_tests_properties("stable-profiles" PROPERTIES LABELS quick)
 
+add_test(
+    NAME "stable-products"
+    COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/test_product_stability.py" "${CMAKE_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/data/product_stability"
+)
+set_tests_properties("stable-products" PROPERTIES LABELS quick)
+
 add_test(
     NAME "machine-only-rules"
     COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/test_machine_only_rules.py" --source_dir "${CMAKE_SOURCE_DIR}" --build_dir "${CMAKE_BINARY_DIR}"

tests/common/stability.py

@@ -0,0 +1,40 @@
+from __future__ import print_function
+
+import sys
+
+
+class Difference(object):
+    def __init__(self):
+        self.added = []
+        self.removed = []
+        self.modified = dict()
+
+    def remove_item_from_comparison(self, item):
+        if item in self.added:
+            self.added.remove(item)
+        if item in self.removed:
+            self.removed.remove(item)
+        if item in self.modified:
+            self.modified.pop(item)
+
+    @property
+    def empty(self):
+        return not (self.added or self.removed or self.modified)
+
+
+def describe_changeset(intro, changeset):
+    if not changeset:
+        return ""
+
+    msg = intro
+    for item in changeset:
+        msg += " - {item}\n".format(item=item)
+    return msg
+
+
+def report_comparison(name, result, message_generator):
+    if not result.empty:
+        msg = message_generator(result, name)
+        print(msg, file=sys.stderr)
+
+

tests/data/product_stability/alinux2.yml

@@ -0,0 +1,72 @@
+aide_bin_path: /usr/sbin/aide
+aide_conf_path: /etc/aide.conf
+audisp_conf_path: /etc/audit
+auid: 1000
+basic_properties_derived: true
+benchmark_id: ALINUX-2
+benchmark_root: ../../linux_os/guide
+chrony_conf_path: /etc/chrony.conf
+cpes:
+- alinux2:
+    check_id: installed_OS_is_alinux2
+    name: cpe:/o:alinux:alibaba_cloud_linux:2
+    title: Alibaba Cloud Linux 2
+cpes_root: ../../shared/applicability
+dconf_gdm_dir: gdm.d
+faillock_path: /var/run/faillock
+full_name: Alibaba Cloud Linux 2
+gid_min: 1000
+groups: {}
+grub2_boot_path: /boot/grub2
+grub2_uefi_boot_path: /boot/grub2
+init_system: systemd
+nobody_gid: 65534
+nobody_uid: 65534
+pkg_manager: yum
+pkg_manager_config_file: /etc/yum.conf
+pkg_system: rpm
+platform_package_overrides:
+  aarch64_arch: null
+  grub2: grub2-common
+  login_defs: shadow-utils
+  no_ovirt: null
+  non-uefi: null
+  not_aarch64_arch: null
+  not_s390x_arch: null
+  ovirt: null
+  s390x_arch: null
+  sssd: sssd-common
+  sssd-ldap: null
+  uefi: null
+  zipl: s390utils-base
+product: alinux2
+profiles_root: ./profiles
+reference_uris:
+  anssi: http://www.ssi.gouv.fr/administration/bonnes-pratiques/
+  app-srg: https://public.cyber.mil/stigs/downloads/?_dl_facet_stigs=application-servers
+  cis: https://www.cisecurity.org/benchmark/aliyun_linux
+  cis-csc: https://www.cisecurity.org/controls/
+  cjis: https://www.fbi.gov/file-repository/cjis-security-policy-v5_5_20160601-2-1.pdf
+  cnss: http://www.cnss.gov/Assets/pdf/CNSSI-1253.pdf
+  cobit5: https://www.isaca.org/resources/cobit
+  cui: http://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-171.pdf
+  dcid: not_officially_available
+  disa: https://public.cyber.mil/stigs/cci/
+  hipaa: https://www.gpo.gov/fdsys/pkg/CFR-2007-title45-vol1/pdf/CFR-2007-title45-vol1-chapA-subchapC.pdf
+  isa-62443-2009: https://www.isa.org/products/isa-62443-2-1-2009-security-for-industrial-automat
+  isa-62443-2013: https://www.isa.org/products/ansi-isa-62443-3-3-99-03-03-2013-security-for-indu
+  ism: https://www.cyber.gov.au/acsc/view-all-content/ism
+  iso27001-2013: https://www.iso.org/standard/54534.html
+  nerc-cip: https://www.nerc.com/pa/Stand/Standard%20Purpose%20Statement%20DL/US_Standard_One-Stop-Shop.xlsx
+  nist: http://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-53r4.pdf
+  nist-csf: https://nvlpubs.nist.gov/nistpubs/CSWP/NIST.CSWP.04162018.pdf
+  os-srg: https://public.cyber.mil/stigs/downloads/?_dl_facet_stigs=operating-systems%2Cgeneral-purpose-os
+  ospp: https://www.niap-ccevs.org/Profile/PP.cfm
+  pcidss: https://www.pcisecuritystandards.org/documents/PCI_DSS_v3-2-1.pdf
+  pcidss4: https://docs-prv.pcisecuritystandards.org/PCI%20DSS/Standard/PCI-DSS-v4_0.pdf
+  stigid: https://public.cyber.mil/stigs/downloads/?_dl_facet_stigs=operating-systems%2Cunix-linux
+  stigref: https://public.cyber.mil/stigs/srg-stig-tools/
+sshd_distributed_config: 'false'
+sysctl_remediate_drop_in_file: 'false'
+type: platform
+uid_min: 1000

tests/data/product_stability/alinux3.yml

@@ -0,0 +1,72 @@
+aide_bin_path: /usr/sbin/aide
+aide_conf_path: /etc/aide.conf
+audisp_conf_path: /etc/audit
+auid: 1000
+basic_properties_derived: true
+benchmark_id: ALINUX-3
+benchmark_root: ../../linux_os/guide
+chrony_conf_path: /etc/chrony.conf
+cpes:
+- alinux3:
+    check_id: installed_OS_is_alinux3
+    name: cpe:/o:alinux:alibaba_cloud_linux:3
+    title: Alibaba Cloud Linux 3
+cpes_root: ../../shared/applicability
+dconf_gdm_dir: gdm.d
+faillock_path: /var/run/faillock
+full_name: Alibaba Cloud Linux 3
+gid_min: 1000
+groups: {}
+grub2_boot_path: /boot/grub2
+grub2_uefi_boot_path: /boot/grub2
+init_system: systemd
+nobody_gid: 65534
+nobody_uid: 65534
+pkg_manager: yum
+pkg_manager_config_file: /etc/yum.conf
+pkg_system: rpm
+platform_package_overrides:
+  aarch64_arch: null
+  grub2: grub2-common
+  login_defs: shadow-utils
+  no_ovirt: null
+  non-uefi: null
+  not_aarch64_arch: null
+  not_s390x_arch: null
+  ovirt: null
+  s390x_arch: null
+  sssd: sssd-common
+  sssd-ldap: null
+  uefi: null
+  zipl: s390utils-base
+product: alinux3
+profiles_root: ./profiles
+reference_uris:
+  anssi: http://www.ssi.gouv.fr/administration/bonnes-pratiques/
+  app-srg: https://public.cyber.mil/stigs/downloads/?_dl_facet_stigs=application-servers
+  cis: https://www.cisecurity.org/benchmark/aliyun_linux
+  cis-csc: https://www.cisecurity.org/controls/
+  cjis: https://www.fbi.gov/file-repository/cjis-security-policy-v5_5_20160601-2-1.pdf
+  cnss: http://www.cnss.gov/Assets/pdf/CNSSI-1253.pdf
+  cobit5: https://www.isaca.org/resources/cobit
+  cui: http://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-171.pdf
+  dcid: not_officially_available
+  disa: https://public.cyber.mil/stigs/cci/
+  hipaa: https://www.gpo.gov/fdsys/pkg/CFR-2007-title45-vol1/pdf/CFR-2007-title45-vol1-chapA-subchapC.pdf
+  isa-62443-2009: https://www.isa.org/products/isa-62443-2-1-2009-security-for-industrial-automat
+  isa-62443-2013: https://www.isa.org/products/ansi-isa-62443-3-3-99-03-03-2013-security-for-indu
+  ism: https://www.cyber.gov.au/acsc/view-all-content/ism
+  iso27001-2013: https://www.iso.org/standard/54534.html
+  nerc-cip: https://www.nerc.com/pa/Stand/Standard%20Purpose%20Statement%20DL/US_Standard_One-Stop-Shop.xlsx
+  nist: http://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-53r4.pdf
+  nist-csf: https://nvlpubs.nist.gov/nistpubs/CSWP/NIST.CSWP.04162018.pdf
+  os-srg: https://public.cyber.mil/stigs/downloads/?_dl_facet_stigs=operating-systems%2Cgeneral-purpose-os
+  ospp: https://www.niap-ccevs.org/Profile/PP.cfm
+  pcidss: https://www.pcisecuritystandards.org/documents/PCI_DSS_v3-2-1.pdf
+  pcidss4: https://docs-prv.pcisecuritystandards.org/PCI%20DSS/Standard/PCI-DSS-v4_0.pdf
+  stigid: https://public.cyber.mil/stigs/downloads/?_dl_facet_stigs=operating-systems%2Cunix-linux
+  stigref: https://public.cyber.mil/stigs/srg-stig-tools/
+sshd_distributed_config: 'false'
+sysctl_remediate_drop_in_file: 'false'
+type: platform
+uid_min: 1000

tests/data/product_stability/anolis8.yml

@@ -0,0 +1,71 @@
+aide_bin_path: /usr/sbin/aide
+aide_conf_path: /etc/aide.conf
+audisp_conf_path: /etc/audit
+auid: 1000
+basic_properties_derived: true
+benchmark_id: ANOLIS-8
+benchmark_root: ../../linux_os/guide
+chrony_conf_path: /etc/chrony.conf
+cpes:
+- anolis8:
+    check_id: installed_OS_is_anolis8
+    name: cpe:/o:anolis:anolis_os:8
+    title: Anolis OS 8
+cpes_root: ../../shared/applicability
+dconf_gdm_dir: gdm.d
+faillock_path: /var/run/faillock
+full_name: Anolis OS 8
+gid_min: 1000
+groups: {}
+grub2_boot_path: /boot/grub2
+grub2_uefi_boot_path: /boot/grub2
+init_system: systemd
+nobody_gid: 65534
+nobody_uid: 65534
+pkg_manager: yum
+pkg_manager_config_file: /etc/yum.conf
+pkg_system: rpm
+platform_package_overrides:
+  aarch64_arch: null
+  grub2: grub2-common
+  login_defs: shadow-utils
+  no_ovirt: null
+  non-uefi: null
+  not_aarch64_arch: null
+  not_s390x_arch: null
+  ovirt: null
+  s390x_arch: null
+  sssd: sssd-common
+  sssd-ldap: null
+  uefi: null
+  zipl: s390utils-base
+product: anolis8
+profiles_root: ./profiles
+reference_uris:
+  anssi: http://www.ssi.gouv.fr/administration/bonnes-pratiques/
+  app-srg: https://public.cyber.mil/stigs/downloads/?_dl_facet_stigs=application-servers
+  cis-csc: https://www.cisecurity.org/controls/
+  cjis: https://www.fbi.gov/file-repository/cjis-security-policy-v5_5_20160601-2-1.pdf
+  cnss: http://www.cnss.gov/Assets/pdf/CNSSI-1253.pdf
+  cobit5: https://www.isaca.org/resources/cobit
+  cui: http://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-171.pdf
+  dcid: not_officially_available
+  disa: https://public.cyber.mil/stigs/cci/
+  hipaa: https://www.gpo.gov/fdsys/pkg/CFR-2007-title45-vol1/pdf/CFR-2007-title45-vol1-chapA-subchapC.pdf
+  isa-62443-2009: https://www.isa.org/products/isa-62443-2-1-2009-security-for-industrial-automat
+  isa-62443-2013: https://www.isa.org/products/ansi-isa-62443-3-3-99-03-03-2013-security-for-indu
+  ism: https://www.cyber.gov.au/acsc/view-all-content/ism
+  iso27001-2013: https://www.iso.org/standard/54534.html
+  nerc-cip: https://www.nerc.com/pa/Stand/Standard%20Purpose%20Statement%20DL/US_Standard_One-Stop-Shop.xlsx
+  nist: http://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-53r4.pdf
+  nist-csf: https://nvlpubs.nist.gov/nistpubs/CSWP/NIST.CSWP.04162018.pdf
+  os-srg: https://public.cyber.mil/stigs/downloads/?_dl_facet_stigs=operating-systems%2Cgeneral-purpose-os
+  ospp: https://www.niap-ccevs.org/Profile/PP.cfm
+  pcidss: https://www.pcisecuritystandards.org/documents/PCI_DSS_v3-2-1.pdf
+  pcidss4: https://docs-prv.pcisecuritystandards.org/PCI%20DSS/Standard/PCI-DSS-v4_0.pdf
+  stigid: https://public.cyber.mil/stigs/downloads/?_dl_facet_stigs=operating-systems%2Cunix-linux
+  stigref: https://public.cyber.mil/stigs/srg-stig-tools/
+sshd_distributed_config: 'false'
+sysctl_remediate_drop_in_file: 'false'
+type: platform
+uid_min: 1000

tests/data/product_stability/chromium.yml

@@ -0,0 +1,67 @@
+aide_bin_path: /usr/sbin/aide
+aide_conf_path: /etc/aide.conf
+audisp_conf_path: /etc/audit
+auid: 1000
+basic_properties_derived: true
+benchmark_id: CHROMIUM
+benchmark_root: ./guide
+chrony_conf_path: /etc/chrony.conf
+cpes:
+- chromium:
+    check_id: installed_app_is_chromium
+    name: cpe:/a:google:chromium-browser
+    title: Google Chromium Browser
+cpes_root: ../../shared/applicability
+dconf_gdm_dir: gdm.d
+faillock_path: /var/run/faillock
+full_name: Chromium
+gid_min: 1000
+groups: {}
+grub2_boot_path: /boot/grub2
+grub2_uefi_boot_path: /boot/grub2
+nobody_gid: 65534
+nobody_uid: 65534
+platform_package_overrides:
+  aarch64_arch: null
+  grub2: grub2-common
+  login_defs: login
+  no_ovirt: null
+  non-uefi: null
+  not_aarch64_arch: null
+  not_s390x_arch: null
+  ovirt: null
+  s390x_arch: null
+  sssd: sssd-common
+  sssd-ldap: null
+  uefi: null
+  zipl: s390utils-base
+product: chromium
+profiles_root: ./profiles
+reference_uris:
+  anssi: http://www.ssi.gouv.fr/administration/bonnes-pratiques/
+  app-srg: https://public.cyber.mil/stigs/downloads/?_dl_facet_stigs=application-servers
+  cis-csc: https://www.cisecurity.org/controls/
+  cjis: https://www.fbi.gov/file-repository/cjis-security-policy-v5_5_20160601-2-1.pdf
+  cnss: http://www.cnss.gov/Assets/pdf/CNSSI-1253.pdf
+  cobit5: https://www.isaca.org/resources/cobit
+  cui: http://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-171.pdf
+  dcid: not_officially_available
+  disa: https://public.cyber.mil/stigs/cci/
+  hipaa: https://www.gpo.gov/fdsys/pkg/CFR-2007-title45-vol1/pdf/CFR-2007-title45-vol1-chapA-subchapC.pdf
+  isa-62443-2009: https://www.isa.org/products/isa-62443-2-1-2009-security-for-industrial-automat
+  isa-62443-2013: https://www.isa.org/products/ansi-isa-62443-3-3-99-03-03-2013-security-for-indu
+  ism: https://www.cyber.gov.au/acsc/view-all-content/ism
+  iso27001-2013: https://www.iso.org/standard/54534.html
+  nerc-cip: https://www.nerc.com/pa/Stand/Standard%20Purpose%20Statement%20DL/US_Standard_One-Stop-Shop.xlsx
+  nist: http://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-53r4.pdf
+  nist-csf: https://nvlpubs.nist.gov/nistpubs/CSWP/NIST.CSWP.04162018.pdf
+  os-srg: https://public.cyber.mil/stigs/downloads/?_dl_facet_stigs=operating-systems%2Cgeneral-purpose-os
+  ospp: https://www.niap-ccevs.org/Profile/PP.cfm
+  pcidss: https://www.pcisecuritystandards.org/documents/PCI_DSS_v3-2-1.pdf
+  pcidss4: https://docs-prv.pcisecuritystandards.org/PCI%20DSS/Standard/PCI-DSS-v4_0.pdf
+  stigid: https://public.cyber.mil/stigs/downloads/?_dl_facet_stigs=operating-systems%2Cunix-linux
+  stigref: https://public.cyber.mil/stigs/srg-stig-tools/
+sshd_distributed_config: 'false'
+sysctl_remediate_drop_in_file: 'false'
+type: product
+uid_min: 1000