Add more integration tests, unskip tests disabled eons ago

pulumi · Oct 8, 2023 · c5939c8 · c5939c8
1 parent 835abb3
commit c5939c8
Show file tree

Hide file tree

Showing 10 changed files with 71 additions and 15 deletions.
diff --git a/tests/integration/config/policy-pack-python/__main__.py b/tests/integration/config/policy-pack-python/__main__.py
@@ -332,17 +332,24 @@ def validate(args, report_violation):
         verify(args.get_config())
 
 
+def remediate(args):
+    if verify is not None:
+        verify(args.get_config())
+
+
 PolicyPack(
     name="config-policy",
-    enforcement_level=EnforcementLevel.MANDATORY,
     policies=[
         ResourceValidationPolicy(
+            enforcement_level=EnforcementLevel.REMEDIATE,
             name="resource-validation",
             description="Verifies policy config during resource validation.",
             validate=validate,
+            remediate=remediate,
             config_schema=schema,
         ),
         StackValidationPolicy(
+            enforcement_level=EnforcementLevel.MANDATORY,
             name="stack-validation",
             description="Verifies policy config during stack validation.",
             validate=validate,

diff --git a/tests/integration/config/policy-pack/index.ts b/tests/integration/config/policy-pack/index.ts
@@ -3,7 +3,15 @@
 import * as assert from "assert";
 
 import * as pulumi from "@pulumi/pulumi";
-import { PolicyConfigSchema, PolicyPack, PolicyPackConfig } from "@pulumi/policy";
+import {
+    PolicyConfigSchema,
+    PolicyPack,
+    PolicyPackConfig,
+    remediateResourceOfType,
+    validateRemediateResourceOfType,
+    validateResourceOfType,
+} from "@pulumi/policy";
+import * as random from "@pulumi/random";
 
 interface TestScenario {
     schema: PolicyConfigSchema;
@@ -350,11 +358,14 @@ new PolicyPack("config-policy", {
         {
             name: "resource-validation",
             description: "Verifies policy config during resource validation.",
-            enforcementLevel: "mandatory",
+            enforcementLevel: "remediate",
             configSchema: scenarios[index].schema,
             validateResource: (args, reportViolation) => {
                 scenarios[index].verify?.(args);
             },
+            remediateResource: (args) => {
+                scenarios[index].verify?.(args);
+            },
         },
         {
             name: "stack-validation",

diff --git a/tests/integration/deserialize/policy-pack-python/__main__.py b/tests/integration/deserialize/policy-pack-python/__main__.py
@@ -54,6 +54,7 @@ def verify(r):
     enforcement_level=EnforcementLevel.MANDATORY,
     policies=[
         ResourceValidationPolicy(
+            enforcement_level=EnforcementLevel.REMEDIATE,
             name="resource-validation",
             description="Verifies deserialized properties during resource validation.",
             validate=validate_resource,

diff --git a/tests/integration/deserialize/policy-pack/index.ts b/tests/integration/deserialize/policy-pack/index.ts
@@ -9,6 +9,7 @@ new PolicyPack("deserialize-policy", {
     enforcementLevel: "mandatory",
     policies: [
         {
+            enforcementLevel: "remediate",
             name: "resource-validation",
             description: "Verifies deserialized properties during resource validation.",
             validateResource: (args, reportViolation) => {

diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go
@@ -458,16 +458,16 @@ func TestValidateStack(t *testing.T) {
 
 // Test that accessing unknown values returns an error during previews.
 func TestUnknownValues(t *testing.T) {
-	t.Skip("https://github.com/pulumi/pulumi-policy/issues/263")
 	runPolicyPackIntegrationTest(t, "unknown_values", NodeJS, map[string]string{
 		"aws:region": "us-west-2",
 	}, []policyTestScenario{
 		{
 			WantErrors: []string{
-				"[advisory]  unknown-values-policy v0.0.1  unknown-values-resource-validation (random:index/randomPet:RandomPet: pet)",
-				"can't run policy 'unknown-values-resource-validation' during preview: string value at .prefix can't be known during preview",
-				"[advisory]  unknown-values-policy v0.0.1  unknown-values-stack-validation",
-				"can't run policy 'unknown-values-stack-validation' during preview: string value at .prefix can't be known during preview",
+				"unknown-values-policy@v0.0.1",
+				"[advisory]  unknown-values-resource-validation  (random:index/randomPet:RandomPet: pet)",
+				"can't run policy 'unknown-values-resource-validation' from policy pack 'unknown-values-policy@v0.0.1' during preview: string value at .prefix can't be known during preview",
+				"[advisory]  unknown-values-stack-validation",
+				"can't run policy 'unknown-values-stack-validation' from policy pack 'unknown-values-policy@v0.0.1' during preview: string value at .prefix can't be known during preview",
 			},
 			Advisory: true,
 		},
@@ -484,7 +484,6 @@ func TestRuntimeData(t *testing.T) {
 
 // Test resource options.
 func TestResourceOptions(t *testing.T) {
-	t.Skip("https://github.com/pulumi/pulumi-policy/issues/291")
 	runPolicyPackIntegrationTest(t, "resource_options", NodeJS, nil, []policyTestScenario{
 		// Test scenario 1: test resource options.
 		{WantErrors: nil},
@@ -495,7 +494,6 @@ func TestResourceOptions(t *testing.T) {
 
 // Test parent and dependencies.
 func TestParentDependencies(t *testing.T) {
-	t.Skip("https://github.com/pulumi/pulumi-policy/issues/292")
 	runPolicyPackIntegrationTest(t, "parent_dependencies", NodeJS, nil, []policyTestScenario{
 		{WantErrors: nil},
 	})
@@ -853,8 +851,9 @@ func TestRemoteComponent(t *testing.T) {
 	runPolicyPackIntegrationTest(t, "remote_component", NodeJS, nil, []policyTestScenario{
 		{
 			WantErrors: []string{
+				"remote-component-policy@v0.0.1",
 				"[advisory]  remote-component-policy v0.0.1  resource-validation (random:index/randomString:RandomString: innerRandom)",
-				"can't run policy 'resource-validation' during preview: string value at .keepers.hi can't be known during preview",
+				"can't run policy 'resource-validation' from policy pack 'remote-component-policy@v0.0.1' during preview: string value at .keepers.hi can't be known during preview",
 			},
 			Advisory: true,
 		},

diff --git a/tests/integration/resource_options/policy-pack-python/__main__.py b/tests/integration/resource_options/policy-pack-python/__main__.py
@@ -20,6 +20,10 @@ def validate_resource(args, report_violation):
     validate(args)
 
 
+def remediate_resource(args):
+    validate(args)
+
+
 def validate_stack(args, report_violation):
     for r in args.resources:
         validate(r)
@@ -121,7 +125,12 @@ def validate_dynamic_resource(r):
             protect=False,
             ignore_changes=[],
             delete_before_replace=None,
-            aliases=[create_urn("pulumi-nodejs:dynamic:Resource", "old-name-for-aliased")],
+            # Note that the engine explicitly does not preserve aliases pointing to resources that no
+            # longer exist. Because we don't actually introduce real aliases here, "old-name-for-aliases"
+            # is not paired up with a resource, and so the aliases array will be empty. If the engine
+            # preserved these aliases, we would have instead checked for:
+            # aliases=[create_urn("pulumi-nodejs:dynamic:Resource", "old-name-for-aliased")],
+            aliases=[],
             additional_secret_outputs=[],
             custom_timeouts=PolicyCustomTimeouts(0, 0, 0),
         ), r.opts)
@@ -193,6 +202,7 @@ def options_equal(expected: PolicyResourceOptions, actual: PolicyResourceOptions
             name="validate-resource",
             description="Validates resource options during `validateResource`.",
             validate=validate_resource,
+            remediate=remediate_resource,
         ),
         StackValidationPolicy(
             name="validate-stack",

diff --git a/tests/integration/resource_options/policy-pack/index.ts b/tests/integration/resource_options/policy-pack/index.ts
@@ -14,6 +14,9 @@ new PolicyPack("resource-options-test-policy", {
             validateResource: (args, reportViolation) => {
                 validate(args);
             },
+            remediateResource: (args) => {
+                validate(args);
+            },
         },
         {
             name: "validate-stack",
@@ -120,7 +123,12 @@ function validateDynamicResource(r: ResourceValidationArgs | PolicyResource) {
 
         case "aliased":
             assert.deepStrictEqual(r.opts, Object.assign({}, defaultOptions, {
-                aliases: [createURN("pulumi-nodejs:dynamic:Resource", "old-name-for-aliased")],
+                // Note that the engine explicitly does not preserve aliases pointing to resources that no
+                // longer exist. Because we don't actually introduce real aliases here, "old-name-for-aliases"
+                // is not paired up with a resource, and so the aliases array will be empty. If the engine
+                // preserved these aliases, we would have instead checked for:
+                // aliases: [createURN("pulumi-nodejs:dynamic:Resource", "old-name-for-aliased")],
+                aliases: [],
             }));
             break;
 

diff --git a/tests/integration/resource_options/program/resource.ts b/tests/integration/resource_options/program/resource.ts
@@ -18,5 +18,8 @@ export class Provider implements pulumi.dynamic.ResourceProvider {
 export class Resource extends pulumi.dynamic.Resource {
     constructor(name: string, opts?: pulumi.CustomResourceOptions) {
         super(Provider.instance, name, {}, opts);
+        if (name === "aliased") {
+            console.log("OPTS = " + JSON.stringify(opts));
+        }
     }
 }
diff --git a/tests/integration/unknown_values/policy-pack-python/__main__.py b/tests/integration/unknown_values/policy-pack-python/__main__.py
@@ -14,6 +14,10 @@ def validate_resource(args, report_violation):
     verify(args)
 
 
+def remediate_resource(args):
+    verify(args)
+
+
 def validate_stack(args, report_violation):
     for r in args.resources:
         verify(r)
@@ -31,14 +35,16 @@ def verify(r):
 
 PolicyPack(
     name="unknown-values-policy",
-    enforcement_level=EnforcementLevel.MANDATORY,
     policies=[
         ResourceValidationPolicy(
+            enforcement_level=EnforcementLevel.REMEDIATE,
             name="unknown-values-resource-validation",
             description="Accessing unknown values during preview results in a violation.",
             validate=validate_resource,
+            remediate=remediate_resource,
         ),
         StackValidationPolicy(
+            enforcement_level=EnforcementLevel.MANDATORY,
             name="unknown-values-stack-validation",
             description="Accessing unknown values during preview results in a violation.",
             validate=validate_stack,

diff --git a/tests/integration/unknown_values/policy-pack/index.ts b/tests/integration/unknown_values/policy-pack/index.ts
@@ -1,6 +1,11 @@
 // Copyright 2016-2020, Pulumi Corporation.  All rights reserved.
 
-import { PolicyPack, validateStackResourcesOfType, validateResourceOfType } from "@pulumi/policy";
+import {
+    PolicyPack,
+    remediateResourceOfType,
+    validateStackResourcesOfType,
+    validateResourceOfType,
+} from "@pulumi/policy";
 
 import * as random from "@pulumi/random";
 
@@ -15,6 +20,11 @@ new PolicyPack("unknown-values-policy", {
                 // during previews given the associated Pulumi program.
                 console.log(pet.prefix);
             }),
+            remediateResource: remediateResourceOfType(random.RandomPet, (pet, args) => {
+                // Accessing `.prefix` is expected to result in a policy violation because its value is unknown
+                // during previews given the associated Pulumi program.
+                console.log(pet.prefix);
+            }),
         },
         {
             name: "unknown-values-stack-validation",