From 693401ab454dfbfa7d5443b61bc175b99d79aa09 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 25 Jan 2023 15:16:14 -0800 Subject: [PATCH 01/28] cty: "Known-ness" represented internally by a Go type, not a Go value Previously the internal definition of an unknown value was any value whose raw value "v" field was exactly equal to the unexported global "unknown". In preparation for supporting optional refinement of unknown values we'll now broaden that definition to any value where "v" is an instance of *unknownType, regardless of exact value. In later commits we'll add optional fields inside struct unknownType to capture the optional additional refinements. "Refinements" means extra constraints on an unknown value that don't affect its type but do constrain the possible concrete values the unknown value could be a placeholder for. --- cty/helper.go | 4 ++-- cty/unknown.go | 14 +++++++++----- cty/value.go | 3 ++- cty/value_ops.go | 4 ++-- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/cty/helper.go b/cty/helper.go index 1b88e9fa..c342f13c 100644 --- a/cty/helper.go +++ b/cty/helper.go @@ -8,7 +8,7 @@ import ( // unknowns, for operations that short-circuit to return unknown in that case. func anyUnknown(values ...Value) bool { for _, val := range values { - if val.v == unknown { + if _, unknown := val.v.(*unknownType); unknown { return true } } @@ -39,7 +39,7 @@ func typeCheck(required Type, ret Type, values ...Value) (shortCircuit *Value, e ) } - if val.v == unknown { + if _, unknown := val.v.(*unknownType); unknown { hasUnknown = true } } diff --git a/cty/unknown.go b/cty/unknown.go index 83893c02..272f6f6e 100644 --- a/cty/unknown.go +++ b/cty/unknown.go @@ -5,9 +5,13 @@ package cty type unknownType struct { } -// unknown is a special value that can be used as the internal value of a -// Value to create a placeholder for a value that isn't yet known. -var unknown interface{} = &unknownType{} +// totallyUnknown is the representation a a value we know nothing about at +// all. Subsequent refinements of an unknown value will cause creation of +// other values of unknownType that can represent additional constraints +// on the unknown value, but all unknown values start as totally unknown +// and we will also typically lose all unknown value refinements when +// round-tripping through serialization formats. +var totallyUnknown interface{} = &unknownType{} // UnknownVal returns an Value that represents an unknown value of the given // type. Unknown values can be used to represent a value that is @@ -19,7 +23,7 @@ var unknown interface{} = &unknownType{} func UnknownVal(t Type) Value { return Value{ ty: t, - v: unknown, + v: totallyUnknown, } } @@ -80,6 +84,6 @@ func init() { } DynamicVal = Value{ ty: DynamicPseudoType, - v: unknown, + v: totallyUnknown, } } diff --git a/cty/value.go b/cty/value.go index f6a25dde..e5b29b60 100644 --- a/cty/value.go +++ b/cty/value.go @@ -48,7 +48,8 @@ func (val Value) IsKnown() bool { if val.IsMarked() { return val.unmarkForce().IsKnown() } - return val.v != unknown + _, unknown := val.v.(*unknownType) + return !unknown } // IsNull returns true if the value is null. Values of any type can be diff --git a/cty/value_ops.go b/cty/value_ops.go index 88b3637c..0e014ac3 100644 --- a/cty/value_ops.go +++ b/cty/value_ops.go @@ -276,7 +276,7 @@ func (val Value) Equals(other Value) Value { // in one are also in the other. for it := s1.Iterator(); it.Next(); { rv := it.Value() - if rv == unknown { // "unknown" is the internal representation of unknown-ness + if _, unknown := rv.(*unknownType); unknown { // "*unknownType" is the internal representation of unknown-ness return UnknownVal(Bool) } if !s2.Has(rv) { @@ -285,7 +285,7 @@ func (val Value) Equals(other Value) Value { } for it := s2.Iterator(); it.Next(); { rv := it.Value() - if rv == unknown { // "unknown" is the internal representation of unknown-ness + if _, unknown := rv.(*unknownType); unknown { // "*unknownType" is the internal representation of unknown-ness return UnknownVal(Bool) } if !s1.Has(rv) { From 017178fd3886cc7168633e2959b1bc363ac8d55f Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 25 Jan 2023 18:17:30 -0800 Subject: [PATCH 02/28] cty: "Refinements" for unknown values This is a new concept allowing for an unknown value to optionally carry some additional information about constraints on the value for which it is a placeholder. Refinements do not affect the type of the value or most equality testing rules. As of this commit they also don't affect any other operations on values, but in future commits we'll use refinements to improve the precision of certain operations on unknown values if the refinements give enough information to still produce a known result. For the moment this is limited just to a few key cases that seem likely to help produce known results in common operations involving unknown values: - Equality tests will null, which are commonly used as part of conditional branches in calling applications. - Numeric bounds, in particular focused on comparison to zero but general enough to help with other numeric comparisons too. - Bounds on collection length, to use in conjunction with the numeric bounds so that asking for the length of an unknown collection can return an unknown number with numeric bounds. - String prefixes, to help with string-prefix-related tests that seem to be a common subset of validation checks in calling applications. Applications will tend to need to do some work to refine their own values in order to benefit fully from this, but we'll also plumb refinements into some of the built-in value operations and stdlib functions over time so that even applications not working directly with refinements can still get some indirect benefit. --- cty/unknown.go | 4 + cty/unknown_refinement.go | 660 ++++++++++++++++++++++++++++++++++++++ cty/value_ops.go | 19 +- cty/value_ops_test.go | 83 ++++- cty/value_range.go | 220 +++++++++++++ 5 files changed, 983 insertions(+), 3 deletions(-) create mode 100644 cty/unknown_refinement.go create mode 100644 cty/value_range.go diff --git a/cty/unknown.go b/cty/unknown.go index 272f6f6e..b3aefa45 100644 --- a/cty/unknown.go +++ b/cty/unknown.go @@ -3,6 +3,10 @@ package cty // unknownType is the placeholder type used for the sigil value representing // "Unknown", to make it unambigiously distinct from any other possible value. type unknownType struct { + // refinement is an optional object which, if present, describes some + // additional constraints we know about the range of real values this + // unknown value could be a placeholder for. + refinement unknownValRefinement } // totallyUnknown is the representation a a value we know nothing about at diff --git a/cty/unknown_refinement.go b/cty/unknown_refinement.go new file mode 100644 index 00000000..84df483f --- /dev/null +++ b/cty/unknown_refinement.go @@ -0,0 +1,660 @@ +package cty + +import ( + "fmt" + "math/big" + "strings" +) + +// Refine creates a [RefinementBuilder] with which to annotate the reciever +// with zero or more additional refinements that constrain the range of +// the value. +// +// Calling methods on a RefinementBuilder for a known value essentially just +// serves as assertions about the range of that value, leading to panics if +// those assertions don't hold in practice. This is mainly supported just to +// make programs that rely on refinements automatically self-check by using +// the refinement codepath unconditionally on both placeholders and final +// values for those placeholders. It's always a bug to refine the range of +// an unknown value and then later substitute an exact value outside of the +// refined range. +// +// Calling methods on a RefinementBuilder for an unknown value is perhaps +// more useful because the newly-refined value will then be a placeholder for +// a smaller range of values and so it may be possible for other operations +// on the unknown value to return a known result despite the exact value not +// yet being known. +// +// It is never valid to refine [DynamicVal], because that value is a +// placeholder for a value about which we knkow absolutely nothing. A value +// must at least have a known root type before it can support further +// refinement. +func (v Value) Refine() *RefinementBuilder { + if unk, isUnk := v.v.(*unknownType); isUnk && unk.refinement != nil { + // We're refining a value that's already been refined before, so + // we'll start from a copy of its existing refinements. + wip := unk.refinement.copy() + return &RefinementBuilder{v, wip} + } + + ty := v.Type() + var wip unknownValRefinement + switch { + case ty == String: + wip = &refinementString{} + case ty == Number: + wip = &refinementNumber{} + case ty.IsCollectionType(): + wip = &refinementCollection{ + // A collection can never have a negative length, so we'll + // start with that already constrained. + minLen: Zero, + minInc: true, + } + case ty == Bool || ty.IsObjectType() || ty.IsTupleType() || ty.IsCapsuleType(): + // For other known types we'll just track nullability + wip = &refinementNullable{} + default: + // we leave "wip" as nil for all other types, representing that + // they don't support refinements at all and so any call on the + // RefinementBuilder should fail. + + // NOTE: We intentionally don't allow any refinements for + // cty.DynamicPseudoType here, even though it could be nice in principle + // to at least track non-nullness for those, because it's historically + // been valid to directly compare values with cty.DynamicVal using + // the Go "==" operator and recording a refinement for an untyped + // unknown value would break existing code relying on that. + } + + return &RefinementBuilder{v, wip} +} + +// RefinementBuilder is a supporting type for the [Value.Refine] method, +// using the builder pattern to apply zero or more constraints before +// constructing a new value with all of those constraints applied. +// +// Most of the methods of this type return the same reciever to allow +// for method call chaining. End call chains with a call to +// [RefinementBuilder.NewValue] to obtain the newly-refined value. +type RefinementBuilder struct { + orig Value + wip unknownValRefinement +} + +func (b *RefinementBuilder) assertRefineable() { + if b.wip == nil { + panic(fmt.Sprintf("cannot refine a %#v value", b.orig.Type())) + } +} + +// NotNull constrains the value as definitely not being null. +// +// NotNull is valid when refining values of the following types: +// - number, boolean, and string values +// - list, set, or map types of any element type +// - values of object types +// - values of collection types +// - values of capsule types +// +// When refining any other type this function will panic. +// +// In particular note that it is not valid to constrain an untyped value +// -- a value whose type is `cty.DynamicPseudoType` -- as being non-null. +// An unknown value of an unknown type is always completely unconstrained. +func (b *RefinementBuilder) NotNull() *RefinementBuilder { + b.assertRefineable() + + if b.orig.IsKnown() && b.orig.IsNull() { + panic("refining null value as non-null") + } + + b.wip.setNull(tristateFalse) + + return b +} + +// Null constrains the value as definitely null. +// +// Null is valid for the same types as [RefinementBuilder.NotNull]. +// When refining any other type this function will panic. +// +// Explicitly cnstraining a value to be null is strange because that suggests +// that the caller does actually know the value -- there is only one null +// value for each type constraint -- but this is here for symmetry with the +// fact that a [ValueRange] can also represent that a value is definitely null. +func (b *RefinementBuilder) Null() *RefinementBuilder { + b.assertRefineable() + + if b.orig.IsKnown() && !b.orig.IsNull() { + panic("refining non-null value as null") + } + + b.wip.setNull(tristateTrue) + + return b +} + +// NumericRange constrains the upper and/or lower bounds of a number value, +// or panics if this builder is not refining a number value. +// +// The two given values are interpreted as inclusive bounds and either one +// may be an unknown number if only one of the two bounds is currently known. +// If either of the given values is not a non-null number value then this +// function will panic. +func (b *RefinementBuilder) NumberRangeInclusive(min, max Value) *RefinementBuilder { + return b.numberRange(min, max, true, true) +} + +// CollectionLengthLowerBound constrains the lower bound of the length of a +// collection value, or panics if this builder is not refining a collection +// value. +// +// The lower bound must be a known, non-null number or this function will +// panic. +func (b *RefinementBuilder) CollectionLengthLowerBound(min Value, inclusive bool) *RefinementBuilder { + b.assertRefineable() + + wip, ok := b.wip.(*refinementCollection) + if !ok { + panic(fmt.Sprintf("cannot refine collection length bounds for a %#v value", b.orig.Type())) + } + + if min.IsNull() { + panic("collection length bound is null") + } + if !min.IsKnown() { + panic("collection length bound is unknown") + } + + if b.orig.IsKnown() { + realLen := b.orig.Length() + if gt := min.GreaterThan(realLen); gt.IsKnown() && gt.True() { + panic(fmt.Sprintf("refining collection of length %#v with minimum bound %#v", realLen, min)) + } + } + + if wip.minLen != NilVal { + var ok bool + if wip.minInc { + ok = min.GreaterThanOrEqualTo(wip.minLen).True() + } else { + ok = min.GreaterThan(wip.minLen).True() + } + if !ok { + panic("refined collection length lower bound is inconsistent with existing lower bound") + } + } + + wip.minLen = min + wip.minInc = inclusive + wip.assertConsistentLengthBounds() + + return b +} + +// CollectionLengthUpperBound constrains the upper bound of the length of a +// collection value, or panics if this builder is not refining a collection +// value. +// +// The upper bound must be a known, non-null number or this function will +// panic. +func (b *RefinementBuilder) CollectionLengthUpperBound(max Value, inclusive bool) *RefinementBuilder { + b.assertRefineable() + + wip, ok := b.wip.(*refinementCollection) + if !ok { + panic(fmt.Sprintf("cannot refine collection length bounds for a %#v value", b.orig.Type())) + } + + if max.IsNull() { + panic("collection length bound is null") + } + if !max.IsKnown() { + panic("collection length bound is unknown") + } + + if b.orig.IsKnown() { + realLen := b.orig.Length() + if gt := max.LessThan(realLen); gt.IsKnown() && gt.True() { + panic(fmt.Sprintf("refining collection of length %#v with maximum bound %#v", realLen, max)) + } + } + + if wip.maxLen != NilVal { + var ok bool + if wip.maxInc { + ok = max.LessThanOrEqualTo(wip.minLen).True() + } else { + ok = max.LessThan(wip.minLen).True() + } + if !ok { + panic("refined collection length upper bound is inconsistent with existing upper bound") + } + } + + wip.maxLen = max + wip.maxInc = inclusive + wip.assertConsistentLengthBounds() + + return b +} + +// StringPrefix constrains the prefix of a string value, or panics if this +// builder is not refining a string value. +// +// The given prefix will be Unicode normalized in the same way that a +// cty.StringVal would be. However, since prefix is just a substring the +// normalization may produce a non-matching prefix string if the given prefix +// splits a sequence of combining characters. For correct results always ensure +// that the prefix ends at a grapheme cluster boundary. +func (b *RefinementBuilder) StringPrefix(prefix string) *RefinementBuilder { + b.assertRefineable() + + wip, ok := b.wip.(*refinementString) + if !ok { + panic(fmt.Sprintf("cannot refine string prefix for a %#v value", b.orig.Type())) + } + + // We must apply the same Unicode processing we'd normally use for a + // cty string so that the prefix will be comparable. + prefix = NormalizeString(prefix) + + // If we have a known string value then the given prefix must actually + // match it. + if b.orig.IsKnown() && !b.orig.IsNull() { + have := b.orig.AsString() + matchLen := len(have) + if l := len(prefix); l < matchLen { + matchLen = l + } + have = have[:matchLen] + new := prefix[:matchLen] + if have != new { + panic("refined prefix is inconsistent with known value") + } + } + + // If we already have a refined prefix then the overlapping parts of that + // and the new prefix must match. + { + matchLen := len(wip.prefix) + if l := len(prefix); l < matchLen { + matchLen = l + } + + have := wip.prefix[:matchLen] + new := prefix[:matchLen] + if have != new { + panic("refined prefix is inconsistent with previous refined prefix") + } + } + + // We'll only save the new prefix if it's longer than the one we already + // had. + if len(prefix) > len(wip.prefix) { + wip.prefix = prefix + } + + return b +} + +func (b *RefinementBuilder) numberRange(min, max Value, minInc, maxInc bool) *RefinementBuilder { + b.assertRefineable() + + wip, ok := b.wip.(*refinementNumber) + if !ok { + panic(fmt.Sprintf("cannot refine numeric range for a %#v value", b.orig.Type())) + } + // After this point b.orig is guaranteed to have type cty.Number + + if min.Type() != Number || max.Type() != Number { + panic("refining numeric range with a non-numeric bound") + } + if min.IsNull() || max.IsNull() { + panic("refining numeric range with a null bound") + } + + uncomparable := func(v Value) bool { + return v.IsNull() || !v.IsKnown() + } + checkMinRangeFunc := func(inclusive bool) func(Value, Value) bool { + if inclusive { + return func(a, b Value) bool { + if uncomparable(a) || uncomparable(b) { + return true // default to valid if we're not sure + } + return a.GreaterThanOrEqualTo(b).True() + } + } else { + return func(a, b Value) bool { + if uncomparable(a) || uncomparable(b) { + return true // default to valid if we're not sure + } + return a.GreaterThan(b).True() + } + } + } + checkMaxRangeFunc := func(inclusive bool) func(Value, Value) bool { + if inclusive { + return func(a, b Value) bool { + if uncomparable(a) || uncomparable(b) { + return true // default to valid if we're not sure + } + return a.LessThanOrEqualTo(b).True() + } + } else { + return func(a, b Value) bool { + if uncomparable(a) || uncomparable(b) { + return true // default to valid if we're not sure + } + return a.LessThan(b).True() + } + } + } + + // If our original value is known then it must be in the given range. + if v := b.orig; v.IsKnown() && !v.IsNull() { + if !checkMinRangeFunc(minInc)(v, min) { + panic(fmt.Sprintf("refining %#v with invalid lower bound %#v", v, min)) + } + if !checkMaxRangeFunc(maxInc)(v, max) { + panic(fmt.Sprintf("refining %#v with invalid upper bound %#v", v, min)) + } + } + + // If we already have bounds then the new bounds must be consistent with them. + if wip.min != NilVal && !checkMinRangeFunc(wip.minInc)(wip.min, min) { + panic(fmt.Sprintf("new refined lower bound %#v conflicts with previous %#v", min, wip.min)) + } + if wip.max != NilVal && !checkMaxRangeFunc(wip.maxInc)(wip.max, max) { + panic(fmt.Sprintf("new refined upper bound %#v conflicts with previous %#v", min, wip.min)) + } + + // We only record known bounds. An unknown value for either bound means + // it's either unbounded or we'll retain a prevously-recorded bound. + if min.IsKnown() { + wip.min = min + wip.minInc = minInc + } + if max.IsKnown() { + wip.max = max + wip.maxInc = maxInc + } + wip.assertConsistentBounds() + + return b +} + +// NewValue completes the refinement process by constructing a new value +// that is guaranteed to meet all of the previously-specified refinements. +// +// If the original value being refined was known then the result is exactly +// that value, because otherwise the previous refinement calls would have +// panicked reporting the refinements as invalid for the value. +// +// If the original value was unknown then the result is typically also unknown +// but may have additional refinements compared to the original. If the applied +// refinements have reduced the range to a single exact value then the result +// might be that known value. +func (b *RefinementBuilder) NewValue() Value { + if b.orig.IsKnown() { + return b.orig + } + + // We have a few cases where the value has been refined enough that we now + // know exactly what the value is, or at least we can produce a more + // detailed approximation of it. + switch b.wip.null() { + case tristateTrue: + // There is only one null value of each type so this is now known. + return NullVal(b.orig.Type()) + case tristateFalse: + // If we know it's definitely not null then we might have enough + // information to construct a known, non-null value. + if rfn, ok := b.wip.(*refinementNumber); ok { + // If both bounds are inclusive and equal then our value can + // only be the same number as the bounds. + if rfn.maxInc && rfn.minInc { + if rfn.min != NilVal && rfn.max != NilVal { + eq := rfn.min.Equals(rfn.max) + if eq.IsKnown() && eq.True() { + return rfn.min + } + } + } + } else if rfn, ok := b.wip.(*refinementCollection); ok { + // If both length bounds are inclusive and equal then we know our + // length is the same number as the bounds. + if rfn.maxInc && rfn.minInc { + if rfn.minLen != NilVal && rfn.maxLen != NilVal { + eq := rfn.minLen.Equals(rfn.maxLen) + if eq.IsKnown() && eq.True() { + knownLen := rfn.minLen + ty := b.orig.Type() + if knownLen == Zero { + // If we know the length is zero then we can construct + // a known value of any collection kind. + switch { + case ty.IsListType(): + return ListValEmpty(ty.ElementType()) + case ty.IsSetType(): + return SetValEmpty(ty.ElementType()) + case ty.IsMapType(): + return MapValEmpty(ty.ElementType()) + } + } else if ty.IsListType() { + // If we know the length of the list then we can + // create a known list with unknown elements instead + // of a wholly-unknown list. + if knownLen, acc := knownLen.AsBigFloat().Int64(); acc == big.Exact { + elems := make([]Value, knownLen) + unk := UnknownVal(ty.ElementType()) + for i := range elems { + elems[i] = unk + } + return ListVal(elems) + } + } else if ty.IsSetType() && knownLen == NumberIntVal(1) { + // If we know we have a one-element set then we + // know the one element can't possibly coalesce with + // anything else and so we can create a known set with + // an unknown element. + return SetVal([]Value{UnknownVal(ty.ElementType())}) + } + } + } + } + } + } + + return Value{ + ty: b.orig.ty, + v: &unknownType{refinement: b.wip}, + } +} + +// unknownValRefinment is an interface pretending to be a sum type representing +// the different kinds of unknown value refinements we support for different +// types of value. +type unknownValRefinement interface { + unknownValRefinementSigil() + copy() unknownValRefinement + null() tristateBool + setNull(tristateBool) + rawEqual(other unknownValRefinement) bool + GoString() string +} + +type refinementString struct { + refinementNullable + prefix string +} + +func (r *refinementString) unknownValRefinementSigil() {} + +func (r *refinementString) copy() unknownValRefinement { + ret := *r + // Everything in refinementString is immutable, so a shallow copy is sufficient. + return &ret +} + +func (r *refinementString) rawEqual(other unknownValRefinement) bool { + { + other, ok := other.(*refinementString) + if !ok { + return false + } + return (r.refinementNullable.rawEqual(&other.refinementNullable) && + r.prefix == other.prefix) + } +} + +func (r *refinementString) GoString() string { + var b strings.Builder + b.WriteString(r.refinementNullable.GoString()) + if r.prefix != "" { + fmt.Fprintf(&b, ".StringPrefix(%q)", r.prefix) + } + return b.String() +} + +type refinementNumber struct { + refinementNullable + min, max Value + minInc, maxInc bool +} + +func (r *refinementNumber) unknownValRefinementSigil() {} + +func (r *refinementNumber) copy() unknownValRefinement { + ret := *r + // Everything in refinementNumber is immutable, so a shallow copy is sufficient. + return &ret +} + +func (r *refinementNumber) rawEqual(other unknownValRefinement) bool { + { + other, ok := other.(*refinementNumber) + if !ok { + return false + } + return (r.refinementNullable.rawEqual(&other.refinementNullable) && + r.min.RawEquals(other.min) && + r.max.RawEquals(other.max) && + r.minInc == other.minInc && + r.maxInc == other.maxInc) + } +} + +func (r *refinementNumber) GoString() string { + var b strings.Builder + b.WriteString(r.refinementNullable.GoString()) + if r.min != NilVal { + fmt.Fprintf(&b, ".NumberLowerBound(%#v, %t)", r.min, r.minInc) + } + if r.max != NilVal { + fmt.Fprintf(&b, ".NumberUpperBound(%#v, %t)", r.max, r.maxInc) + } + return b.String() +} + +func (r *refinementNumber) assertConsistentBounds() { + if r.min != NilVal && r.max != NilVal && r.max.LessThan(r.min).True() { + panic("number upper bound is less than lower bound") + } +} + +type refinementCollection struct { + refinementNullable + minLen, maxLen Value + minInc, maxInc bool +} + +func (r *refinementCollection) unknownValRefinementSigil() {} + +func (r *refinementCollection) copy() unknownValRefinement { + ret := *r + // Everything in refinementCollection is immutable, so a shallow copy is sufficient. + return &ret +} + +func (r *refinementCollection) rawEqual(other unknownValRefinement) bool { + { + other, ok := other.(*refinementCollection) + if !ok { + return false + } + return (r.refinementNullable.rawEqual(&other.refinementNullable) && + r.minLen.RawEquals(other.minLen) && + r.maxLen.RawEquals(other.maxLen) && + r.minInc == other.minInc && + r.maxInc == other.maxInc) + } +} + +func (r *refinementCollection) GoString() string { + var b strings.Builder + b.WriteString(r.refinementNullable.GoString()) + if r.minLen != NilVal && r.minLen != Zero { + // (a lower bound of zero is the default) + fmt.Fprintf(&b, ".CollectionLengthLowerBound(%#v, %t)", r.minLen, r.minInc) + } + if r.maxLen != NilVal { + fmt.Fprintf(&b, ".CollectionLengthUpperBound(%#v, %t)", r.maxLen, r.maxInc) + } + return b.String() +} + +func (r *refinementCollection) assertConsistentLengthBounds() { + if r.minLen != NilVal && r.maxLen != NilVal && r.maxLen.LessThan(r.minLen).True() { + panic("collection length upper bound is less than lower bound") + } +} + +type refinementNullable struct { + isNull tristateBool +} + +func (r *refinementNullable) unknownValRefinementSigil() {} + +func (r *refinementNullable) copy() unknownValRefinement { + ret := *r + // Everything in refinementJustNull is immutable, so a shallow copy is sufficient. + return &ret +} + +func (r *refinementNullable) null() tristateBool { + return r.isNull +} + +func (r *refinementNullable) setNull(v tristateBool) { + r.isNull = v +} + +func (r *refinementNullable) rawEqual(other unknownValRefinement) bool { + { + other, ok := other.(*refinementNullable) + if !ok { + return false + } + return r.isNull == other.isNull + } +} + +func (r *refinementNullable) GoString() string { + switch r.isNull { + case tristateFalse: + return ".NotNull()" + case tristateTrue: + return ".Null()" + default: + return "" + } +} + +type tristateBool rune + +const tristateTrue tristateBool = 'T' +const tristateFalse tristateBool = 'F' +const tristateUnknown tristateBool = 0 diff --git a/cty/value_ops.go b/cty/value_ops.go index 0e014ac3..83466afc 100644 --- a/cty/value_ops.go +++ b/cty/value_ops.go @@ -33,7 +33,12 @@ func (val Value) GoString() string { return "cty.DynamicVal" } if !val.IsKnown() { - return fmt.Sprintf("cty.UnknownVal(%#v)", val.ty) + rfn := val.v.(*unknownType).refinement + var suffix string + if rfn != nil { + suffix = ".Refine()" + rfn.GoString() + ".NewValue()" + } + return fmt.Sprintf("cty.UnknownVal(%#v)%s", val.ty, suffix) } // By the time we reach here we've dealt with all of the exceptions around @@ -393,7 +398,17 @@ func (val Value) RawEquals(other Value) bool { other = other.unmarkForce() if (!val.IsKnown()) && (!other.IsKnown()) { - return true + // If either unknown value has refinements then they must match. + valRfn := val.v.(*unknownType).refinement + otherRfn := other.v.(*unknownType).refinement + switch { + case (valRfn == nil) != (otherRfn == nil): + return false + case valRfn != nil: + return valRfn.rawEqual(otherRfn) + default: + return true + } } if (val.IsKnown() && !other.IsKnown()) || (other.IsKnown() && !val.IsKnown()) { return false diff --git a/cty/value_ops_test.go b/cty/value_ops_test.go index 1d41183b..2c6dca03 100644 --- a/cty/value_ops_test.go +++ b/cty/value_ops_test.go @@ -1595,6 +1595,67 @@ func TestValueRawEquals(t *testing.T) { }), false, }, + { + UnknownVal(Bool).Refine().NotNull().NewValue(), + UnknownVal(Bool), + false, + }, + { + UnknownVal(Bool), + UnknownVal(Bool).Refine().NotNull().NewValue(), + false, + }, + { + UnknownVal(Number).Refine().NumberRangeInclusive(Zero, NumberIntVal(1)).NewValue(), + UnknownVal(Number).Refine().NumberRangeInclusive(Zero, NumberIntVal(2)).NewValue(), + false, + }, + { + UnknownVal(Number).Refine().NumberRangeInclusive(Zero, NumberIntVal(1)).NewValue(), + UnknownVal(Number).Refine().NumberRangeInclusive(Zero, NumberIntVal(1)).NewValue(), + true, + }, + { + UnknownVal(String), + UnknownVal(String).Refine().StringPrefix("foo").NewValue(), + false, + }, + { + UnknownVal(String).Refine().StringPrefix("foo").NewValue(), + UnknownVal(String).Refine().StringPrefix("foo").NewValue(), + true, + }, + { + UnknownVal(String).Refine().NotNull().StringPrefix("foo").NewValue(), + UnknownVal(String).Refine().StringPrefix("foo").NewValue(), + false, + }, + { + UnknownVal(String).Refine().StringPrefix("foo").NewValue(), + UnknownVal(String).Refine().StringPrefix("bar").NewValue(), + false, + }, + { + UnknownVal(String).Refine().Null().NewValue(), + NullVal(String), + true, // The refinement expression collapses into a simple null + }, + { + UnknownVal(Number).Refine().NumberRangeInclusive(Zero, Zero).NewValue(), + Zero, + false, // Refinement can't collapse to zero because it might be null + }, + { + UnknownVal(Number).Refine().NotNull().NumberRangeInclusive(Zero, Zero).NewValue(), + Zero, + true, // Refinement collapses to zero because it's not null and the two bounds are equal + }, + { + UnknownVal(List(String)).Refine().NotNull().CollectionLengthUpperBound(Zero, true).NewValue(), + ListValEmpty(String), + true, // Colection length lower bound is always at least zero so this refinement collapses to an empty list + }, + // Marks { StringVal("a").Mark(1), @@ -1617,7 +1678,7 @@ func TestValueRawEquals(t *testing.T) { t.Run(fmt.Sprintf("%#v.RawEquals(%#v)", test.LHS, test.RHS), func(t *testing.T) { got := test.LHS.RawEquals(test.RHS) if !got == test.Expected { - t.Fatalf("wrong Equals result\ngot: %#v\nwant: %#v", got, test.Expected) + t.Fatalf("wrong RawEquals result\ngot: %#v\nwant: %#v", got, test.Expected) } }) } @@ -3299,6 +3360,26 @@ func TestValueGoString(t *testing.T) { UnknownVal(Tuple([]Type{String, Bool})), `cty.UnknownVal(cty.Tuple([]cty.Type{cty.String, cty.Bool}))`, }, + { + UnknownVal(String).Refine().NotNull().NewValue(), + `cty.UnknownVal(cty.String).Refine().NotNull().NewValue()`, + }, + { + UnknownVal(String).Refine().NotNull().StringPrefix("a").NewValue(), + `cty.UnknownVal(cty.String).Refine().NotNull().StringPrefix("a").NewValue()`, + }, + { + UnknownVal(Bool).Refine().NotNull().NewValue(), + `cty.UnknownVal(cty.Bool).Refine().NotNull().NewValue()`, + }, + { + UnknownVal(Number).Refine().NumberRangeInclusive(Zero, UnknownVal(Number)).NewValue(), + `cty.UnknownVal(cty.Number).Refine().NumberLowerBound(cty.NumberIntVal(0), true).NewValue()`, + }, + { + UnknownVal(Number).Refine().NumberRangeInclusive(Zero, NumberIntVal(1)).NewValue(), + `cty.UnknownVal(cty.Number).Refine().NumberLowerBound(cty.NumberIntVal(0), true).NumberUpperBound(cty.NumberIntVal(1), true).NewValue()`, + }, { StringVal(""), diff --git a/cty/value_range.go b/cty/value_range.go new file mode 100644 index 00000000..e9967203 --- /dev/null +++ b/cty/value_range.go @@ -0,0 +1,220 @@ +package cty + +import ( + "fmt" +) + +// Range returns an object that offers partial information about the range +// of the receiver. +// +// This is most relevant for unknown values, because it gives access to any +// optional additional constraints on the final value (specified by the source +// of the value using "refinements") beyond what we can assume from the value's +// type. +// +// Calling Range for a known value is a little strange, but it's supported by +// returning a [ValueRange] object that describes the exact value as closely +// as possible. Typically a caller should work directly with the exact value +// in that case, but some purposes might only need the level of detail +// offered by ranges and so can share code between both known and unknown +// values. +func (v Value) Range() ValueRange { + // For an unknown value we just use its own refinements. + if unk, isUnk := v.v.(*unknownType); isUnk { + refinement := unk.refinement + if refinement == nil { + // We'll generate an unconstrained refinement, just to + // simplify the code in ValueRange methods which can + // therefore assume that there's always a refinement. + refinement = &refinementNullable{isNull: tristateUnknown} + } + return ValueRange{v.Type(), refinement} + } + + if v.IsNull() { + // If we know a value is null then we'll just report that, + // since no other refinements make sense for a definitely-null value. + return ValueRange{ + v.Type(), + &refinementNullable{isNull: tristateTrue}, + } + } + + // For a known value we construct synthetic refinements that match + // the value, just as a convenience for callers that want to share + // codepaths between both known and unknown values. + ty := v.Type() + var synth unknownValRefinement + switch { + case ty == String: + synth = &refinementString{ + prefix: v.AsString(), + } + case ty == Number: + synth = &refinementNumber{ + min: v, + max: v, + minInc: true, + maxInc: true, + } + case ty.IsCollectionType(): + synth = &refinementCollection{ + minLen: v.Length(), + maxLen: v.Length(), + minInc: true, + maxInc: true, + } + default: + // If we don't have anything else to say then we can at least + // guarantee that the value isn't null. + synth = &refinementNullable{} + } + + // If we get down here then the value is definitely not null + synth.setNull(tristateFalse) + + return ValueRange{ty, synth} +} + +// ValueRange offers partial information about the range of a value. +// +// This is primarily interesting for unknown values, because it provides access +// to any additional known constraints (specified using "refinements") on the +// range of the value beyond what is represented by the value's type. +type ValueRange struct { + ty Type + raw unknownValRefinement +} + +// TypeConstraint returns a type constraint describing the value's type as +// precisely as possible with the available information. +func (r ValueRange) TypeConstraint() Type { + return r.ty +} + +// CouldBeNull returns true unless the value being described is definitely +// known to represent a non-null value. +func (r ValueRange) CouldBeNull() bool { + if r.raw == nil { + // A totally-unconstrained unknown value could be null + return true + } + return r.raw.null() != tristateFalse +} + +// NumberLowerBound returns information about the lower bound of the range of +// a number value, or panics if the value is definitely not a number. +// +// If the value is nullable then the result represents the range of the number +// only if it turns out not to be null. +// +// The resulting value might itself be an unknown number if there is no +// known lower bound. In that case the "inclusive" flag is meaningless. +func (r ValueRange) NumberLowerBound() (min Value, inclusive bool) { + if r.ty == DynamicPseudoType { + // We don't even know if this is a number yet. + return UnknownVal(Number), false + } + if r.ty != Number { + panic(fmt.Sprintf("NumberLowerBound for %#v", r.ty)) + } + if rfn, ok := r.raw.(*refinementNumber); ok && rfn.min != NilVal { + return rfn.min, rfn.minInc + } + return UnknownVal(Number), false +} + +// NumberUpperBound returns information about the upper bound of the range of +// a number value, or panics if the value is definitely not a number. +// +// If the value is nullable then the result represents the range of the number +// only if it turns out not to be null. +// +// The resulting value might itself be an unknown number if there is no +// known upper bound. In that case the "inclusive" flag is meaningless. +func (r ValueRange) NumberUpperBound() (max Value, inclusive bool) { + if r.ty == DynamicPseudoType { + // We don't even know if this is a number yet. + return UnknownVal(Number), false + } + if r.ty != Number { + panic(fmt.Sprintf("NumberUpperBound for %#v", r.ty)) + } + if rfn, ok := r.raw.(*refinementNumber); ok && rfn.max != NilVal { + return rfn.max, rfn.maxInc + } + return UnknownVal(Number), false +} + +// StringPrefix returns a string that is guaranteed to be the prefix of +// the string value being described, or panics if the value is definitely not +// a string. +// +// If the value is nullable then the result represents the prefix of the string +// only if it turns out to not be null. +// +// If the resulting value is zero-length then the value could potentially be +// a string but it has no known prefix. +// +// cty.String values always contain normalized UTF-8 sequences; the result is +// also guaranteed to be a normalized UTF-8 sequence so the result also +// represents the exact bytes of the string value's prefix. +func (r ValueRange) StringPrefix() string { + if r.ty == DynamicPseudoType { + // We don't even know if this is a string yet. + return "" + } + if r.ty != String { + panic(fmt.Sprintf("StringPrefix for %#v", r.ty)) + } + if rfn, ok := r.raw.(*refinementString); ok { + return rfn.prefix + } + return "" +} + +// LengthLowerBound returns information about the lower bound of the length of +// a collection-typed value, or panics if the value is definitely not a +// collection. +// +// If the value is nullable then the result represents the range of the length +// only if the value turns out not to be null. +// +// The resulting value might itself be an unknown number if there is no +// known lower bound. In that case the "inclusive" flag is meaningless. +func (r ValueRange) LengthLowerBound() (min Value, inclusive bool) { + if r.ty == DynamicPseudoType { + // We don't even know if this is a collection yet. + return UnknownVal(Number), false + } + if !r.ty.IsCollectionType() { + panic(fmt.Sprintf("LengthLowerBound for %#v", r.ty)) + } + if rfn, ok := r.raw.(*refinementCollection); ok && rfn.minLen != NilVal { + return rfn.minLen, rfn.minInc + } + return UnknownVal(Number), false +} + +// LengthUpperBound returns information about the upper bound of the length of +// a collection-typed value, or panics if the value is definitely not a +// collection. +// +// If the value is nullable then the result represents the range of the length +// only if the value turns out not to be null. +// +// The resulting value might itself be an unknown number if there is no +// known upper bound. In that case the "inclusive" flag is meaningless. +func (r ValueRange) LengthUpperBound() (min Value, inclusive bool) { + if r.ty == DynamicPseudoType { + // We don't even know if this is a collection yet. + return UnknownVal(Number), false + } + if !r.ty.IsCollectionType() { + panic(fmt.Sprintf("LengthUpperBound for %#v", r.ty)) + } + if rfn, ok := r.raw.(*refinementCollection); ok && rfn.maxLen != NilVal { + return rfn.maxLen, rfn.maxInc + } + return UnknownVal(Number), false +} From 223d67e08c2cb37cef165ac4ab84019869ac3f03 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 26 Jan 2023 17:58:55 -0800 Subject: [PATCH 03/28] cty: Value.Equal result is never null, and comparisons to non-null Using the "refinements" feature we can refine the range of any unknown values from Equal to exclude cty.NullVal(cty.Bool), since we know that equality tests can never return null. We can also return a known False if the test is between a null value and an unknown value which has been refined as non-null. --- cty/function/stdlib/general_test.go | 8 +-- cty/value_ops.go | 41 +++++++++------ cty/value_ops_test.go | 80 ++++++++++++++++++----------- cty/value_range.go | 22 ++++++++ 4 files changed, 103 insertions(+), 48 deletions(-) diff --git a/cty/function/stdlib/general_test.go b/cty/function/stdlib/general_test.go index e9300582..d1dad2f7 100644 --- a/cty/function/stdlib/general_test.go +++ b/cty/function/stdlib/general_test.go @@ -36,22 +36,22 @@ func TestEqual(t *testing.T) { { cty.NumberIntVal(1), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).Refine().NotNull().NewValue(), }, { cty.UnknownVal(cty.Number), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).Refine().NotNull().NewValue(), }, { cty.NumberIntVal(1), cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).Refine().NotNull().NewValue(), }, { cty.DynamicVal, cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).Refine().NotNull().NewValue(), }, } diff --git a/cty/value_ops.go b/cty/value_ops.go index 83466afc..aaa47b1e 100644 --- a/cty/value_ops.go +++ b/cty/value_ops.go @@ -130,13 +130,24 @@ func (val Value) Equals(other Value) Value { return val.Equals(other).WithMarks(valMarks, otherMarks) } - // Start by handling Unknown values before considering types. - // This needs to be done since Null values are always equal regardless of - // type. + // Some easy cases with comparisons to null. + switch { + case val.IsNull() && definitelyNotNull(other): + return False + case other.IsNull() && definitelyNotNull(val): + return False + } + + // We need to deal with unknown values before anything else with nulls + // because any unknown value that hasn't yet been refined as non-null + // could become null, and nulls of any types are equal to one another. + unknownResult := func() Value { + return UnknownVal(Bool).Refine().NotNull().NewValue() + } switch { case !val.IsKnown() && !other.IsKnown(): // both unknown - return UnknownVal(Bool) + return unknownResult() case val.IsKnown() && !other.IsKnown(): switch { case val.IsNull(), other.ty.HasDynamicTypes(): @@ -144,13 +155,13 @@ func (val Value) Equals(other Value) Value { // nulls of any type are equal. // An unknown with a dynamic type compares as unknown, which we need // to check before the type comparison below. - return UnknownVal(Bool) + return unknownResult() case !val.ty.Equals(other.ty): // There is no null comparison or dynamic types, so unequal types // will never be equal. return False default: - return UnknownVal(Bool) + return unknownResult() } case other.IsKnown() && !val.IsKnown(): switch { @@ -159,13 +170,13 @@ func (val Value) Equals(other Value) Value { // nulls of any type are equal. // An unknown with a dynamic type compares as unknown, which we need // to check before the type comparison below. - return UnknownVal(Bool) + return unknownResult() case !other.ty.Equals(val.ty): // There's no null comparison or dynamic types, so unequal types // will never be equal. return False default: - return UnknownVal(Bool) + return unknownResult() } } @@ -187,7 +198,7 @@ func (val Value) Equals(other Value) Value { return BoolVal(false) } - return UnknownVal(Bool) + return unknownResult() } if !val.ty.Equals(other.ty) { @@ -221,7 +232,7 @@ func (val Value) Equals(other Value) Value { } eq := lhs.Equals(rhs) if !eq.IsKnown() { - return UnknownVal(Bool) + return unknownResult() } if eq.False() { result = false @@ -242,7 +253,7 @@ func (val Value) Equals(other Value) Value { } eq := lhs.Equals(rhs) if !eq.IsKnown() { - return UnknownVal(Bool) + return unknownResult() } if eq.False() { result = false @@ -264,7 +275,7 @@ func (val Value) Equals(other Value) Value { } eq := lhs.Equals(rhs) if !eq.IsKnown() { - return UnknownVal(Bool) + return unknownResult() } if eq.False() { result = false @@ -282,7 +293,7 @@ func (val Value) Equals(other Value) Value { for it := s1.Iterator(); it.Next(); { rv := it.Value() if _, unknown := rv.(*unknownType); unknown { // "*unknownType" is the internal representation of unknown-ness - return UnknownVal(Bool) + return unknownResult() } if !s2.Has(rv) { equal = false @@ -291,7 +302,7 @@ func (val Value) Equals(other Value) Value { for it := s2.Iterator(); it.Next(); { rv := it.Value() if _, unknown := rv.(*unknownType); unknown { // "*unknownType" is the internal representation of unknown-ness - return UnknownVal(Bool) + return unknownResult() } if !s1.Has(rv) { equal = false @@ -318,7 +329,7 @@ func (val Value) Equals(other Value) Value { } eq := lhs.Equals(rhs) if !eq.IsKnown() { - return UnknownVal(Bool) + return unknownResult() } if eq.False() { result = false diff --git a/cty/value_ops_test.go b/cty/value_ops_test.go index 2c6dca03..9c1459c1 100644 --- a/cty/value_ops_test.go +++ b/cty/value_ops_test.go @@ -11,6 +11,8 @@ func TestValueEquals(t *testing.T) { capsuleB := CapsuleVal(capsuleTestType1, &capsuleTestType1Native{"capsuleB"}) capsuleC := CapsuleVal(capsuleTestType2, &capsuleTestType2Native{"capsuleC"}) + unknownResult := UnknownVal(Bool).Refine().NotNull().NewValue() + tests := []struct { LHS Value RHS Value @@ -250,47 +252,47 @@ func TestValueEquals(t *testing.T) { { TupleVal([]Value{UnknownVal(Number)}), TupleVal([]Value{NumberIntVal(1)}), - UnknownVal(Bool), + unknownResult, }, { TupleVal([]Value{UnknownVal(Number)}), TupleVal([]Value{UnknownVal(Number)}), - UnknownVal(Bool), + unknownResult, }, { TupleVal([]Value{NumberIntVal(1)}), TupleVal([]Value{UnknownVal(Number)}), - UnknownVal(Bool), + unknownResult, }, { TupleVal([]Value{NumberIntVal(1)}), TupleVal([]Value{DynamicVal}), - UnknownVal(Bool), + unknownResult, }, { TupleVal([]Value{DynamicVal}), TupleVal([]Value{NumberIntVal(1)}), - UnknownVal(Bool), + unknownResult, }, { TupleVal([]Value{NumberIntVal(1)}), UnknownVal(Tuple([]Type{Number})), - UnknownVal(Bool), + unknownResult, }, { UnknownVal(Tuple([]Type{Number})), TupleVal([]Value{NumberIntVal(1)}), - UnknownVal(Bool), + unknownResult, }, { DynamicVal, TupleVal([]Value{NumberIntVal(1)}), - UnknownVal(Bool), + unknownResult, }, { TupleVal([]Value{NumberIntVal(1)}), DynamicVal, - UnknownVal(Bool), + unknownResult, }, // Lists @@ -532,7 +534,7 @@ func TestValueEquals(t *testing.T) { SetVal([]Value{ UnknownVal(Number), }), - UnknownVal(Bool), + unknownResult, }, { SetVal([]Value{ @@ -542,7 +544,7 @@ func TestValueEquals(t *testing.T) { NumberIntVal(1), UnknownVal(Number), }), - UnknownVal(Bool), + unknownResult, }, { SetVal([]Value{ @@ -552,7 +554,7 @@ func TestValueEquals(t *testing.T) { SetVal([]Value{ NumberIntVal(1), }), - UnknownVal(Bool), + unknownResult, }, // Capsules @@ -574,7 +576,7 @@ func TestValueEquals(t *testing.T) { { capsuleA, UnknownVal(capsuleTestType1), // same type - UnknownVal(Bool), + unknownResult, }, { capsuleA, @@ -586,22 +588,22 @@ func TestValueEquals(t *testing.T) { { NumberIntVal(2), UnknownVal(Number), - UnknownVal(Bool), + unknownResult, }, { NumberIntVal(1), DynamicVal, - UnknownVal(Bool), + unknownResult, }, { DynamicVal, BoolVal(true), - UnknownVal(Bool), + unknownResult, }, { DynamicVal, DynamicVal, - UnknownVal(Bool), + unknownResult, }, { ListVal([]Value{ @@ -612,7 +614,7 @@ func TestValueEquals(t *testing.T) { StringVal("hi"), DynamicVal, }), - UnknownVal(Bool), + unknownResult, }, { ListVal([]Value{ @@ -623,7 +625,7 @@ func TestValueEquals(t *testing.T) { StringVal("hi"), UnknownVal(String), }), - UnknownVal(Bool), + unknownResult, }, { MapVal(map[string]Value{ @@ -634,7 +636,7 @@ func TestValueEquals(t *testing.T) { "static": StringVal("hi"), "dynamic": DynamicVal, }), - UnknownVal(Bool), + unknownResult, }, { MapVal(map[string]Value{ @@ -645,7 +647,7 @@ func TestValueEquals(t *testing.T) { "static": StringVal("hi"), "dynamic": UnknownVal(String), }), - UnknownVal(Bool), + unknownResult, }, { NullVal(String), @@ -660,7 +662,7 @@ func TestValueEquals(t *testing.T) { { UnknownVal(String), UnknownVal(Number), - UnknownVal(Bool), + unknownResult, }, { StringVal(""), @@ -675,7 +677,7 @@ func TestValueEquals(t *testing.T) { { StringVal(""), UnknownVal(String), - UnknownVal(Bool), + unknownResult, }, { NullVal(DynamicPseudoType), @@ -685,17 +687,17 @@ func TestValueEquals(t *testing.T) { { NullVal(String), UnknownVal(Number), - UnknownVal(Bool), // because second operand might eventually be null + unknownResult, // because second operand might eventually be null }, { UnknownVal(String), NullVal(Number), - UnknownVal(Bool), // because first operand might eventually be null + unknownResult, // because first operand might eventually be null }, { UnknownVal(String), UnknownVal(Number), - UnknownVal(Bool), // because both operands might eventually be null + unknownResult, // because both operands might eventually be null }, { StringVal("hello"), @@ -759,7 +761,7 @@ func TestValueEquals(t *testing.T) { ObjectVal(map[string]Value{ "a": DynamicVal, }), - UnknownVal(Bool), + unknownResult, }, { ObjectVal(map[string]Value{ @@ -779,7 +781,27 @@ func TestValueEquals(t *testing.T) { ObjectVal(map[string]Value{ "a": UnknownVal(List(List(DynamicPseudoType))), }), - UnknownVal(Bool), + unknownResult, + }, + { + NullVal(String), + UnknownVal(String).Refine().NotNull().NewValue(), + False, + }, + { + UnknownVal(String).Refine().NotNull().NewValue(), + NullVal(String), + False, + }, + { + UnknownVal(String).Refine().Null().NewValue(), + NullVal(String), + True, // NOTE: The refinement should collapse to NullVal(String) + }, + { + NullVal(String), + UnknownVal(String).Refine().Null().NewValue(), + True, // NOTE: The refinement should collapse to NullVal(String) }, // Marks @@ -814,7 +836,7 @@ func TestValueEquals(t *testing.T) { t.Run(fmt.Sprintf("%#v.Equals(%#v)", test.LHS, test.RHS), func(t *testing.T) { got := test.LHS.Equals(test.RHS) if !got.RawEquals(test.Expected) { - t.Fatalf("wrong Equals result\ngot: %#v\nwant: %#v", got, test.Expected) + t.Fatalf("wrong Equals result\nLHS: %#v\nRHS: %#v\ngot: %#v\nwant: %#v", test.LHS, test.RHS, got, test.Expected) } }) } diff --git a/cty/value_range.go b/cty/value_range.go index e9967203..8c32d2bf 100644 --- a/cty/value_range.go +++ b/cty/value_range.go @@ -102,6 +102,15 @@ func (r ValueRange) CouldBeNull() bool { return r.raw.null() != tristateFalse } +// DefinitelyNotNull returns true if there are no null values in the range. +func (r ValueRange) DefinitelyNotNull() bool { + if r.raw == nil { + // A totally-unconstrained unknown value could be null + return false + } + return r.raw.null() == tristateFalse +} + // NumberLowerBound returns information about the lower bound of the range of // a number value, or panics if the value is definitely not a number. // @@ -218,3 +227,16 @@ func (r ValueRange) LengthUpperBound() (min Value, inclusive bool) { } return UnknownVal(Number), false } + +// definitelyNotNull is a convenient helper for the common situation of checking +// whether a value could possibly be null. +// +// Returns true if the given value is either a known value that isn't null +// or an unknown value that has been refined to exclude null values from its +// range. +func definitelyNotNull(v Value) bool { + if v.IsKnown() { + return !v.IsNull() + } + return v.Range().DefinitelyNotNull() +} From 3cec2c0bcfb5dfde906b941278c5dce4de7c5f0b Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 26 Jan 2023 18:18:43 -0800 Subject: [PATCH 04/28] cty: Value.Length refines its unknown results The length of a collection is never null and always at least zero. For sets containing unknown values in particular we also know that they must have at least one element (otherwise there wouldn't be an unknown value) and that they can't have any more elements than currently stored (because coalescing can't ever generate new elements). --- cty/function/stdlib/collection.go | 1 + cty/function/stdlib/collection_test.go | 15 ++++++++++++--- cty/value_ops.go | 10 +++++++--- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/cty/function/stdlib/collection.go b/cty/function/stdlib/collection.go index 0573e74e..d666b30b 100644 --- a/cty/function/stdlib/collection.go +++ b/cty/function/stdlib/collection.go @@ -114,6 +114,7 @@ var LengthFunc = function.New(&function.Spec{ Name: "collection", Type: cty.DynamicPseudoType, AllowDynamicType: true, + AllowUnknown: true, AllowMarked: true, }, }, diff --git a/cty/function/stdlib/collection_test.go b/cty/function/stdlib/collection_test.go index eac92d5d..8f7f5a5d 100644 --- a/cty/function/stdlib/collection_test.go +++ b/cty/function/stdlib/collection_test.go @@ -947,7 +947,9 @@ func TestLength(t *testing.T) { }, { cty.SetVal([]cty.Value{cty.True, cty.UnknownVal(cty.Bool)}), - cty.UnknownVal(cty.Number), // Don't know if the unknown in the input represents cty.True or cty.False + // Don't know if the unknown in the input represents cty.True or cty.False, + // so it may or may not coalesce with the one known value. + cty.UnknownVal(cty.Number).Refine().NotNull().NumberRangeInclusive(cty.NumberIntVal(1), cty.NumberIntVal(2)).NewValue(), }, { cty.SetVal([]cty.Value{cty.UnknownVal(cty.Bool)}), @@ -971,11 +973,18 @@ func TestLength(t *testing.T) { }, { cty.UnknownVal(cty.List(cty.Bool)), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).Refine().NotNull().NumberRangeInclusive(cty.Zero, cty.UnknownVal(cty.Number)).NewValue(), }, { cty.DynamicVal, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).Refine().NotNull().NumberRangeInclusive(cty.Zero, cty.UnknownVal(cty.Number)).NewValue(), + }, + { + // TODO: This one should really preserve the length bounds as the + // numeric bounds of its result, but cty.Value.Length isn't yet + // able to do that. + cty.UnknownVal(cty.List(cty.Bool)).Refine().CollectionLengthUpperBound(cty.NumberIntVal(2), true).NewValue(), + cty.UnknownVal(cty.Number).Refine().NotNull().NumberRangeInclusive(cty.Zero, cty.UnknownVal(cty.Number)).NewValue(), }, { // Marked collections return a marked length cty.ListVal([]cty.Value{ diff --git a/cty/value_ops.go b/cty/value_ops.go index aaa47b1e..281c7570 100644 --- a/cty/value_ops.go +++ b/cty/value_ops.go @@ -1038,7 +1038,9 @@ func (val Value) Length() Value { } if !val.IsKnown() { - return UnknownVal(Number) + // TODO: If the unknown value has been refined with explicit length + // bounds then we should use those as the refined range of this result. + return UnknownVal(Number).Refine().NotNull().NumberRangeInclusive(Zero, UnknownVal(Number)).NewValue() } if val.Type().IsSetType() { // The Length rules are a little different for sets because if any @@ -1056,8 +1058,10 @@ func (val Value) Length() Value { // unknown value cannot represent more than one known value. return NumberIntVal(storeLength) } - // Otherwise, we cannot predict the length. - return UnknownVal(Number) + // Otherwise, we cannot predict the length exactly but we can at + // least constrain both bounds of its range, because value coalescing + // can only ever reduce the number of elements in the set. + return UnknownVal(Number).Refine().NotNull().NumberRangeInclusive(NumberIntVal(1), NumberIntVal(storeLength)).NewValue() } return NumberIntVal(int64(val.LengthInt())) From 1113690e07d1e5764fcc5a807d2bf07e36956905 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 26 Jan 2023 18:37:57 -0800 Subject: [PATCH 05/28] cty: Arithmetic operations never return null We can refine the unknown value results for all of the arithmetic functions to represent that they cannot possibly be null. The Value.Absolute method is additionally guaranteed to return a value greater than or equal to zero, by definition. --- cty/unknown_refinement.go | 7 +++++ cty/value_ops.go | 21 +++++++++------ cty/value_ops_test.go | 56 +++++++++++++++++++-------------------- 3 files changed, 48 insertions(+), 36 deletions(-) diff --git a/cty/unknown_refinement.go b/cty/unknown_refinement.go index 84df483f..95bc579b 100644 --- a/cty/unknown_refinement.go +++ b/cty/unknown_refinement.go @@ -70,6 +70,13 @@ func (v Value) Refine() *RefinementBuilder { return &RefinementBuilder{v, wip} } +// RefineNotNull is a shorthand for Value.Refine().NotNull().NewValue(), because +// declaring that a unknown value isn't null is by far the most common use of +// refinements. +func (v Value) RefineNotNull() Value { + return v.Refine().NotNull().NewValue() +} + // RefinementBuilder is a supporting type for the [Value.Refine] method, // using the builder pattern to apply zero or more constraints before // constructing a new value with all of those constraints applied. diff --git a/cty/value_ops.go b/cty/value_ops.go index 281c7570..8a495c4c 100644 --- a/cty/value_ops.go +++ b/cty/value_ops.go @@ -36,7 +36,12 @@ func (val Value) GoString() string { rfn := val.v.(*unknownType).refinement var suffix string if rfn != nil { - suffix = ".Refine()" + rfn.GoString() + ".NewValue()" + calls := rfn.GoString() + if calls == ".NotNull()" { + suffix = ".RefineNotNull()" + } else { + suffix = ".Refine()" + rfn.GoString() + ".NewValue()" + } } return fmt.Sprintf("cty.UnknownVal(%#v)%s", val.ty, suffix) } @@ -574,7 +579,7 @@ func (val Value) Add(other Value) Value { if shortCircuit := mustTypeCheck(Number, Number, val, other); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Number) - return *shortCircuit + return (*shortCircuit).RefineNotNull() } ret := new(big.Float) @@ -593,7 +598,7 @@ func (val Value) Subtract(other Value) Value { if shortCircuit := mustTypeCheck(Number, Number, val, other); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Number) - return *shortCircuit + return (*shortCircuit).RefineNotNull() } return val.Add(other.Negate()) @@ -609,7 +614,7 @@ func (val Value) Negate() Value { if shortCircuit := mustTypeCheck(Number, Number, val); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Number) - return *shortCircuit + return (*shortCircuit).RefineNotNull() } ret := new(big.Float).Neg(val.v.(*big.Float)) @@ -627,7 +632,7 @@ func (val Value) Multiply(other Value) Value { if shortCircuit := mustTypeCheck(Number, Number, val, other); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Number) - return *shortCircuit + return (*shortCircuit).RefineNotNull() } // find the larger precision of the arguments @@ -672,7 +677,7 @@ func (val Value) Divide(other Value) Value { if shortCircuit := mustTypeCheck(Number, Number, val, other); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Number) - return *shortCircuit + return (*shortCircuit).RefineNotNull() } ret := new(big.Float) @@ -704,7 +709,7 @@ func (val Value) Modulo(other Value) Value { if shortCircuit := mustTypeCheck(Number, Number, val, other); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Number) - return *shortCircuit + return (*shortCircuit).RefineNotNull() } // We cheat a bit here with infinities, just abusing the Multiply operation @@ -742,7 +747,7 @@ func (val Value) Absolute() Value { if shortCircuit := mustTypeCheck(Number, Number, val); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Number) - return *shortCircuit + return (*shortCircuit).Refine().NotNull().NumberRangeInclusive(Zero, UnknownVal(Number)).NewValue() } ret := (&big.Float{}).Abs(val.v.(*big.Float)) diff --git a/cty/value_ops_test.go b/cty/value_ops_test.go index 9c1459c1..e75a16fa 100644 --- a/cty/value_ops_test.go +++ b/cty/value_ops_test.go @@ -11,7 +11,7 @@ func TestValueEquals(t *testing.T) { capsuleB := CapsuleVal(capsuleTestType1, &capsuleTestType1Native{"capsuleB"}) capsuleC := CapsuleVal(capsuleTestType2, &capsuleTestType2Native{"capsuleC"}) - unknownResult := UnknownVal(Bool).Refine().NotNull().NewValue() + unknownResult := UnknownVal(Bool).RefineNotNull() tests := []struct { LHS Value @@ -1730,22 +1730,22 @@ func TestValueAdd(t *testing.T) { { NumberIntVal(1), UnknownVal(Number), - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { UnknownVal(Number), UnknownVal(Number), - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { NumberIntVal(1), DynamicVal, - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { DynamicVal, DynamicVal, - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { Zero.Mark(1), @@ -1798,22 +1798,22 @@ func TestValueSubtract(t *testing.T) { { NumberIntVal(1), UnknownVal(Number), - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { UnknownVal(Number), UnknownVal(Number), - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { NumberIntVal(1), DynamicVal, - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { DynamicVal, DynamicVal, - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { Zero.Mark(1), @@ -1857,11 +1857,11 @@ func TestValueNegate(t *testing.T) { }, { UnknownVal(Number), - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { DynamicVal, - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { Zero.Mark(1), @@ -1903,22 +1903,22 @@ func TestValueMultiply(t *testing.T) { { NumberIntVal(1), UnknownVal(Number), - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { UnknownVal(Number), UnknownVal(Number), - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { NumberIntVal(1), DynamicVal, - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { DynamicVal, DynamicVal, - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { Zero.Mark(1), @@ -1991,22 +1991,22 @@ func TestValueDivide(t *testing.T) { { NumberIntVal(1), UnknownVal(Number), - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { UnknownVal(Number), UnknownVal(Number), - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { NumberIntVal(1), DynamicVal, - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { DynamicVal, DynamicVal, - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { Zero.Mark(1), @@ -2089,22 +2089,22 @@ func TestValueModulo(t *testing.T) { { NumberIntVal(1), UnknownVal(Number), - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { UnknownVal(Number), UnknownVal(Number), - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { NumberIntVal(1), DynamicVal, - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { DynamicVal, DynamicVal, - UnknownVal(Number), + UnknownVal(Number).RefineNotNull(), }, { NumberIntVal(10).Mark(1), @@ -2169,11 +2169,11 @@ func TestValueAbsolute(t *testing.T) { }, { UnknownVal(Number), - UnknownVal(Number), + UnknownVal(Number).Refine().NotNull().NumberRangeInclusive(Zero, UnknownVal(Number)).NewValue(), }, { DynamicVal, - UnknownVal(Number), + UnknownVal(Number).Refine().NotNull().NumberRangeInclusive(Zero, UnknownVal(Number)).NewValue(), }, { NumberIntVal(-1).Mark(1), @@ -2185,7 +2185,7 @@ func TestValueAbsolute(t *testing.T) { t.Run(fmt.Sprintf("%#v.Absolute()", test.Receiver), func(t *testing.T) { got := test.Receiver.Absolute() if !got.RawEquals(test.Expected) { - t.Fatalf("Absolute returned %#v; want %#v", got, test.Expected) + t.Fatalf("wrong result\ngot: %#v\nwant: %#v", got, test.Expected) } }) } @@ -3384,7 +3384,7 @@ func TestValueGoString(t *testing.T) { }, { UnknownVal(String).Refine().NotNull().NewValue(), - `cty.UnknownVal(cty.String).Refine().NotNull().NewValue()`, + `cty.UnknownVal(cty.String).RefineNotNull()`, }, { UnknownVal(String).Refine().NotNull().StringPrefix("a").NewValue(), @@ -3392,7 +3392,7 @@ func TestValueGoString(t *testing.T) { }, { UnknownVal(Bool).Refine().NotNull().NewValue(), - `cty.UnknownVal(cty.Bool).Refine().NotNull().NewValue()`, + `cty.UnknownVal(cty.Bool).RefineNotNull()`, }, { UnknownVal(Number).Refine().NumberRangeInclusive(Zero, UnknownVal(Number)).NewValue(), From 9a36ada35e42070516b5ee08e00ca109a4be6da1 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 26 Jan 2023 18:48:03 -0800 Subject: [PATCH 06/28] cty: Non-null refinements for all other Value operation methods Many of our operation methods are guaranteed to never return null, so we'll refine the unknown values they return to specify that. --- cty/value_ops.go | 32 +++++++-------- cty/value_ops_test.go | 96 +++++++++++++++++++++---------------------- 2 files changed, 64 insertions(+), 64 deletions(-) diff --git a/cty/value_ops.go b/cty/value_ops.go index 8a495c4c..729d87f5 100644 --- a/cty/value_ops.go +++ b/cty/value_ops.go @@ -920,23 +920,23 @@ func (val Value) HasIndex(key Value) Value { } if val.ty == DynamicPseudoType { - return UnknownVal(Bool) + return UnknownVal(Bool).RefineNotNull() } switch { case val.Type().IsListType(): if key.Type() == DynamicPseudoType { - return UnknownVal(Bool) + return UnknownVal(Bool).RefineNotNull() } if key.Type() != Number { return False } if !key.IsKnown() { - return UnknownVal(Bool) + return UnknownVal(Bool).RefineNotNull() } if !val.IsKnown() { - return UnknownVal(Bool) + return UnknownVal(Bool).RefineNotNull() } index, accuracy := key.v.(*big.Float).Int64() @@ -947,17 +947,17 @@ func (val Value) HasIndex(key Value) Value { return BoolVal(int(index) < len(val.v.([]interface{})) && index >= 0) case val.Type().IsMapType(): if key.Type() == DynamicPseudoType { - return UnknownVal(Bool) + return UnknownVal(Bool).RefineNotNull() } if key.Type() != String { return False } if !key.IsKnown() { - return UnknownVal(Bool) + return UnknownVal(Bool).RefineNotNull() } if !val.IsKnown() { - return UnknownVal(Bool) + return UnknownVal(Bool).RefineNotNull() } keyStr := key.v.(string) @@ -966,14 +966,14 @@ func (val Value) HasIndex(key Value) Value { return BoolVal(exists) case val.Type().IsTupleType(): if key.Type() == DynamicPseudoType { - return UnknownVal(Bool) + return UnknownVal(Bool).RefineNotNull() } if key.Type() != Number { return False } if !key.IsKnown() { - return UnknownVal(Bool) + return UnknownVal(Bool).RefineNotNull() } index, accuracy := key.v.(*big.Float).Int64() @@ -1008,10 +1008,10 @@ func (val Value) HasElement(elem Value) Value { panic("not a set type") } if !val.IsKnown() || !elem.IsKnown() { - return UnknownVal(Bool) + return UnknownVal(Bool).RefineNotNull() } if val.IsNull() { - panic("can't call HasElement on a nil value") + panic("can't call HasElement on a null value") } if !ty.ElementType().Equals(elem.Type()) { return False @@ -1202,7 +1202,7 @@ func (val Value) Not() Value { if shortCircuit := mustTypeCheck(Bool, Bool, val); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Bool) - return *shortCircuit + return (*shortCircuit).RefineNotNull() } return BoolVal(!val.v.(bool)) @@ -1219,7 +1219,7 @@ func (val Value) And(other Value) Value { if shortCircuit := mustTypeCheck(Bool, Bool, val, other); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Bool) - return *shortCircuit + return (*shortCircuit).RefineNotNull() } return BoolVal(val.v.(bool) && other.v.(bool)) @@ -1236,7 +1236,7 @@ func (val Value) Or(other Value) Value { if shortCircuit := mustTypeCheck(Bool, Bool, val, other); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Bool) - return *shortCircuit + return (*shortCircuit).RefineNotNull() } return BoolVal(val.v.(bool) || other.v.(bool)) @@ -1253,7 +1253,7 @@ func (val Value) LessThan(other Value) Value { if shortCircuit := mustTypeCheck(Number, Bool, val, other); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Bool) - return *shortCircuit + return (*shortCircuit).RefineNotNull() } return BoolVal(val.v.(*big.Float).Cmp(other.v.(*big.Float)) < 0) @@ -1270,7 +1270,7 @@ func (val Value) GreaterThan(other Value) Value { if shortCircuit := mustTypeCheck(Number, Bool, val, other); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Bool) - return *shortCircuit + return (*shortCircuit).RefineNotNull() } return BoolVal(val.v.(*big.Float).Cmp(other.v.(*big.Float)) > 0) diff --git a/cty/value_ops_test.go b/cty/value_ops_test.go index e75a16fa..4bb59266 100644 --- a/cty/value_ops_test.go +++ b/cty/value_ops_test.go @@ -2419,17 +2419,17 @@ func TestValueHasIndex(t *testing.T) { { ListVal([]Value{StringVal("hello")}), UnknownVal(Number), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { ListVal([]Value{StringVal("hello")}), DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { UnknownVal(List(String)), NumberIntVal(0), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { UnknownVal(List(String)), @@ -2469,17 +2469,17 @@ func TestValueHasIndex(t *testing.T) { { MapVal(map[string]Value{"greeting": StringVal("hello")}), UnknownVal(String), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { MapVal(map[string]Value{"greeting": StringVal("hello")}), DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { UnknownVal(Map(String)), StringVal("hello"), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { UnknownVal(Map(String)), @@ -2524,7 +2524,7 @@ func TestValueHasIndex(t *testing.T) { { TupleVal([]Value{StringVal("hello")}), UnknownVal(Number), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { UnknownVal(Tuple([]Type{String})), @@ -2534,17 +2534,17 @@ func TestValueHasIndex(t *testing.T) { { TupleVal([]Value{StringVal("hello")}), DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { DynamicVal, StringVal("hello"), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { DynamicVal, NumberIntVal(0), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { ListVal([]Value{StringVal("hello")}).Mark(1), @@ -2753,11 +2753,11 @@ func TestValueNot(t *testing.T) { }, { UnknownVal(Bool), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { True.Mark(1), @@ -2804,32 +2804,32 @@ func TestValueAnd(t *testing.T) { { UnknownVal(Bool), UnknownVal(Bool), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { True, UnknownVal(Bool), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { UnknownVal(Bool), True, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { DynamicVal, DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { True, DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { DynamicVal, True, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { True.Mark(1), @@ -2887,32 +2887,32 @@ func TestValueOr(t *testing.T) { { UnknownVal(Bool), UnknownVal(Bool), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { True, UnknownVal(Bool), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { UnknownVal(Bool), True, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { DynamicVal, DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { True, DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { DynamicVal, True, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { True.Mark(1), @@ -2990,32 +2990,32 @@ func TestLessThan(t *testing.T) { { UnknownVal(Number), UnknownVal(Number), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { NumberIntVal(1), UnknownVal(Number), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { UnknownVal(Number), NumberIntVal(1), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { DynamicVal, DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { NumberIntVal(1), DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { DynamicVal, NumberIntVal(1), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { NumberIntVal(0).Mark(1), @@ -3093,32 +3093,32 @@ func TestGreaterThan(t *testing.T) { { UnknownVal(Number), UnknownVal(Number), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { NumberIntVal(1), UnknownVal(Number), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { UnknownVal(Number), NumberIntVal(1), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { DynamicVal, DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { NumberIntVal(1), DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { DynamicVal, NumberIntVal(1), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { NumberIntVal(1).Mark(1), @@ -3196,32 +3196,32 @@ func TestLessThanOrEqualTo(t *testing.T) { { UnknownVal(Number), UnknownVal(Number), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { NumberIntVal(1), UnknownVal(Number), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { UnknownVal(Number), NumberIntVal(1), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { DynamicVal, DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { NumberIntVal(1), DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { DynamicVal, NumberIntVal(1), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { NumberIntVal(0).Mark(1), @@ -3299,32 +3299,32 @@ func TestGreaterThanOrEqualTo(t *testing.T) { { UnknownVal(Number), UnknownVal(Number), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { NumberIntVal(1), UnknownVal(Number), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { UnknownVal(Number), NumberIntVal(1), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { DynamicVal, DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { NumberIntVal(1), DynamicVal, - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { DynamicVal, NumberIntVal(1), - UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), }, { NumberIntVal(0).Mark(1), From 41d10a56e15c3d5441608e62adf28ae7f23ddcfc Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 31 Jan 2023 15:57:51 -0800 Subject: [PATCH 07/28] function/stdlib: non-null refinements for functions where that is true This slightly reduces the result range described by unknown results from these functions so that downstream operations on the results have more chance of producing known results even though their input is unknown. --- cty/function/function.go | 29 +++++++ cty/function/stdlib/bool.go | 9 ++- cty/function/stdlib/bool_test.go | 20 ++--- cty/function/stdlib/bytes.go | 6 +- cty/function/stdlib/collection.go | 19 ++++- cty/function/stdlib/collection_test.go | 38 ++++----- cty/function/stdlib/conversion.go | 33 ++++++++ cty/function/stdlib/csv.go | 1 + cty/function/stdlib/datetime.go | 3 +- cty/function/stdlib/format.go | 8 +- cty/function/stdlib/format_test.go | 24 +++--- cty/function/stdlib/general.go | 11 ++- cty/function/stdlib/general_test.go | 14 ++-- cty/function/stdlib/json.go | 3 +- cty/function/stdlib/json_test.go | 6 +- cty/function/stdlib/number.go | 58 +++++++++----- cty/function/stdlib/number_test.go | 106 ++++++++++++------------- cty/function/stdlib/regexp.go | 2 + cty/function/stdlib/regexp_test.go | 10 +-- cty/function/stdlib/sequence.go | 4 +- cty/function/stdlib/set.go | 15 ++-- cty/function/stdlib/set_test.go | 14 ++-- cty/function/stdlib/string.go | 45 +++++++---- cty/function/stdlib/string_replace.go | 6 +- cty/function/stdlib/string_test.go | 16 ++-- cty/unknown_refinement.go | 47 ++++++++++- 26 files changed, 365 insertions(+), 182 deletions(-) diff --git a/cty/function/function.go b/cty/function/function.go index c4d99f6c..1cfbf2b7 100644 --- a/cty/function/function.go +++ b/cty/function/function.go @@ -39,6 +39,19 @@ type Spec struct { // depending on its arguments. Type TypeFunc + // RefineResult is an optional callback for describing additional + // refinements for the result value beyond what can be described using + // a type constraint. + // + // A refinement callback should always return the same builder it was + // given, typically after modifying it using the methods of + // [cty.RefinementBuilder]. + // + // Any refinements described by this callback must hold for the entire + // range of results from the function. For refinements that only apply + // to certain results, use direct refinement within [Impl] instead. + RefineResult func(*cty.RefinementBuilder) *cty.RefinementBuilder + // Impl is the ImplFunc that implements the function's behavior. // // Functions are expected to behave as pure functions, and not create @@ -233,6 +246,22 @@ func (f Function) Call(args []cty.Value) (val cty.Value, err error) { return cty.NilVal, err } + if refineResult := f.spec.RefineResult; refineResult != nil { + // If this function has a refinement callback then we'll refine + // our result value in the same way regardless of how we return. + // It's the function author's responsibility to ensure that the + // refinements they specify are valid for the full range of possible + // return values from the function. If not, this will panic when + // detecting an inconsistency. + defer func() { + if val != cty.NilVal { + if val.IsKnown() || val.Type() != cty.DynamicPseudoType { + val = val.RefineWith(refineResult) + } + } + }() + } + // Type checking already dealt with most situations relating to our // parameter specification, but we still need to deal with unknown // values and marked values. diff --git a/cty/function/stdlib/bool.go b/cty/function/stdlib/bool.go index 8192d8ce..2826bf6e 100644 --- a/cty/function/stdlib/bool.go +++ b/cty/function/stdlib/bool.go @@ -15,7 +15,8 @@ var NotFunc = function.New(&function.Spec{ AllowMarked: true, }, }, - Type: function.StaticReturnType(cty.Bool), + Type: function.StaticReturnType(cty.Bool), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { return args[0].Not(), nil }, @@ -37,7 +38,8 @@ var AndFunc = function.New(&function.Spec{ AllowMarked: true, }, }, - Type: function.StaticReturnType(cty.Bool), + Type: function.StaticReturnType(cty.Bool), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { return args[0].And(args[1]), nil }, @@ -59,7 +61,8 @@ var OrFunc = function.New(&function.Spec{ AllowMarked: true, }, }, - Type: function.StaticReturnType(cty.Bool), + Type: function.StaticReturnType(cty.Bool), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { return args[0].Or(args[1]), nil }, diff --git a/cty/function/stdlib/bool_test.go b/cty/function/stdlib/bool_test.go index 9a2179f7..e170b2e3 100644 --- a/cty/function/stdlib/bool_test.go +++ b/cty/function/stdlib/bool_test.go @@ -22,11 +22,11 @@ func TestNot(t *testing.T) { }, { cty.UnknownVal(cty.Bool), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.True.Mark(1), @@ -78,22 +78,22 @@ func TestAnd(t *testing.T) { { cty.True, cty.UnknownVal(cty.Bool), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.UnknownVal(cty.Bool), cty.UnknownVal(cty.Bool), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.True, cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.DynamicVal, cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, } @@ -141,22 +141,22 @@ func TestOr(t *testing.T) { { cty.True, cty.UnknownVal(cty.Bool), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.UnknownVal(cty.Bool), cty.UnknownVal(cty.Bool), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.True, cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.DynamicVal, cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, } diff --git a/cty/function/stdlib/bytes.go b/cty/function/stdlib/bytes.go index 3fe600ff..fe67e6f3 100644 --- a/cty/function/stdlib/bytes.go +++ b/cty/function/stdlib/bytes.go @@ -38,7 +38,8 @@ var BytesLenFunc = function.New(&function.Spec{ AllowDynamicType: true, }, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { bufPtr := args[0].EncapsulatedValue().(*[]byte) return cty.NumberIntVal(int64(len(*bufPtr))), nil @@ -65,7 +66,8 @@ var BytesSliceFunc = function.New(&function.Spec{ AllowDynamicType: true, }, }, - Type: function.StaticReturnType(Bytes), + Type: function.StaticReturnType(Bytes), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { bufPtr := args[0].EncapsulatedValue().(*[]byte) diff --git a/cty/function/stdlib/collection.go b/cty/function/stdlib/collection.go index d666b30b..7095c5f6 100644 --- a/cty/function/stdlib/collection.go +++ b/cty/function/stdlib/collection.go @@ -32,6 +32,7 @@ var HasIndexFunc = function.New(&function.Spec{ } return cty.Bool, nil }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { return args[0].HasIndex(args[1]), nil }, @@ -125,6 +126,7 @@ var LengthFunc = function.New(&function.Spec{ } return cty.Number, nil }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { return args[0].Length(), nil }, @@ -252,6 +254,7 @@ var CoalesceListFunc = function.New(&function.Spec{ return last, nil }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { for _, arg := range args { if !arg.IsKnown() { @@ -284,7 +287,8 @@ var CompactFunc = function.New(&function.Spec{ Type: cty.List(cty.String), }, }, - Type: function.StaticReturnType(cty.List(cty.String)), + Type: function.StaticReturnType(cty.List(cty.String)), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { listVal := args[0] if !listVal.IsWhollyKnown() { @@ -325,7 +329,8 @@ var ContainsFunc = function.New(&function.Spec{ Type: cty.DynamicPseudoType, }, }, - Type: function.StaticReturnType(cty.Bool), + Type: function.StaticReturnType(cty.Bool), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { arg := args[0] ty := arg.Type() @@ -383,6 +388,7 @@ var DistinctFunc = function.New(&function.Spec{ Type: func(args []cty.Value) (cty.Type, error) { return args[0].Type(), nil }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { listVal := args[0] @@ -427,6 +433,7 @@ var ChunklistFunc = function.New(&function.Spec{ Type: func(args []cty.Value) (cty.Type, error) { return cty.List(args[0].Type()), nil }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { listVal := args[0] sizeVal := args[1] @@ -514,6 +521,7 @@ var FlattenFunc = function.New(&function.Spec{ } return cty.Tuple(tys), nil }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { inputList := args[0] @@ -612,6 +620,7 @@ var KeysFunc = function.New(&function.Spec{ return cty.DynamicPseudoType, function.NewArgErrorf(0, "must have map or object type") } }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { // We must unmark the value before we can use ElementIterator on it, and // then re-apply the same marks (possibly none) when we return. Since we @@ -833,6 +842,7 @@ var MergeFunc = function.New(&function.Spec{ return cty.Object(attrs), nil }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { outputMap := make(map[string]cty.Value) var markses []cty.ValueMarks // remember any marked maps/objects we find @@ -892,6 +902,7 @@ var ReverseListFunc = function.New(&function.Spec{ return cty.NilType, function.NewArgErrorf(0, "can only reverse list or tuple values, not %s", argTy.FriendlyName()) } }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { in, marks := args[0].Unmark() inVals := in.AsValueSlice() @@ -965,6 +976,7 @@ var SetProductFunc = function.New(&function.Spec{ } return cty.Set(cty.Tuple(elemTys)), nil }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { ety := retType.ElementType() var retMarks cty.ValueMarks @@ -1102,6 +1114,7 @@ var SliceFunc = function.New(&function.Spec{ } return cty.Tuple(argTy.TupleElementTypes()[startIndex:endIndex]), nil }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { inputList, marks := args[0].Unmark() @@ -1216,6 +1229,7 @@ var ValuesFunc = function.New(&function.Spec{ } return cty.NilType, errors.New("values() requires a map as the first argument") }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { mapVar := args[0] @@ -1304,6 +1318,7 @@ var ZipmapFunc = function.New(&function.Spec{ return cty.NilType, errors.New("values argument must be a list or tuple value") } }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { keys := args[0] values := args[1] diff --git a/cty/function/stdlib/collection_test.go b/cty/function/stdlib/collection_test.go index 8f7f5a5d..5c001810 100644 --- a/cty/function/stdlib/collection_test.go +++ b/cty/function/stdlib/collection_test.go @@ -57,22 +57,22 @@ func TestHasIndex(t *testing.T) { { cty.ListValEmpty(cty.Number), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.UnknownVal(cty.List(cty.Bool)), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.ListValEmpty(cty.Number), cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.DynamicVal, cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, } @@ -107,7 +107,7 @@ func TestChunklist(t *testing.T) { { cty.UnknownVal(cty.List(cty.String)), cty.NumberIntVal(2), - cty.UnknownVal(cty.List(cty.List(cty.String))), + cty.UnknownVal(cty.List(cty.List(cty.String))).RefineNotNull(), ``, }, { @@ -359,7 +359,7 @@ func TestContains(t *testing.T) { { listWithUnknown, cty.StringVal("orange"), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), false, }, { @@ -404,7 +404,7 @@ func TestContains(t *testing.T) { cty.StringVal("fox"), }), cty.StringVal("quick"), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), false, }, { // set val @@ -424,7 +424,7 @@ func TestContains(t *testing.T) { cty.StringVal("fox"), }), cty.StringVal("quick"), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), false, }, { // nested unknown @@ -436,7 +436,7 @@ func TestContains(t *testing.T) { cty.ObjectVal(map[string]cty.Value{ "a": cty.StringVal("b"), }), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), false, }, { // tuple val @@ -557,7 +557,7 @@ func TestMerge(t *testing.T) { "c": cty.StringVal("d"), }), }, - cty.UnknownVal(cty.Map(cty.String)), + cty.UnknownVal(cty.Map(cty.String)).RefineNotNull(), false, }, { // handle dynamic unknown @@ -1398,7 +1398,7 @@ func TestValues(t *testing.T) { }, { cty.UnknownVal(cty.Map(cty.String)), - cty.UnknownVal(cty.List(cty.String)), + cty.UnknownVal(cty.List(cty.String)).RefineNotNull(), ``, }, { @@ -1443,12 +1443,12 @@ func TestValues(t *testing.T) { }, { cty.UnknownVal(cty.EmptyObject), - cty.UnknownVal(cty.EmptyTuple), + cty.UnknownVal(cty.EmptyTuple).RefineNotNull(), ``, }, { cty.UnknownVal(cty.Object(map[string]cty.Type{"a": cty.String})), - cty.UnknownVal(cty.Tuple([]cty.Type{cty.String})), + cty.UnknownVal(cty.Tuple([]cty.Type{cty.String})).RefineNotNull(), ``, }, { // The object itself is not marked, just an inner attribute value. @@ -1524,19 +1524,19 @@ func TestZipMap(t *testing.T) { { cty.UnknownVal(cty.List(cty.String)), cty.UnknownVal(cty.List(cty.String)), - cty.UnknownVal(cty.Map(cty.String)), + cty.UnknownVal(cty.Map(cty.String)).RefineNotNull(), ``, }, { cty.UnknownVal(cty.List(cty.String)), cty.ListValEmpty(cty.String), - cty.UnknownVal(cty.Map(cty.String)), + cty.UnknownVal(cty.Map(cty.String)).RefineNotNull(), ``, }, { cty.ListValEmpty(cty.String), cty.UnknownVal(cty.List(cty.String)), - cty.UnknownVal(cty.Map(cty.String)), + cty.UnknownVal(cty.Map(cty.String)).RefineNotNull(), ``, }, { @@ -1635,7 +1635,7 @@ func TestZipMap(t *testing.T) { { cty.ListValEmpty(cty.String), cty.UnknownVal(cty.EmptyTuple), - cty.UnknownVal(cty.EmptyObject), + cty.UnknownVal(cty.EmptyObject).RefineNotNull(), ``, }, { @@ -2498,7 +2498,7 @@ func TestSetproduct(t *testing.T) { cty.SetVal([]cty.Value{cty.StringVal("x"), cty.UnknownVal(cty.String)}).Mark("a"), cty.SetVal([]cty.Value{cty.True, cty.False}).Mark("b"), }, - cty.UnknownVal(cty.Set(cty.Tuple([]cty.Type{cty.String, cty.Bool}))).WithMarks(cty.NewValueMarks("a", "b")), + cty.UnknownVal(cty.Set(cty.Tuple([]cty.Type{cty.String, cty.Bool}))).RefineNotNull().WithMarks(cty.NewValueMarks("a", "b")), ``, }, } @@ -2548,7 +2548,7 @@ func TestReverseList(t *testing.T) { }, { cty.UnknownVal(cty.List(cty.String)), - cty.UnknownVal(cty.List(cty.String)), + cty.UnknownVal(cty.List(cty.String)).RefineNotNull(), ``, }, { // marks on list elements diff --git a/cty/function/stdlib/conversion.go b/cty/function/stdlib/conversion.go index f61b5340..5d06a451 100644 --- a/cty/function/stdlib/conversion.go +++ b/cty/function/stdlib/conversion.go @@ -87,3 +87,36 @@ func MakeToFunc(wantTy cty.Type) function.Function { }, }) } + +// AssertNotNullFunc is a function which does nothing except return an error +// if the argument given to it is null. +// +// This could be useful in some cases where the automatic refinment of +// nullability isn't precise enough, because the result is guaranteed to not +// be null and can therefore allow downstream comparisons to null to return +// a known value even if the value is otherwise unknown. +var AssertNotNullFunc = function.New(&function.Spec{ + Description: "Returns the given value varbatim if it is non-null, or raises an error if it's null.", + Params: []function.Parameter{ + { + Name: "v", + Type: cty.DynamicPseudoType, + // NOTE: We intentionally don't set AllowNull here, and so + // the function system will automatically reject a null argument + // for us before calling Impl. + }, + }, + Type: func(args []cty.Value) (cty.Type, error) { + return args[0].Type(), nil + }, + RefineResult: refineNonNull, + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + // Our argument doesn't set AllowNull: true, so we're guaranteed to + // have a non-null value in args[0]. + return args[0], nil + }, +}) + +func AssertNotNull(v cty.Value) (cty.Value, error) { + return AssertNotNullFunc.Call([]cty.Value{v}) +} diff --git a/cty/function/stdlib/csv.go b/cty/function/stdlib/csv.go index 20d82bcd..e854e817 100644 --- a/cty/function/stdlib/csv.go +++ b/cty/function/stdlib/csv.go @@ -43,6 +43,7 @@ var CSVDecodeFunc = function.New(&function.Spec{ } return cty.List(cty.Object(atys)), nil }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { ety := retType.ElementType() atys := ety.AttributeTypes() diff --git a/cty/function/stdlib/datetime.go b/cty/function/stdlib/datetime.go index e668403a..85f58d4c 100644 --- a/cty/function/stdlib/datetime.go +++ b/cty/function/stdlib/datetime.go @@ -23,7 +23,8 @@ var FormatDateFunc = function.New(&function.Spec{ Type: cty.String, }, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { formatStr := args[0].AsString() timeStr := args[1].AsString() diff --git a/cty/function/stdlib/format.go b/cty/function/stdlib/format.go index ca163a87..68ceae98 100644 --- a/cty/function/stdlib/format.go +++ b/cty/function/stdlib/format.go @@ -30,7 +30,8 @@ var FormatFunc = function.New(&function.Spec{ Type: cty.DynamicPseudoType, AllowNull: true, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { for _, arg := range args[1:] { if !arg.IsWhollyKnown() { @@ -59,7 +60,8 @@ var FormatListFunc = function.New(&function.Spec{ AllowNull: true, AllowUnknown: true, }, - Type: function.StaticReturnType(cty.List(cty.String)), + Type: function.StaticReturnType(cty.List(cty.String)), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { fmtVal := args[0] args = args[1:] @@ -164,7 +166,7 @@ var FormatListFunc = function.New(&function.Spec{ // We require all nested values to be known because the only // thing we can do for a collection/structural type is print // it as JSON and that requires it to be wholly known. - ret = append(ret, cty.UnknownVal(cty.String)) + ret = append(ret, cty.UnknownVal(cty.String).RefineNotNull()) continue Results } } diff --git a/cty/function/stdlib/format_test.go b/cty/function/stdlib/format_test.go index aad73a8f..261f92c4 100644 --- a/cty/function/stdlib/format_test.go +++ b/cty/function/stdlib/format_test.go @@ -99,7 +99,7 @@ func TestFormat(t *testing.T) { []cty.Value{cty.TupleVal([]cty.Value{ cty.UnknownVal(cty.String), })}, - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), ``, }, { @@ -445,7 +445,7 @@ func TestFormat(t *testing.T) { { cty.UnknownVal(cty.String), []cty.Value{cty.True}, - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), ``, }, { @@ -457,13 +457,13 @@ func TestFormat(t *testing.T) { { cty.StringVal("Hello, %s!"), []cty.Value{cty.UnknownVal(cty.String)}, - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), ``, }, { cty.StringVal("Hello, %[2]s!"), []cty.Value{cty.UnknownVal(cty.String), cty.StringVal("Ermintrude")}, - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), ``, }, @@ -752,7 +752,7 @@ func TestFormatList(t *testing.T) { []cty.Value{ cty.True, }, - cty.UnknownVal(cty.List(cty.String)), + cty.UnknownVal(cty.List(cty.String)).RefineNotNull(), ``, }, 15: { @@ -761,7 +761,7 @@ func TestFormatList(t *testing.T) { cty.UnknownVal(cty.String), }, cty.ListVal([]cty.Value{ - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), }), ``, }, @@ -780,7 +780,7 @@ func TestFormatList(t *testing.T) { []cty.Value{ cty.UnknownVal(cty.List(cty.String)), }, - cty.UnknownVal(cty.List(cty.String)), + cty.UnknownVal(cty.List(cty.String)).RefineNotNull(), ``, }, 18: { @@ -794,7 +794,7 @@ func TestFormatList(t *testing.T) { }, cty.ListVal([]cty.Value{ cty.StringVal(`["hello"]`), - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), cty.StringVal(`["world"]`), }), ``, @@ -804,7 +804,7 @@ func TestFormatList(t *testing.T) { []cty.Value{ cty.UnknownVal(cty.Tuple([]cty.Type{cty.String})), }, - cty.UnknownVal(cty.List(cty.String)), + cty.UnknownVal(cty.List(cty.String)).RefineNotNull(), ``, }, 20: { @@ -813,7 +813,7 @@ func TestFormatList(t *testing.T) { cty.UnknownVal(cty.Tuple([]cty.Type{cty.String})), cty.UnknownVal(cty.Tuple([]cty.Type{cty.String, cty.String})), }, - cty.UnknownVal(cty.List(cty.String)), + cty.UnknownVal(cty.List(cty.String)).RefineNotNull(), `argument 2 has length 2, which is inconsistent with argument 1 of length 1`, }, 21: { @@ -822,7 +822,7 @@ func TestFormatList(t *testing.T) { cty.ListVal([]cty.Value{cty.StringVal("hi")}), cty.UnknownVal(cty.Tuple([]cty.Type{cty.String, cty.String})), }, - cty.UnknownVal(cty.List(cty.String)), + cty.UnknownVal(cty.List(cty.String)).RefineNotNull(), `argument 2 has length 2, which is inconsistent with argument 1 of length 1`, }, 22: { @@ -833,7 +833,7 @@ func TestFormatList(t *testing.T) { cty.UnknownVal(cty.String), }), }, - cty.UnknownVal(cty.List(cty.String)), + cty.UnknownVal(cty.List(cty.String)).RefineNotNull(), ``, }, 23: { diff --git a/cty/function/stdlib/general.go b/cty/function/stdlib/general.go index 4f70fff9..627b55a5 100644 --- a/cty/function/stdlib/general.go +++ b/cty/function/stdlib/general.go @@ -26,7 +26,8 @@ var EqualFunc = function.New(&function.Spec{ AllowNull: true, }, }, - Type: function.StaticReturnType(cty.Bool), + Type: function.StaticReturnType(cty.Bool), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { return args[0].Equals(args[1]), nil }, @@ -50,7 +51,8 @@ var NotEqualFunc = function.New(&function.Spec{ AllowNull: true, }, }, - Type: function.StaticReturnType(cty.Bool), + Type: function.StaticReturnType(cty.Bool), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { return args[0].Equals(args[1]).Not(), nil }, @@ -77,6 +79,7 @@ var CoalesceFunc = function.New(&function.Spec{ } return retType, nil }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { for _, argVal := range args { if !argVal.IsKnown() { @@ -92,6 +95,10 @@ var CoalesceFunc = function.New(&function.Spec{ }, }) +func refineNonNull(b *cty.RefinementBuilder) *cty.RefinementBuilder { + return b.NotNull() +} + // Equal determines whether the two given values are equal, returning a // bool value. func Equal(a cty.Value, b cty.Value) (cty.Value, error) { diff --git a/cty/function/stdlib/general_test.go b/cty/function/stdlib/general_test.go index d1dad2f7..d95fd21b 100644 --- a/cty/function/stdlib/general_test.go +++ b/cty/function/stdlib/general_test.go @@ -36,22 +36,22 @@ func TestEqual(t *testing.T) { { cty.NumberIntVal(1), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Bool).Refine().NotNull().NewValue(), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.UnknownVal(cty.Number), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Bool).Refine().NotNull().NewValue(), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.NumberIntVal(1), cty.DynamicVal, - cty.UnknownVal(cty.Bool).Refine().NotNull().NewValue(), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.DynamicVal, cty.DynamicVal, - cty.UnknownVal(cty.Bool).Refine().NotNull().NewValue(), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, } @@ -97,15 +97,15 @@ func TestCoalesce(t *testing.T) { }, { []cty.Value{cty.UnknownVal(cty.Bool), cty.True}, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { []cty.Value{cty.UnknownVal(cty.Bool), cty.StringVal("hello")}, - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), }, { []cty.Value{cty.DynamicVal, cty.True}, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { []cty.Value{cty.DynamicVal}, diff --git a/cty/function/stdlib/json.go b/cty/function/stdlib/json.go index 63dd320e..30de8a51 100644 --- a/cty/function/stdlib/json.go +++ b/cty/function/stdlib/json.go @@ -16,7 +16,8 @@ var JSONEncodeFunc = function.New(&function.Spec{ AllowNull: true, }, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { val := args[0] if !val.IsWhollyKnown() { diff --git a/cty/function/stdlib/json_test.go b/cty/function/stdlib/json_test.go index 5c1348fc..9b8e6f86 100644 --- a/cty/function/stdlib/json_test.go +++ b/cty/function/stdlib/json_test.go @@ -42,15 +42,15 @@ func TestJSONEncode(t *testing.T) { }, { cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), }, { cty.ObjectVal(map[string]cty.Value{"dunno": cty.UnknownVal(cty.Bool), "false": cty.False}), - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), }, { cty.DynamicVal, - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), }, { cty.NullVal(cty.String), diff --git a/cty/function/stdlib/number.go b/cty/function/stdlib/number.go index ce737513..7005f746 100644 --- a/cty/function/stdlib/number.go +++ b/cty/function/stdlib/number.go @@ -20,7 +20,8 @@ var AbsoluteFunc = function.New(&function.Spec{ AllowMarked: true, }, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { return args[0].Absolute(), nil }, @@ -40,7 +41,8 @@ var AddFunc = function.New(&function.Spec{ AllowDynamicType: true, }, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { // big.Float.Add can panic if the input values are opposing infinities, // so we must catch that here in order to remain within @@ -74,7 +76,8 @@ var SubtractFunc = function.New(&function.Spec{ AllowDynamicType: true, }, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { // big.Float.Sub can panic if the input values are infinities, // so we must catch that here in order to remain within @@ -108,7 +111,8 @@ var MultiplyFunc = function.New(&function.Spec{ AllowDynamicType: true, }, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { // big.Float.Mul can panic if the input values are both zero or both // infinity, so we must catch that here in order to remain within @@ -143,7 +147,8 @@ var DivideFunc = function.New(&function.Spec{ AllowDynamicType: true, }, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { // big.Float.Quo can panic if the input values are both zero or both // infinity, so we must catch that here in order to remain within @@ -178,7 +183,8 @@ var ModuloFunc = function.New(&function.Spec{ AllowDynamicType: true, }, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { // big.Float.Mul can panic if the input values are both zero or both // infinity, so we must catch that here in order to remain within @@ -215,7 +221,8 @@ var GreaterThanFunc = function.New(&function.Spec{ AllowMarked: true, }, }, - Type: function.StaticReturnType(cty.Bool), + Type: function.StaticReturnType(cty.Bool), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { return args[0].GreaterThan(args[1]), nil }, @@ -237,7 +244,8 @@ var GreaterThanOrEqualToFunc = function.New(&function.Spec{ AllowMarked: true, }, }, - Type: function.StaticReturnType(cty.Bool), + Type: function.StaticReturnType(cty.Bool), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { return args[0].GreaterThanOrEqualTo(args[1]), nil }, @@ -259,7 +267,8 @@ var LessThanFunc = function.New(&function.Spec{ AllowMarked: true, }, }, - Type: function.StaticReturnType(cty.Bool), + Type: function.StaticReturnType(cty.Bool), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { return args[0].LessThan(args[1]), nil }, @@ -281,7 +290,8 @@ var LessThanOrEqualToFunc = function.New(&function.Spec{ AllowMarked: true, }, }, - Type: function.StaticReturnType(cty.Bool), + Type: function.StaticReturnType(cty.Bool), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { return args[0].LessThanOrEqualTo(args[1]), nil }, @@ -297,7 +307,8 @@ var NegateFunc = function.New(&function.Spec{ AllowMarked: true, }, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { return args[0].Negate(), nil }, @@ -311,7 +322,8 @@ var MinFunc = function.New(&function.Spec{ Type: cty.Number, AllowDynamicType: true, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { if len(args) == 0 { return cty.NilVal, fmt.Errorf("must pass at least one number") @@ -336,7 +348,8 @@ var MaxFunc = function.New(&function.Spec{ Type: cty.Number, AllowDynamicType: true, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { if len(args) == 0 { return cty.NilVal, fmt.Errorf("must pass at least one number") @@ -362,7 +375,8 @@ var IntFunc = function.New(&function.Spec{ AllowDynamicType: true, }, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { bf := args[0].AsBigFloat() if bf.IsInt() { @@ -384,7 +398,8 @@ var CeilFunc = function.New(&function.Spec{ Type: cty.Number, }, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { f := args[0].AsBigFloat() @@ -414,7 +429,8 @@ var FloorFunc = function.New(&function.Spec{ Type: cty.Number, }, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { f := args[0].AsBigFloat() @@ -447,7 +463,8 @@ var LogFunc = function.New(&function.Spec{ Type: cty.Number, }, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { var num float64 if err := gocty.FromCtyValue(args[0], &num); err != nil { @@ -476,7 +493,8 @@ var PowFunc = function.New(&function.Spec{ Type: cty.Number, }, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { var num float64 if err := gocty.FromCtyValue(args[0], &num); err != nil { @@ -502,7 +520,8 @@ var SignumFunc = function.New(&function.Spec{ Type: cty.Number, }, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { var num int if err := gocty.FromCtyValue(args[0], &num); err != nil { @@ -539,6 +558,7 @@ var ParseIntFunc = function.New(&function.Spec{ } return cty.Number, nil }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { var numstr string diff --git a/cty/function/stdlib/number_test.go b/cty/function/stdlib/number_test.go index 1286e14e..9df55fd2 100644 --- a/cty/function/stdlib/number_test.go +++ b/cty/function/stdlib/number_test.go @@ -36,11 +36,11 @@ func TestAbsolute(t *testing.T) { }, { cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.DynamicVal, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, } @@ -73,22 +73,22 @@ func TestAdd(t *testing.T) { { cty.NumberIntVal(1), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.UnknownVal(cty.Number), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.NumberIntVal(1), cty.DynamicVal, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.DynamicVal, cty.DynamicVal, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, } @@ -121,22 +121,22 @@ func TestSubtract(t *testing.T) { { cty.NumberIntVal(1), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.UnknownVal(cty.Number), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.NumberIntVal(1), cty.DynamicVal, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.DynamicVal, cty.DynamicVal, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, } @@ -169,22 +169,22 @@ func TestMultiply(t *testing.T) { { cty.NumberIntVal(1), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.UnknownVal(cty.Number), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.NumberIntVal(1), cty.DynamicVal, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.DynamicVal, cty.DynamicVal, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, } @@ -237,22 +237,22 @@ func TestDivide(t *testing.T) { { cty.NumberIntVal(1), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.UnknownVal(cty.Number), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.NumberIntVal(1), cty.DynamicVal, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.DynamicVal, cty.DynamicVal, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, } @@ -305,22 +305,22 @@ func TestModulo(t *testing.T) { { cty.NumberIntVal(1), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.UnknownVal(cty.Number), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.NumberIntVal(1), cty.DynamicVal, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.DynamicVal, cty.DynamicVal, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, } @@ -350,11 +350,11 @@ func TestNegate(t *testing.T) { }, { cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.DynamicVal, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, } @@ -397,22 +397,22 @@ func TestLessThan(t *testing.T) { { cty.NumberIntVal(1), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.UnknownVal(cty.Number), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.NumberIntVal(1), cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.DynamicVal, cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, } @@ -455,22 +455,22 @@ func TestLessThanOrEqualTo(t *testing.T) { { cty.NumberIntVal(1), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.UnknownVal(cty.Number), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.NumberIntVal(1), cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.DynamicVal, cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, } @@ -513,22 +513,22 @@ func TestGreaterThan(t *testing.T) { { cty.NumberIntVal(1), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.UnknownVal(cty.Number), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.NumberIntVal(1), cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.DynamicVal, cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, } @@ -571,22 +571,22 @@ func TestGreaterThanOrEqualTo(t *testing.T) { { cty.NumberIntVal(1), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.UnknownVal(cty.Number), cty.UnknownVal(cty.Number), - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.NumberIntVal(1), cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, { cty.DynamicVal, cty.DynamicVal, - cty.UnknownVal(cty.Bool), + cty.UnknownVal(cty.Bool).RefineNotNull(), }, } @@ -640,11 +640,11 @@ func TestMin(t *testing.T) { }, { []cty.Value{cty.PositiveInfinity, cty.UnknownVal(cty.Number)}, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { []cty.Value{cty.PositiveInfinity, cty.DynamicVal}, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { []cty.Value{cty.Zero.Mark(1), cty.NumberIntVal(1)}, @@ -702,11 +702,11 @@ func TestMax(t *testing.T) { }, { []cty.Value{cty.PositiveInfinity, cty.UnknownVal(cty.Number)}, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { []cty.Value{cty.PositiveInfinity, cty.DynamicVal}, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, } @@ -1170,55 +1170,55 @@ func TestParseInt(t *testing.T) { { cty.StringVal("FF"), cty.NumberIntVal(10), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), true, }, { cty.StringVal("00FF"), cty.NumberIntVal(10), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), true, }, { cty.StringVal("-00FF"), cty.NumberIntVal(10), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), true, }, { cty.NumberIntVal(2), cty.NumberIntVal(10), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), true, }, { cty.StringVal("1"), cty.NumberIntVal(63), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), true, }, { cty.StringVal("1"), cty.NumberIntVal(-1), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), true, }, { cty.StringVal("1"), cty.NumberIntVal(1), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), true, }, { cty.StringVal("1"), cty.NumberIntVal(0), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), true, }, { cty.StringVal("1.2"), cty.NumberIntVal(10), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), true, }, } diff --git a/cty/function/stdlib/regexp.go b/cty/function/stdlib/regexp.go index ab4257b6..24654442 100644 --- a/cty/function/stdlib/regexp.go +++ b/cty/function/stdlib/regexp.go @@ -33,6 +33,7 @@ var RegexFunc = function.New(&function.Spec{ } return retTy, err }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { if retType == cty.DynamicPseudoType { return cty.DynamicVal, nil @@ -79,6 +80,7 @@ var RegexAllFunc = function.New(&function.Spec{ } return cty.List(retTy), err }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { ety := retType.ElementType() if ety == cty.DynamicPseudoType { diff --git a/cty/function/stdlib/regexp_test.go b/cty/function/stdlib/regexp_test.go index d2fcb593..6bba0648 100644 --- a/cty/function/stdlib/regexp_test.go +++ b/cty/function/stdlib/regexp_test.go @@ -43,14 +43,14 @@ func TestRegex(t *testing.T) { cty.UnknownVal(cty.Tuple([]cty.Type{ cty.String, cty.String, - })), + })).RefineNotNull(), }, { cty.StringVal("(?P[0-9]*)"), cty.UnknownVal(cty.String), cty.UnknownVal(cty.Object(map[string]cty.Type{ "num": cty.String, - })), + })).RefineNotNull(), }, { cty.UnknownVal(cty.String), @@ -134,19 +134,19 @@ func TestRegexAll(t *testing.T) { cty.UnknownVal(cty.List(cty.Tuple([]cty.Type{ cty.String, cty.String, - }))), + }))).RefineNotNull(), }, { cty.StringVal("(?P[0-9]*)"), cty.UnknownVal(cty.String), cty.UnknownVal(cty.List(cty.Object(map[string]cty.Type{ "num": cty.String, - }))), + }))).RefineNotNull(), }, { cty.UnknownVal(cty.String), cty.StringVal("135abc456def"), - cty.UnknownVal(cty.List(cty.DynamicPseudoType)), + cty.UnknownVal(cty.List(cty.DynamicPseudoType)).RefineNotNull(), }, } diff --git a/cty/function/stdlib/sequence.go b/cty/function/stdlib/sequence.go index 6b2d97b4..009949d4 100644 --- a/cty/function/stdlib/sequence.go +++ b/cty/function/stdlib/sequence.go @@ -74,6 +74,7 @@ var ConcatFunc = function.New(&function.Spec{ } return cty.Tuple(etys), nil }, + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { switch { case retType.IsListType(): @@ -143,7 +144,8 @@ var RangeFunc = function.New(&function.Spec{ Name: "params", Type: cty.Number, }, - Type: function.StaticReturnType(cty.List(cty.Number)), + Type: function.StaticReturnType(cty.List(cty.Number)), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { var start, end, step cty.Value switch len(args) { diff --git a/cty/function/stdlib/set.go b/cty/function/stdlib/set.go index 15f4c05e..6da22919 100644 --- a/cty/function/stdlib/set.go +++ b/cty/function/stdlib/set.go @@ -23,7 +23,8 @@ var SetHasElementFunc = function.New(&function.Spec{ AllowDynamicType: true, }, }, - Type: function.StaticReturnType(cty.Bool), + Type: function.StaticReturnType(cty.Bool), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { return args[0].HasElement(args[1]), nil }, @@ -43,7 +44,8 @@ var SetUnionFunc = function.New(&function.Spec{ Type: cty.Set(cty.DynamicPseudoType), AllowDynamicType: true, }, - Type: setOperationReturnType, + Type: setOperationReturnType, + RefineResult: refineNonNull, Impl: setOperationImpl(func(s1, s2 cty.ValueSet) cty.ValueSet { return s1.Union(s2) }, true), @@ -63,7 +65,8 @@ var SetIntersectionFunc = function.New(&function.Spec{ Type: cty.Set(cty.DynamicPseudoType), AllowDynamicType: true, }, - Type: setOperationReturnType, + Type: setOperationReturnType, + RefineResult: refineNonNull, Impl: setOperationImpl(func(s1, s2 cty.ValueSet) cty.ValueSet { return s1.Intersection(s2) }, false), @@ -83,7 +86,8 @@ var SetSubtractFunc = function.New(&function.Spec{ AllowDynamicType: true, }, }, - Type: setOperationReturnType, + Type: setOperationReturnType, + RefineResult: refineNonNull, Impl: setOperationImpl(func(s1, s2 cty.ValueSet) cty.ValueSet { return s1.Subtract(s2) }, false), @@ -103,7 +107,8 @@ var SetSymmetricDifferenceFunc = function.New(&function.Spec{ Type: cty.Set(cty.DynamicPseudoType), AllowDynamicType: true, }, - Type: setOperationReturnType, + Type: setOperationReturnType, + RefineResult: refineNonNull, Impl: setOperationImpl(func(s1, s2 cty.ValueSet) cty.ValueSet { return s1.SymmetricDifference(s2) }, false), diff --git a/cty/function/stdlib/set_test.go b/cty/function/stdlib/set_test.go index 71c8a204..9dce2783 100644 --- a/cty/function/stdlib/set_test.go +++ b/cty/function/stdlib/set_test.go @@ -81,7 +81,7 @@ func TestSetUnion(t *testing.T) { cty.SetVal([]cty.Value{cty.StringVal("5")}), cty.UnknownVal(cty.Set(cty.Number)), }, - cty.UnknownVal(cty.Set(cty.String)), + cty.UnknownVal(cty.Set(cty.String)).RefineNotNull(), }, { []cty.Value{ @@ -178,14 +178,14 @@ func TestSetIntersection(t *testing.T) { cty.SetVal([]cty.Value{cty.StringVal("5")}), cty.UnknownVal(cty.Set(cty.Number)), }, - cty.UnknownVal(cty.Set(cty.String)), + cty.UnknownVal(cty.Set(cty.String)).RefineNotNull(), }, { []cty.Value{ cty.SetVal([]cty.Value{cty.StringVal("5")}), cty.SetVal([]cty.Value{cty.UnknownVal(cty.String)}), }, - cty.UnknownVal(cty.Set(cty.String)), + cty.UnknownVal(cty.Set(cty.String)).RefineNotNull(), }, } @@ -257,12 +257,12 @@ func TestSetSubtract(t *testing.T) { { cty.SetVal([]cty.Value{cty.StringVal("5")}), cty.UnknownVal(cty.Set(cty.Number)), - cty.UnknownVal(cty.Set(cty.String)), + cty.UnknownVal(cty.Set(cty.String)).RefineNotNull(), }, { cty.SetVal([]cty.Value{cty.StringVal("5")}), cty.SetVal([]cty.Value{cty.UnknownVal(cty.String)}), - cty.UnknownVal(cty.Set(cty.String)), + cty.UnknownVal(cty.Set(cty.String)).RefineNotNull(), }, } @@ -334,12 +334,12 @@ func TestSetSymmetricDifference(t *testing.T) { { cty.SetVal([]cty.Value{cty.StringVal("5")}), cty.UnknownVal(cty.Set(cty.Number)), - cty.UnknownVal(cty.Set(cty.String)), + cty.UnknownVal(cty.Set(cty.String)).RefineNotNull(), }, { cty.SetVal([]cty.Value{cty.StringVal("5")}), cty.SetVal([]cty.Value{cty.UnknownVal(cty.Number)}), - cty.UnknownVal(cty.Set(cty.String)), + cty.UnknownVal(cty.Set(cty.String)).RefineNotNull(), }, } diff --git a/cty/function/stdlib/string.go b/cty/function/stdlib/string.go index f340ef74..08b30535 100644 --- a/cty/function/stdlib/string.go +++ b/cty/function/stdlib/string.go @@ -22,7 +22,8 @@ var UpperFunc = function.New(&function.Spec{ AllowDynamicType: true, }, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { in := args[0].AsString() out := strings.ToUpper(in) @@ -39,7 +40,8 @@ var LowerFunc = function.New(&function.Spec{ AllowDynamicType: true, }, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { in := args[0].AsString() out := strings.ToLower(in) @@ -56,7 +58,8 @@ var ReverseFunc = function.New(&function.Spec{ AllowDynamicType: true, }, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { in := []byte(args[0].AsString()) out := make([]byte, len(in)) @@ -84,7 +87,8 @@ var StrlenFunc = function.New(&function.Spec{ AllowDynamicType: true, }, }, - Type: function.StaticReturnType(cty.Number), + Type: function.StaticReturnType(cty.Number), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { in := args[0].AsString() l := 0 @@ -122,7 +126,8 @@ var SubstrFunc = function.New(&function.Spec{ AllowDynamicType: true, }, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { in := []byte(args[0].AsString()) var offset, length int @@ -218,7 +223,8 @@ var JoinFunc = function.New(&function.Spec{ Description: "One or more lists of strings to join.", Type: cty.List(cty.String), }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { sep := args[0].AsString() listVals := args[1:] @@ -262,7 +268,8 @@ var SortFunc = function.New(&function.Spec{ Type: cty.List(cty.String), }, }, - Type: function.StaticReturnType(cty.List(cty.String)), + Type: function.StaticReturnType(cty.List(cty.String)), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { listVal := args[0] @@ -307,7 +314,8 @@ var SplitFunc = function.New(&function.Spec{ Type: cty.String, }, }, - Type: function.StaticReturnType(cty.List(cty.String)), + Type: function.StaticReturnType(cty.List(cty.String)), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { sep := args[0].AsString() str := args[1].AsString() @@ -333,7 +341,8 @@ var ChompFunc = function.New(&function.Spec{ Type: cty.String, }, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { newlines := regexp.MustCompile(`(?:\r\n?|\n)*\z`) return cty.StringVal(newlines.ReplaceAllString(args[0].AsString(), "")), nil @@ -356,7 +365,8 @@ var IndentFunc = function.New(&function.Spec{ Type: cty.String, }, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { var spaces int if err := gocty.FromCtyValue(args[0], &spaces); err != nil { @@ -378,7 +388,8 @@ var TitleFunc = function.New(&function.Spec{ Type: cty.String, }, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { return cty.StringVal(strings.Title(args[0].AsString())), nil }, @@ -394,7 +405,8 @@ var TrimSpaceFunc = function.New(&function.Spec{ Type: cty.String, }, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { return cty.StringVal(strings.TrimSpace(args[0].AsString())), nil }, @@ -416,7 +428,8 @@ var TrimFunc = function.New(&function.Spec{ Type: cty.String, }, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { str := args[0].AsString() cutset := args[1].AsString() @@ -443,7 +456,8 @@ var TrimPrefixFunc = function.New(&function.Spec{ Type: cty.String, }, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { str := args[0].AsString() prefix := args[1].AsString() @@ -467,7 +481,8 @@ var TrimSuffixFunc = function.New(&function.Spec{ Type: cty.String, }, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { str := args[0].AsString() cutset := args[1].AsString() diff --git a/cty/function/stdlib/string_replace.go b/cty/function/stdlib/string_replace.go index 573083bc..25a821bb 100644 --- a/cty/function/stdlib/string_replace.go +++ b/cty/function/stdlib/string_replace.go @@ -30,7 +30,8 @@ var ReplaceFunc = function.New(&function.Spec{ Type: cty.String, }, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { str := args[0].AsString() substr := args[1].AsString() @@ -59,7 +60,8 @@ var RegexReplaceFunc = function.New(&function.Spec{ Type: cty.String, }, }, - Type: function.StaticReturnType(cty.String), + Type: function.StaticReturnType(cty.String), + RefineResult: refineNonNull, Impl: func(args []cty.Value, retType cty.Type) (ret cty.Value, err error) { str := args[0].AsString() substr := args[1].AsString() diff --git a/cty/function/stdlib/string_test.go b/cty/function/stdlib/string_test.go index 05fe9416..9778aff9 100644 --- a/cty/function/stdlib/string_test.go +++ b/cty/function/stdlib/string_test.go @@ -49,11 +49,11 @@ func TestUpper(t *testing.T) { }, { cty.UnknownVal(cty.String), - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), }, { cty.DynamicVal, - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), }, { cty.StringVal("hello").Mark(1), @@ -103,11 +103,11 @@ func TestLower(t *testing.T) { }, { cty.UnknownVal(cty.String), - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), }, { cty.DynamicVal, - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), }, } @@ -174,11 +174,11 @@ func TestReverse(t *testing.T) { }, { cty.UnknownVal(cty.String), - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), }, { cty.DynamicVal, - cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), }, } @@ -245,11 +245,11 @@ func TestStrlen(t *testing.T) { }, { cty.UnknownVal(cty.String), - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, { cty.DynamicVal, - cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.Number).RefineNotNull(), }, } diff --git a/cty/unknown_refinement.go b/cty/unknown_refinement.go index 95bc579b..9878a85e 100644 --- a/cty/unknown_refinement.go +++ b/cty/unknown_refinement.go @@ -40,6 +40,8 @@ func (v Value) Refine() *RefinementBuilder { ty := v.Type() var wip unknownValRefinement switch { + case ty == DynamicPseudoType && !v.IsKnown(): + panic("cannot refine an unknown value of an unknown type") case ty == String: wip = &refinementString{} case ty == Number: @@ -54,13 +56,22 @@ func (v Value) Refine() *RefinementBuilder { case ty == Bool || ty.IsObjectType() || ty.IsTupleType() || ty.IsCapsuleType(): // For other known types we'll just track nullability wip = &refinementNullable{} + case ty == DynamicPseudoType && v.IsNull(): + // It's okay in principle to refine a null value of unknown type, + // although all we can refine about it is that it's definitely null and + // so this is pretty pointless and only supported to avoid callers + // always needing to treat this situation as a special case to avoid + // panic. + wip = &refinementNullable{ + isNull: tristateTrue, + } default: // we leave "wip" as nil for all other types, representing that // they don't support refinements at all and so any call on the // RefinementBuilder should fail. // NOTE: We intentionally don't allow any refinements for - // cty.DynamicPseudoType here, even though it could be nice in principle + // cty.DynamicVal here, even though it could be nice in principle // to at least track non-nullness for those, because it's historically // been valid to directly compare values with cty.DynamicVal using // the Go "==" operator and recording a refinement for an untyped @@ -70,6 +81,38 @@ func (v Value) Refine() *RefinementBuilder { return &RefinementBuilder{v, wip} } +// RefineWith is a variant of Refine which uses callback functions instead of +// the builder pattern. +// +// The result is equivalent to passing the return value of [Value.Refine] to the +// first callback, and then continue passing the builder through any other +// callbacks in turn, and then calling [RefinementBuilder.NewValue] on the +// final result. +// +// The builder pattern approach of [Value.Refine] is more convenient for inline +// annotation of refinements when constructing a value, but this alternative +// approach may be more convenient when applying pre-defined collections of +// refinements, or when refinements are defined separately from the values +// they will apply to. +// +// Each refiner callback should return the same pointer that it was given, +// typically after having mutated it using the [RefinementBuilder] methods. +// It's invalid to return a different builder. +func (v Value) RefineWith(refiners ...func(*RefinementBuilder) *RefinementBuilder) Value { + if len(refiners) == 0 { + return v + } + origBuilder := v.Refine() + builder := origBuilder + for _, refiner := range refiners { + builder = refiner(builder) + if builder != origBuilder { + panic("refiner callback returned a different builder") + } + } + return builder.NewValue() +} + // RefineNotNull is a shorthand for Value.Refine().NotNull().NewValue(), because // declaring that a unknown value isn't null is by far the most common use of // refinements. @@ -478,7 +521,7 @@ func (b *RefinementBuilder) NewValue() Value { return Value{ ty: b.orig.ty, v: &unknownType{refinement: b.wip}, - } + }.WithSameMarks(b.orig) } // unknownValRefinment is an interface pretending to be a sum type representing From ad5f1d2b153c6ee8edc619de6923fc34c688a2ee Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 31 Jan 2023 18:33:19 -0800 Subject: [PATCH 08/28] cty: Simplify the collection length refinement model I initially just copied the refinement model for numeric range, but in retrospect that was silly because we know that collection lengths are always integers and so there's no need to model unknown values and fractional values and whatever else. Instead, we now just have an integer lower and upper bound, always known but when unconstrained they are the smallest and largest values of int respectively. --- cty/function/stdlib/collection_test.go | 2 +- cty/unknown_refinement.go | 154 +++++++++---------------- cty/value_ops_test.go | 2 +- cty/value_range.go | 41 ++++--- 4 files changed, 80 insertions(+), 119 deletions(-) diff --git a/cty/function/stdlib/collection_test.go b/cty/function/stdlib/collection_test.go index 5c001810..5be0d30d 100644 --- a/cty/function/stdlib/collection_test.go +++ b/cty/function/stdlib/collection_test.go @@ -983,7 +983,7 @@ func TestLength(t *testing.T) { // TODO: This one should really preserve the length bounds as the // numeric bounds of its result, but cty.Value.Length isn't yet // able to do that. - cty.UnknownVal(cty.List(cty.Bool)).Refine().CollectionLengthUpperBound(cty.NumberIntVal(2), true).NewValue(), + cty.UnknownVal(cty.List(cty.Bool)).Refine().CollectionLengthUpperBound(2).NewValue(), cty.UnknownVal(cty.Number).Refine().NotNull().NumberRangeInclusive(cty.Zero, cty.UnknownVal(cty.Number)).NewValue(), }, { // Marked collections return a marked length diff --git a/cty/unknown_refinement.go b/cty/unknown_refinement.go index 9878a85e..42965b0c 100644 --- a/cty/unknown_refinement.go +++ b/cty/unknown_refinement.go @@ -2,7 +2,7 @@ package cty import ( "fmt" - "math/big" + "math" "strings" ) @@ -50,8 +50,8 @@ func (v Value) Refine() *RefinementBuilder { wip = &refinementCollection{ // A collection can never have a negative length, so we'll // start with that already constrained. - minLen: Zero, - minInc: true, + minLen: 0, + maxLen: math.MaxInt, } case ty == Bool || ty.IsObjectType() || ty.IsTupleType() || ty.IsCapsuleType(): // For other known types we'll just track nullability @@ -199,10 +199,7 @@ func (b *RefinementBuilder) NumberRangeInclusive(min, max Value) *RefinementBuil // CollectionLengthLowerBound constrains the lower bound of the length of a // collection value, or panics if this builder is not refining a collection // value. -// -// The lower bound must be a known, non-null number or this function will -// panic. -func (b *RefinementBuilder) CollectionLengthLowerBound(min Value, inclusive bool) *RefinementBuilder { +func (b *RefinementBuilder) CollectionLengthLowerBound(min int) *RefinementBuilder { b.assertRefineable() wip, ok := b.wip.(*refinementCollection) @@ -210,34 +207,19 @@ func (b *RefinementBuilder) CollectionLengthLowerBound(min Value, inclusive bool panic(fmt.Sprintf("cannot refine collection length bounds for a %#v value", b.orig.Type())) } - if min.IsNull() { - panic("collection length bound is null") - } - if !min.IsKnown() { - panic("collection length bound is unknown") - } - + minVal := NumberIntVal(int64(min)) if b.orig.IsKnown() { realLen := b.orig.Length() - if gt := min.GreaterThan(realLen); gt.IsKnown() && gt.True() { - panic(fmt.Sprintf("refining collection of length %#v with minimum bound %#v", realLen, min)) + if gt := minVal.GreaterThan(realLen); gt.IsKnown() && gt.True() { + panic(fmt.Sprintf("refining collection of length %#v with lower bound %#v", realLen, min)) } } - if wip.minLen != NilVal { - var ok bool - if wip.minInc { - ok = min.GreaterThanOrEqualTo(wip.minLen).True() - } else { - ok = min.GreaterThan(wip.minLen).True() - } - if !ok { - panic("refined collection length lower bound is inconsistent with existing lower bound") - } + if wip.minLen > min { + panic(fmt.Sprintf("refined collection length lower bound %d is inconsistent with existing lower bound %d", min, wip.minLen)) } wip.minLen = min - wip.minInc = inclusive wip.assertConsistentLengthBounds() return b @@ -249,7 +231,7 @@ func (b *RefinementBuilder) CollectionLengthLowerBound(min Value, inclusive bool // // The upper bound must be a known, non-null number or this function will // panic. -func (b *RefinementBuilder) CollectionLengthUpperBound(max Value, inclusive bool) *RefinementBuilder { +func (b *RefinementBuilder) CollectionLengthUpperBound(max int) *RefinementBuilder { b.assertRefineable() wip, ok := b.wip.(*refinementCollection) @@ -257,34 +239,19 @@ func (b *RefinementBuilder) CollectionLengthUpperBound(max Value, inclusive bool panic(fmt.Sprintf("cannot refine collection length bounds for a %#v value", b.orig.Type())) } - if max.IsNull() { - panic("collection length bound is null") - } - if !max.IsKnown() { - panic("collection length bound is unknown") - } - if b.orig.IsKnown() { + maxVal := NumberIntVal(int64(max)) realLen := b.orig.Length() - if gt := max.LessThan(realLen); gt.IsKnown() && gt.True() { - panic(fmt.Sprintf("refining collection of length %#v with maximum bound %#v", realLen, max)) + if lt := maxVal.LessThan(realLen); lt.IsKnown() && lt.True() { + panic(fmt.Sprintf("refining collection of length %#v with upper bound %#v", realLen, max)) } } - if wip.maxLen != NilVal { - var ok bool - if wip.maxInc { - ok = max.LessThanOrEqualTo(wip.minLen).True() - } else { - ok = max.LessThan(wip.minLen).True() - } - if !ok { - panic("refined collection length upper bound is inconsistent with existing upper bound") - } + if wip.maxLen < max { + panic(fmt.Sprintf("refined collection length upper bound %d is inconsistent with existing upper bound %d", max, wip.maxLen)) } wip.maxLen = max - wip.maxInc = inclusive wip.assertConsistentLengthBounds() return b @@ -474,45 +441,38 @@ func (b *RefinementBuilder) NewValue() Value { } } } else if rfn, ok := b.wip.(*refinementCollection); ok { - // If both length bounds are inclusive and equal then we know our - // length is the same number as the bounds. - if rfn.maxInc && rfn.minInc { - if rfn.minLen != NilVal && rfn.maxLen != NilVal { - eq := rfn.minLen.Equals(rfn.maxLen) - if eq.IsKnown() && eq.True() { - knownLen := rfn.minLen - ty := b.orig.Type() - if knownLen == Zero { - // If we know the length is zero then we can construct - // a known value of any collection kind. - switch { - case ty.IsListType(): - return ListValEmpty(ty.ElementType()) - case ty.IsSetType(): - return SetValEmpty(ty.ElementType()) - case ty.IsMapType(): - return MapValEmpty(ty.ElementType()) - } - } else if ty.IsListType() { - // If we know the length of the list then we can - // create a known list with unknown elements instead - // of a wholly-unknown list. - if knownLen, acc := knownLen.AsBigFloat().Int64(); acc == big.Exact { - elems := make([]Value, knownLen) - unk := UnknownVal(ty.ElementType()) - for i := range elems { - elems[i] = unk - } - return ListVal(elems) - } - } else if ty.IsSetType() && knownLen == NumberIntVal(1) { - // If we know we have a one-element set then we - // know the one element can't possibly coalesce with - // anything else and so we can create a known set with - // an unknown element. - return SetVal([]Value{UnknownVal(ty.ElementType())}) - } + // If both of the bounds are equal then we know the length is + // the same number as the bounds. + if rfn.minLen == rfn.maxLen { + knownLen := rfn.minLen + ty := b.orig.Type() + if knownLen == 0 { + // If we know the length is zero then we can construct + // a known value of any collection kind. + switch { + case ty.IsListType(): + return ListValEmpty(ty.ElementType()) + case ty.IsSetType(): + return SetValEmpty(ty.ElementType()) + case ty.IsMapType(): + return MapValEmpty(ty.ElementType()) + } + } else if ty.IsListType() { + // If we know the length of the list then we can + // create a known list with unknown elements instead + // of a wholly-unknown list. + elems := make([]Value, knownLen) + unk := UnknownVal(ty.ElementType()) + for i := range elems { + elems[i] = unk } + return ListVal(elems) + } else if ty.IsSetType() && knownLen == 1 { + // If we know we have a one-element set then we + // know the one element can't possibly coalesce with + // anything else and so we can create a known set with + // an unknown element. + return SetVal([]Value{UnknownVal(ty.ElementType())}) } } } @@ -617,8 +577,7 @@ func (r *refinementNumber) assertConsistentBounds() { type refinementCollection struct { refinementNullable - minLen, maxLen Value - minInc, maxInc bool + minLen, maxLen int } func (r *refinementCollection) unknownValRefinementSigil() {} @@ -636,29 +595,26 @@ func (r *refinementCollection) rawEqual(other unknownValRefinement) bool { return false } return (r.refinementNullable.rawEqual(&other.refinementNullable) && - r.minLen.RawEquals(other.minLen) && - r.maxLen.RawEquals(other.maxLen) && - r.minInc == other.minInc && - r.maxInc == other.maxInc) + r.minLen == other.minLen && + r.maxLen == other.maxLen) } } func (r *refinementCollection) GoString() string { var b strings.Builder b.WriteString(r.refinementNullable.GoString()) - if r.minLen != NilVal && r.minLen != Zero { - // (a lower bound of zero is the default) - fmt.Fprintf(&b, ".CollectionLengthLowerBound(%#v, %t)", r.minLen, r.minInc) + if r.minLen != 0 { + fmt.Fprintf(&b, ".CollectionLengthLowerBound(%d)", r.minLen) } - if r.maxLen != NilVal { - fmt.Fprintf(&b, ".CollectionLengthUpperBound(%#v, %t)", r.maxLen, r.maxInc) + if r.maxLen != math.MaxInt { + fmt.Fprintf(&b, ".CollectionLengthUpperBound(%d)", r.maxLen) } return b.String() } func (r *refinementCollection) assertConsistentLengthBounds() { - if r.minLen != NilVal && r.maxLen != NilVal && r.maxLen.LessThan(r.minLen).True() { - panic("collection length upper bound is less than lower bound") + if r.maxLen < r.minLen { + panic(fmt.Sprintf("collection length upper bound %d is less than lower bound %d", r.maxLen, r.minLen)) } } diff --git a/cty/value_ops_test.go b/cty/value_ops_test.go index 4bb59266..bd4b6383 100644 --- a/cty/value_ops_test.go +++ b/cty/value_ops_test.go @@ -1673,7 +1673,7 @@ func TestValueRawEquals(t *testing.T) { true, // Refinement collapses to zero because it's not null and the two bounds are equal }, { - UnknownVal(List(String)).Refine().NotNull().CollectionLengthUpperBound(Zero, true).NewValue(), + UnknownVal(List(String)).Refine().NotNull().CollectionLengthUpperBound(0).NewValue(), ListValEmpty(String), true, // Colection length lower bound is always at least zero so this refinement collapses to an empty list }, diff --git a/cty/value_range.go b/cty/value_range.go index 8c32d2bf..46f9d275 100644 --- a/cty/value_range.go +++ b/cty/value_range.go @@ -2,6 +2,7 @@ package cty import ( "fmt" + "math" ) // Range returns an object that offers partial information about the range @@ -58,12 +59,19 @@ func (v Value) Range() ValueRange { maxInc: true, } case ty.IsCollectionType(): - synth = &refinementCollection{ - minLen: v.Length(), - maxLen: v.Length(), - minInc: true, - maxInc: true, + if lenVal := v.Length(); lenVal.IsKnown() { + l, _ := lenVal.AsBigFloat().Int64() + synth = &refinementCollection{ + minLen: int(l), + maxLen: int(l), + } + } else { + synth = &refinementCollection{ + minLen: 0, + maxLen: math.MaxInt, + } } + default: // If we don't have anything else to say then we can at least // guarantee that the value isn't null. @@ -188,21 +196,18 @@ func (r ValueRange) StringPrefix() string { // // If the value is nullable then the result represents the range of the length // only if the value turns out not to be null. -// -// The resulting value might itself be an unknown number if there is no -// known lower bound. In that case the "inclusive" flag is meaningless. -func (r ValueRange) LengthLowerBound() (min Value, inclusive bool) { +func (r ValueRange) LengthLowerBound() int { if r.ty == DynamicPseudoType { // We don't even know if this is a collection yet. - return UnknownVal(Number), false + return 0 } if !r.ty.IsCollectionType() { panic(fmt.Sprintf("LengthLowerBound for %#v", r.ty)) } - if rfn, ok := r.raw.(*refinementCollection); ok && rfn.minLen != NilVal { - return rfn.minLen, rfn.minInc + if rfn, ok := r.raw.(*refinementCollection); ok { + return rfn.minLen } - return UnknownVal(Number), false + return 0 } // LengthUpperBound returns information about the upper bound of the length of @@ -214,18 +219,18 @@ func (r ValueRange) LengthLowerBound() (min Value, inclusive bool) { // // The resulting value might itself be an unknown number if there is no // known upper bound. In that case the "inclusive" flag is meaningless. -func (r ValueRange) LengthUpperBound() (min Value, inclusive bool) { +func (r ValueRange) LengthUpperBound() int { if r.ty == DynamicPseudoType { // We don't even know if this is a collection yet. - return UnknownVal(Number), false + return math.MaxInt } if !r.ty.IsCollectionType() { panic(fmt.Sprintf("LengthUpperBound for %#v", r.ty)) } - if rfn, ok := r.raw.(*refinementCollection); ok && rfn.maxLen != NilVal { - return rfn.maxLen, rfn.maxInc + if rfn, ok := r.raw.(*refinementCollection); ok { + return rfn.maxLen } - return UnknownVal(Number), false + return math.MaxInt } // definitelyNotNull is a convenient helper for the common situation of checking From f09db2e4c666c53f7378a4a8efde22365c35890e Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 31 Jan 2023 18:37:36 -0800 Subject: [PATCH 09/28] msgpack: Round-trip some unknown value refinements The new concept of "refinements" for unknown values allows us to track some additional constraints on the range of an unknown value. Normally those constraints would get lost during serialization, but for msgpack in particular we can use an extension code to represent an unknown value that has refinements and then include a serialization of the refinements inside its body. This then allows preserving the refinements when communicating over wire protocols that use msgpack as their base serialization. However, as of this commit the unmarshal step isn't quite complete becausethe refinement API for numeric range is incomplete. We'll address both the refinement API and the msgpack package's use of it in a future commit. This required upgrading to a newer major version of the msgpack library because v4 did not have all of the functionality required to encode a raw extension object's payload. --- cty/msgpack/dynamic.go | 2 +- cty/msgpack/marshal.go | 11 +- cty/msgpack/roundtrip_test.go | 53 ++++++ cty/msgpack/type_implied.go | 4 +- cty/msgpack/unknown.go | 296 +++++++++++++++++++++++++++++++++- cty/msgpack/unmarshal.go | 7 +- go.mod | 8 +- go.sum | 38 ++--- 8 files changed, 375 insertions(+), 44 deletions(-) diff --git a/cty/msgpack/dynamic.go b/cty/msgpack/dynamic.go index 9a4e94c2..95d9160f 100644 --- a/cty/msgpack/dynamic.go +++ b/cty/msgpack/dynamic.go @@ -3,7 +3,7 @@ package msgpack import ( "bytes" - "github.com/vmihailenco/msgpack/v4" + "github.com/vmihailenco/msgpack/v5" "github.com/zclconf/go-cty/cty" ) diff --git a/cty/msgpack/marshal.go b/cty/msgpack/marshal.go index 2c4da8b5..27da762a 100644 --- a/cty/msgpack/marshal.go +++ b/cty/msgpack/marshal.go @@ -5,7 +5,7 @@ import ( "math/big" "sort" - "github.com/vmihailenco/msgpack/v4" + "github.com/vmihailenco/msgpack/v5" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/convert" ) @@ -31,7 +31,8 @@ func Marshal(val cty.Value, ty cty.Type) ([]byte, error) { var path cty.Path var buf bytes.Buffer enc := msgpack.NewEncoder(&buf) - enc.UseCompactEncoding(true) + enc.UseCompactInts(true) + enc.UseCompactFloats(true) err := marshal(val, ty, path, enc) if err != nil { @@ -53,11 +54,7 @@ func marshal(val cty.Value, ty cty.Type, path cty.Path, enc *msgpack.Encoder) er } if !val.IsKnown() { - err := enc.Encode(unknownVal) - if err != nil { - return path.NewError(err) - } - return nil + return marshalUnknownValue(val.Range(), path, enc) } if val.IsNull() { err := enc.EncodeNil() diff --git a/cty/msgpack/roundtrip_test.go b/cty/msgpack/roundtrip_test.go index 23de4e6b..41f4d438 100644 --- a/cty/msgpack/roundtrip_test.go +++ b/cty/msgpack/roundtrip_test.go @@ -38,6 +38,18 @@ func TestRoundTrip(t *testing.T) { cty.UnknownVal(cty.String), cty.String, }, + { + cty.UnknownVal(cty.String).RefineNotNull(), + cty.String, + }, + { + cty.UnknownVal(cty.String).Refine().StringPrefix("foo-").NewValue(), + cty.String, + }, + { + cty.UnknownVal(cty.String).Refine().NotNull().StringPrefix("foo-").NewValue(), + cty.String, + }, { cty.True, @@ -55,6 +67,10 @@ func TestRoundTrip(t *testing.T) { cty.UnknownVal(cty.Bool), cty.Bool, }, + { + cty.UnknownVal(cty.Bool).RefineNotNull(), + cty.Bool, + }, { cty.NumberIntVal(1), @@ -80,6 +96,14 @@ func TestRoundTrip(t *testing.T) { cty.NegativeInfinity, cty.Number, }, + { + cty.UnknownVal(cty.Number), + cty.Number, + }, + { + cty.UnknownVal(cty.Number).RefineNotNull(), + cty.Number, + }, { cty.ListVal([]cty.Value{ @@ -107,6 +131,35 @@ func TestRoundTrip(t *testing.T) { cty.ListValEmpty(cty.String), cty.List(cty.String), }, + { + cty.UnknownVal(cty.List(cty.String)), + cty.List(cty.String), + }, + { + cty.UnknownVal(cty.List(cty.String)).RefineNotNull(), + cty.List(cty.String), + }, + { + cty.UnknownVal(cty.List(cty.String)).Refine().CollectionLengthLowerBound(1).NewValue(), + cty.List(cty.String), + }, + { + cty.UnknownVal(cty.List(cty.String)).Refine().CollectionLengthUpperBound(1).NewValue(), + cty.List(cty.String), + }, + { + cty.UnknownVal(cty.List(cty.String)).Refine().CollectionLengthLowerBound(1).CollectionLengthUpperBound(2).NewValue(), + cty.List(cty.String), + }, + { + // NOTE: This refinement should collapse to a known 2-element list with unknown elements + cty.UnknownVal(cty.List(cty.String)).Refine().CollectionLengthLowerBound(2).CollectionLengthUpperBound(2).NewValue(), + cty.List(cty.String), + }, + { + cty.UnknownVal(cty.List(cty.String)).Refine().CollectionLengthUpperBound(1).NotNull().NewValue(), + cty.List(cty.String), + }, { cty.SetVal([]cty.Value{ diff --git a/cty/msgpack/type_implied.go b/cty/msgpack/type_implied.go index a169f28f..48400637 100644 --- a/cty/msgpack/type_implied.go +++ b/cty/msgpack/type_implied.go @@ -5,8 +5,8 @@ import ( "fmt" "io" - "github.com/vmihailenco/msgpack/v4" - msgpackcodes "github.com/vmihailenco/msgpack/v4/codes" + "github.com/vmihailenco/msgpack/v5" + msgpackcodes "github.com/vmihailenco/msgpack/v5/msgpcode" "github.com/zclconf/go-cty/cty" ) diff --git a/cty/msgpack/unknown.go b/cty/msgpack/unknown.go index 6507bc4b..03ffa974 100644 --- a/cty/msgpack/unknown.go +++ b/cty/msgpack/unknown.go @@ -1,16 +1,310 @@ package msgpack +import ( + "bytes" + "io" + "math" + "unicode/utf8" + + "github.com/vmihailenco/msgpack/v5" + "github.com/zclconf/go-cty/cty" +) + type unknownType struct{} var unknownVal = unknownType{} // unknownValBytes is the raw bytes of the msgpack fixext1 value we -// write to represent an unknown value. It's an extension value of +// write to represent a totally unknown value. It's an extension value of // type zero whose value is irrelevant. Since it's irrelevant, we // set it to a single byte whose value is also zero, since that's // the most compact possible representation. +// +// The representation of a refined unknown value is different. See +// marshalUnknownValue for more details. var unknownValBytes = []byte{0xd4, 0, 0} func (uv unknownType) MarshalMsgpack() ([]byte, error) { return unknownValBytes, nil } + +const unknownWithRefinementsExt = 0x0c + +type unknownValRefinementKey int64 + +const unknownValNullness unknownValRefinementKey = 1 +const unknownValStringPrefix unknownValRefinementKey = 2 +const unknownValNumberMin unknownValRefinementKey = 3 +const unknownValNumberMax unknownValRefinementKey = 4 +const unknownValLengthMin unknownValRefinementKey = 5 +const unknownValLengthMax unknownValRefinementKey = 6 + +func marshalUnknownValue(rng cty.ValueRange, path cty.Path, enc *msgpack.Encoder) error { + if rng.TypeConstraint() == cty.DynamicPseudoType { + // cty.DynamicVal can never have refinements + err := enc.Encode(unknownVal) + if err != nil { + return path.NewError(err) + } + return nil + } + + var refnBuf bytes.Buffer + refnEnc := msgpack.NewEncoder(&refnBuf) + mapLen := 0 + + if rng.DefinitelyNotNull() { + mapLen++ + refnEnc.EncodeInt(int64(unknownValNullness)) + refnEnc.EncodeBool(false) + } + switch { + case rng.TypeConstraint() == cty.Number: + lower, lowerInc := rng.NumberLowerBound() + upper, upperInc := rng.NumberUpperBound() + boundTy := cty.Tuple([]cty.Type{cty.Number, cty.Bool}) + if lower.IsKnown() { + mapLen++ + refnEnc.EncodeInt(int64(unknownValNumberMin)) + marshal( + cty.TupleVal([]cty.Value{lower, cty.BoolVal(lowerInc)}), + boundTy, + nil, + refnEnc, + ) + } + if upper.IsKnown() { + mapLen++ + refnEnc.EncodeInt(int64(unknownValNumberMax)) + marshal( + cty.TupleVal([]cty.Value{upper, cty.BoolVal(upperInc)}), + boundTy, + nil, + refnEnc, + ) + } + case rng.TypeConstraint() == cty.String: + if prefix := rng.StringPrefix(); prefix != "" { + mapLen++ + refnEnc.EncodeInt(int64(unknownValStringPrefix)) + refnEnc.EncodeString(prefix) + } + case rng.TypeConstraint().IsCollectionType(): + lower := rng.LengthLowerBound() + upper := rng.LengthUpperBound() + if lower != 0 { + mapLen++ + refnEnc.EncodeInt(int64(unknownValLengthMin)) + refnEnc.EncodeInt(int64(lower)) + } + if upper != math.MaxInt { + mapLen++ + refnEnc.EncodeInt(int64(unknownValLengthMax)) + refnEnc.EncodeInt(int64(upper)) + } + } + + if mapLen == 0 { + // No refinements to encode, so we'll use the old compact representation. + err := enc.Encode(unknownVal) + if err != nil { + return path.NewError(err) + } + return nil + } + + // If we have at least one refinement to encode then we'll use the new + // representation of unknown values where refinement information is in the + // extension payload. + var lenBuf bytes.Buffer + lenEnc := msgpack.NewEncoder(&lenBuf) + lenEnc.EncodeMapLen(mapLen) + + err := enc.EncodeExtHeader(unknownWithRefinementsExt, lenBuf.Len()+refnBuf.Len()) + if err != nil { + return path.NewErrorf("failed to write unknown value: %s", err) + } + _, err = enc.Writer().Write(lenBuf.Bytes()) + if err != nil { + return path.NewErrorf("failed to write unknown value: %s", err) + } + _, err = enc.Writer().Write(refnBuf.Bytes()) + if err != nil { + return path.NewErrorf("failed to write unknown value: %s", err) + } + return nil +} + +func unmarshalUnknownValue(dec *msgpack.Decoder, ty cty.Type, path cty.Path) (cty.Value, error) { + // The next item in the stream should be a msgpack extension value, + // which might be zero-length for a totally unknown value, or it might + // contain a mapping describing some type-specific refinements. + typeCode, extLen, err := dec.DecodeExtHeader() + if err != nil { + return cty.DynamicVal, path.NewErrorf("extension code is required for unknown value") + } + + if extLen <= 1 { + // Zero-length or one-length extension represents an unknown value with + // no refinements. (msgpack's serialization of a zero-length extension + // is one byte longer than a one-byte extension, so the encoder uses + // one nul byte as its "totally unknown" encoding. + + if extLen > 0 { + // We need to skip the body, then. + body := make([]byte, extLen) + _, err = io.ReadAtLeast(dec.Buffered(), body, len(body)) + if err != nil { + return cty.DynamicVal, path.NewErrorf("failed to read msgpack extension body: %s", err) + } + } + return cty.UnknownVal(ty), nil + } + + if typeCode != unknownWithRefinementsExt { + // If there's a non-zero length then we require a specific type code + // as an additional signal that the body is intended to be a refinement map. + return cty.DynamicVal, path.NewErrorf("unsupported extension type 0x%02x with len %d", typeCode, extLen) + } + + if extLen > 1024 { + // A refinement description greater than 1 kiB is unreasonable and + // might be an abusive attempt to allocate large amounts of memory + // in a system consuming this input. + return cty.DynamicVal, path.NewErrorf("oversize unknown value refinement") + } + + // If we get here then typeCode == 0xc and we have a non-zero length. + // We expect to find a msgpack-encoded map in the payload which describes + // any refinements to add to the result. + body := make([]byte, extLen) + _, err = io.ReadAtLeast(dec.Buffered(), body, len(body)) + if err != nil { + return cty.DynamicVal, path.NewErrorf("failed to read msgpack extension body: %s", err) + } + + rfnDec := msgpack.NewDecoder(bytes.NewReader(body)) + entryCount, err := rfnDec.DecodeMapLen() + if err != nil { + return cty.DynamicVal, path.NewErrorf("failed to decode msgpack extension body: not a map") + } + + if ty == cty.DynamicPseudoType { + // We'll silently ignore all refinements for DynamicPseudoType for now, + // since we know that's invalid today but we might find a way to + // support it in the future and if so will want to introduce that + // in a backward-compatible way. + return cty.UnknownVal(ty), nil + } + + builder := cty.UnknownVal(ty).Refine() + for i := 0; i < entryCount; i++ { + // Our refinement encoding format uses compact msgpack primitives to + // minimize the encoding size of refinements, which could otherwise + // add up to be quite large for a payload containing lots of unknown + // values. The keys are small integers to fit in the positive fixint + // encoding scheme. The values are encoded differently depending on + // the key but also aim for compactness. + // The smallest possible non-empty refinement map is three bytes: + // - one byte to encode that it's a one-element map + // - one byte to encode the key + // - at least one byte to encode the value associated with that key + // Encoders should avoid encoding zero-length maps and prefer to + // leave the payload zero bytes long in that case. + + keyCode, err := rfnDec.DecodeInt64() + if err != nil { + return cty.DynamicVal, path.NewErrorf("failed to decode msgpack extension body: non-integer key in map") + } + + // Exactly which keys are supported depends on the destination type. + // We'll reject keys that we know can't possibly apply to the given + // type, but we'll ignore keys we haven't seen before to allow for + // future expansion of the possible refinements. + // These keys all have intentionally-short names + switch keyCode := unknownValRefinementKey(keyCode); keyCode { + case unknownValNullness: + isNull, err := rfnDec.DecodeBool() + if err != nil { + return cty.DynamicVal, path.NewErrorf("failed to decode msgpack extension body: null refinement is not boolean") + } + // The presence of this key means we're refining the null-ness one + // way or another. If nullness is unknown then this key should not + // be present at all. + if isNull { + // it'd be weird to actually serialize a refinement like + // this because trying to apply this refinement in the first + // place should've collapsed into a known null value. But we'll + // allow it anyway just for complete encoding of the current + // refinement model. + builder = builder.Null() + } else { + builder = builder.NotNull() + } + case unknownValStringPrefix: + if ty != cty.String { + return cty.DynamicVal, path.NewErrorf("failed to decode msgpack extension body: string prefix refinement for non-string type") + } + prefixStr, err := rfnDec.DecodeString() + if err != nil { + return cty.DynamicVal, path.NewErrorf("failed to decode msgpack extension body: string prefix refinement is not string") + } + if !utf8.ValidString(prefixStr) { + return cty.DynamicVal, path.NewErrorf("failed to decode msgpack extension body: string prefix refinement is not valid UTF-8") + } + builder = builder.StringPrefix(prefixStr) + case unknownValLengthMin, unknownValLengthMax: + if !ty.IsCollectionType() { + return cty.DynamicVal, path.NewErrorf("failed to decode msgpack extension body: length lower bound refinement for non-collection type") + } + + bound, err := rfnDec.DecodeInt() + if err != nil { + return cty.DynamicVal, path.NewErrorf("failed to decode msgpack extension body: length bound refinement must be integer or [integer, bool] array") + } + switch keyCode { + case unknownValLengthMin: + builder = builder.CollectionLengthLowerBound(bound) + case unknownValLengthMax: + builder = builder.CollectionLengthUpperBound(bound) + default: + panic("unsupported keyCode") // should not get here + } + case unknownValNumberMin, unknownValNumberMax: + if ty != cty.Number { + return cty.DynamicVal, path.NewErrorf("failed to decode msgpack extension body: numeric bound refinement for non-number type") + } + // We want to support all of the same various number encodings we + // support for normal numbers, so here we'll cheat a bit and decode + // using our own value unmarshal function. + rawBound, err := unmarshal(rfnDec, cty.Tuple([]cty.Type{cty.Number, cty.Bool}), nil) + if err != nil || rawBound.IsNull() || !rawBound.IsKnown() { + return cty.DynamicVal, path.NewErrorf("failed to decode msgpack extension body: length bound refinement must be [number, bool] array") + } + boundVal := rawBound.Index(cty.Zero) + isIncVal := rawBound.Index(cty.NumberIntVal(1)) + if boundVal.Type() != cty.Number || !boundVal.IsKnown() || boundVal.IsNull() { + return cty.DynamicVal, path.NewErrorf("failed to decode msgpack extension body: length bound refinement must be [number, bool] array") + } + if isIncVal.Type() != cty.Bool || !isIncVal.IsKnown() || isIncVal.IsNull() { + return cty.DynamicVal, path.NewErrorf("failed to decode msgpack extension body: length bound refinement must be [number, bool] array") + } + //isInc := isIncVal.True() + switch keyCode { + case unknownValNumberMin: + // TODO: builder doesn't have a NumberRangeLowerBound method yet + case unknownValNumberMax: + // TODO: builder doesn't have a NumberRangeUpperBound method yet + default: + panic("unsupported keyCode") // should not get here + } + } + } + + // NOTE: We intentionally ignore any trailing bytes after the extension + // map in case we want to pack something else in there later or in case + // a future version wants to use padding to optimize storage. Current + // encoders should not add any extra content there, though. + + return builder.NewValue(), nil +} diff --git a/cty/msgpack/unmarshal.go b/cty/msgpack/unmarshal.go index 1ea0b0a2..08da1f73 100644 --- a/cty/msgpack/unmarshal.go +++ b/cty/msgpack/unmarshal.go @@ -3,8 +3,8 @@ package msgpack import ( "bytes" - "github.com/vmihailenco/msgpack/v4" - msgpackCodes "github.com/vmihailenco/msgpack/v4/codes" + "github.com/vmihailenco/msgpack/v5" + msgpackCodes "github.com/vmihailenco/msgpack/v5/msgpcode" "github.com/zclconf/go-cty/cty" ) @@ -30,8 +30,7 @@ func unmarshal(dec *msgpack.Decoder, ty cty.Type, path cty.Path) (cty.Value, err if msgpackCodes.IsExt(peek) { // We just assume _all_ extensions are unknown values, // since we don't have any other extensions. - dec.Skip() // skip what we've peeked - return cty.UnknownVal(ty), nil + return unmarshalUnknownValue(dec, ty, path) } if ty == cty.DynamicPseudoType { return unmarshalDynamic(dec, path) diff --git a/go.mod b/go.mod index 8f2173c5..40fde375 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,13 @@ module github.com/zclconf/go-cty require ( github.com/apparentlymart/go-textseg/v13 v13.0.0 github.com/google/go-cmp v0.3.1 - github.com/vmihailenco/msgpack/v4 v4.3.12 + github.com/vmihailenco/msgpack/v5 v5.3.5 golang.org/x/text v0.3.7 ) require ( - github.com/golang/protobuf v1.3.4 // indirect - github.com/vmihailenco/tagparser v0.1.1 // indirect - golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect - google.golang.org/appengine v1.6.5 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect ) go 1.18 diff --git a/go.sum b/go.sum index 35938e75..8859777d 100644 --- a/go.sum +++ b/go.sum @@ -1,31 +1,21 @@ github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= -github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= -github.com/vmihailenco/tagparser v0.1.1 h1:quXMXlA39OCbd2wAdTsGDlK9RkOk6Wuw+x37wVyIuWY= -github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 7e0b8ac7a2134d4337922fb002c878b2d2f4d0eb Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 1 Feb 2023 14:38:37 -0800 Subject: [PATCH 10/28] cty: RefinementBuilder numeric bounds The API for declaring numeric bounds was previously rather awkward and inconsistent with the API for inspecting them in the ValueRange type. Now we use separate methods for defining the lower bound and upper bound, allowing the inclusiveness of each one to be independent. We retain the previous NumberRangeInclusive helper for setting both bounds at once, but it's now just a thin wrapper around the other two. --- cty/unknown_refinement.go | 196 +++++++++++++++++++++----------------- 1 file changed, 106 insertions(+), 90 deletions(-) diff --git a/cty/unknown_refinement.go b/cty/unknown_refinement.go index 42965b0c..ede30622 100644 --- a/cty/unknown_refinement.go +++ b/cty/unknown_refinement.go @@ -193,7 +193,101 @@ func (b *RefinementBuilder) Null() *RefinementBuilder { // If either of the given values is not a non-null number value then this // function will panic. func (b *RefinementBuilder) NumberRangeInclusive(min, max Value) *RefinementBuilder { - return b.numberRange(min, max, true, true) + return b.NumberRangeLowerBound(min, true).NumberRangeUpperBound(max, true) +} + +// NumberRangeLowerBound constraints the lower bound of a number value, or +// panics if this builder is not refining a number value. +func (b *RefinementBuilder) NumberRangeLowerBound(min Value, inclusive bool) *RefinementBuilder { + b.assertRefineable() + + wip, ok := b.wip.(*refinementNumber) + if !ok { + panic(fmt.Sprintf("cannot refine numeric bounds for a %#v value", b.orig.Type())) + } + + if !min.IsKnown() { + // Nothing to do if the lower bound is unknown. + return b + } + if min.IsNull() { + panic("number range lower bound must not be null") + } + + if inclusive { + if gt := min.GreaterThan(b.orig); gt.IsKnown() && gt.True() { + panic(fmt.Sprintf("refining %#v to be >= %#v", b.orig, min)) + } + } else { + if gt := min.GreaterThanOrEqualTo(b.orig); gt.IsKnown() && gt.True() { + panic(fmt.Sprintf("refining %#v to be > %#v", b.orig, min)) + } + } + + if wip.min != NilVal { + var ok Value + if inclusive && !wip.minInc { + ok = min.GreaterThan(wip.min) + } else { + ok = min.GreaterThanOrEqualTo(wip.min) + } + if ok.IsKnown() && ok.False() { + panic("refined number lower bound is inconsistent with existing lower bound") + } + } + + wip.min = min + wip.minInc = inclusive + + wip.assertConsistentBounds() + return b +} + +// NumberRangeUpperBound constraints the upper bound of a number value, or +// panics if this builder is not refining a number value. +func (b *RefinementBuilder) NumberRangeUpperBound(max Value, inclusive bool) *RefinementBuilder { + b.assertRefineable() + + wip, ok := b.wip.(*refinementNumber) + if !ok { + panic(fmt.Sprintf("cannot refine numeric bounds for a %#v value", b.orig.Type())) + } + + if !max.IsKnown() { + // Nothing to do if the upper bound is unknown. + return b + } + if max.IsNull() { + panic("number range upper bound must not be null") + } + + if inclusive { + if lt := max.LessThan(b.orig); lt.IsKnown() && lt.True() { + panic(fmt.Sprintf("refining %#v to be <= %#v", b.orig, max)) + } + } else { + if lt := max.LessThanOrEqualTo(b.orig); lt.IsKnown() && lt.True() { + panic(fmt.Sprintf("refining %#v to be < %#v", b.orig, max)) + } + } + + if wip.max != NilVal { + var ok Value + if inclusive && !wip.maxInc { + ok = max.LessThan(wip.max) + } else { + ok = max.LessThanOrEqualTo(wip.max) + } + if ok.IsKnown() && ok.False() { + panic("refined number upper bound is inconsistent with existing upper bound") + } + } + + wip.max = max + wip.maxInc = inclusive + + wip.assertConsistentBounds() + return b } // CollectionLengthLowerBound constrains the lower bound of the length of a @@ -316,93 +410,6 @@ func (b *RefinementBuilder) StringPrefix(prefix string) *RefinementBuilder { return b } -func (b *RefinementBuilder) numberRange(min, max Value, minInc, maxInc bool) *RefinementBuilder { - b.assertRefineable() - - wip, ok := b.wip.(*refinementNumber) - if !ok { - panic(fmt.Sprintf("cannot refine numeric range for a %#v value", b.orig.Type())) - } - // After this point b.orig is guaranteed to have type cty.Number - - if min.Type() != Number || max.Type() != Number { - panic("refining numeric range with a non-numeric bound") - } - if min.IsNull() || max.IsNull() { - panic("refining numeric range with a null bound") - } - - uncomparable := func(v Value) bool { - return v.IsNull() || !v.IsKnown() - } - checkMinRangeFunc := func(inclusive bool) func(Value, Value) bool { - if inclusive { - return func(a, b Value) bool { - if uncomparable(a) || uncomparable(b) { - return true // default to valid if we're not sure - } - return a.GreaterThanOrEqualTo(b).True() - } - } else { - return func(a, b Value) bool { - if uncomparable(a) || uncomparable(b) { - return true // default to valid if we're not sure - } - return a.GreaterThan(b).True() - } - } - } - checkMaxRangeFunc := func(inclusive bool) func(Value, Value) bool { - if inclusive { - return func(a, b Value) bool { - if uncomparable(a) || uncomparable(b) { - return true // default to valid if we're not sure - } - return a.LessThanOrEqualTo(b).True() - } - } else { - return func(a, b Value) bool { - if uncomparable(a) || uncomparable(b) { - return true // default to valid if we're not sure - } - return a.LessThan(b).True() - } - } - } - - // If our original value is known then it must be in the given range. - if v := b.orig; v.IsKnown() && !v.IsNull() { - if !checkMinRangeFunc(minInc)(v, min) { - panic(fmt.Sprintf("refining %#v with invalid lower bound %#v", v, min)) - } - if !checkMaxRangeFunc(maxInc)(v, max) { - panic(fmt.Sprintf("refining %#v with invalid upper bound %#v", v, min)) - } - } - - // If we already have bounds then the new bounds must be consistent with them. - if wip.min != NilVal && !checkMinRangeFunc(wip.minInc)(wip.min, min) { - panic(fmt.Sprintf("new refined lower bound %#v conflicts with previous %#v", min, wip.min)) - } - if wip.max != NilVal && !checkMaxRangeFunc(wip.maxInc)(wip.max, max) { - panic(fmt.Sprintf("new refined upper bound %#v conflicts with previous %#v", min, wip.min)) - } - - // We only record known bounds. An unknown value for either bound means - // it's either unbounded or we'll retain a prevously-recorded bound. - if min.IsKnown() { - wip.min = min - wip.minInc = minInc - } - if max.IsKnown() { - wip.max = max - wip.maxInc = maxInc - } - wip.assertConsistentBounds() - - return b -} - // NewValue completes the refinement process by constructing a new value // that is guaranteed to meet all of the previously-specified refinements. // @@ -570,8 +577,17 @@ func (r *refinementNumber) GoString() string { } func (r *refinementNumber) assertConsistentBounds() { - if r.min != NilVal && r.max != NilVal && r.max.LessThan(r.min).True() { - panic("number upper bound is less than lower bound") + if r.min == NilVal || r.max == NilVal { + return // If only one bound is constrained then there's nothing to be inconsistent with + } + var ok Value + if r.minInc != r.maxInc { + ok = r.min.LessThan(r.max) + } else { + ok = r.min.LessThanOrEqualTo(r.max) + } + if ok.IsKnown() && ok.False() { + panic(fmt.Sprintf("number lower bound %#v is greater than upper bound %#v", r.min, r.max)) } } From d0d411f0b23100fd9617687711f11e3e81b558c8 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 1 Feb 2023 14:52:36 -0800 Subject: [PATCH 11/28] cty: Ignore numeric range refinements that don't add information Previously we considered it a usage error to assert a bound that was outside of the existing refined range. However, if we already know that a number value is less than 2 then it isn't really an error to assert that it's also less than 4; the second assertion just doesn't give us any new information and so we should silently discard it and retain the original assertion. It's still an error to make assertions that directly contradict existing refinements, such as asserting that an unknown value isn't null and then later asserting that it's null. --- cty/unknown_refinement.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cty/unknown_refinement.go b/cty/unknown_refinement.go index ede30622..13966783 100644 --- a/cty/unknown_refinement.go +++ b/cty/unknown_refinement.go @@ -158,6 +158,9 @@ func (b *RefinementBuilder) NotNull() *RefinementBuilder { if b.orig.IsKnown() && b.orig.IsNull() { panic("refining null value as non-null") } + if b.wip.null() == tristateTrue { + panic("refining null value as non-null") + } b.wip.setNull(tristateFalse) @@ -179,6 +182,9 @@ func (b *RefinementBuilder) Null() *RefinementBuilder { if b.orig.IsKnown() && !b.orig.IsNull() { panic("refining non-null value as null") } + if b.wip.null() == tristateFalse { + panic("refining non-null value as null") + } b.wip.setNull(tristateTrue) @@ -232,7 +238,7 @@ func (b *RefinementBuilder) NumberRangeLowerBound(min Value, inclusive bool) *Re ok = min.GreaterThanOrEqualTo(wip.min) } if ok.IsKnown() && ok.False() { - panic("refined number lower bound is inconsistent with existing lower bound") + return b // Our existing refinement is more constrained } } @@ -279,7 +285,7 @@ func (b *RefinementBuilder) NumberRangeUpperBound(max Value, inclusive bool) *Re ok = max.LessThanOrEqualTo(wip.max) } if ok.IsKnown() && ok.False() { - panic("refined number upper bound is inconsistent with existing upper bound") + return b // Our existing refinement is more constrained } } @@ -310,7 +316,7 @@ func (b *RefinementBuilder) CollectionLengthLowerBound(min int) *RefinementBuild } if wip.minLen > min { - panic(fmt.Sprintf("refined collection length lower bound %d is inconsistent with existing lower bound %d", min, wip.minLen)) + return b // Our existing refinement is more constrained } wip.minLen = min @@ -342,7 +348,7 @@ func (b *RefinementBuilder) CollectionLengthUpperBound(max int) *RefinementBuild } if wip.maxLen < max { - panic(fmt.Sprintf("refined collection length upper bound %d is inconsistent with existing upper bound %d", max, wip.maxLen)) + return b // Our existing refinement is more constrained } wip.maxLen = max From 4d84cb3ba7fcdeca9106ed643e03a35233dee6a9 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 1 Feb 2023 14:53:14 -0800 Subject: [PATCH 12/28] msgpack: Can now round-trip unknown value refinements for number range --- cty/msgpack/roundtrip_test.go | 20 ++++++++++++++++++++ cty/msgpack/unknown.go | 6 +++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/cty/msgpack/roundtrip_test.go b/cty/msgpack/roundtrip_test.go index 41f4d438..d82f126a 100644 --- a/cty/msgpack/roundtrip_test.go +++ b/cty/msgpack/roundtrip_test.go @@ -104,6 +104,26 @@ func TestRoundTrip(t *testing.T) { cty.UnknownVal(cty.Number).RefineNotNull(), cty.Number, }, + { + cty.UnknownVal(cty.Number).Refine().NumberRangeLowerBound(cty.Zero, true).NewValue(), + cty.Number, + }, + { + cty.UnknownVal(cty.Number).Refine().NumberRangeLowerBound(cty.Zero, false).NewValue(), + cty.Number, + }, + { + cty.UnknownVal(cty.Number).Refine().NumberRangeUpperBound(cty.Zero, true).NewValue(), + cty.Number, + }, + { + cty.UnknownVal(cty.Number).Refine().NumberRangeUpperBound(cty.Zero, false).NewValue(), + cty.Number, + }, + { + cty.UnknownVal(cty.Number).Refine().NumberRangeInclusive(cty.Zero, cty.NumberIntVal(1)).NewValue(), + cty.Number, + }, { cty.ListVal([]cty.Value{ diff --git a/cty/msgpack/unknown.go b/cty/msgpack/unknown.go index 03ffa974..dc55a2ea 100644 --- a/cty/msgpack/unknown.go +++ b/cty/msgpack/unknown.go @@ -289,12 +289,12 @@ func unmarshalUnknownValue(dec *msgpack.Decoder, ty cty.Type, path cty.Path) (ct if isIncVal.Type() != cty.Bool || !isIncVal.IsKnown() || isIncVal.IsNull() { return cty.DynamicVal, path.NewErrorf("failed to decode msgpack extension body: length bound refinement must be [number, bool] array") } - //isInc := isIncVal.True() + isInc := isIncVal.True() switch keyCode { case unknownValNumberMin: - // TODO: builder doesn't have a NumberRangeLowerBound method yet + builder = builder.NumberRangeLowerBound(boundVal, isInc) case unknownValNumberMax: - // TODO: builder doesn't have a NumberRangeUpperBound method yet + builder = builder.NumberRangeUpperBound(boundVal, isInc) default: panic("unsupported keyCode") // should not get here } From 5ce6a9c33377a9f7df5dbf8e0634b11937631081 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 1 Feb 2023 15:27:48 -0800 Subject: [PATCH 13/28] cty: Value.Length constrains range of unknown results When we're being asked for the length of an unknown collection we can always put at least _some_ bounds on the range of the result, because we know that a collection can't have a negative length and it cannot have a length that is bigger than the current machine's integer size because otherwise we wouldn't be able to index into it. For some collections we may have further refinements on the length of the unknown collection, in which case we'll also transfer those to being refinements on the range of the numeric result. --- cty/function/stdlib/collection_test.go | 9 +++------ cty/value_ops.go | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/cty/function/stdlib/collection_test.go b/cty/function/stdlib/collection_test.go index 5be0d30d..3fb92535 100644 --- a/cty/function/stdlib/collection_test.go +++ b/cty/function/stdlib/collection_test.go @@ -973,18 +973,15 @@ func TestLength(t *testing.T) { }, { cty.UnknownVal(cty.List(cty.Bool)), - cty.UnknownVal(cty.Number).Refine().NotNull().NumberRangeInclusive(cty.Zero, cty.UnknownVal(cty.Number)).NewValue(), + cty.UnknownVal(cty.Number).Refine().NotNull().NumberRangeInclusive(cty.Zero, cty.NumberIntVal(int64(math.MaxInt))).NewValue(), }, { cty.DynamicVal, - cty.UnknownVal(cty.Number).Refine().NotNull().NumberRangeInclusive(cty.Zero, cty.UnknownVal(cty.Number)).NewValue(), + cty.UnknownVal(cty.Number).Refine().NotNull().NumberRangeInclusive(cty.Zero, cty.NumberIntVal(int64(math.MaxInt))).NewValue(), }, { - // TODO: This one should really preserve the length bounds as the - // numeric bounds of its result, but cty.Value.Length isn't yet - // able to do that. cty.UnknownVal(cty.List(cty.Bool)).Refine().CollectionLengthUpperBound(2).NewValue(), - cty.UnknownVal(cty.Number).Refine().NotNull().NumberRangeInclusive(cty.Zero, cty.UnknownVal(cty.Number)).NewValue(), + cty.UnknownVal(cty.Number).Refine().NotNull().NumberRangeInclusive(cty.Zero, cty.NumberIntVal(2)).NewValue(), }, { // Marked collections return a marked length cty.ListVal([]cty.Value{ diff --git a/cty/value_ops.go b/cty/value_ops.go index 729d87f5..5d0179d6 100644 --- a/cty/value_ops.go +++ b/cty/value_ops.go @@ -1043,9 +1043,10 @@ func (val Value) Length() Value { } if !val.IsKnown() { - // TODO: If the unknown value has been refined with explicit length - // bounds then we should use those as the refined range of this result. - return UnknownVal(Number).Refine().NotNull().NumberRangeInclusive(Zero, UnknownVal(Number)).NewValue() + // If the whole collection isn't known then the length isn't known + // either, but we can still put some bounds on the range of the result. + rng := val.Range() + return UnknownVal(Number).RefineWith(valueRefineLengthResult(rng)) } if val.Type().IsSetType() { // The Length rules are a little different for sets because if any @@ -1072,6 +1073,17 @@ func (val Value) Length() Value { return NumberIntVal(int64(val.LengthInt())) } +func valueRefineLengthResult(collRng ValueRange) func(*RefinementBuilder) *RefinementBuilder { + return func(b *RefinementBuilder) *RefinementBuilder { + return b. + NotNull(). + NumberRangeInclusive( + NumberIntVal(int64(collRng.LengthLowerBound())), + NumberIntVal(int64(collRng.LengthUpperBound())), + ) + } +} + // LengthInt is like Length except it returns an int. It has the same behavior // as Length except that it will panic if the receiver is unknown. // From 0a2e9084d9b6b14d98556328da3edf872e1fd130 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 1 Feb 2023 15:58:44 -0800 Subject: [PATCH 14/28] cty: Value.LessThan and Value.GreaterThan use number range refinements This allows us to return a known result in more cases than before by making deductions based on the known numeric range. This first implementation is a little more conservative than it strictly needs to be because it just treats all bounds as exclusive rather than taking into account inclusive bounds. This makes the implementation a bit simpler for this initial round though, and should never generate an incorrect known result. --- cty/function/stdlib/number.go | 8 ++++++ cty/function/stdlib/number_test.go | 5 ++++ cty/value_ops.go | 44 ++++++++++++++++++++++++++++++ cty/value_ops_test.go | 20 ++++++++++++++ 4 files changed, 77 insertions(+) diff --git a/cty/function/stdlib/number.go b/cty/function/stdlib/number.go index 7005f746..73ef32f1 100644 --- a/cty/function/stdlib/number.go +++ b/cty/function/stdlib/number.go @@ -211,12 +211,14 @@ var GreaterThanFunc = function.New(&function.Spec{ { Name: "a", Type: cty.Number, + AllowUnknown: true, AllowDynamicType: true, AllowMarked: true, }, { Name: "b", Type: cty.Number, + AllowUnknown: true, AllowDynamicType: true, AllowMarked: true, }, @@ -234,12 +236,14 @@ var GreaterThanOrEqualToFunc = function.New(&function.Spec{ { Name: "a", Type: cty.Number, + AllowUnknown: true, AllowDynamicType: true, AllowMarked: true, }, { Name: "b", Type: cty.Number, + AllowUnknown: true, AllowDynamicType: true, AllowMarked: true, }, @@ -257,12 +261,14 @@ var LessThanFunc = function.New(&function.Spec{ { Name: "a", Type: cty.Number, + AllowUnknown: true, AllowDynamicType: true, AllowMarked: true, }, { Name: "b", Type: cty.Number, + AllowUnknown: true, AllowDynamicType: true, AllowMarked: true, }, @@ -280,12 +286,14 @@ var LessThanOrEqualToFunc = function.New(&function.Spec{ { Name: "a", Type: cty.Number, + AllowUnknown: true, AllowDynamicType: true, AllowMarked: true, }, { Name: "b", Type: cty.Number, + AllowUnknown: true, AllowDynamicType: true, AllowMarked: true, }, diff --git a/cty/function/stdlib/number_test.go b/cty/function/stdlib/number_test.go index 9df55fd2..bb12821f 100644 --- a/cty/function/stdlib/number_test.go +++ b/cty/function/stdlib/number_test.go @@ -399,6 +399,11 @@ func TestLessThan(t *testing.T) { cty.UnknownVal(cty.Number), cty.UnknownVal(cty.Bool).RefineNotNull(), }, + { + cty.NumberIntVal(1), + cty.UnknownVal(cty.Number).Refine().NumberRangeLowerBound(cty.NumberIntVal(2), true).NewValue(), + cty.True, // deduced from refinement + }, { cty.UnknownVal(cty.Number), cty.UnknownVal(cty.Number), diff --git a/cty/value_ops.go b/cty/value_ops.go index 5d0179d6..aad2cc54 100644 --- a/cty/value_ops.go +++ b/cty/value_ops.go @@ -1264,6 +1264,28 @@ func (val Value) LessThan(other Value) Value { } if shortCircuit := mustTypeCheck(Number, Bool, val, other); shortCircuit != nil { + // We might be able to return a known answer even with unknown inputs. + // FIXME: This is more conservative than it needs to be, because it + // treats all bounds as exclusive bounds. + valRng := val.Range() + otherRng := other.Range() + if valRng.TypeConstraint() == Number && other.Range().TypeConstraint() == Number { + valMax, _ := valRng.NumberUpperBound() + otherMin, _ := otherRng.NumberLowerBound() + if valMax.IsKnown() && otherMin.IsKnown() { + if r := valMax.LessThan(otherMin); r.True() { + return True + } + } + valMin, _ := valRng.NumberLowerBound() + otherMax, _ := otherRng.NumberUpperBound() + if valMin.IsKnown() && otherMax.IsKnown() { + if r := valMin.GreaterThan(otherMax); r.True() { + return False + } + } + } + shortCircuit = forceShortCircuitType(shortCircuit, Bool) return (*shortCircuit).RefineNotNull() } @@ -1281,6 +1303,28 @@ func (val Value) GreaterThan(other Value) Value { } if shortCircuit := mustTypeCheck(Number, Bool, val, other); shortCircuit != nil { + // We might be able to return a known answer even with unknown inputs. + // FIXME: This is more conservative than it needs to be, because it + // treats all bounds as exclusive bounds. + valRng := val.Range() + otherRng := other.Range() + if valRng.TypeConstraint() == Number && other.Range().TypeConstraint() == Number { + valMin, _ := valRng.NumberLowerBound() + otherMax, _ := otherRng.NumberUpperBound() + if valMin.IsKnown() && otherMax.IsKnown() { + if r := valMin.GreaterThan(otherMax); r.True() { + return True + } + } + valMax, _ := valRng.NumberUpperBound() + otherMin, _ := otherRng.NumberLowerBound() + if valMax.IsKnown() && otherMin.IsKnown() { + if r := valMax.LessThan(otherMin); r.True() { + return False + } + } + } + shortCircuit = forceShortCircuitType(shortCircuit, Bool) return (*shortCircuit).RefineNotNull() } diff --git a/cty/value_ops_test.go b/cty/value_ops_test.go index bd4b6383..8d694ffa 100644 --- a/cty/value_ops_test.go +++ b/cty/value_ops_test.go @@ -3002,6 +3002,16 @@ func TestLessThan(t *testing.T) { NumberIntVal(1), UnknownVal(Bool).RefineNotNull(), }, + { + UnknownVal(Number).Refine().NumberRangeUpperBound(Zero, true).NewValue(), + NumberIntVal(1), + True, // Deduction from the refinement + }, + { + UnknownVal(Number).Refine().NumberRangeLowerBound(NumberIntVal(2), true).NewValue(), + NumberIntVal(1), + False, // Deduction from the refinement + }, { DynamicVal, DynamicVal, @@ -3105,6 +3115,16 @@ func TestGreaterThan(t *testing.T) { NumberIntVal(1), UnknownVal(Bool).RefineNotNull(), }, + { + UnknownVal(Number).Refine().NumberRangeLowerBound(NumberIntVal(2), true).NewValue(), + NumberIntVal(1), + True, // Deduction based on the refinements + }, + { + UnknownVal(Number).Refine().NumberRangeUpperBound(NumberIntVal(0), true).NewValue(), + NumberIntVal(1), + False, // Deduction based on the refinements + }, { DynamicVal, DynamicVal, From 6af71e228fd57dc6db263e7e66aa50f50532def1 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 1 Feb 2023 16:26:57 -0800 Subject: [PATCH 15/28] function/stdlib: Strlen considers string prefix refinements When given an unknown value, the Strlen function will now return a refined unknown value with an inclusive lower bound. This bound is always at least zero because a string can't have a negative length, but if we also know a prefix for the string then we can refine the result even more to take that into account. --- cty/function/stdlib/string.go | 40 ++++++++++++++++++++++-------- cty/function/stdlib/string_test.go | 8 ++++-- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/cty/function/stdlib/string.go b/cty/function/stdlib/string.go index 08b30535..04033026 100644 --- a/cty/function/stdlib/string.go +++ b/cty/function/stdlib/string.go @@ -84,26 +84,46 @@ var StrlenFunc = function.New(&function.Spec{ { Name: "str", Type: cty.String, + AllowUnknown: true, AllowDynamicType: true, }, }, - Type: function.StaticReturnType(cty.Number), - RefineResult: refineNonNull, + Type: function.StaticReturnType(cty.Number), + RefineResult: func(b *cty.RefinementBuilder) *cty.RefinementBuilder { + // String length is never null and never negative. + // (We might refine the lower bound even more inside Impl.) + return b.NotNull().NumberRangeLowerBound(cty.NumberIntVal(0), true) + }, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { - in := args[0].AsString() - l := 0 - - inB := []byte(in) - for i := 0; i < len(in); { - d, _, _ := textseg.ScanGraphemeClusters(inB[i:], true) - l++ - i += d + if !args[0].IsKnown() { + ret := cty.UnknownVal(cty.Number) + // We may be able to still return a constrained result based on the + // refined range of the unknown value. + inRng := args[0].Range() + if inRng.TypeConstraint() == cty.String { + prefixLen := int64(graphemeClusterCount(inRng.StringPrefix())) + ret = ret.Refine().NumberRangeLowerBound(cty.NumberIntVal(prefixLen), true).NewValue() + } + return ret, nil } + in := args[0].AsString() + l := graphemeClusterCount(in) return cty.NumberIntVal(int64(l)), nil }, }) +func graphemeClusterCount(in string) int { + l := 0 + inB := []byte(in) + for i := 0; i < len(in); { + d, _, _ := textseg.ScanGraphemeClusters(inB[i:], true) + l++ + i += d + } + return l +} + var SubstrFunc = function.New(&function.Spec{ Description: "Extracts a substring from the given string.", Params: []function.Parameter{ diff --git a/cty/function/stdlib/string_test.go b/cty/function/stdlib/string_test.go index 9778aff9..3f27addb 100644 --- a/cty/function/stdlib/string_test.go +++ b/cty/function/stdlib/string_test.go @@ -245,11 +245,15 @@ func TestStrlen(t *testing.T) { }, { cty.UnknownVal(cty.String), - cty.UnknownVal(cty.Number).RefineNotNull(), + cty.UnknownVal(cty.Number).Refine().NotNull().NumberRangeLowerBound(cty.Zero, true).NewValue(), + }, + { + cty.UnknownVal(cty.String).Refine().StringPrefix("wé́́é́́é́́-").NewValue(), + cty.UnknownVal(cty.Number).Refine().NotNull().NumberRangeLowerBound(cty.NumberIntVal(5), true).NewValue(), }, { cty.DynamicVal, - cty.UnknownVal(cty.Number).RefineNotNull(), + cty.UnknownVal(cty.Number).Refine().NotNull().NumberRangeLowerBound(cty.Zero, true).NewValue(), }, } From 85bd94887f4fe107713a2be89dbd5c7146ebc1c1 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 1 Feb 2023 17:23:43 -0800 Subject: [PATCH 16/28] cty: Value.Equals disqualifies unknown values based on range When comparing a known value to an unknown value we may be able to return a definitive False if we can prove that the known value isn't in the range of the unknown value. --- cty/value_ops.go | 14 +++++++ cty/value_ops_test.go | 35 ++++++++++++++++ cty/value_range.go | 94 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+) diff --git a/cty/value_ops.go b/cty/value_ops.go index aad2cc54..85dac769 100644 --- a/cty/value_ops.go +++ b/cty/value_ops.go @@ -142,6 +142,20 @@ func (val Value) Equals(other Value) Value { case other.IsNull() && definitelyNotNull(val): return False } + // If we have one known value and one unknown value then we may be + // able to quickly disqualify equality based on the range of the unknown + // value. + if val.IsKnown() && !other.IsKnown() { + otherRng := other.Range() + if ok := otherRng.Includes(val); ok.IsKnown() && ok.False() { + return False + } + } else if other.IsKnown() && !val.IsKnown() { + valRng := val.Range() + if ok := valRng.Includes(other); ok.IsKnown() && ok.False() { + return False + } + } // We need to deal with unknown values before anything else with nulls // because any unknown value that hasn't yet been refined as non-null diff --git a/cty/value_ops_test.go b/cty/value_ops_test.go index 8d694ffa..55f8d41f 100644 --- a/cty/value_ops_test.go +++ b/cty/value_ops_test.go @@ -595,6 +595,16 @@ func TestValueEquals(t *testing.T) { DynamicVal, unknownResult, }, + { + NumberIntVal(2), + UnknownVal(Number).Refine().NumberRangeLowerBound(Zero, true).NewValue(), + unknownResult, + }, + { + NumberIntVal(2), + UnknownVal(Number).Refine().NumberRangeLowerBound(NumberIntVal(4), true).NewValue(), + False, // deduction from refinement + }, { DynamicVal, BoolVal(true), @@ -627,6 +637,11 @@ func TestValueEquals(t *testing.T) { }), unknownResult, }, + { + UnknownVal(List(String)).Refine().CollectionLengthLowerBound(1).NewValue(), + ListValEmpty(String), + False, // deduction from refinement + }, { MapVal(map[string]Value{ "static": StringVal("hi"), @@ -803,6 +818,26 @@ func TestValueEquals(t *testing.T) { UnknownVal(String).Refine().Null().NewValue(), True, // NOTE: The refinement should collapse to NullVal(String) }, + { + UnknownVal(String).Refine().StringPrefix("foo-").NewValue(), + StringVal("notfoo-bar"), + False, // Deduction from refinement + }, + { + StringVal(""), + UnknownVal(String).Refine().StringPrefix("foo-").NewValue(), + False, // Deduction from refinement + }, + { + StringVal("").Mark("a"), + UnknownVal(String).Mark("b").Refine().StringPrefix("foo-").NewValue(), + False.Mark("a").Mark("b"), // Deduction from refinement + }, + { + UnknownVal(String).Refine().StringPrefix("foo-").NewValue(), + StringVal("foo-bar"), + unknownResult, + }, // Marks { diff --git a/cty/value_range.go b/cty/value_range.go index 46f9d275..9fd5ec42 100644 --- a/cty/value_range.go +++ b/cty/value_range.go @@ -3,6 +3,7 @@ package cty import ( "fmt" "math" + "strings" ) // Range returns an object that offers partial information about the range @@ -233,6 +234,99 @@ func (r ValueRange) LengthUpperBound() int { return math.MaxInt } +// Includes determines whether the given value is in the receiving range. +// +// It can return only three possible values: +// - [cty.True] if the range definitely includes the value +// - [cty.False] if the range definitely does not include the value +// - An unknown value of [cty.Bool] if there isn't enough information to decide. +// +// This function is not fully comprehensive: it may return an unknown value +// in some cases where a definitive value could be computed in principle, and +// those same situations may begin returning known values in later releases as +// the rules are refined to be more complete. Currently the rules focus mainly +// on answering [cty.False], because disproving membership tends to be more +// useful than proving membership. +func (r ValueRange) Includes(v Value) Value { + unknownResult := UnknownVal(Bool).RefineNotNull() + + if r.raw.null() == tristateTrue { + if v.IsNull() { + return True + } else { + return False + } + } + if r.raw.null() == tristateFalse { + if v.IsNull() { + return False + } + // A definitely-not-null value could potentially match + // but we won't know until we do some more checks below. + } + // If our range includes both null and non-null values and the value is + // null then it's definitely in range. + if v.IsNull() { + return True + } + if len(v.Type().TestConformance(r.TypeConstraint())) != 0 { + // If the value doesn't conform to the type constraint then it's + // definitely not in the range. + return False + } + if v.Type() == DynamicPseudoType { + // If it's an unknown value of an unknown type then there's no + // further tests we can make. + return unknownResult + } + + switch r.raw.(type) { + case *refinementString: + if v.IsKnown() { + prefix := r.StringPrefix() + got := v.AsString() + fmt.Printf("prefix %q got %q\n", prefix, got) + + if !strings.HasPrefix(got, prefix) { + return False + } + } + case *refinementCollection: + lenVal := v.Length() + minLen := NumberIntVal(int64(r.LengthLowerBound())) + maxLen := NumberIntVal(int64(r.LengthUpperBound())) + if minOk := lenVal.GreaterThanOrEqualTo(minLen); minOk.IsKnown() && minOk.False() { + return False + } + if maxOk := lenVal.LessThanOrEqualTo(maxLen); maxOk.IsKnown() && maxOk.False() { + return False + } + case *refinementNumber: + minVal, minInc := r.NumberLowerBound() + maxVal, maxInc := r.NumberUpperBound() + var minOk, maxOk Value + if minInc { + minOk = v.GreaterThanOrEqualTo(minVal) + } else { + minOk = v.GreaterThan(minVal) + } + if maxInc { + maxOk = v.LessThanOrEqualTo(maxVal) + } else { + maxOk = v.LessThan(maxVal) + } + if minOk.IsKnown() && minOk.False() { + return False + } + if maxOk.IsKnown() && maxOk.False() { + return False + } + } + + // If we fall out here then we don't have enough information to decide. + return unknownResult +} + // definitelyNotNull is a convenient helper for the common situation of checking // whether a value could possibly be null. // From 800485d4238dbaf44d1d9b3eec880aa5b40f11f7 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 2 Feb 2023 13:01:50 -0800 Subject: [PATCH 17/28] cty: Refine string prefixes more conservatively Because cty strings are unicode texts following both the normalization and text segmentation rules, we need to be cautious in how we model prefix matching of strings in the refinement system. To avoid problems caused by combining character sequences, we'll trim off characters from the end of a prefix if we cannot guarantee that they won't combine with something that follows. This then avoids us overpromising what the prefix will be in situations where the last character might change after appending the rest of the string. For applications that fully control their values and can guarantee that no combining will occur at the boundary, the new function StringPrefixFull retains the previous behavior of StringPrefix. The idea here is to be conservative by default but allow applications to be less conservative if they are able to offer greater guarantees. --- cty/ctystrings/doc.go | 26 +++++ cty/ctystrings/normalize.go | 14 +++ cty/ctystrings/prefix.go | 139 +++++++++++++++++++++++ cty/ctystrings/prefix_test.go | 208 ++++++++++++++++++++++++++++++++++ cty/msgpack/unknown.go | 5 +- cty/unknown_refinement.go | 42 ++++++- cty/value_init.go | 5 +- cty/value_ops_test.go | 8 +- cty/value_range.go | 1 - 9 files changed, 436 insertions(+), 12 deletions(-) create mode 100644 cty/ctystrings/doc.go create mode 100644 cty/ctystrings/normalize.go create mode 100644 cty/ctystrings/prefix.go create mode 100644 cty/ctystrings/prefix_test.go diff --git a/cty/ctystrings/doc.go b/cty/ctystrings/doc.go new file mode 100644 index 00000000..0ea7f984 --- /dev/null +++ b/cty/ctystrings/doc.go @@ -0,0 +1,26 @@ +// Package ctystrings is a collection of string manipulation utilities which +// intend to help application developers implement string-manipulation +// functionality in a way that respects the cty model of strings, even when +// they are working in the realm of Go strings. +// +// cty strings are, internally, NFC-normalized as defined in Unicode Standard +// Annex #15 and encoded as UTF-8. +// +// When working with [cty.Value] of string type cty manages this +// automatically as an implementation detail, but when applications call +// [Value.AsString] they will receive a value that has been subjected to that +// normalization, and so may need to take that normalization into account when +// manipulating the resulting string or comparing it with other Go strings +// that did not originate in a [cty.Value]. +// +// Although the core representation of [cty.String] only considers whole +// strings, it's also conventional in other locations such as the standard +// library functions to consider strings as being sequences of grapheme +// clusters as defined by Unicode Standard Annex #29, which adds further +// rules about combining multiple consecutive codepoints together into a +// single user-percieved character. Functions that work with substrings should +// always use grapheme clusters as their smallest unit of splitting strings, +// and never break strings in the middle of a grapheme cluster. The functions +// in this package respect that convention unless otherwise stated in their +// documentation. +package ctystrings diff --git a/cty/ctystrings/normalize.go b/cty/ctystrings/normalize.go new file mode 100644 index 00000000..9b3bce90 --- /dev/null +++ b/cty/ctystrings/normalize.go @@ -0,0 +1,14 @@ +package ctystrings + +import ( + "golang.org/x/text/unicode/norm" +) + +// Normalize applies NFC normalization to the given string, returning the +// transformed string. +// +// This function achieves the same effect as wrapping a string in a value +// using [cty.StringVal] and then unwrapping it again using [Value.AsString]. +func Normalize(str string) string { + return norm.NFC.String(str) +} diff --git a/cty/ctystrings/prefix.go b/cty/ctystrings/prefix.go new file mode 100644 index 00000000..1d9f5c5a --- /dev/null +++ b/cty/ctystrings/prefix.go @@ -0,0 +1,139 @@ +package ctystrings + +import ( + "fmt" + "unicode/utf8" + + "github.com/apparentlymart/go-textseg/v13/textseg" + "golang.org/x/text/unicode/norm" +) + +// SafeKnownPrefix takes a string intended to represent a known prefix of +// another string and modifies it so that it would be safe to use with +// byte-based prefix matching against another NFC-normalized string. It +// also takes into account grapheme cluster boundaries and trims off any +// suffix that could potentially be an incomplete grapheme cluster. +// +// Specifically, SafeKnownPrefix first applies NFC normalization to the prefix +// and then trims off one or more characters from the end of the string which +// could potentially be transformed into a different character if another +// string were appended to it. For example, a trailing latin letter will +// typically be trimmed because appending a combining diacritic mark would +// transform it into a different character. +// +// This transformation is important whenever the remainder of the string is +// arbitrary user input not directly controlled by the application. If an +// application can guarantee that the remainder of the string will not begin +// with combining marks then it is safe to instead just normalize the prefix +// string with [Normalize]. +// +// Note that this function only takes into account normalization boundaries +// and does _not_ take into account grapheme cluster boundaries as defined +// by Unicode Standard Annex #29. +func SafeKnownPrefix(prefix string) string { + prefix = Normalize(prefix) + + // Our starting approach here is essentially what a streaming parser would + // do when consuming a Unicode string in chunks and needing to determine + // what prefix of the current buffer is safe to process without waiting for + // more information, which is described in TR15 section 13.1 + // "Buffering with Unicode Normalization": + // https://unicode.org/reports/tr15/#Buffering_with_Unicode_Normalization + // + // The general idea here is to find the last character in the string that + // could potentially start a sequence of codepoints that would combine + // together, and then truncate the string to exclude that character and + // everything after it. + + form := norm.NFC + lastBoundary := form.LastBoundary([]byte(prefix)) + if lastBoundary != -1 && lastBoundary != len(prefix) { + prefix = prefix[:lastBoundary] + // If we get here then we've already shortened the prefix and so + // further analysis below is unnecessary because it would be relying + // on an incomplete prefix anyway. + return prefix + } + + // Now we'll use the textseg package's grapheme cluster scanner to scan + // as far through the string as we can without the scanner telling us + // that it would need more bytes to decide. + // + // This step is conservative because the grapheme cluster rules are not + // designed with prefix-matching in mind. In the base case we'll just + // always discard the last grapheme cluster, although we do have some + // special cases for trailing codepoints that can't possibly combine with + // subsequent codepoints to form a single grapheme cluster and which seem + // likely to arise often in practical use. + remain := []byte(prefix) + prevBoundary := 0 + thisBoundary := 0 + for len(remain) > 0 { + advance, _, err := textseg.ScanGraphemeClusters(remain, false) + if err != nil { + // ScanGraphemeClusters should never return an error because + // any sequence of valid UTF-8 encodings is valid input. + panic(fmt.Sprintf("textseg.ScanGraphemeClusters returned error: %s", err)) + } + if advance == 0 { + // If we have at least one byte remaining but the scanner cannot + // advance then that means the remainder might be an incomplete + // grapheme cluster and so we need to stop here, discarding the + // rest of the input. However, we do now know that we can safely + // include what we found on the previous iteration of this loop. + prevBoundary = thisBoundary + break + } + prevBoundary = thisBoundary + thisBoundary += advance + remain = remain[advance:] + } + + // This is our heuristic for detecting cases where we can be sure that + // the above algorithm was too conservative because the last segment + // we found is definitely not subject to the grapheme cluster "do not split" + // rules. + suspect := prefix[prevBoundary:thisBoundary] + if sequenceMustEndGraphemeCluster(suspect) { + prevBoundary = thisBoundary + } + + return prefix[:prevBoundary] +} + +// sequenceMustEndGraphemeCluster is a heuristic we use to avoid discarding +// the final grapheme cluster of a prefix in SafeKnownPrefix by recognizing +// that a particular sequence is one known to not be subject to any of +// the UAX29 "do not break" rules. +// +// If this function returns true then it is safe to include the given byte +// sequence at the end of a safe prefix. Otherwise we don't know whether or +// not it is safe. +func sequenceMustEndGraphemeCluster(s string) bool { + // For now we're only considering sequences that represent a single + // codepoint. We'll assume that any sequence of two or more codepoints + // that could be a grapheme cluster might be extendable. + if utf8.RuneCountInString(s) != 1 { + return false + } + + r, _ := utf8.DecodeRuneInString(s) + + // Our initial ruleset is focused on characters that are commonly used + // as delimiters in text intended for both human and machine use, such + // as JSON documents. + // + // We don't include any letters or digits of any script here intentionally + // because those are the ones most likely to be subject to combining rules + // in either current or future Unicode specifications. + // + // We can safely grow this set over time, but we should be very careful + // about shrinking it because it could cause value refinements to loosen + // and thus cause results that were once known to become unknown. + switch r { + case '-', '_', ':', ';', '/', '\\', ',', '.', '(', ')', '{', '}', '[', ']', '|', '?', '!', '~', ' ', '\t', '@', '#', '$', '%', '^', '&', '*', '+', '"', '\'': + return true + default: + return false + } +} diff --git a/cty/ctystrings/prefix_test.go b/cty/ctystrings/prefix_test.go new file mode 100644 index 00000000..7ca3024b --- /dev/null +++ b/cty/ctystrings/prefix_test.go @@ -0,0 +1,208 @@ +package ctystrings + +import ( + "testing" +) + +func TestSafeKnownPrefix(t *testing.T) { + tests := []struct { + Input, Want string + }{ + // NOTE: Under future improvements to SafeKnownPrefix the "Want" + // results for all of these tests can safely get longer, thereby + // describing a more precise constraint, but we should avoid making + // them shorter because that will weaken existing constraints from + // older versions. + // (We might make exceptions for behaviors that are found to be + // clearly wrong, but consider the consequences carefully.) + + { + "", + "", + }, + { + "a", + "", // The "a" is discarded because it might combine with diacritics to follow + }, + { + "boo", + "bo", // The final o is discarded because it might combine with diacritics to follow + }, + { + "boop\r", + "boop", // The final \r is discarded because it could combine with \r\n to produce a single grapheme cluster + }, + { + "hello 가", + "hello ", // Hangul syllables can combine arbitrarily, so we must trim of trailing ones + }, + { + "hello 🤷🏽‍♂️", + "hello ", // We conservatively trim the whole emoji sequence because other emoji modifiers might come in later unicode specs + }, + { + "hello 🤷🏽‍♂️ ", + "hello 🤷🏽‍♂️ ", // A subsequent character avoids the need to trim + }, + { + "hello 🤷", + "hello ", // "Person Shrugging" can potentially combine with subsequent skin tone modifiers or ZWJ followed by gender presentation modifiers + }, + { + "hello 🤷 ", + "hello 🤷 ", // A subsequent character avoids the need to trim + }, + { + "hello 🤷\u200d", // U+200D is "zero width joiner" + "hello ", // The "Person Shrugging" followed by zero with joiner anticipates a subsequent modifier to join with + }, + { + "hello \U0001f1e6", // This is the beginning of a "regional indicator symbol", which are supposed to appear in pairs but we only have one here + "hello ", // The symbol was discarded because we can't know what character it represents until we have both parts + }, + { + "hello \U0001f1e6\U0001f1e6", // This is a regional indicator symbol "AA", which happens to be Aruba but it's not important exactly which country we're encoding + "hello ", // The text segmentation spec allows any number of consecutive regional indicators, so we must always discard any number of them at the end. + }, + { + "hello \U0001f1e6\U0001f1e6 ", + "hello \U0001f1e6\U0001f1e6 ", // A subsequent character avoids the need to trim + }, + + // The following all rely on our additional heuristic about certain + // commonly-used delimiters that we know can never be the beginning + // of a combined grapheme cluster sequence. We make these exceptions + // because cty tends to be used more often for constructing strings + // for use by machines than for constructing text for human consumption. + { + "ami-", // e.g. prefix of an Amazon EC2 object identifier + "ami-", + }, + { + "foo_", // e.g. prefix of a variable name + "foo_", + }, + { + `{"foo":`, // e.g. prefix of a JSON object + `{"foo":`, + }, + { + `beep();`, // e.g. prefix of a program in a C-like language? + `beep();`, + }, + { + `https://`, // e.g. prefix of a URL with a known scheme + `https://`, + }, + { + `c:\`, // e.g. windows filesystem path with a known drive letter + `c:\`, + }, + { + `["foo",`, // e.g. prefix of a JSON document that includes a partially-known array + `["foo",`, + }, + { + `foo.bar.`, // e.g. prefix of a traversal through attributes + `foo.bar.`, + }, + { + `beep(`, // e.g. prefix of a program in a C-like language? + `beep(`, + }, + { + `beep()`, // e.g. prefix of a program in a C-like language? + `beep()`, + }, + { + `{`, // e.g. prefix of a JSON object + `{`, + }, + { + `[{}`, // e.g. fragment of JSON + `[{}`, + }, + { + `[`, // e.g. prefix of a JSON array + `[`, + }, + { + `[[]`, // e.g. fragment of JSON + `[[]`, + }, + { + `whatever |`, // e.g. partial Unix-style command line + `whatever |`, + }, + { + `https://example.com/foo?`, // e.g. prefix of a URL with a query string + `https://example.com/foo?`, + }, + { + `boop!`, // dunno but seems weird to have ? without ! + `boop!`, + }, + { + `ls ~`, // A reference to somebody's home directory + `ls ~`, + }, + { + `a `, // A space always disambiguates whether our suffix is safe + `a `, + }, + { + "a\t", // A tab always disambiguates whether our suffix is safe + "a\t", + }, + { + `username@`, // e.g. incomplete email address + `username@`, + }, + { + `#`, // e.g. start of a single-linecomment in some machine languages, or a "hashtag" + `#`, + }, + { + `print $`, // e.g. start of a reference to a Perl scalar + `print $`, + }, + { + `print %`, // e.g. start of a reference to a Perl hash + `print %`, + }, + { + `^`, // e.g. start of a pessimistic version constraint in some version constraint syntaxes + `^`, + }, + { + `foo(&`, // e.g. the "address of" operator in some programming languages + `foo(&`, + }, + { + `foo *`, // e.g. multiplying by something + `foo *`, + }, + { + `foo +`, // e.g. addition + `foo +`, + }, + { + `["`, // e.g. we know it's a JSON string but we don't know the content yet + `["`, + }, + { + `['`, // e.g. a string in a JSON-like language that also supports single quotes! + `['`, + }, + } + + for _, test := range tests { + t.Run(test.Input, func(t *testing.T) { + got := SafeKnownPrefix(test.Input) + + if got != test.Want { + t.Errorf("wrong result\ninput: %q\ngot: %q\nwant: %q", test.Input, got, test.Want) + } + }) + } +} diff --git a/cty/msgpack/unknown.go b/cty/msgpack/unknown.go index dc55a2ea..667bef8a 100644 --- a/cty/msgpack/unknown.go +++ b/cty/msgpack/unknown.go @@ -252,7 +252,10 @@ func unmarshalUnknownValue(dec *msgpack.Decoder, ty cty.Type, path cty.Path) (ct if !utf8.ValidString(prefixStr) { return cty.DynamicVal, path.NewErrorf("failed to decode msgpack extension body: string prefix refinement is not valid UTF-8") } - builder = builder.StringPrefix(prefixStr) + // We assume that the original creator of this value already took + // care of making sure the prefix is safe, so we don't need to + // constrain it any further. + builder = builder.StringPrefixFull(prefixStr) case unknownValLengthMin, unknownValLengthMax: if !ty.IsCollectionType() { return cty.DynamicVal, path.NewErrorf("failed to decode msgpack extension body: length lower bound refinement for non-collection type") diff --git a/cty/unknown_refinement.go b/cty/unknown_refinement.go index 13966783..83c968e9 100644 --- a/cty/unknown_refinement.go +++ b/cty/unknown_refinement.go @@ -4,6 +4,8 @@ import ( "fmt" "math" "strings" + + "github.com/zclconf/go-cty/cty/ctystrings" ) // Refine creates a [RefinementBuilder] with which to annotate the reciever @@ -361,11 +363,41 @@ func (b *RefinementBuilder) CollectionLengthUpperBound(max int) *RefinementBuild // builder is not refining a string value. // // The given prefix will be Unicode normalized in the same way that a -// cty.StringVal would be. However, since prefix is just a substring the -// normalization may produce a non-matching prefix string if the given prefix -// splits a sequence of combining characters. For correct results always ensure -// that the prefix ends at a grapheme cluster boundary. +// cty.StringVal would be. +// +// Due to Unicode normalization and grapheme cluster rules, appending new +// characters to a string can change the meaning of earlier characters. +// StringPrefix may discard one or more characters from the end of the given +// prefix to avoid that problem. +// +// Although cty cannot check this automatically, applications should avoid +// relying on the discarding of the suffix for correctness. For example, if the +// prefix ends with an emoji base character then StringPrefix will discard it +// in case subsequent characters include emoji modifiers, but it's still +// incorrect for the final string to use an entirely different base character. +// +// Applications which fully control the final result and can guarantee the +// subsequent characters will not combine with the prefix may be able to use +// [RefinementBuilder.StringPrefixFull] instead, after carefully reviewing +// the constraints described in its documentation. func (b *RefinementBuilder) StringPrefix(prefix string) *RefinementBuilder { + return b.StringPrefixFull(ctystrings.SafeKnownPrefix(prefix)) +} + +// StringPrefixFull is a variant of StringPrefix that will never shorten the +// given prefix to take into account the possibility of the next character +// combining with the end of the prefix. +// +// Applications which fully control the subsequent characters can use this +// as long as they guarantee that the characters added later cannot possibly +// combine with characters at the end of the prefix to form a single grapheme +// cluster. For example, it would be unsafe to use the full prefix "hello" if +// there is any chance that the final string will add a combining diacritic +// character after the "o", because that would then change the final character. +// +// Use [RefinementBuilder.StringPrefix] instead if an application cannot fully +// control the final result to avoid violating this rule. +func (b *RefinementBuilder) StringPrefixFull(prefix string) *RefinementBuilder { b.assertRefineable() wip, ok := b.wip.(*refinementString) @@ -537,7 +569,7 @@ func (r *refinementString) GoString() string { var b strings.Builder b.WriteString(r.refinementNullable.GoString()) if r.prefix != "" { - fmt.Fprintf(&b, ".StringPrefix(%q)", r.prefix) + fmt.Fprintf(&b, ".StringPrefixFull(%q)", r.prefix) } return b.String() } diff --git a/cty/value_init.go b/cty/value_init.go index 6dcae273..a1743a09 100644 --- a/cty/value_init.go +++ b/cty/value_init.go @@ -5,8 +5,7 @@ import ( "math/big" "reflect" - "golang.org/x/text/unicode/norm" - + "github.com/zclconf/go-cty/cty/ctystrings" "github.com/zclconf/go-cty/cty/set" ) @@ -107,7 +106,7 @@ func StringVal(v string) Value { // A return value from this function can be meaningfully compared byte-for-byte // with a Value.AsString result. func NormalizeString(s string) string { - return norm.NFC.String(s) + return ctystrings.Normalize(s) } // ObjectVal returns a Value of an object type whose structure is defined diff --git a/cty/value_ops_test.go b/cty/value_ops_test.go index 55f8d41f..0714d4ec 100644 --- a/cty/value_ops_test.go +++ b/cty/value_ops_test.go @@ -3442,8 +3442,12 @@ func TestValueGoString(t *testing.T) { `cty.UnknownVal(cty.String).RefineNotNull()`, }, { - UnknownVal(String).Refine().NotNull().StringPrefix("a").NewValue(), - `cty.UnknownVal(cty.String).Refine().NotNull().StringPrefix("a").NewValue()`, + UnknownVal(String).Refine().NotNull().StringPrefix("a-").NewValue(), + `cty.UnknownVal(cty.String).Refine().NotNull().StringPrefixFull("a-").NewValue()`, + }, + { + UnknownVal(String).Refine().NotNull().StringPrefix("foo").NewValue(), // The last character of the prefix gets discarded in case the next character is a combining diacritic + `cty.UnknownVal(cty.String).Refine().NotNull().StringPrefixFull("fo").NewValue()`, }, { UnknownVal(Bool).Refine().NotNull().NewValue(), diff --git a/cty/value_range.go b/cty/value_range.go index 9fd5ec42..8c9e6954 100644 --- a/cty/value_range.go +++ b/cty/value_range.go @@ -285,7 +285,6 @@ func (r ValueRange) Includes(v Value) Value { if v.IsKnown() { prefix := r.StringPrefix() got := v.AsString() - fmt.Printf("prefix %q got %q\n", prefix, got) if !strings.HasPrefix(got, prefix) { return False From caa7fcce31e57394553014e382b8164348a54982 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 2 Feb 2023 15:02:22 -0800 Subject: [PATCH 18/28] function/stdlib: JSON functions can use string refinements When encoding a JSON string we will often be able to predict the first character of the result based just on the argument type. That wouldn't be worthwhile alone, but we can also use a known first character of a JSON document to infer a specific result type even for an unknown value, and to raise an error immediately if the known prefix cannot possibly be a JSON value of any type. Unfortunately due to how JSON interacts with the cty type system the only case where type information truly carries through in a round-trip is encoding an unknown string and then decoding that unknown result, but JSON may be produced and consumed by components other than this pair of functions and they may be able to reach more definitive conclusions based on this additional information. --- cty/function/stdlib/json.go | 70 ++++++++++++++++++++++++++- cty/function/stdlib/json_test.go | 82 ++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 2 deletions(-) diff --git a/cty/function/stdlib/json.go b/cty/function/stdlib/json.go index 30de8a51..65597765 100644 --- a/cty/function/stdlib/json.go +++ b/cty/function/stdlib/json.go @@ -1,6 +1,10 @@ package stdlib import ( + "bytes" + "strings" + "unicode/utf8" + "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/function" "github.com/zclconf/go-cty/cty/json" @@ -12,6 +16,7 @@ var JSONEncodeFunc = function.New(&function.Spec{ { Name: "val", Type: cty.DynamicPseudoType, + AllowUnknown: true, AllowDynamicType: true, AllowNull: true, }, @@ -23,8 +28,28 @@ var JSONEncodeFunc = function.New(&function.Spec{ if !val.IsWhollyKnown() { // We can't serialize unknowns, so if the value is unknown or // contains any _nested_ unknowns then our result must be - // unknown. - return cty.UnknownVal(retType), nil + // unknown. However, we might still be able to at least constrain + // the prefix of our string so that downstreams can sniff for + // whether it's valid JSON and what result types it could have. + + valRng := val.Range() + if valRng.CouldBeNull() { + // If null is possible then we can't constrain the result + // beyond the type constraint, because the very first character + // of the string is what distinguishes a null. + return cty.UnknownVal(retType), nil + } + b := cty.UnknownVal(retType).Refine() + ty := valRng.TypeConstraint() + switch { + case ty == cty.String: + b = b.StringPrefixFull(`"`) + case ty.IsObjectType() || ty.IsMapType(): + b = b.StringPrefixFull("{") + case ty.IsTupleType() || ty.IsListType() || ty.IsSetType(): + b = b.StringPrefixFull("[") + } + return b.NewValue(), nil } if val.IsNull() { @@ -36,6 +61,11 @@ var JSONEncodeFunc = function.New(&function.Spec{ return cty.NilVal, err } + // json.Marshal should already produce a trimmed string, but we'll + // make sure it always is because our unknown value refinements above + // assume there will be no leading whitespace before the value. + buf = bytes.TrimSpace(buf) + return cty.StringVal(string(buf)), nil }, }) @@ -51,6 +81,42 @@ var JSONDecodeFunc = function.New(&function.Spec{ Type: func(args []cty.Value) (cty.Type, error) { str := args[0] if !str.IsKnown() { + // If the string isn't known then we can't fully parse it, but + // if the value has been refined with a prefix then we may at + // least be able to reject obviously-invalid syntax and maybe + // even predict the result type. It's safe to return a specific + // result type only if parsing a full document with this prefix + // would return exactly that type or fail with a syntax error. + rng := str.Range() + if prefix := strings.TrimSpace(rng.StringPrefix()); prefix != "" { + // If we know at least one character then it should be one + // of the few characters that can introduce a JSON value. + switch r, _ := utf8.DecodeRuneInString(prefix); r { + case '{', '[': + // These can start object values and array values + // respectively, but we can't actually form a full + // object type constraint or tuple type constraint + // without knowing all of the attributes, so we + // will still return DynamicPseudoType in this case. + case '"': + // This means that the result will either be a string + // or parsing will fail. + return cty.String, nil + case 't', 'f': + // Must either be a boolean value or a syntax error. + return cty.Bool, nil + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.': + // These characters would all start the "number" production. + return cty.Number, nil + case 'n': + // n is valid to begin the keyword "null" but that doesn't + // give us any extra type information. + default: + // No other characters are valid as the beginning of a + // JSON value, so we can safely return an early error. + return cty.NilType, function.NewArgErrorf(0, "a JSON document cannot begin with the character %q", r) + } + } return cty.DynamicPseudoType, nil } diff --git a/cty/function/stdlib/json_test.go b/cty/function/stdlib/json_test.go index 9b8e6f86..3d182a93 100644 --- a/cty/function/stdlib/json_test.go +++ b/cty/function/stdlib/json_test.go @@ -46,6 +46,26 @@ func TestJSONEncode(t *testing.T) { }, { cty.ObjectVal(map[string]cty.Value{"dunno": cty.UnknownVal(cty.Bool), "false": cty.False}), + cty.UnknownVal(cty.String).Refine().NotNull().StringPrefixFull("{").NewValue(), + }, + { + cty.ListVal([]cty.Value{cty.UnknownVal(cty.String)}), + cty.UnknownVal(cty.String).Refine().NotNull().StringPrefixFull("[").NewValue(), + }, + { + cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String).RefineNotNull(), // Can't refine the prefix because the input might be null + }, + { + cty.UnknownVal(cty.String).RefineNotNull(), + cty.UnknownVal(cty.String).Refine().NotNull().StringPrefixFull(`"`).NewValue(), + }, + { + cty.UnknownVal(cty.Number), + cty.UnknownVal(cty.String).RefineNotNull(), + }, + { + cty.UnknownVal(cty.Bool), cty.UnknownVal(cty.String).RefineNotNull(), }, { @@ -106,6 +126,38 @@ func TestJSONDecode(t *testing.T) { cty.UnknownVal(cty.String), cty.DynamicVal, // need to know the value to determine the type }, + { + cty.UnknownVal(cty.String).Refine().StringPrefixFull("1").NewValue(), + cty.UnknownVal(cty.Number), // deduced from refinement + }, + { + cty.UnknownVal(cty.String).Refine().StringPrefixFull("-").NewValue(), + cty.UnknownVal(cty.Number), // deduced from refinement + }, + { + cty.UnknownVal(cty.String).Refine().StringPrefixFull(".").NewValue(), + cty.UnknownVal(cty.Number), // deduced from refinement + }, + { + cty.UnknownVal(cty.String).Refine().StringPrefixFull("t").NewValue(), + cty.UnknownVal(cty.Bool), // deduced from refinement + }, + { + cty.UnknownVal(cty.String).Refine().StringPrefixFull("f").NewValue(), + cty.UnknownVal(cty.Bool), // deduced from refinement + }, + { + cty.UnknownVal(cty.String).Refine().StringPrefixFull(`"blurt`).NewValue(), + cty.UnknownVal(cty.String), // deduced from refinement + }, + { + cty.UnknownVal(cty.String).Refine().StringPrefixFull(`{`).NewValue(), + cty.DynamicVal, // can't deduce the result type, but potentially valid syntax + }, + { + cty.UnknownVal(cty.String).Refine().StringPrefixFull(`[`).NewValue(), + cty.DynamicVal, // can't deduce the result type, but potentially valid syntax + }, { cty.DynamicVal, cty.DynamicVal, @@ -129,4 +181,34 @@ func TestJSONDecode(t *testing.T) { } }) } + + errorTests := []struct { + Input cty.Value + WantError string + }{ + { + cty.StringVal("aaaa"), + `invalid character 'a' looking for beginning of value`, + }, + { + cty.StringVal("nope"), + `invalid character 'o' in literal null (expecting 'u')`, // (the 'n' looked like the beginning of 'null') + }, + { + cty.UnknownVal(cty.String).Refine().StringPrefixFull(`a`).NewValue(), + `a JSON document cannot begin with the character 'a'`, // error deduced from refinement, despite full value being unknown + }, + } + for _, test := range errorTests { + t.Run(fmt.Sprintf("JSONDecode(%#v)", test.Input), func(t *testing.T) { + _, err := JSONDecode(test.Input) + if err == nil { + t.Fatal("unexpected success") + } + + if got, want := err.Error(), test.WantError; got != want { + t.Errorf("wrong error\ngot: %s\nwant: %s", got, want) + } + }) + } } From 570b530ddfaad19d7f0cde540e22f8c5774e490b Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 2 Feb 2023 15:27:20 -0800 Subject: [PATCH 19/28] function/stdlib: Format can refine unknown results It's common for format strings to begin with at least a few literal characters before the first verb, and if so we can predict the prefix of the result even if some of the remaining arguments are unknown. --- cty/function/stdlib/format.go | 15 ++++++++++++--- cty/function/stdlib/format_test.go | 19 +++++++++++++++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/cty/function/stdlib/format.go b/cty/function/stdlib/format.go index 68ceae98..d04a5eec 100644 --- a/cty/function/stdlib/format.go +++ b/cty/function/stdlib/format.go @@ -26,9 +26,10 @@ var FormatFunc = function.New(&function.Spec{ }, }, VarParam: &function.Parameter{ - Name: "args", - Type: cty.DynamicPseudoType, - AllowNull: true, + Name: "args", + Type: cty.DynamicPseudoType, + AllowNull: true, + AllowUnknown: true, }, Type: function.StaticReturnType(cty.String), RefineResult: refineNonNull, @@ -38,6 +39,14 @@ var FormatFunc = function.New(&function.Spec{ // We require all nested values to be known because the only // thing we can do for a collection/structural type is print // it as JSON and that requires it to be wholly known. + // However, we might be able to refine the result with a + // known prefix, if there are literal characters before the + // first formatting verb. + f := args[0].AsString() + if idx := strings.IndexByte(f, '%'); idx > 0 { + prefix := f[:idx] + return cty.UnknownVal(cty.String).Refine().StringPrefix(prefix).NewValue(), nil + } return cty.UnknownVal(cty.String), nil } } diff --git a/cty/function/stdlib/format_test.go b/cty/function/stdlib/format_test.go index 261f92c4..95c9dcc5 100644 --- a/cty/function/stdlib/format_test.go +++ b/cty/function/stdlib/format_test.go @@ -99,7 +99,7 @@ func TestFormat(t *testing.T) { []cty.Value{cty.TupleVal([]cty.Value{ cty.UnknownVal(cty.String), })}, - cty.UnknownVal(cty.String).RefineNotNull(), + cty.UnknownVal(cty.String).Refine().NotNull().StringPrefixFull("tuple with unknown ").NewValue(), ``, }, { @@ -457,12 +457,27 @@ func TestFormat(t *testing.T) { { cty.StringVal("Hello, %s!"), []cty.Value{cty.UnknownVal(cty.String)}, - cty.UnknownVal(cty.String).RefineNotNull(), + cty.UnknownVal(cty.String).Refine().NotNull().StringPrefixFull("Hello, ").NewValue(), + ``, + }, + { + cty.StringVal("Hello%s"), + []cty.Value{cty.UnknownVal(cty.String)}, + // We lose the trailing "o" in the prefix here because the unknown + // value could potentially start with a combining diacritic, which + // would therefore combine into a different character. + cty.UnknownVal(cty.String).Refine().NotNull().StringPrefixFull("Hell").NewValue(), ``, }, { cty.StringVal("Hello, %[2]s!"), []cty.Value{cty.UnknownVal(cty.String), cty.StringVal("Ermintrude")}, + cty.UnknownVal(cty.String).Refine().NotNull().StringPrefixFull("Hello, ").NewValue(), + ``, + }, + { + cty.StringVal("%s!"), + []cty.Value{cty.UnknownVal(cty.String)}, cty.UnknownVal(cty.String).RefineNotNull(), ``, }, From d0b223388ff5e7fbc49acf5cc6540a2a3b44bfc9 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 2 Feb 2023 16:19:32 -0800 Subject: [PATCH 20/28] cty: Refining must preserve marks from the input value Previously we were preserving marks only in the simple cases, and not properly dealing with already-marked known values when checking refinements. Now we'll always unmark a value on entry into Refine, and then reapply the same marks on all return paths out of RefinementBuilder.NewValue. --- cty/marks.go | 3 +++ cty/unknown_refinement.go | 20 ++++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/cty/marks.go b/cty/marks.go index b889e73f..e747503e 100644 --- a/cty/marks.go +++ b/cty/marks.go @@ -190,6 +190,9 @@ func (val Value) HasSameMarks(other Value) bool { // An application that never calls this method does not need to worry about // handling marked values. func (val Value) Mark(mark interface{}) Value { + if _, ok := mark.(ValueMarks); ok { + panic("cannot call Value.Mark with a ValueMarks value (use WithMarks instead)") + } var newMarker marker newMarker.realV = val.v if mr, ok := val.v.(marker); ok { diff --git a/cty/unknown_refinement.go b/cty/unknown_refinement.go index 83c968e9..c531d0bd 100644 --- a/cty/unknown_refinement.go +++ b/cty/unknown_refinement.go @@ -32,11 +32,12 @@ import ( // must at least have a known root type before it can support further // refinement. func (v Value) Refine() *RefinementBuilder { + v, marks := v.Unmark() if unk, isUnk := v.v.(*unknownType); isUnk && unk.refinement != nil { // We're refining a value that's already been refined before, so // we'll start from a copy of its existing refinements. wip := unk.refinement.copy() - return &RefinementBuilder{v, wip} + return &RefinementBuilder{v, marks, wip} } ty := v.Type() @@ -80,7 +81,7 @@ func (v Value) Refine() *RefinementBuilder { // unknown value would break existing code relying on that. } - return &RefinementBuilder{v, wip} + return &RefinementBuilder{v, marks, wip} } // RefineWith is a variant of Refine which uses callback functions instead of @@ -130,8 +131,9 @@ func (v Value) RefineNotNull() Value { // for method call chaining. End call chains with a call to // [RefinementBuilder.NewValue] to obtain the newly-refined value. type RefinementBuilder struct { - orig Value - wip unknownValRefinement + orig Value + marks ValueMarks + wip unknownValRefinement } func (b *RefinementBuilder) assertRefineable() { @@ -459,7 +461,13 @@ func (b *RefinementBuilder) StringPrefixFull(prefix string) *RefinementBuilder { // but may have additional refinements compared to the original. If the applied // refinements have reduced the range to a single exact value then the result // might be that known value. -func (b *RefinementBuilder) NewValue() Value { +func (b *RefinementBuilder) NewValue() (ret Value) { + defer func() { + // Regardless of how we return, the new value should have the same + // marks as our original value. + ret = ret.WithMarks(b.marks) + }() + if b.orig.IsKnown() { return b.orig } @@ -526,7 +534,7 @@ func (b *RefinementBuilder) NewValue() Value { return Value{ ty: b.orig.ty, v: &unknownType{refinement: b.wip}, - }.WithSameMarks(b.orig) + } } // unknownValRefinment is an interface pretending to be a sum type representing From 40efdff4e9fe8d24172c31db9208003bc3db5236 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 2 Feb 2023 20:09:49 -0800 Subject: [PATCH 21/28] convert: Refine conversions from structural to collection When we're converting from a structural type to a collection type the source type gives us information to refine the range of the resulting collection even if the input value is unknown. Similarly when converting from one collection type to another we can transfer the length range constraints because, aside from set element coalescing, conversions cannot change the length of a collection. --- cty/convert/conversion.go | 63 ++++++++++++++- cty/convert/public_test.go | 155 +++++++++++++++++++++++++++++++++++++ cty/unknown_refinement.go | 15 ++++ 3 files changed, 232 insertions(+), 1 deletion(-) diff --git a/cty/convert/conversion.go b/cty/convert/conversion.go index 541b9a49..bc79df8c 100644 --- a/cty/convert/conversion.go +++ b/cty/convert/conversion.go @@ -43,7 +43,7 @@ func getConversion(in cty.Type, out cty.Type, unsafe bool) conversion { out = out.WithoutOptionalAttributesDeep() if !isKnown { - return cty.UnknownVal(dynamicReplace(in.Type(), out)), nil + return prepareUnknownResult(in.Range(), dynamicReplace(in.Type(), out)), nil } if isNull { @@ -199,3 +199,64 @@ func retConversion(conv conversion) Conversion { return conv(in, cty.Path(nil)) } } + +// prepareUnknownResult can apply value refinements to a returned unknown value +// in certain cases where characteristics of the source value or type can +// transfer into range constraints on the result value. +func prepareUnknownResult(sourceRange cty.ValueRange, targetTy cty.Type) cty.Value { + sourceTy := sourceRange.TypeConstraint() + + ret := cty.UnknownVal(targetTy) + if sourceRange.DefinitelyNotNull() { + ret = ret.RefineNotNull() + } + + switch { + case sourceTy.IsObjectType() && targetTy.IsMapType(): + // A map built from an object type always has the same number of + // elements as the source type has attributes. + return ret.Refine().CollectionLength(len(sourceTy.AttributeTypes())).NewValue() + case sourceTy.IsTupleType() && targetTy.IsListType(): + // A list built from a typle type always has the same number of + // elements as the source type has elements. + return ret.Refine().CollectionLength(sourceTy.Length()).NewValue() + case sourceTy.IsTupleType() && targetTy.IsSetType(): + // When building a set from a tuple type we can't exactly constrain + // the length because some elements might coalesce, but we can + // guarantee an upper limit. We can also guarantee at least one + // element if the tuple isn't empty. + switch l := sourceTy.Length(); l { + case 0, 1: + return ret.Refine().CollectionLength(l).NewValue() + default: + return ret.Refine(). + CollectionLengthLowerBound(1). + CollectionLengthUpperBound(sourceTy.Length()). + NewValue() + } + case sourceTy.IsCollectionType() && targetTy.IsCollectionType(): + // NOTE: We only reach this function if there is an available + // conversion between the source and target type, so we don't + // need to repeat element type compatibility checks and such here. + // + // If the source value already has a refined length then we'll + // transfer those refinements to the result, because conversion + // does not change length (aside from set element coalescing). + b := ret.Refine() + if targetTy.IsSetType() { + if sourceRange.LengthLowerBound() > 0 { + // If the source has at least one element then the result + // must always have at least one too, because value coalescing + // cannot totally empty the set. + b = b.CollectionLengthLowerBound(1) + } + } else { + b = b.CollectionLengthLowerBound(sourceRange.LengthLowerBound()) + } + b = b.CollectionLengthUpperBound(sourceRange.LengthUpperBound()) + return b.NewValue() + default: + return ret + } + +} diff --git a/cty/convert/public_test.go b/cty/convert/public_test.go index 44be53be..85e92195 100644 --- a/cty/convert/public_test.go +++ b/cty/convert/public_test.go @@ -1615,6 +1615,161 @@ func TestConvert(t *testing.T) { })), }), }, + + // Object to map refinements + { + Value: cty.UnknownVal(cty.EmptyObject), + Type: cty.Map(cty.String), + Want: cty.UnknownVal(cty.Map(cty.String)).Refine(). + CollectionLength(0). + NewValue(), + }, + { + Value: cty.UnknownVal(cty.EmptyObject).RefineNotNull(), + Type: cty.Map(cty.String), + Want: cty.MapValEmpty(cty.String), + }, + { + Value: cty.UnknownVal(cty.Object(map[string]cty.Type{"a": cty.String})), + Type: cty.Map(cty.String), + Want: cty.UnknownVal(cty.Map(cty.String)).Refine(). + CollectionLength(1). + NewValue(), + }, + { + Value: cty.UnknownVal(cty.Object(map[string]cty.Type{"a": cty.String})).RefineNotNull(), + Type: cty.Map(cty.String), + Want: cty.UnknownVal(cty.Map(cty.String)).Refine(). + NotNull(). + CollectionLength(1). + NewValue(), + }, + + // Tuple to list refinements + { + Value: cty.UnknownVal(cty.EmptyTuple), + Type: cty.List(cty.String), + Want: cty.UnknownVal(cty.List(cty.String)).Refine(). + CollectionLength(0). + NewValue(), + }, + { + Value: cty.UnknownVal(cty.EmptyTuple).RefineNotNull(), + Type: cty.List(cty.String), + Want: cty.ListValEmpty(cty.String), + }, + { + Value: cty.UnknownVal(cty.Tuple([]cty.Type{cty.String})), + Type: cty.List(cty.String), + Want: cty.UnknownVal(cty.List(cty.String)).Refine(). + CollectionLength(1). + NewValue(), + }, + { + Value: cty.UnknownVal(cty.Tuple([]cty.Type{cty.String})).RefineNotNull(), + Type: cty.List(cty.String), + Want: cty.ListVal([]cty.Value{cty.UnknownVal(cty.String)}), + }, + + // Tuple to set refinements + { + Value: cty.UnknownVal(cty.EmptyTuple), + Type: cty.Set(cty.String), + Want: cty.UnknownVal(cty.Set(cty.String)).Refine(). + CollectionLength(0). + NewValue(), + }, + { + Value: cty.UnknownVal(cty.EmptyTuple).RefineNotNull(), + Type: cty.Set(cty.String), + Want: cty.SetValEmpty(cty.String), + }, + { + Value: cty.UnknownVal(cty.Tuple([]cty.Type{cty.String})), + Type: cty.Set(cty.String), + Want: cty.UnknownVal(cty.Set(cty.String)).Refine(). + CollectionLength(1). + NewValue(), + }, + { + Value: cty.UnknownVal(cty.Tuple([]cty.Type{cty.String})).RefineNotNull(), + Type: cty.Set(cty.String), + Want: cty.SetVal([]cty.Value{cty.UnknownVal(cty.String)}), + }, + { + Value: cty.UnknownVal(cty.Tuple([]cty.Type{cty.String, cty.String})), + Type: cty.Set(cty.String), + Want: cty.UnknownVal(cty.Set(cty.String)).Refine(). + CollectionLengthLowerBound(1). + CollectionLengthUpperBound(2). + NewValue(), + }, + { + Value: cty.UnknownVal(cty.Tuple([]cty.Type{cty.String, cty.String})).RefineNotNull(), + Type: cty.Set(cty.String), + Want: cty.UnknownVal(cty.Set(cty.String)).Refine(). + NotNull(). + CollectionLengthLowerBound(1). + CollectionLengthUpperBound(2). + NewValue(), + }, + + // Collection to collection refinements + { + Value: cty.UnknownVal(cty.List(cty.String)).Refine(). + CollectionLengthLowerBound(2). + CollectionLengthUpperBound(4). + NewValue(), + Type: cty.Set(cty.String), + Want: cty.UnknownVal(cty.Set(cty.String)).Refine(). + CollectionLengthLowerBound(1). + CollectionLengthUpperBound(4). + NewValue(), + }, + { + Value: cty.UnknownVal(cty.List(cty.String)).Refine(). + NotNull(). + CollectionLengthLowerBound(2). + CollectionLengthUpperBound(4). + NewValue(), + Type: cty.Set(cty.String), + Want: cty.UnknownVal(cty.Set(cty.String)).Refine(). + NotNull(). + CollectionLengthLowerBound(1). + CollectionLengthUpperBound(4). + NewValue(), + }, + { + Value: cty.UnknownVal(cty.Set(cty.String)).Refine(). + CollectionLengthLowerBound(2). + CollectionLengthUpperBound(4). + NewValue(), + Type: cty.List(cty.String), + Want: cty.UnknownVal(cty.List(cty.String)).Refine(). + CollectionLengthLowerBound(2). + CollectionLengthUpperBound(4). + NewValue(), + }, + { + Value: cty.UnknownVal(cty.Set(cty.String)).Refine(). + NotNull(). + CollectionLengthLowerBound(2). + CollectionLengthUpperBound(4). + NewValue(), + Type: cty.List(cty.String), + Want: cty.UnknownVal(cty.List(cty.String)).Refine(). + NotNull(). + CollectionLengthLowerBound(2). + CollectionLengthUpperBound(4). + NewValue(), + }, + + // General unknown value refinements + { + Value: cty.UnknownVal(cty.Bool).RefineNotNull(), + Type: cty.String, + Want: cty.UnknownVal(cty.String).RefineNotNull(), + }, } for _, test := range tests { diff --git a/cty/unknown_refinement.go b/cty/unknown_refinement.go index c531d0bd..2e1fd7af 100644 --- a/cty/unknown_refinement.go +++ b/cty/unknown_refinement.go @@ -361,6 +361,21 @@ func (b *RefinementBuilder) CollectionLengthUpperBound(max int) *RefinementBuild return b } +// CollectionLength is a shorthand for passing the same length to both +// [CollectionLengthLowerBound] and [CollectionLengthUpperBound]. +// +// A collection with a refined length with equal bounds can sometimes collapse +// to a known value. Refining to length zero always produces a known value. +// The behavior for other lengths varies by collection type kind. +// +// If the unknown value is of a set type, it's only valid to use this method +// if the caller knows that there will be the given number of _unique_ values +// in the set. If any values might potentially coalesce together once known, +// use [CollectionLengthUpperBound] instead. +func (b *RefinementBuilder) CollectionLength(length int) *RefinementBuilder { + return b.CollectionLengthLowerBound(length).CollectionLengthUpperBound(length) +} + // StringPrefix constrains the prefix of a string value, or panics if this // builder is not refining a string value. // From b95d73b00799d57dfe5c13d0b35043c220c27f82 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 3 Feb 2023 11:39:54 -0800 Subject: [PATCH 22/28] function/stdlib: SetProduct refines the length of unknown results We know that the maximum possible cardinality for a resulting set is the product of the cardinalities of all of the input sets. We might not know the exact cardinalities of all of the input sets either, but if we do know an upper bound then we can use that to calculate the maximum cardinality of the result instead. --- cty/function/stdlib/collection.go | 68 +++++++++++++++++-- cty/function/stdlib/collection_test.go | 94 ++++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 6 deletions(-) diff --git a/cty/function/stdlib/collection.go b/cty/function/stdlib/collection.go index 7095c5f6..1816bb9c 100644 --- a/cty/function/stdlib/collection.go +++ b/cty/function/stdlib/collection.go @@ -931,10 +931,11 @@ var SetProductFunc = function.New(&function.Spec{ Description: `Calculates the cartesian product of two or more sets.`, Params: []function.Parameter{}, VarParam: &function.Parameter{ - Name: "sets", - Description: "The sets to consider. Also accepts lists and tuples, and if all arguments are of list or tuple type then the result will preserve the input ordering", - Type: cty.DynamicPseudoType, - AllowMarked: true, + Name: "sets", + Description: "The sets to consider. Also accepts lists and tuples, and if all arguments are of list or tuple type then the result will preserve the input ordering", + Type: cty.DynamicPseudoType, + AllowMarked: true, + AllowUnknown: true, }, Type: func(args []cty.Value) (retType cty.Type, err error) { if len(args) < 2 { @@ -989,7 +990,7 @@ var SetProductFunc = function.New(&function.Spec{ // Continue processing after we find an argument with unknown // length to ensure that we cover all the marks - if !arg.Length().IsKnown() { + if !(arg.IsKnown() && arg.Length().IsKnown()) { hasUnknownLength = true continue } @@ -1001,7 +1002,62 @@ var SetProductFunc = function.New(&function.Spec{ } if hasUnknownLength { - return cty.UnknownVal(retType).WithMarks(retMarks), nil + defer func() { + // We're definitely going to return from somewhere in this + // branch and however we do it we must reapply the marks + // on the way out. + ret = ret.WithMarks(retMarks) + }() + ret := cty.UnknownVal(retType) + + // Even if we don't know the exact length we may be able to + // constrain the upper and lower bounds of the resulting length. + maxLength := 1 + for _, arg := range args { + arg, _ := arg.Unmark() // safe to discard marks because "retMarks" already contains them all + argRng := arg.Range() + ty := argRng.TypeConstraint() + var argMaxLen int + if ty.IsCollectionType() { + argMaxLen = argRng.LengthUpperBound() + } else if ty.IsTupleType() { + argMaxLen = ty.Length() + } else { + // Should not get here but if we do then we'll just + // bail out with an unrefined unknown value. + return ret, nil + } + // The upper bound of a totally-unrefined collection is + // math.MaxInt, which will quickly get us to integer overflow + // here, and so out of pragmatism we'll just impose a reasonable + // upper limit on what is a useful bound to track and return + // unrefined for unusually-large input. + if argMaxLen > 1024 { // arbitrarily-decided threshold + return ret, nil + } + maxLength *= argMaxLen + if maxLength > 2048 { // arbitrarily-decided threshold + return ret, nil + } + if maxLength < 0 { // Seems like we already overflowed, then. + return ret, nil + } + } + + if maxLength == 0 { + // This refinement will typically allow the unknown value to + // collapse into a known empty collection. + ret = ret.Refine().CollectionLength(0).NewValue() + } else { + // If we know there's a nonzero maximum number of elements then + // set element coalescing cannot reduce to fewer than one + // element. + ret = ret.Refine(). + CollectionLengthLowerBound(1). + CollectionLengthUpperBound(maxLength). + NewValue() + } + return ret, nil } if total == 0 { diff --git a/cty/function/stdlib/collection_test.go b/cty/function/stdlib/collection_test.go index 3fb92535..9221bfef 100644 --- a/cty/function/stdlib/collection_test.go +++ b/cty/function/stdlib/collection_test.go @@ -2498,6 +2498,100 @@ func TestSetproduct(t *testing.T) { cty.UnknownVal(cty.Set(cty.Tuple([]cty.Type{cty.String, cty.Bool}))).RefineNotNull().WithMarks(cty.NewValueMarks("a", "b")), ``, }, + + // If the inputs have unknown lengths but have length refinements then + // we can potentially refine our unknown result too. + { + []cty.Value{ + cty.UnknownVal(cty.Set(cty.String)).Refine().CollectionLengthUpperBound(2).NewValue(), + cty.UnknownVal(cty.Set(cty.Number)).Refine().CollectionLengthUpperBound(3).NewValue(), + }, + cty.UnknownVal(cty.Set(cty.Tuple([]cty.Type{cty.String, cty.Number}))).Refine(). + NotNull(). + CollectionLengthLowerBound(1). + CollectionLengthUpperBound(6). + NewValue(), + ``, + }, + { + []cty.Value{ + cty.UnknownVal(cty.Set(cty.String)).Refine().CollectionLengthUpperBound(2).NewValue(), + cty.SetValEmpty(cty.Number), + }, + cty.SetValEmpty(cty.Tuple([]cty.Type{cty.String, cty.Number})), // deduced from refinements + ``, + }, + { + // If we have any input with a very large maximum element count then we'll + // just leave the result length unrefined to reduce the risk of integer overflow. + []cty.Value{ + cty.UnknownVal(cty.Set(cty.String)).Refine().CollectionLengthUpperBound(2).NewValue(), + cty.UnknownVal(cty.Set(cty.Number)).Refine().CollectionLengthUpperBound(4096).NewValue(), + }, + cty.UnknownVal(cty.Set(cty.Tuple([]cty.Type{cty.String, cty.Number}))).RefineNotNull(), + ``, + }, + { + []cty.Value{ + cty.UnknownVal(cty.List(cty.String)).Refine().CollectionLengthUpperBound(2).NewValue(), + cty.UnknownVal(cty.List(cty.Number)).Refine().CollectionLengthUpperBound(3).NewValue(), + }, + // NOTE: When the result is a list rather than a set there is no + // coalescing and so we could potentially also calculate a more + // refined lower bound on the collection length, but since + // this function is primarily for sets for now we just accept a + // set-oriented refinement. If we find that it would be productive + // to further constrain the range of a list result then we can + // make this more precise later. + cty.UnknownVal(cty.List(cty.Tuple([]cty.Type{cty.String, cty.Number}))).Refine(). + NotNull(). + CollectionLengthLowerBound(1). + CollectionLengthUpperBound(6). + NewValue(), + ``, + }, + { + []cty.Value{ + cty.UnknownVal(cty.List(cty.String)).Refine().CollectionLengthUpperBound(2).NewValue(), + cty.ListValEmpty(cty.Number), + }, + cty.ListValEmpty(cty.Tuple([]cty.Type{cty.String, cty.Number})), // deduced from refinements + ``, + }, + { + []cty.Value{ + cty.UnknownVal(cty.Tuple([]cty.Type{cty.String, cty.String})), + cty.UnknownVal(cty.Tuple([]cty.Type{cty.Number, cty.Number, cty.Number})), + }, + // NOTE: When the result is a list rather than a set there is no + // coalescing and so we could potentially also calculate a more + // refined lower bound on the collection length, but since + // this function is primarily for sets for now we just accept a + // set-oriented refinement. If we find that it would be productive + // to further constrain the range of a list result then we can + // make this more precise later. + cty.UnknownVal(cty.List(cty.Tuple([]cty.Type{cty.String, cty.Number}))).Refine(). + NotNull(). + CollectionLengthLowerBound(1). + CollectionLengthUpperBound(6). + NewValue(), + ``, + }, + { + []cty.Value{ + cty.UnknownVal(cty.Tuple([]cty.Type{cty.String, cty.String})), + cty.EmptyTupleVal, + }, + // NOTE: When the result is a list rather than a set there is no + // coalescing and so we could potentially also calculate a more + // refined lower bound on the collection length, but since + // this function is primarily for sets for now we just accept a + // set-oriented refinement. If we find that it would be productive + // to further constrain the range of a list result then we can + // make this more precise later. + cty.ListValEmpty(cty.Tuple([]cty.Type{cty.String, cty.DynamicPseudoType})), + ``, + }, } for _, test := range tests { From 448ca74bd3f66560c5df9888f57b7bb22037705a Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 3 Feb 2023 15:53:12 -0800 Subject: [PATCH 23/28] cty: Treat unrefined numeric ranges as infinities Previously we were treating these as unknown, which is also a reasonable way to model a lack of bounds but is less convenient when we want to do arithmetic against the bounds. --- cty/unknown_refinement.go | 12 ++++++++---- cty/value_range.go | 6 ++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cty/unknown_refinement.go b/cty/unknown_refinement.go index 2e1fd7af..bd2c4493 100644 --- a/cty/unknown_refinement.go +++ b/cty/unknown_refinement.go @@ -246,8 +246,10 @@ func (b *RefinementBuilder) NumberRangeLowerBound(min Value, inclusive bool) *Re } } - wip.min = min - wip.minInc = inclusive + if min != NegativeInfinity { + wip.min = min + wip.minInc = inclusive + } wip.assertConsistentBounds() return b @@ -293,8 +295,10 @@ func (b *RefinementBuilder) NumberRangeUpperBound(max Value, inclusive bool) *Re } } - wip.max = max - wip.maxInc = inclusive + if max != PositiveInfinity { + wip.max = max + wip.maxInc = inclusive + } wip.assertConsistentBounds() return b diff --git a/cty/value_range.go b/cty/value_range.go index 8c9e6954..e512e365 100644 --- a/cty/value_range.go +++ b/cty/value_range.go @@ -137,6 +137,9 @@ func (r ValueRange) NumberLowerBound() (min Value, inclusive bool) { panic(fmt.Sprintf("NumberLowerBound for %#v", r.ty)) } if rfn, ok := r.raw.(*refinementNumber); ok && rfn.min != NilVal { + if !rfn.min.IsKnown() { + return NegativeInfinity, true + } return rfn.min, rfn.minInc } return UnknownVal(Number), false @@ -159,6 +162,9 @@ func (r ValueRange) NumberUpperBound() (max Value, inclusive bool) { panic(fmt.Sprintf("NumberUpperBound for %#v", r.ty)) } if rfn, ok := r.raw.(*refinementNumber); ok && rfn.max != NilVal { + if !rfn.max.IsKnown() { + return PositiveInfinity, true + } return rfn.max, rfn.maxInc } return UnknownVal(Number), false From 7416265fc1303d4673ba85ae3d480be2ebe481a4 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 3 Feb 2023 17:20:20 -0800 Subject: [PATCH 24/28] cty: Refine the ranges of arithmetic results If we are performing addition, subtraction, or multiplication on unknown numbers with known numeric bounds then we can propagate bounds to the result by performing interval arithmetic. This is not as complete as it could be because of trying to share a single implementation across all of the functions while still dealing with all of their panic edge cases. --- cty/msgpack/unknown.go | 4 +- cty/unknown_refinement.go | 4 +- cty/value_ops.go | 12 ++- cty/value_ops_test.go | 192 +++++++++++++++++++++++++++++++++++++- cty/value_range.go | 66 ++++++++++++- 5 files changed, 264 insertions(+), 14 deletions(-) diff --git a/cty/msgpack/unknown.go b/cty/msgpack/unknown.go index 667bef8a..b189ae8d 100644 --- a/cty/msgpack/unknown.go +++ b/cty/msgpack/unknown.go @@ -63,7 +63,7 @@ func marshalUnknownValue(rng cty.ValueRange, path cty.Path, enc *msgpack.Encoder lower, lowerInc := rng.NumberLowerBound() upper, upperInc := rng.NumberUpperBound() boundTy := cty.Tuple([]cty.Type{cty.Number, cty.Bool}) - if lower.IsKnown() { + if lower.IsKnown() && lower != cty.NegativeInfinity { mapLen++ refnEnc.EncodeInt(int64(unknownValNumberMin)) marshal( @@ -73,7 +73,7 @@ func marshalUnknownValue(rng cty.ValueRange, path cty.Path, enc *msgpack.Encoder refnEnc, ) } - if upper.IsKnown() { + if upper.IsKnown() && upper != cty.PositiveInfinity { mapLen++ refnEnc.EncodeInt(int64(unknownValNumberMax)) marshal( diff --git a/cty/unknown_refinement.go b/cty/unknown_refinement.go index bd2c4493..d90bcbc3 100644 --- a/cty/unknown_refinement.go +++ b/cty/unknown_refinement.go @@ -632,10 +632,10 @@ func (r *refinementNumber) rawEqual(other unknownValRefinement) bool { func (r *refinementNumber) GoString() string { var b strings.Builder b.WriteString(r.refinementNullable.GoString()) - if r.min != NilVal { + if r.min != NilVal && r.min != NegativeInfinity { fmt.Fprintf(&b, ".NumberLowerBound(%#v, %t)", r.min, r.minInc) } - if r.max != NilVal { + if r.max != NilVal && r.max != PositiveInfinity { fmt.Fprintf(&b, ".NumberUpperBound(%#v, %t)", r.max, r.maxInc) } return b.String() diff --git a/cty/value_ops.go b/cty/value_ops.go index 85dac769..171a2391 100644 --- a/cty/value_ops.go +++ b/cty/value_ops.go @@ -593,7 +593,8 @@ func (val Value) Add(other Value) Value { if shortCircuit := mustTypeCheck(Number, Number, val, other); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Number) - return (*shortCircuit).RefineNotNull() + ret := shortCircuit.RefineWith(numericRangeArithmetic(Value.Add, val.Range(), other.Range())) + return ret.RefineNotNull() } ret := new(big.Float) @@ -612,7 +613,8 @@ func (val Value) Subtract(other Value) Value { if shortCircuit := mustTypeCheck(Number, Number, val, other); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Number) - return (*shortCircuit).RefineNotNull() + ret := shortCircuit.RefineWith(numericRangeArithmetic(Value.Subtract, val.Range(), other.Range())) + return ret.RefineNotNull() } return val.Add(other.Negate()) @@ -646,7 +648,8 @@ func (val Value) Multiply(other Value) Value { if shortCircuit := mustTypeCheck(Number, Number, val, other); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Number) - return (*shortCircuit).RefineNotNull() + ret := shortCircuit.RefineWith(numericRangeArithmetic(Value.Multiply, val.Range(), other.Range())) + return ret.RefineNotNull() } // find the larger precision of the arguments @@ -691,6 +694,9 @@ func (val Value) Divide(other Value) Value { if shortCircuit := mustTypeCheck(Number, Number, val, other); shortCircuit != nil { shortCircuit = forceShortCircuitType(shortCircuit, Number) + // TODO: We could potentially refine the range of the result here, but + // we don't right now because our division operation is not monotone + // if the denominator could potentially be zero. return (*shortCircuit).RefineNotNull() } diff --git a/cty/value_ops_test.go b/cty/value_ops_test.go index 0714d4ec..7a765a09 100644 --- a/cty/value_ops_test.go +++ b/cty/value_ops_test.go @@ -1767,6 +1767,66 @@ func TestValueAdd(t *testing.T) { UnknownVal(Number), UnknownVal(Number).RefineNotNull(), }, + { + NumberIntVal(1), + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), false). + NewValue(), + UnknownVal(Number).Refine(). + NotNull(). + NumberRangeLowerBound(NumberIntVal(3), true). + NewValue(), + }, + { + Zero, + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), false). + NewValue(), + UnknownVal(Number).Refine(). + NotNull(). + NumberRangeLowerBound(NumberIntVal(2), true). + NewValue(), + }, + { + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), false). + NewValue(), + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), false). + NewValue(), + UnknownVal(Number).Refine(). + NotNull(). + NumberRangeLowerBound(NumberIntVal(4), true). + NewValue(), + }, + { + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(1), true). + NumberRangeUpperBound(NumberIntVal(2), false). + NewValue(), + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), false). + NewValue(), + UnknownVal(Number).Refine(). + NotNull(). + NumberRangeLowerBound(NumberIntVal(3), true). + NewValue(), + }, + { + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(1), true). + NumberRangeUpperBound(NumberIntVal(2), false). + NewValue(), + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), false). + NumberRangeUpperBound(NumberIntVal(3), false). + NewValue(), + UnknownVal(Number).Refine(). + NotNull(). + NumberRangeLowerBound(NumberIntVal(3), true). + NumberRangeUpperBound(NumberIntVal(5), true). + NewValue(), + }, { UnknownVal(Number), UnknownVal(Number), @@ -1803,7 +1863,7 @@ func TestValueAdd(t *testing.T) { t.Run(fmt.Sprintf("%#v.Add(%#v)", test.LHS, test.RHS), func(t *testing.T) { got := test.LHS.Add(test.RHS) if !got.RawEquals(test.Expected) { - t.Fatalf("Add returned %#v; want %#v", got, test.Expected) + t.Fatalf("Wrong result\ngot: %#v\nwant: %#v", got, test.Expected) } }) } @@ -1840,6 +1900,63 @@ func TestValueSubtract(t *testing.T) { UnknownVal(Number), UnknownVal(Number).RefineNotNull(), }, + { + NumberIntVal(1), + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), true). + NewValue(), + UnknownVal(Number).Refine(). + NotNull(). + NumberRangeUpperBound(NumberIntVal(-1), true). + NewValue(), + }, + { + Zero, + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), true). + NewValue(), + UnknownVal(Number).Refine(). + NotNull(). + NumberRangeUpperBound(NumberIntVal(-2), true). + NewValue(), + }, + { + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), true). + NewValue(), + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), true). + NewValue(), + UnknownVal(Number).RefineNotNull(), // We don't currently refine this case + }, + { + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(1), true). + NumberRangeUpperBound(NumberIntVal(2), false). + NewValue(), + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), true). + NewValue(), + UnknownVal(Number).Refine(). + NotNull(). + NumberRangeUpperBound(NumberIntVal(0), true). + NewValue(), + }, + { + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(1), true). + NumberRangeUpperBound(NumberIntVal(2), false). + NewValue(), + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), false). + NumberRangeUpperBound(NumberIntVal(3), false). + NewValue(), + UnknownVal(Number).Refine(). + NotNull(). + NumberRangeLowerBound(NumberIntVal(-2), true). + NumberRangeUpperBound(NumberIntVal(0), true). + NewValue(), + }, { NumberIntVal(1), DynamicVal, @@ -1871,7 +1988,7 @@ func TestValueSubtract(t *testing.T) { t.Run(fmt.Sprintf("%#v.Subtract(%#v)", test.LHS, test.RHS), func(t *testing.T) { got := test.LHS.Subtract(test.RHS) if !got.RawEquals(test.Expected) { - t.Fatalf("Subtract returned %#v; want %#v", got, test.Expected) + t.Fatalf("wrong result\ngot: %#v\nwant: %#v", got, test.Expected) } }) } @@ -1908,7 +2025,7 @@ func TestValueNegate(t *testing.T) { t.Run(fmt.Sprintf("%#v.Negate()", test.Receiver), func(t *testing.T) { got := test.Receiver.Negate() if !got.RawEquals(test.Expected) { - t.Fatalf("Negate returned %#v; want %#v", got, test.Expected) + t.Fatalf("wrong result\ngot: %#v\nwant: %#v", got, test.Expected) } }) } @@ -1945,6 +2062,71 @@ func TestValueMultiply(t *testing.T) { UnknownVal(Number), UnknownVal(Number).RefineNotNull(), }, + { + NumberIntVal(3), + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), false). + NewValue(), + UnknownVal(Number).Refine(). + NotNull(). + NumberRangeLowerBound(NumberIntVal(6), true). + NewValue(), + }, + { + Zero, + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), false). + NewValue(), + UnknownVal(Number).RefineNotNull(), // We can't currently refine this case + }, + { + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), false). + NewValue(), + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(4), false). + NewValue(), + UnknownVal(Number).Refine(). + NotNull(). + NumberRangeLowerBound(NumberIntVal(8), true). + NewValue(), + }, + { + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(3), true). + NumberRangeUpperBound(NumberIntVal(4), false). + NewValue(), + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), false). + NewValue(), + UnknownVal(Number).Refine(). + NotNull(). + NumberRangeLowerBound(NumberIntVal(6), true). + NewValue(), + }, + { + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(1), true). + NumberRangeUpperBound(NumberIntVal(2), false). + NewValue(), + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(2), false). + NumberRangeUpperBound(NumberIntVal(3), false). + NewValue(), + UnknownVal(Number).Refine(). + NotNull(). + NumberRangeLowerBound(NumberIntVal(2), true). + NumberRangeUpperBound(NumberIntVal(6), true). + NewValue(), + }, + { + UnknownVal(Number).Refine(). + NumberRangeLowerBound(NumberIntVal(1), true). + NumberRangeUpperBound(NumberIntVal(2), false). + NewValue(), + Zero, + Zero, // deduced by refinement + }, { NumberIntVal(1), DynamicVal, @@ -1986,7 +2168,7 @@ func TestValueMultiply(t *testing.T) { t.Run(fmt.Sprintf("%#v.Multiply(%#v)", test.LHS, test.RHS), func(t *testing.T) { got := test.LHS.Multiply(test.RHS) if !got.RawEquals(test.Expected) { - t.Fatalf("Multiply returned %#v; want %#v", got, test.Expected) + t.Fatalf("wrong result\ngot: %#v\nwant: %#v", got, test.Expected) } }) } @@ -2064,7 +2246,7 @@ func TestValueDivide(t *testing.T) { t.Run(fmt.Sprintf("%#v.Divide(%#v)", test.LHS, test.RHS), func(t *testing.T) { got := test.LHS.Divide(test.RHS) if !got.RawEquals(test.Expected) { - t.Fatalf("Divide returned %#v; want %#v", got, test.Expected) + t.Fatalf("wrong result\ngot: %#v\nwant: %#v", got, test.Expected) } }) } diff --git a/cty/value_range.go b/cty/value_range.go index e512e365..36f21946 100644 --- a/cty/value_range.go +++ b/cty/value_range.go @@ -142,7 +142,7 @@ func (r ValueRange) NumberLowerBound() (min Value, inclusive bool) { } return rfn.min, rfn.minInc } - return UnknownVal(Number), false + return NegativeInfinity, false } // NumberUpperBound returns information about the upper bound of the range of @@ -167,7 +167,7 @@ func (r ValueRange) NumberUpperBound() (max Value, inclusive bool) { } return rfn.max, rfn.maxInc } - return UnknownVal(Number), false + return PositiveInfinity, false } // StringPrefix returns a string that is guaranteed to be the prefix of @@ -332,6 +332,68 @@ func (r ValueRange) Includes(v Value) Value { return unknownResult } +// numericRangeArithmetic is a helper we use to calculate derived numeric ranges +// for arithmetic on refined numeric values. +// +// op must be a monotone operation. numericRangeArithmetic adapts that operation +// into the equivalent interval arithmetic operation. +// +// The result is a superset of the range of the given operation against the +// given input ranges, if it's possible to calculate that without encountering +// an invalid operation. Currently the result is inexact due to ignoring +// the inclusiveness of the input bounds and just always returning inclusive +// bounds. +func numericRangeArithmetic(op func(a, b Value) Value, a, b ValueRange) func(*RefinementBuilder) *RefinementBuilder { + wrapOp := func(a, b Value) (ret Value) { + // Our functions have various panicking edge cases involving incompatible + // uses of infinities. To keep things simple here we'll catch those + // and just return an unconstrained number. + defer func() { + if v := recover(); v != nil { + ret = UnknownVal(Number) + } + }() + return op(a, b) + } + + return func(builder *RefinementBuilder) *RefinementBuilder { + aMin, _ := a.NumberLowerBound() + aMax, _ := a.NumberUpperBound() + bMin, _ := b.NumberLowerBound() + bMax, _ := b.NumberUpperBound() + + v1 := wrapOp(aMin, bMin) + v2 := wrapOp(aMin, bMax) + v3 := wrapOp(aMax, bMin) + v4 := wrapOp(aMax, bMax) + + newMin := mostNumberValue(Value.LessThan, v1, v2, v3, v4) + newMax := mostNumberValue(Value.GreaterThan, v1, v2, v3, v4) + + if isInf := newMin.Equals(NegativeInfinity); isInf.IsKnown() && isInf.False() { + builder = builder.NumberRangeLowerBound(newMin, true) + } + if isInf := newMax.Equals(PositiveInfinity); isInf.IsKnown() && isInf.False() { + builder = builder.NumberRangeUpperBound(newMax, true) + } + return builder + } +} + +func mostNumberValue(op func(i, j Value) Value, v1 Value, vN ...Value) Value { + r := v1 + for _, v := range vN { + more := op(v, r) + if !more.IsKnown() { + return UnknownVal(Number) + } + if more.True() { + r = v + } + } + return r +} + // definitelyNotNull is a convenient helper for the common situation of checking // whether a value could possibly be null. // From 129105783d6fb3140881d11bf5f717c0f969e10e Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Mon, 6 Feb 2023 12:18:28 -0800 Subject: [PATCH 25/28] docs: New documentation about the "Refinements" concept --- docs/concepts.md | 5 + docs/marks.md | 9 ++ docs/refinements.md | 291 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 305 insertions(+) create mode 100644 docs/refinements.md diff --git a/docs/concepts.md b/docs/concepts.md index c70f80a0..2f66b1d8 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -75,6 +75,11 @@ promises to never produce an unknown value for an operation unless one of the operands is itself unknown, and so applications can opt out of this additional complexity by never providing unknown values as operands. +At minimum an unknown value has a type constraint which describes a set of +types that the final value could possibly have once known. In some cases we +can refine an unknown value with additional dynamic information, using +[Value Refinements](refinements.md). + ## Type Equality and Type Conformance Two types are said to be equal if they are exactly equivalent. Each type kind diff --git a/docs/marks.md b/docs/marks.md index 686b2eea..f136e64e 100644 --- a/docs/marks.md +++ b/docs/marks.md @@ -106,3 +106,12 @@ unmark the whole data structure first (e.g. using `Value.UnmarkDeep`) and then decide what to do with those marks in order to ensure that if it makes sense to propagate them through the serialization then they will get represented somehow. + +## Relationship to "Refinements" + +The idea of annotating a value with additional information has some overlap +with the concept of [Refinements](refinements.md). However, the two have +different purposes and so different design details and tradeoffs. + +For more details, see +[the corresponding section in the Refinements documentation](refinements.md#relationship-to-marks). diff --git a/docs/refinements.md b/docs/refinements.md new file mode 100644 index 00000000..ac474510 --- /dev/null +++ b/docs/refinements.md @@ -0,0 +1,291 @@ +# Value Refinements + +_Refinements_ are dynamic annotations associated with unknown values that +each shrink the range of possible values futher than can be represented by +type constraint alone. + +When an unknown value is refined, it allows certain operations against that +unknown value to produce a known result, and allows some operations to fail +earlier than they would with a fully-unknown value by detecting that a valid +result is impossible using just the refinement information. + +Refinements always _shrink_ the range of an unknown value, and never grow it. +That makes it valid for some operations to ignore refinements and just treat +an unknown value as representing any possible value of its type constraint, +which is important to avoid burdening all downstream callers of `cty` from +handling all refinements and from immediately adding support for new kinds of +refinement if this model gets extended in future releases. + +However, note that `Value.RawEquals` _does_ take into account refinements, so +any tests that assert against the exact final value of an operation may need +to be updated after adopting a new version of `cty` which makes increased use +of refinements. `Value.RawEquals` is not intended as part of the _user model_ +of `cty` and so this should not negatively impact the end-user-visible behavior +of an application using `cty`, although of course they might benefit from +more specific results from operations that can now take refinements into +account. + +## How to refine a value + +You can derive a more refined value from a less refined value by using the +`Value.Refine` method to obtain a _refinement builder_, which uses the +builder pattern to construct a new value with one or more extra refinements. + +```go +val := cty.UnknownVal(cty.String).Refine(). + NotNull(). + StringPrefix("https://"). + NewValue() +``` + +The above snippet would produce a refined local value whose range is limited +only to non-null strings which start with the prefix `"https://"`. This +information can, in theory, allow `val.Equals(cty.NullVal(cty.String))` to +return `cty.False` rather than `cty.UnknownVal(cty.Bool)`, and allow a prefix +match against the string to return a known result. + +In practice not all operations against unknown values can make full use of +unknown value refinements, but hopefully the coverage will increase over time. + +Only unknown values can have refinements, because known values are already +refined by their concrete value: simple values like `cty.Zero` are constrained +to exactly one value, while some values like `cty.ListValEmpty(cty.DynamicPseudoType)` +represent a set of possible values -- all empty lists of any element type, in +this case. + +However, the `Refine` operation _is_ also supported for known values and in that +case acts as a self-checking assertion that the known value does actually +meet the requirements. If you write your codepaths to unconditionally assign +refinements regardless of whether the value is known then your code will +self-check and raise a panic if the final known value doesn't match the +previously-promised refinements. + +A similar rule applies to applying new refinements already-refined values: it's +fine to describe a less specific refinement, which will therefore be ignored +because it adds no new information. It's an application bug to describe a +contradictory refinement, such as a new string prefix that doesn't match one +previously assigned. + +## Value ranges + +The `Refine()` method described above constructs a value with refinements. To +access the information from those refinements, use the `Value.Range` method to +obtain a `cty.ValueRange` object, which describes a superset of all of the +values that a particular value could have. + +For example, you can use `val.Range().DefinitelyNotNull()` to test whether a +particular value is guaranteed to be non-null once it is finally known. This +again works for both known and unknown values, so e.g. +`cty.StringVal("foo").Range().DefinitelyNotNull()` returns `true` because +a known, non-null string value is _definitely not null_. + +When writing operations that depend only on information that can be determined +from refinements it's valid to depend exclusively on `Value.Range` and rely on +the fact that the range of an already-known value is just a very narrow range +that covers only what that specific value covers. + +The model of value ranges is imprecise, though: it's limited only to information +we can track for unknown values through refinements. Many operations will still +need a special codepath to handle the unknown case vs. the known case so they +can take into account the additional detail from the exact value once known. + +## Available Refinements + +The set of possible refinement types might grow over time, but the initial set +is focused on a narrow set of possibilities that seems likely to allow a number +of other operations to either produce known results from unknown input or to +rule that particular input is invalid despite not yet being known. + +The most notable restriction on refinements is that the available refinements +vary depending on the type constraint of the value being refined. + +The least flexible case is `cty.DynamicVal` -- an unknown value of an unknown +type -- which is the one value that cannot be refined at all and will cause +a panic if you try. This is a pragmatic compromise for backward compatibility: +existing callers use patterns like `val == cty.DynamicVal` to test for this +specific special value, and any refinements of that value would make it no +longer equal. + +Unknown values of built-in exact types, and also unknown values whose type +_kind_ is constrained even if the element/attribute types are not, can at +least be refined as being non-null, and because that is a common situation +there is a shorthand for it which avoids using the builder pattern: +`val.RefineNotNull()`. + +All other possible refinements are type-constraint-specific: + +* `cty.String` + + For strings we can refine a known prefix of the string, which is intended + for situations where the string represents some microsyntax with a + known prefix, such as a URL of a particular known scheme. + + * `.StringPrefix(string)` specifies a known prefix of the final string. + + By default an unknown string has no known prefix, which is the same + as the prefix being the empty string. + + Because `cty`'s model of strings is a sequence of Unicode grapheme + clusters, `.StringPrefix` will quietly disregard trailing Unicode + code units of the given prefix that might combine with other code + units to form a new combined grapheme. This is a good safe default + behavior for situations where the remainder of the string is under + end-user control and might begin with combining diacritics or + emoji variation sequences. Applications should not rely on the + details of this heuristic because it may become more precise in + later releases. + + * `.StringPrefixFull(string)` is like `.StringPrefix` but does not trim + possibly-combining code units from the end of the given string. + + Applications must use this with care, making sure that they control + the final string enough to guarantee that the subsequent additional + code units will never combine with any characters in the given prefix. + +* `cty.Number` + + For numbers we can refine both the lower and upper bound of possible values, + with each boundary being either inclusive or exclusive. + + * `.NumberRangeLowerBound(cty.Value, bool)` refines the lower bound of + possible values for an unknown number. The boolean argument represents + whether the bound is _inclusive_. + + The given value must be a non-null `cty.Number` value. An unrefined + number effectively has a lower bound of `(cty.NegativeInfinity, true)`. + + * `.NumberRangeUpperBound(cty.Value, bool)` refines the upper bound of + possible values for an unknown number. The boolean argument represents + whether the bound is _inclusive_. + + The given value must be a non-null `cty.Number` value. An unrefined + number effectively has an upper bound of `(cty.PositiveInfinity, true)`. + + * `.NumberRangeInclusive(min, max cty.Value)` is a helper wrapper around + the previous two methods that declares both an upper and lower bound + at the same time, while specifying that both are inclusive bounds. + +* `cty.List`, `cty.Set`, and `cty.Map` types + + For all collection types we can refine the lower and upper bound of the + length of the collection. The boundaries on length are always inclusive + and are integers, because it isn't possible to have a fraction of an + element. + + * `.CollectionLengthLowerBound(int)` refines the lower bound of possible + lengths for an unknown collection. + + An unrefined collection effectively has a lower bound of zero, because + it's not possible for a collection to have a negative length. + + * `.CollectionLengthUpperBound(int)` refines the upper bound of possible + lengths for an unknown collection. + + An unrefined collection has an upper bound that matches the largest + valid Go slice index on the current platform, because `cty`'s + collections are implemented in terms of Go's collection types. + However, applications should typically not expose that specific value + to users (it's an implementation detail) and should instead present + the maximum value as an unconstrained length. + + * `.CollectionLength(int)` is a shorthand that refines both the lower and + upper bounds to the same value. This is a helpful requirement to make + whenever possible because it will often allow the final value to be + a known collection with unknown elements, as described in + [Refinement Value Collapse](#refinement-value-collapse). + +Some built-in operations will automatically take into account refinements from +their input operands and propagate them in a suitable way to the result. +However, that is not a guarantee for all operations and so should be treated +as a "best effort" behavior which will hopefully become more precise in future +versions. + +Behaviors implemented in downstream applications, such as custom functions +using [the function system](functions.md), might also take into account +refinements. If they do their work using only _operation methods_ on `Value` +then the handling of refinements might come for free. If they do work using +_integration methods_ instead then they will need to explicitly handle +refinements if desired. If they don't then by default the result from an +unknown input will be a totally-unrefined unknown value, though will hopefully +still have a useful type constraint. + +## Refinement Value Collapse + +For some kinds of refinement it's possible to constrain the range so much that +only one possible value remains. In that case, the `.NewValue()` method of the +refinement builder might return a known value instead of an unknown value. + +For example, if the lower bound and upper bound of a collection's length are +equal then the length of the collection is effectively known. For some lengths +of some collection kinds the refinement can collapse into a known collection +containing unknown values. For example, an unknown list that's known to have +exactly two values can be represented equivalently as a known list of length +two where both elements are unknown themselves. + +The exact details of how refinement collapse is decided might change in future +versions, but only in ways that can make results "more known". It would be a +breaking change to weaken a rule to produce unknown values in more cases, so +that kind of change would be reserved only for fixing an important bug or +design error. + +## Refinements are Dynamic Only + +Refinements belong to unknown values rather than to type constraints, and so +refining an unknown value does not change its type constraint. + +This design is a tradeoff: making the refinements dynamic and implicit means +that it's possible to add more detailed refinements over type without making +breaking changes to explicit type information, but the downside is that +it isn't possible to represent refinements in any situation that is only +aware of types. + +For example, it isn't currently possible to represent the idea of an unknown +map whose elements each have a further refinement applied, because the +refinements apply to the map itself and there are not yet any specific element +values for the element refinements to attach to. + +(It would be possible in theory to allow refining an unknown collection with +meta-refinements about its hypothetical elements, but that is not currently +supported because it would mean that refinements would need to be resolved +recursively and that would be considerably more complex and expensive than +the current single-value-only refinements structure.) + +## Refinements Under Serialization + +Refinements are intentionally designed so that they only constrain the range +of an unknown value, and never expand it. This means that it should typically +be safe to discard refinements in situations like serialization where there +may not be any way to represent the refinements. After decoding the unknown +value now has a wider range but it should still be a superset of the true +range of the value. This is an example of the general rule that no operation +on an unknown value is _guaranteed_ to fully preserve the input refinements +or to consider them when calculating the result. + +The official MessagePack serialization in particular does have some support +for retaining approximations of refinements as part of its serialization of +unknown values, using a MessagePack extension value. Some detail may still +be lost under round-tripping but the output range should always be a superset +of the input range. As long as both the serializer and deserializer are using +the `cty/msgpack` sub-package unknown values will propagate automatically +without any additional caller effort. + +## Relationship to "Marks" + +The idea of annotating a value with additional information has some overlap +with the concept of [Marks](marks.md). However, the two have different purposes +and so different design details and tradeoffs. + +Marks should typically be used for additional information that is independent +of the specific type and value, such as marking a value as having come from +a sensitive location. The marking then propagates to all results from operations +on that value, usually without changing the behavior of that operation. In a +sense the mark represents the _origin_ of the value rather than the value +itself. + +Refinements are instead directly part of the value. By reducing the possible +range of an unknown value placeholder, other downstream operations can in turn +produce a more refined result, or possibly even a known result from unknown +inputs. Refinements do not naively propagate from one value to the next, but +some operations will use the refinements of their operands to calculate a new +set of refiments for their result, with the rules varying on a case-by-case +basis depending on what calculation the operation represents. From 199911c1d3edf3c9c7b3a8cecd01c31d13bfb649 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 7 Feb 2023 16:25:57 -0800 Subject: [PATCH 26/28] cty: Known result special cases for Multiply, And, and Or Now that we're in the business of trying to produce known values from operations on unknown inputs where possible, these are some simple cases that hold regardless of the range/refinements of the unknown operands and can help take a particular unknown result out of consideration when evaluating a broader expression. --- cty/value_ops.go | 17 +++++++++++++ cty/value_ops_test.go | 56 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/cty/value_ops.go b/cty/value_ops.go index 171a2391..c4584bd9 100644 --- a/cty/value_ops.go +++ b/cty/value_ops.go @@ -647,6 +647,11 @@ func (val Value) Multiply(other Value) Value { } if shortCircuit := mustTypeCheck(Number, Number, val, other); shortCircuit != nil { + // If either value is exactly zero then the result must either be + // zero or an error. + if val == Zero || other == Zero { + return Zero + } shortCircuit = forceShortCircuitType(shortCircuit, Number) ret := shortCircuit.RefineWith(numericRangeArithmetic(Value.Multiply, val.Range(), other.Range())) return ret.RefineNotNull() @@ -1250,6 +1255,12 @@ func (val Value) And(other Value) Value { } if shortCircuit := mustTypeCheck(Bool, Bool, val, other); shortCircuit != nil { + // If either value is known to be exactly False then it doesn't + // matter what the other value is, because the final result must + // either be False or an error. + if val == False || other == False { + return False + } shortCircuit = forceShortCircuitType(shortCircuit, Bool) return (*shortCircuit).RefineNotNull() } @@ -1267,6 +1278,12 @@ func (val Value) Or(other Value) Value { } if shortCircuit := mustTypeCheck(Bool, Bool, val, other); shortCircuit != nil { + // If either value is known to be exactly True then it doesn't + // matter what the other value is, because the final result must + // either be True or an error. + if val == True || other == True { + return True + } shortCircuit = forceShortCircuitType(shortCircuit, Bool) return (*shortCircuit).RefineNotNull() } diff --git a/cty/value_ops_test.go b/cty/value_ops_test.go index 7a765a09..d3ad9027 100644 --- a/cty/value_ops_test.go +++ b/cty/value_ops_test.go @@ -2072,12 +2072,22 @@ func TestValueMultiply(t *testing.T) { NumberRangeLowerBound(NumberIntVal(6), true). NewValue(), }, + { + Zero, + UnknownVal(Number), + Zero, + }, + { + UnknownVal(Number), + Zero, + Zero, + }, { Zero, UnknownVal(Number).Refine(). NumberRangeLowerBound(NumberIntVal(2), false). NewValue(), - UnknownVal(Number).RefineNotNull(), // We can't currently refine this case + Zero, }, { UnknownVal(Number).Refine(). @@ -3033,6 +3043,16 @@ func TestValueAnd(t *testing.T) { True, UnknownVal(Bool).RefineNotNull(), }, + { + False, + UnknownVal(Bool), + False, + }, + { + UnknownVal(Bool), + False, + False, + }, { DynamicVal, DynamicVal, @@ -3048,6 +3068,16 @@ func TestValueAnd(t *testing.T) { True, UnknownVal(Bool).RefineNotNull(), }, + { + False, + DynamicVal, + False, + }, + { + DynamicVal, + False, + False, + }, { True.Mark(1), True, @@ -3109,11 +3139,21 @@ func TestValueOr(t *testing.T) { { True, UnknownVal(Bool), - UnknownVal(Bool).RefineNotNull(), + True, }, { UnknownVal(Bool), True, + True, + }, + { + False, + UnknownVal(Bool), + UnknownVal(Bool).RefineNotNull(), + }, + { + UnknownVal(Bool), + False, UnknownVal(Bool).RefineNotNull(), }, { @@ -3124,11 +3164,21 @@ func TestValueOr(t *testing.T) { { True, DynamicVal, - UnknownVal(Bool).RefineNotNull(), + True, }, { DynamicVal, True, + True, + }, + { + False, + DynamicVal, + UnknownVal(Bool).RefineNotNull(), + }, + { + DynamicVal, + False, UnknownVal(Bool).RefineNotNull(), }, { From 6ff38c3f27d4f2396f71c332d91a92a7ce373c9b Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 7 Feb 2023 17:15:02 -0800 Subject: [PATCH 27/28] function/stdlib: SortFunc always preserves the length of its input If the list argument is an unknown value or contains unknown values then we can't possibly return a fully-known result, but we do at least know that sorting will never change the number of elements and so we can refine our unknown result using the range of the input value. The refinements system automatically collapses an unknown list collection whose upper and lower length bounds are equal into a known list where all elements are unknown, so this automatically preserves the known-ness of the input length in the case where we're given a known list with unknown elements, without needing to handle that as a special case here. --- cty/function/stdlib/string.go | 18 +++++-- cty/function/stdlib/string_test.go | 80 ++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/cty/function/stdlib/string.go b/cty/function/stdlib/string.go index 04033026..57ebce1b 100644 --- a/cty/function/stdlib/string.go +++ b/cty/function/stdlib/string.go @@ -284,8 +284,9 @@ var SortFunc = function.New(&function.Spec{ Description: "Applies a lexicographic sort to the elements of the given list.", Params: []function.Parameter{ { - Name: "list", - Type: cty.List(cty.String), + Name: "list", + Type: cty.List(cty.String), + AllowUnknown: true, }, }, Type: function.StaticReturnType(cty.List(cty.String)), @@ -295,8 +296,17 @@ var SortFunc = function.New(&function.Spec{ if !listVal.IsWhollyKnown() { // If some of the element values aren't known yet then we - // can't yet predict the order of the result. - return cty.UnknownVal(retType), nil + // can't yet predict the order of the result, but we can be + // sure that the length won't change. + ret := cty.UnknownVal(retType) + if listVal.Type().IsListType() { + rng := listVal.Range() + ret = ret.Refine(). + CollectionLengthLowerBound(rng.LengthLowerBound()). + CollectionLengthUpperBound(rng.LengthUpperBound()). + NewValue() + } + return ret, nil } if listVal.LengthInt() == 0 { // Easy path return listVal, nil diff --git a/cty/function/stdlib/string_test.go b/cty/function/stdlib/string_test.go index 3f27addb..254a2e73 100644 --- a/cty/function/stdlib/string_test.go +++ b/cty/function/stdlib/string_test.go @@ -1,6 +1,7 @@ package stdlib import ( + "fmt" "testing" "github.com/zclconf/go-cty/cty" @@ -477,3 +478,82 @@ func TestJoin(t *testing.T) { }) } } + +func TestSort(t *testing.T) { + tests := []struct { + Input cty.Value + Want cty.Value + WantErr string + }{ + { + cty.ListValEmpty(cty.String), + cty.ListValEmpty(cty.String), + ``, + }, + { + cty.ListVal([]cty.Value{cty.StringVal("a")}), + cty.ListVal([]cty.Value{cty.StringVal("a")}), + ``, + }, + { + cty.ListVal([]cty.Value{cty.StringVal("b"), cty.StringVal("a")}), + cty.ListVal([]cty.Value{cty.StringVal("a"), cty.StringVal("b")}), + ``, + }, + { + cty.ListVal([]cty.Value{cty.StringVal("b"), cty.StringVal("a"), cty.StringVal("c")}), + cty.ListVal([]cty.Value{cty.StringVal("a"), cty.StringVal("b"), cty.StringVal("c")}), + ``, + }, + { + cty.UnknownVal(cty.List(cty.String)), + cty.UnknownVal(cty.List(cty.String)).RefineNotNull(), + ``, + }, + { + // If the list contains any unknown values then we can still + // preserve the length of the list by generating a known list + // with unknown elements, because sort can never change the length. + cty.ListVal([]cty.Value{cty.StringVal("b"), cty.UnknownVal(cty.String)}), + cty.ListVal([]cty.Value{cty.UnknownVal(cty.String), cty.UnknownVal(cty.String)}), + ``, + }, + { + // For a completely unknown list we can still preserve any + // refinements it had for its length, because sorting can never + // change the length. + cty.UnknownVal(cty.List(cty.String)).Refine(). + CollectionLengthLowerBound(1). + CollectionLengthUpperBound(2). + NewValue(), + cty.UnknownVal(cty.List(cty.String)).Refine(). + NotNull(). + CollectionLengthLowerBound(1). + CollectionLengthUpperBound(2). + NewValue(), + ``, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("Sort(%#v)", test.Input), func(t *testing.T) { + got, err := Sort(test.Input) + + if test.WantErr != "" { + errStr := fmt.Sprintf("%s", err) + if errStr != test.WantErr { + t.Errorf("wrong error\ngot: %s\nwant: %s", errStr, test.WantErr) + } + return + } + + if err != nil { + t.Fatalf("unexpected error: %s", err.Error()) + } + + if !got.RawEquals(test.Want) { + t.Errorf("wrong result\ninput: %#v\ngot: %#v\nwant: %#v", test.Input, got, test.Want) + } + }) + } +} From 007cb634961cb09661821fc878537702d02c591b Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 21 Feb 2023 10:29:34 -0800 Subject: [PATCH 28/28] docs: An explicit policy for what "backward compatible" means So far I've just left this implied because mostly I was the one responsible for upgrading this library in various applications that use it (by virtue of an employment coincidence). However, the (indirect) use of this library has broadened lately, so it's worth being clear about what does not constitute a breaking change in a minor release so that application developers can have some guidance on what to do when they see a test failure after upgrading and so that hopefully they can avoid depending on implementation details in their main code, outside of unit tests. --- COMPATIBILITY.md | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 3 ++ 2 files changed, 91 insertions(+) create mode 100644 COMPATIBILITY.md diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md new file mode 100644 index 00000000..02cbe65a --- /dev/null +++ b/COMPATIBILITY.md @@ -0,0 +1,88 @@ +# `cty` backward-compatibility policy + +This library includes a number of behaviors that aim to support "best effort" +partial evaluation in the presence of wholly- or partially-unknown inputs. +Over time we've improved the accuracy of those analyses, but doing so changes +the specific results returned by certain operations. + +This document aims to describe what sorts of changes are allowed in new minor +releases and how those changes might affect the behavior of dependents after +upgrading. + +Where possible we'll avoid making changes like these in _patch_ releases, which +focus instead only on correcting incorrect behavior. An exception would be if +a minor release introduced an incorrect behavior and then a patch release +repaired it to either restore the previous correct behavior or implement a new +compromise correct behavior. + +## Unknown Values can become "more known" + +The most significant policy is that any operation that was previously returning +an unknown value may return either a known value or a _more refined_ unknown +value in later releases, as long as the new result is a subset of the range +of the previous result. + +When using only the _operation methods_ and functionality derived from them, +`cty` will typically handle these deductions automatically and return the most +specific result it is able to. In those cases we expect that these changes will +be seen as an improvement for end-users, and not require significant changes +to calling applications to pass on those benefits. + +When working with _integration methods_ (those which return results using +"normal" Go types rather than `cty.Value`) these changes can be more sigificant, +because applications can therefore observe the differences more readily. +For example, if an unknown value is replaced with a known value of the same +type then `Value.IsKnown` will begin returning `true` where it previously +returned `false`. Applications should be designed to avoid depending on +specific implementation details like these and instead aim to be more general +to handle both known and unknown values. + +A specific sensitive area for compatibility is the `Value.RawEquals` method, +which is sensitive to all of the possible variations in values. Applications +should not use this method for normal application code to avoid exposing +implementation details to end-users, but might use it to assert exact expected +results in unit tests. Such test cases may begin failing after upgrading, and +application developers should carefully consider whether the new results conform +to these rules and update the tests to match as part of their upgrade if so. If +the changed result seems _not_ to conform to these rules then that might be a +bug; please report it! + +## Error situations may begin succeeding + +Over time the valid inputs or other constraints on functionality might be +loosened to support new capabilities. Any operation or function that returned +an error in a previous release can begin succeeding with any valid result in +a new release. + +## Error message text might change + +This library aims to generate good, actionable error messages for user-facing +problems and to give sufficient information to a calling application to generate +its own high-quality error messages in situations where `cty` is not directly +"talking to" an end-user. + +This means that in later releases the exact text of error messages in certain +situations may change, typically to add additional context or increase +precision. + +If a function is documented as returning a particular error type in a certain +situation then that should be preserved in future releases, but if there is +no explicit documentation then calling applications should not depend on the +dynamic type of any `error` result, or should at least do so cautiously with +a fallback to a general error handler. + +## Passing on changes to Go standard library + +Some parts of `cty` are wrappers around functionality implemented in the Go +standard library. If the underlying packages change in newer versions of Go +then we may or may not pass on the change through the `cty` API, depending on +the circumstances. + +A specific notable example is Unicode support: this library depends on various +Unicode algorithms and data tables indirectly through its dependencies, +including some in the Go standard library, and so its exact treatment of strings +is likely to vary between releases as the Unicode standard grows. We aim to +follow the version of Unicode supported in the latest version of the Go standard +library, although we may lag behind slightly after new Go releases due to the +need to update other libraries that implement other parts of the Unicode +specifications. diff --git a/README.md b/README.md index d0b48a89..4ab44e1b 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,9 @@ For more details, see the following documentation: * [Conversion to and from native Go values](./docs/gocty.md) * [JSON serialization](./docs/json.md) * [`cty` Functions system](./docs/functions.md) +* [Compatibility Policy for future Minor Releases](./COMPATIBILITY.md): please + review this before using `cty` in your application to avoid depending on + implementation details that may change. ---