Add utility to "normalize" numeric string values on a Document

source/JsMaterialX/JsMaterialXCore/JsDocument.cpp

@@ -103,6 +103,7 @@ EMSCRIPTEN_BINDINGS(document)
         .function("hasColorManagementConfig", &mx::Document::hasColorManagementConfig)
         .function("getColorManagementConfig", &mx::Document::getColorManagementConfig)
         .function("invalidateCache", &mx::Document::invalidateCache)
+        .function("normalizeValueStrings", &mx::Document::normalizeValueStrings)
         .class_property("CATEGORY", &mx::Document::CATEGORY)
         .class_property("CMS_ATTRIBUTE", &mx::Document::CMS_ATTRIBUTE)
         .class_property("CMS_CONFIG_ATTRIBUTE", &mx::Document::CMS_CONFIG_ATTRIBUTE);

source/JsMaterialX/JsMaterialXCore/JsUtil.cpp

@@ -47,4 +47,6 @@ EMSCRIPTEN_BINDINGS(util)
     ems::function("splitNamePath", &mx::splitNamePath);
     ems::function("createNamePath", &mx::createNamePath);
     ems::function("parentNamePath", &mx::parentNamePath);
+
+    ems::function("normalizeValueString", &mx::normalizeValueString);
 }

source/MaterialXCore/Document.cpp

@@ -4,6 +4,7 @@
 //
 
 #include <MaterialXCore/Document.h>
+#include <MaterialXCore/Util.h>
 
 #include <mutex>
 
@@ -401,6 +402,30 @@ void Document::invalidateCache()
     _cache->valid = false;
 }
 
+void Document::normalizeValueStrings()
+{
+    for (ElementPtr elem : traverseTree())
+    {
+        ValueElementPtr valueElem = elem->asA<ValueElement>();
+        if (!valueElem)
+        {
+            continue;
+        }
+
+        const string& originalString = valueElem->getValueString();
+        if (originalString.empty())
+        {
+            continue;
+        }
+
+        string normalizeString = normalizeValueString(originalString, valueElem->getType());
+        if (normalizeString != originalString)
+        {
+            valueElem->setValueString(normalizeString);
+        }
+    }
+}
+
 //
 // Deprecated methods
 //

source/MaterialXCore/Document.h

@@ -663,6 +663,9 @@ class MX_CORE_API Document : public GraphElement
     /// Invalidate cached data for optimized lookups within the given document.
     void invalidateCache();
 
+    /// Normalize value strings for all numeric values attributes
+    void normalizeValueStrings();
+
     /// @}
 
     //

source/MaterialXCore/Util.cpp

@@ -6,6 +6,8 @@
 #include <MaterialXCore/Types.h>
 
 #include <cctype>
+#include <sstream>
+#include <iomanip>
 
 MATERIALX_NAMESPACE_BEGIN
 
@@ -179,4 +181,89 @@ string parentNamePath(const string& namePath)
     return EMPTY_STRING;
 }
 
+
+std::string normalizeValueString(const std::string& str, const std::string valueType)
+{
+    // Currently supported value types.
+    StringSet supportedTypes = { "integer", "float", "vector2", "vector3", "vector4", "color3", "color4", "matrix33", "matrix44"};
+    if (supportedTypes.end() == supportedTypes.find(valueType))
+    {
+        return str;
+    }
+
+    const string TOKEN_SEPARATOR = ", ";
+
+    std::string result;
+    std::string token;
+
+    bool isInteger = (valueType == "integer");
+    if (isInteger)
+    {
+        // Remove leading and trailing spaces
+        result = str;
+        result.erase(0, result.find_first_not_of(' '));
+        result.erase(result.find_last_not_of(' ') + 1);
+    }
+    else
+    {
+        std::stringstream ss(str);
+        while (std::getline(ss, token, ','))
+        {
+            // Remove leading and trailing spaces
+            token.erase(0, token.find_first_not_of(' '));
+            token.erase(token.find_last_not_of(' ') + 1);
+
+            // Skip if the token is empty. Note that this is
+            // invalid in MaterialX but can still be stored this way. 
+            if (!token.empty())
+            {
+                // Remove leading zeros
+                token.erase(0, token.find_first_not_of('0'));
+
+                // Preserve 0 values
+                if (token.empty() || token[0] == '.')
+                {
+                    token = "0" + token;
+                }
+
+                // Check if string token has a 'e' character
+                // implying it is a floating point number in scientific notation.
+                // If it does not have an 'e' character, remove trailing zeros.
+                //
+                size_t ePos = token.find('e');
+                if (ePos == std::string::npos)
+                {
+                    // If the token contains a decimal point, remove trailing zeros
+                    size_t decimalPos = token.find('.');
+                    if (decimalPos != std::string::npos)
+                    {
+                        token.erase(token.find_last_not_of('0') + 1);
+
+                        // If the token ends with a decimal point after removing trailing zeros, remove it
+                        if (token.back() == '.')
+                        {
+                            token.pop_back();
+                        }
+                    }
+                }
+            }
+
+            // Append the formatted token to the result with a comma
+            result += token + TOKEN_SEPARATOR;
+        }
+
+        // Remove the last separator
+        if (!result.empty())
+        {
+            for (size_t i = 0; i < TOKEN_SEPARATOR.size(); i++)
+            {
+                result.pop_back();
+            }
+        }
+    }
+
+    return result;
+}
+
+
 MATERIALX_NAMESPACE_END

source/MaterialXCore/Util.h

@@ -69,6 +69,10 @@ MX_CORE_API string createNamePath(const StringVec& nameVec);
 /// Given a name path, return the parent name path
 MX_CORE_API string parentNamePath(const string& namePath);
 
+/// Normalize a string containing a value.
+/// Currently single floating point numbers or an array of comma seperated numbers is supported.
+MX_CORE_API string normalizeValueString(const string& str, const std::string valueType);
+
 MATERIALX_NAMESPACE_END
 
 #endif

source/MaterialXTest/MaterialXCore/CoreUtil.cpp

@@ -7,6 +7,7 @@
 
 #include <MaterialXCore/Util.h>
 #include <MaterialXCore/Document.h>
+#include <iostream>
 
 namespace mx = MaterialX;
 
@@ -38,6 +39,64 @@ TEST_CASE("String utilities", "[coreutil]")
     REQUIRE(!mx::stringEndsWith("testName", "test"));
 }
 
+TEST_CASE("Value normalization", "[coreutil]")
+{
+    std::string inputInteger = "  12  ";
+    std::string resultInteger = mx::normalizeValueString(inputInteger, "integer");
+    REQUIRE(resultInteger == "12");
+
+    std::string inputScalar = "  1.2e-10  ";
+    std::string resultScalar = mx::normalizeValueString(inputScalar, "float");
+    REQUIRE(resultScalar == "1.2e-10");
+    std::string inputScalar1 = "  00.1000  ";
+    std::string resultScalar1 = mx::normalizeValueString(inputScalar1, "float");
+    REQUIRE(resultScalar1 == "0.1");
+    std::string inputScalar2 = ".000000";
+    std::string resultScalar2 = mx::normalizeValueString(inputScalar2, "float");
+    REQUIRE(resultScalar2 == "0");
+    std::string inputScalar3 = "000.";
+    std::string resultScalar3 = mx::normalizeValueString(inputScalar3, "float");
+    REQUIRE(resultScalar3 == "0");
+    std::string inputScalar4 = "000.01";
+    std::string resultScalar4 = mx::normalizeValueString(inputScalar4, "float");
+    REQUIRE(resultScalar4 == "0.01");
+
+    std::string inputVector1 = "1.0, 2.0,  0000.231";
+    std::string resultVector1 = mx::normalizeValueString(inputVector1, "vector3");
+    REQUIRE(resultVector1 == "1, 2, 0.231");
+    resultVector1 = mx::normalizeValueString(inputVector1, "color3");
+    REQUIRE(resultVector1 == "1, 2, 0.231");
+
+    std::string inputVector2 = "0001.2000, 0000.00010";
+    std::string resultVector2 = mx::normalizeValueString(inputVector2, "vector2");
+    REQUIRE(resultVector2 == "1.2, 0.0001");
+
+    std::string inputVector3 = "01.0,         1.2e-10 ,  0000.2310, 0.1";
+    std::string resultVector3 = mx::normalizeValueString(inputVector3, "vector4");
+    REQUIRE(resultVector3 == "1, 1.2e-10, 0.231, 0.1");
+    resultVector3 = mx::normalizeValueString(inputVector3, "color4");
+    REQUIRE(resultVector3 == "1, 1.2e-10, 0.231, 0.1");
+
+    std::string inputVector4 = "01.0,  ,  0000.2310, 0.1";
+    std::string resultVector4 = mx::normalizeValueString(inputVector4, "vector4");
+    REQUIRE(resultVector4 == "1, , 0.231, 0.1");
+
+    std::string inputMatrix3 = "01.0,         2.0,  0000.2310, "
+        "   01.0,         2.0,  0000.2310, "
+        "01.0,         2.0,  0000.2310       ";
+    std::string resultMatrix3 = mx::normalizeValueString(inputMatrix3, "matrix33");
+    std::string compareMatrix3 = "1, 2, 0.231, 1, 2, 0.231, 1, 2, 0.231";
+    REQUIRE(resultMatrix3 == compareMatrix3);
+
+    std::string inputMatrix = "01.0,         2.0,  0000.2310,  0.100,"
+                              "01.0,         2.0,  0000.2310, 0.100,"
+                              "01.0,         2.0,  0000.2310, 0.100,"
+                              "01.0,         2.0,  0000.2310, 0.100";
+    std::string resultMatrix = mx::normalizeValueString(inputMatrix, "matrix44");
+    std::string compareMatrix = "1, 2, 0.231, 0.1, 1, 2, 0.231, 0.1, 1, 2, 0.231, 0.1, 1, 2, 0.231, 0.1";
+    REQUIRE(resultMatrix == compareMatrix);
+}
+
 TEST_CASE("Print utilities", "[coreutil]")
 {
     // Create a document.

source/MaterialXTest/MaterialXCore/Document.cpp

@@ -10,6 +10,8 @@
 #include <MaterialXFormat/Util.h>
 #include <MaterialXFormat/XmlIo.h>
 
+#include <map>
+
 namespace mx = MaterialX;
 
 TEST_CASE("Document", "[document]")
@@ -116,3 +118,88 @@ TEST_CASE("Document", "[document]")
     // Validate the combined document.
     REQUIRE(doc->validate());
 }
+
+#include <iostream>
+
+TEST_CASE("Document equivalence", "[document]")
+{
+    mx::DocumentPtr doc = mx::createDocument();
+    std::multimap<std::string, std::string> inputMap;
+
+    inputMap.insert({ "color3", "  1.0,   +2.0,  3.0   " });
+    inputMap.insert({ "color4", "1.0,   2.00, 0.3000, -4" });
+    inputMap.insert({ "float", "  1.2e-10  " });
+    inputMap.insert({ "float", "  00.1000  " });
+    inputMap.insert({ "integer", "  12 " });
+    inputMap.insert({ "matrix33", 
+                      "01.0,         2.0,  0000.2310, "
+                      "   01.0,         2.0,  0000.2310, "
+                      "01.0,         2.0,  0000.2310       " });
+    inputMap.insert({ "matrix44",
+                      "01.0,         2.0,  0000.2310, 0.100, "
+                      "01.0,         2.0,  0000.2310, 0.100, "
+                      "01.0,         2.0,  0000.2310, 0.100, "
+                      "01.0,         2.0,  0000.2310, 0.100" });
+    inputMap.insert({ "vector2", "1.0,   0.012345678" });
+    inputMap.insert({ "vector3", "  1.0,   +2.0,  3.0   " });
+    inputMap.insert({ "vector4", "1.0,   2.00, 0.3000, -4" });
+    inputMap.insert({ "string", "mystring" });
+    inputMap.insert({ "boolean", "false" });
+    inputMap.insert({ "filename", "filename1" });
+
+    unsigned int index = 0;
+    for (auto it = inputMap.begin(); it != inputMap.end(); ++it)
+    {
+        mx::InputPtr input= doc->addInput("input" + std::to_string(index), (*it).first);
+        input->setValueString((*it).second);
+        index++;
+    }
+
+    mx::DocumentPtr doc2 = mx::createDocument();
+    std::multimap<std::string, std::string> inputMap2;
+    inputMap2.insert({ "color3", "1, 2, 3" });
+    inputMap2.insert({ "color4", "1, 2, 0.3, -4" });
+    inputMap2.insert({ "float", "1.2e-10" });
+    inputMap2.insert({ "float", "0.1" });
+    inputMap2.insert({ "integer", "12" });
+    inputMap2.insert({ "matrix33", "1, 2, 0.231,  1, 2, 0.231,  1, 2, 0.231,  1, 2, 0.231" });
+    inputMap2.insert({ "matrix44", "1, 2, 0.231, 0.1, 1, 2, 0.231, 0.1, 1, 2, 0.231, 0.1, 1, 2, 0.231, 0.1" });
+    inputMap2.insert({ "vector2", "1, 0.0123456" });
+    inputMap2.insert({ "vector3", "1, 2, 3" });
+    inputMap2.insert({ "vector4", "1, 2, 0.3, -4" });
+    inputMap2.insert({ "string", "mystring" });
+    inputMap2.insert({ "boolean", "false" });
+    inputMap2.insert({ "filename", "filename1" });    
+
+    index = 0;
+    for (auto it = inputMap.begin(); it != inputMap.end(); ++it)
+    {
+        mx::InputPtr input = doc2->addInput("input" + std::to_string(index), (*it).first);
+        input->setValueString((*it).second);
+        index++;
+    }
+
+    std::string status;
+    doc->normalizeValueStrings();
+    bool valid = doc->validate(&status);
+    {
+        std::cout << "input doc status: " << status << std::endl;
+    }
+    REQUIRE(valid);
+
+    doc2->normalizeValueStrings();
+    valid = doc2->validate(&status);
+    {
+        std::cout << "input doc 2 status: " << status << std::endl;
+    }
+    REQUIRE(valid);
+
+    // Note: do not check doc2 == doc as that is a pointer comparison
+    bool equivalent = (*doc2 == *doc);
+    if (!equivalent)
+    {
+        std::cout << "doc 1: " << mx::prettyPrint(doc) << std::endl;
+        std::cout << "doc 2: " << mx::prettyPrint(doc2) << std::endl;
+    }
+    REQUIRE(equivalent);
+}

source/PyMaterialX/PyMaterialXCore/PyDocument.cpp

@@ -116,5 +116,6 @@ void bindPyDocument(py::module& mod)
         .def("getColorManagementSystem", &mx::Document::getColorManagementSystem)
         .def("setColorManagementConfig", &mx::Document::setColorManagementConfig)
         .def("hasColorManagementConfig", &mx::Document::hasColorManagementConfig)
-        .def("getColorManagementConfig", &mx::Document::getColorManagementConfig);
+        .def("getColorManagementConfig", &mx::Document::getColorManagementConfig)
+        .def("normalizeValueStrings", &mx::Document::normalizeValueStrings);
 }

source/PyMaterialX/PyMaterialXCore/PyUtil.cpp

@@ -28,4 +28,5 @@ void bindPyUtil(py::module& mod)
     mod.def("splitNamePath", &mx::splitNamePath);
     mod.def("createNamePath", &mx::createNamePath);
     mod.def("parentNamePath", &mx::parentNamePath);
+    mod.def("normalizeValueString", &mx::normalizeValueString);
 }