From 9e2c7be7d3c6a380c5f61074d9a5a690b617c3dc Mon Sep 17 00:00:00 2001 From: Andy LoPresto Date: Wed, 6 Sep 2017 16:45:13 -0700 Subject: [PATCH] NIFI-4353 - Added XmlUtils class. - Added unit test. - Added XXE test resource. - Refactored JAXB unmarshalling globally to prevent XXE attacks. - Refactored duplicated/legacy code. - Cleaned up commented code. - Switched from FileInputStream back to StreamSource in AuthorizerFactoryBean. - This closes #2134 --- nifi-commons/nifi-security-utils/pom.xml | 13 +++ .../apache/nifi/security/xml/XmlUtils.java | 44 +++++++++ .../nifi/security/xml/XmlUtilsTest.groovy | 96 +++++++++++++++++++ .../src/test/resources/xxe_template.xml | 7 ++ .../cluster/ClusterNodeInformation.java | 13 ++- .../nifi-framework/nifi-authorizer/pom.xml | 4 + .../authorization/AuthorizerFactoryBean.java | 42 ++++---- .../FileAccessPolicyProvider.java | 90 +++++++++-------- .../authorization/FileUserGroupProvider.java | 85 +++++++++------- .../cluster/protocol/HeartbeatPayload.java | 19 ++-- .../protocol/jaxb/JaxbProtocolContext.java | 12 ++- .../ClusterProtocolHeartbeatMonitor.java | 34 ++----- .../apache/nifi/cluster/BulletinsPayload.java | 17 ++-- .../apache/nifi/controller/TemplateUtils.java | 59 ++++-------- .../StandardSnippetDeserializer.java | 11 ++- .../persistence/TemplateDeserializer.java | 21 ++-- .../persistence/TemplateSerializerTest.java | 12 +-- .../nifi/web/api/ProcessGroupResource.java | 6 +- .../nifi-web/nifi-web-security/pom.xml | 4 + .../LoginIdentityProviderFactoryBean.java | 5 +- .../nifi-update-attribute-model/pom.xml | 6 ++ .../attributes/serde/CriteriaSerDe.java | 16 ++-- 22 files changed, 396 insertions(+), 220 deletions(-) create mode 100644 nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XmlUtils.java create mode 100644 nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/xml/XmlUtilsTest.groovy create mode 100644 nifi-commons/nifi-security-utils/src/test/resources/xxe_template.xml diff --git a/nifi-commons/nifi-security-utils/pom.xml b/nifi-commons/nifi-security-utils/pom.xml index 5c9acc3c2cf1..3f4a088f23e8 100644 --- a/nifi-commons/nifi-security-utils/pom.xml +++ b/nifi-commons/nifi-security-utils/pom.xml @@ -60,5 +60,18 @@ nifi-properties + + + + org.apache.rat + apache-rat-plugin + + + src/test/resources/xxe_template.xml + + + + + diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XmlUtils.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XmlUtils.java new file mode 100644 index 000000000000..99c90a6d530f --- /dev/null +++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/xml/XmlUtils.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.security.xml; + +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.transform.stream.StreamSource; +import java.io.InputStream; + +public class XmlUtils { + + public static XMLStreamReader createSafeReader(InputStream inputStream) throws XMLStreamException { + if (inputStream == null) { + throw new IllegalArgumentException("The provided input stream cannot be null"); + } + return createSafeReader(new StreamSource(inputStream)); + } + + public static XMLStreamReader createSafeReader(StreamSource source) throws XMLStreamException { + if (source == null) { + throw new IllegalArgumentException("The provided source cannot be null"); + } + + XMLInputFactory xif = XMLInputFactory.newFactory(); + xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); + return xif.createXMLStreamReader(source); + } +} diff --git a/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/xml/XmlUtilsTest.groovy b/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/xml/XmlUtilsTest.groovy new file mode 100644 index 000000000000..6a1286fff203 --- /dev/null +++ b/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/xml/XmlUtilsTest.groovy @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.security.xml + +import org.junit.After +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.xml.bind.JAXBContext +import javax.xml.bind.UnmarshalException +import javax.xml.bind.Unmarshaller +import javax.xml.bind.annotation.XmlAccessType +import javax.xml.bind.annotation.XmlAccessorType +import javax.xml.bind.annotation.XmlAttribute +import javax.xml.bind.annotation.XmlRootElement +import javax.xml.stream.XMLStreamReader + +import static groovy.test.GroovyAssert.shouldFail + +@RunWith(JUnit4.class) +class XmlUtilsTest { + private static final Logger logger = LoggerFactory.getLogger(XmlUtilsTest.class) + + @BeforeClass + static void setUpOnce() throws Exception { + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + void setUp() throws Exception { + + } + + @After + void tearDown() throws Exception { + + } + + @Test + void testShouldHandleXXEInUnmarshal() { + // Arrange + final String XXE_TEMPLATE_FILEPATH = "src/test/resources/xxe_template.xml" + InputStream templateStream = new File(XXE_TEMPLATE_FILEPATH).newInputStream() + + JAXBContext context = JAXBContext.newInstance(XmlObject.class) + + // Act + def msg = shouldFail(UnmarshalException) { + Unmarshaller unmarshaller = context.createUnmarshaller() + XMLStreamReader xsr = XmlUtils.createSafeReader(templateStream) + def parsed = unmarshaller.unmarshal(xsr, XmlObject.class) + logger.info("Unmarshalled ${parsed.toString()}") + } + + // Assert + logger.expected(msg) + assert msg =~ "XMLStreamException: ParseError " + } +} + +@XmlAccessorType( XmlAccessType.NONE ) +@XmlRootElement(name = "object") +class XmlObject { + @XmlAttribute + String name + + @XmlAttribute + String description + + @XmlAttribute + String groupId + + @XmlAttribute + String timestamp +} diff --git a/nifi-commons/nifi-security-utils/src/test/resources/xxe_template.xml b/nifi-commons/nifi-security-utils/src/test/resources/xxe_template.xml new file mode 100644 index 000000000000..372d5070a1f0 --- /dev/null +++ b/nifi-commons/nifi-security-utils/src/test/resources/xxe_template.xml @@ -0,0 +1,7 @@ +]> + + &xxe; + Arbitrary XML that has an XXE attack present. + 3a204982-015e-1000-eaa2-19d352ec8394 + 09/05/2017 14:51:01 PDT + diff --git a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/ClusterNodeInformation.java b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/ClusterNodeInformation.java index 1bc83b9a7773..5a0264e4d467 100644 --- a/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/ClusterNodeInformation.java +++ b/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/cluster/ClusterNodeInformation.java @@ -19,13 +19,15 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.Collection; - import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import org.apache.nifi.security.xml.XmlUtils; @XmlRootElement public class ClusterNodeInformation { @@ -61,7 +63,12 @@ public void marshal(final OutputStream os) throws JAXBException { } public static ClusterNodeInformation unmarshal(final InputStream is) throws JAXBException { - final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); - return (ClusterNodeInformation) unmarshaller.unmarshal(is); + try { + final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); + final XMLStreamReader xsr = XmlUtils.createSafeReader(is); + return (ClusterNodeInformation) unmarshaller.unmarshal(xsr); + } catch (XMLStreamException e) { + throw new JAXBException("Error unmarshalling the cluster node information", e); + } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/pom.xml index b0a64d98ea02..41d23c00d8e6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/pom.xml @@ -87,6 +87,10 @@ org.apache.nifi nifi-framework-authorization + + org.apache.nifi + nifi-security-utils + \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java index fdc04a0c11b5..746c0eda1a82 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-authorizer/src/main/java/org/apache/nifi/authorization/AuthorizerFactoryBean.java @@ -16,6 +16,25 @@ */ package org.apache.nifi.authorization; +import java.io.File; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.xml.XMLConstants; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBElement; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Unmarshaller; +import javax.xml.stream.XMLStreamReader; +import javax.xml.transform.stream.StreamSource; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.authorization.annotation.AuthorizerContext; import org.apache.nifi.authorization.exception.AuthorizationAccessException; @@ -25,6 +44,7 @@ import org.apache.nifi.authorization.generated.Property; import org.apache.nifi.bundle.Bundle; import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.util.file.classloader.ClassLoaderUtils; import org.slf4j.Logger; @@ -33,25 +53,6 @@ import org.springframework.beans.factory.FactoryBean; import org.xml.sax.SAXException; -import javax.xml.XMLConstants; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBElement; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Unmarshaller; -import javax.xml.transform.stream.StreamSource; -import javax.xml.validation.Schema; -import javax.xml.validation.SchemaFactory; -import java.io.File; -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - /** * Factory bean for loading the configured authorizer. */ @@ -168,9 +169,10 @@ private Authorizers loadAuthorizersConfiguration() throws Exception { final Schema schema = schemaFactory.newSchema(Authorizers.class.getResource(AUTHORIZERS_XSD)); // attempt to unmarshal + final XMLStreamReader xsr = XmlUtils.createSafeReader(new StreamSource(authorizersConfigurationFile)); final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); unmarshaller.setSchema(schema); - final JAXBElement element = unmarshaller.unmarshal(new StreamSource(authorizersConfigurationFile), Authorizers.class); + final JAXBElement element = unmarshaller.unmarshal(xsr, Authorizers.class); return element.getValue(); } catch (SAXException | JAXBException e) { throw new Exception("Unable to load the authorizer configuration file at: " + authorizersConfigurationFile.getAbsolutePath(), e); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAccessPolicyProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAccessPolicyProvider.java index 653a94997eb0..f71ad716c711 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAccessPolicyProvider.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAccessPolicyProvider.java @@ -16,6 +16,39 @@ */ package org.apache.nifi.authorization; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.xml.XMLConstants; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBElement; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.stream.XMLStreamWriter; +import javax.xml.transform.stream.StreamSource; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.authorization.annotation.AuthorizerContext; import org.apache.nifi.authorization.exception.AuthorizationAccessException; @@ -30,6 +63,7 @@ import org.apache.nifi.authorization.util.IdentityMapping; import org.apache.nifi.authorization.util.IdentityMappingUtil; import org.apache.nifi.components.PropertyValue; +import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.user.generated.Users; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.util.file.FileUtils; @@ -42,39 +76,6 @@ import org.w3c.dom.NodeList; import org.xml.sax.SAXException; -import javax.xml.XMLConstants; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBElement; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Marshaller; -import javax.xml.bind.Unmarshaller; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.stream.XMLOutputFactory; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamWriter; -import javax.xml.transform.stream.StreamSource; -import javax.xml.validation.Schema; -import javax.xml.validation.SchemaFactory; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.io.StringWriter; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvider { private static final Logger logger = LoggerFactory.getLogger(FileAccessPolicyProvider.class); @@ -528,11 +529,17 @@ private void saveAuthorizations(final Authorizations authorizations) throws JAXB } private Authorizations unmarshallAuthorizations() throws JAXBException { - final Unmarshaller unmarshaller = JAXB_AUTHORIZATIONS_CONTEXT.createUnmarshaller(); - unmarshaller.setSchema(authorizationsSchema); + try { + final XMLStreamReader xsr = XmlUtils.createSafeReader(new StreamSource(authorizationsFile)); + final Unmarshaller unmarshaller = JAXB_AUTHORIZATIONS_CONTEXT.createUnmarshaller(); + unmarshaller.setSchema(authorizationsSchema); - final JAXBElement element = unmarshaller.unmarshal(new StreamSource(authorizationsFile), Authorizations.class); - return element.getValue(); + final JAXBElement element = unmarshaller.unmarshal(xsr, Authorizations.class); + return element.getValue(); + } catch (XMLStreamException e) { + logger.error("Encountered an error reading authorizations file: ", e); + throw new JAXBException("Error reading authorizations file", e); + } } /** @@ -626,8 +633,15 @@ private void convertLegacyAuthorizedUsers(final Authorizations authorizations) t final Unmarshaller unmarshaller = JAXB_USERS_CONTEXT.createUnmarshaller(); unmarshaller.setSchema(usersSchema); + final XMLStreamReader xsr; + try { + xsr = XmlUtils.createSafeReader(new StreamSource(authorizedUsersFile)); + } catch (XMLStreamException e) { + logger.error("Encountered an error reading authorized users file: ", e); + throw new JAXBException("Error reading authorized users file", e); + } final JAXBElement element = unmarshaller.unmarshal( - new StreamSource(authorizedUsersFile), org.apache.nifi.user.generated.Users.class); + xsr, org.apache.nifi.user.generated.Users.class); final org.apache.nifi.user.generated.Users users = element.getValue(); if (users.getUser().isEmpty()) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileUserGroupProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileUserGroupProvider.java index 59c829c84832..edcfa51420d2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileUserGroupProvider.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileUserGroupProvider.java @@ -16,6 +16,39 @@ */ package org.apache.nifi.authorization; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.xml.XMLConstants; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBElement; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.stream.XMLStreamWriter; +import javax.xml.transform.stream.StreamSource; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.authorization.annotation.AuthorizerContext; import org.apache.nifi.authorization.exception.AuthorizationAccessException; @@ -28,6 +61,7 @@ import org.apache.nifi.authorization.util.IdentityMapping; import org.apache.nifi.authorization.util.IdentityMappingUtil; import org.apache.nifi.components.PropertyValue; +import org.apache.nifi.security.xml.XmlUtils; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.util.file.FileUtils; import org.slf4j.Logger; @@ -38,39 +72,6 @@ import org.w3c.dom.NodeList; import org.xml.sax.SAXException; -import javax.xml.XMLConstants; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBElement; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Marshaller; -import javax.xml.bind.Unmarshaller; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.stream.XMLOutputFactory; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamWriter; -import javax.xml.transform.stream.StreamSource; -import javax.xml.validation.Schema; -import javax.xml.validation.SchemaFactory; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.io.StringWriter; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - public class FileUserGroupProvider implements ConfigurableUserGroupProvider { private static final Logger logger = LoggerFactory.getLogger(FileUserGroupProvider.class); @@ -665,8 +666,13 @@ private Tenants unmarshallTenants() throws JAXBException { final Unmarshaller unmarshaller = JAXB_TENANTS_CONTEXT.createUnmarshaller(); unmarshaller.setSchema(tenantsSchema); - final JAXBElement element = unmarshaller.unmarshal(new StreamSource(tenantsFile), Tenants.class); - return element.getValue(); + try { + final XMLStreamReader xsr = XmlUtils.createSafeReader(new StreamSource(tenantsFile)); + final JAXBElement element = unmarshaller.unmarshal(xsr, Tenants.class); + return element.getValue(); + } catch (XMLStreamException e) { + throw new JAXBException("Error unmarshalling tenants", e); + } } private void populateInitialUsers(final Tenants tenants) { @@ -688,11 +694,18 @@ private void convertLegacyAuthorizedUsers(final Tenants tenants) throws Authoriz throw new AuthorizerCreationException("Legacy Authorized Users File '" + legacyAuthorizedUsersFile + "' does not exists"); } + XMLStreamReader xsr; + try { + xsr = XmlUtils.createSafeReader(new StreamSource(authorizedUsersFile)); + } catch (XMLStreamException e) { + throw new AuthorizerCreationException("Error converting the legacy authorizers file", e); + } + final Unmarshaller unmarshaller = JAXB_USERS_CONTEXT.createUnmarshaller(); unmarshaller.setSchema(usersSchema); final JAXBElement element = unmarshaller.unmarshal( - new StreamSource(authorizedUsersFile), org.apache.nifi.user.generated.Users.class); + xsr, org.apache.nifi.user.generated.Users.class); final org.apache.nifi.user.generated.Users users = element.getValue(); if (users.getUser().isEmpty()) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/HeartbeatPayload.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/HeartbeatPayload.java index 8363a2013435..20848be599bb 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/HeartbeatPayload.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/HeartbeatPayload.java @@ -21,14 +21,15 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.List; - import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; import javax.xml.bind.annotation.XmlRootElement; - +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import org.apache.nifi.cluster.coordination.node.NodeConnectionStatus; +import org.apache.nifi.security.xml.XmlUtils; /** * The payload of the heartbeat. The payload contains status to inform the cluster manager the current workload of this node. @@ -111,18 +112,14 @@ public static void marshal(final HeartbeatPayload payload, final OutputStream os public static HeartbeatPayload unmarshal(final InputStream is) throws ProtocolException { try { final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); - return (HeartbeatPayload) unmarshaller.unmarshal(is); - } catch (final JAXBException je) { - throw new ProtocolException(je); + final XMLStreamReader xsr = XmlUtils.createSafeReader(is); + return (HeartbeatPayload) unmarshaller.unmarshal(xsr); + } catch (final JAXBException | XMLStreamException e) { + throw new ProtocolException(e); } } public static HeartbeatPayload unmarshal(final byte[] bytes) throws ProtocolException { - try { - final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); - return (HeartbeatPayload) unmarshaller.unmarshal(new ByteArrayInputStream(bytes)); - } catch (final JAXBException je) { - throw new ProtocolException(je); - } + return unmarshal(new ByteArrayInputStream(bytes)); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/jaxb/JaxbProtocolContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/jaxb/JaxbProtocolContext.java index 4d44b4e86a34..23d45d11f5b3 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/jaxb/JaxbProtocolContext.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/jaxb/JaxbProtocolContext.java @@ -25,15 +25,16 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; - import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; - +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import org.apache.nifi.cluster.protocol.ProtocolContext; import org.apache.nifi.cluster.protocol.ProtocolMessageMarshaller; import org.apache.nifi.cluster.protocol.ProtocolMessageUnmarshaller; +import org.apache.nifi.security.xml.XmlUtils; /** * Implements a context for communicating internally amongst the cluster using @@ -135,10 +136,11 @@ public T unmarshal(final InputStream is) throws IOException { final Unmarshaller unmarshaller = jaxbCtx.createUnmarshaller(); final byte[] msg = new byte[totalBytesRead]; buffer.get(msg); - return (T) unmarshaller.unmarshal(new ByteArrayInputStream(msg)); + final XMLStreamReader xsr = XmlUtils.createSafeReader(new ByteArrayInputStream(msg)); + return (T) unmarshaller.unmarshal(xsr); - } catch (final JAXBException je) { - throw new IOException("Failed unmarshalling protocol message due to: " + je, je); + } catch (final JAXBException | XMLStreamException e) { + throw new IOException("Failed unmarshalling protocol message due to: " + e, e); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/heartbeat/ClusterProtocolHeartbeatMonitor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/heartbeat/ClusterProtocolHeartbeatMonitor.java index 78ec8dffe909..2d6f02327e1f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/heartbeat/ClusterProtocolHeartbeatMonitor.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/heartbeat/ClusterProtocolHeartbeatMonitor.java @@ -16,6 +16,16 @@ */ package org.apache.nifi.cluster.coordination.heartbeat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.nifi.cluster.coordination.ClusterCoordinator; import org.apache.nifi.cluster.coordination.node.NodeConnectionState; import org.apache.nifi.cluster.coordination.node.NodeConnectionStatus; @@ -36,19 +46,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.Unmarshaller; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.function.Function; -import java.util.stream.Collectors; - /** * Uses Apache ZooKeeper to advertise the address to send heartbeats to, and * then relies on the NiFi Cluster Protocol to receive heartbeat messages from @@ -63,17 +60,6 @@ public class ClusterProtocolHeartbeatMonitor extends AbstractHeartbeatMonitor im private volatile long purgeTimestamp = System.currentTimeMillis(); - protected static final Unmarshaller nodeIdentifierUnmarshaller; - - static { - try { - final JAXBContext jaxbContext = JAXBContext.newInstance(NodeIdentifier.class); - nodeIdentifierUnmarshaller = jaxbContext.createUnmarshaller(); - } catch (final Exception e) { - throw new RuntimeException("Failed to create an Unmarshaller for unmarshalling Node Identifier", e); - } - } - public ClusterProtocolHeartbeatMonitor(final ClusterCoordinator clusterCoordinator, final ProtocolListener protocolListener, final NiFiProperties nifiProperties) { super(clusterCoordinator, nifiProperties); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/cluster/BulletinsPayload.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/cluster/BulletinsPayload.java index 77d66205dcfc..e8a33a5b5553 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/cluster/BulletinsPayload.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/cluster/BulletinsPayload.java @@ -27,9 +27,12 @@ import javax.xml.bind.Unmarshaller; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import org.apache.nifi.cluster.protocol.ProtocolException; import org.apache.nifi.jaxb.BulletinAdapter; import org.apache.nifi.reporting.Bulletin; +import org.apache.nifi.security.xml.XmlUtils; /** * The payload of the bulletins. @@ -77,18 +80,14 @@ public static void marshal(final BulletinsPayload payload, final OutputStream os public static BulletinsPayload unmarshal(final InputStream is) throws ProtocolException { try { final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); - return (BulletinsPayload) unmarshaller.unmarshal(is); - } catch (final JAXBException je) { - throw new ProtocolException(je); + final XMLStreamReader xsr = XmlUtils.createSafeReader(is); + return (BulletinsPayload) unmarshaller.unmarshal(xsr); + } catch (final JAXBException | XMLStreamException e) { + throw new ProtocolException(e); } } public static BulletinsPayload unmarshal(final byte[] bytes) throws ProtocolException { - try { - final Unmarshaller unmarshaller = JAXB_CONTEXT.createUnmarshaller(); - return (BulletinsPayload) unmarshaller.unmarshal(new ByteArrayInputStream(bytes)); - } catch (final JAXBException je) { - throw new ProtocolException(je); - } + return unmarshal(new ByteArrayInputStream(bytes)); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/TemplateUtils.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/TemplateUtils.java index be27c5e35dcf..9583bbb04193 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/TemplateUtils.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/TemplateUtils.java @@ -18,7 +18,6 @@ package org.apache.nifi.controller; import org.apache.nifi.persistence.TemplateDeserializer; -import org.apache.nifi.stream.io.StreamUtils; import org.apache.nifi.web.api.dto.ConnectableDTO; import org.apache.nifi.web.api.dto.ConnectionDTO; import org.apache.nifi.web.api.dto.ControllerServiceDTO; @@ -32,16 +31,15 @@ import org.apache.nifi.web.api.dto.TemplateDTO; import org.w3c.dom.Element; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.Unmarshaller; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; import java.io.ByteArrayInputStream; -import java.io.DataInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; import java.util.Collection; -import java.util.List; import java.util.Map; import java.util.Set; @@ -49,9 +47,18 @@ public class TemplateUtils { public static TemplateDTO parseDto(final Element templateElement) { try { - JAXBContext context = JAXBContext.newInstance(TemplateDTO.class); - Unmarshaller unmarshaller = context.createUnmarshaller(); - return unmarshaller.unmarshal(new DOMSource(templateElement), TemplateDTO.class).getValue(); + final DOMSource domSource = new DOMSource(templateElement); + + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + final StreamResult streamResult = new StreamResult(baos); + + // need to stream the template element as the TemplateDeserializer.deserialize operation needs to re-parse + // in order to apply explicit properties on the XMLInputFactory + final TransformerFactory transformerFactory = TransformerFactory.newInstance(); + final Transformer transformer = transformerFactory.newTransformer(); + transformer.transform(domSource, streamResult); + + return parseDto(baos.toByteArray()); } catch (final Exception e) { throw new RuntimeException("Could not parse XML as a valid template", e); } @@ -61,42 +68,10 @@ public static TemplateDTO parseDto(final byte[] bytes) { try (final InputStream in = new ByteArrayInputStream(bytes)) { return TemplateDeserializer.deserialize(in); } catch (final IOException ioe) { - throw new RuntimeException("Could not parse bytes as template", ioe); // won't happen because of the types of streams being used + throw new RuntimeException("Could not parse bytes as template", ioe); } } - public static List