diff --git a/CHANGELOG.md b/CHANGELOG.md index 64e5c5b8ff..fbde757ff0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ Usage: ``` ### 0.2.1-SNAPSHOT * Fix #53: Renamed plugins to openshift/kubernetes-maven-plugin keeping acronym (oc/k8s) for goal +* Support for setting BuildConfig memory/cpu request and limits, Ported PR fabric8io/fabric8-maven-plugin#1772 ### 0.2.0 (05-03-2020) * Fix #71: script to extract changelog information for notifications diff --git a/jkube-kit/common/src/main/java/org/eclipse/jkube/kit/common/util/KubernetesHelper.java b/jkube-kit/common/src/main/java/org/eclipse/jkube/kit/common/util/KubernetesHelper.java index afeb687ed5..6c102f35ed 100644 --- a/jkube-kit/common/src/main/java/org/eclipse/jkube/kit/common/util/KubernetesHelper.java +++ b/jkube-kit/common/src/main/java/org/eclipse/jkube/kit/common/util/KubernetesHelper.java @@ -28,6 +28,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedList; @@ -37,6 +38,7 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.CountDownLatch; +import java.util.regex.Pattern; import io.fabric8.kubernetes.api.model.Config; import io.fabric8.kubernetes.api.model.Container; @@ -58,6 +60,7 @@ import io.fabric8.kubernetes.api.model.PodList; import io.fabric8.kubernetes.api.model.PodSpec; import io.fabric8.kubernetes.api.model.PodStatus; +import io.fabric8.kubernetes.api.model.Quantity; import io.fabric8.kubernetes.api.model.ReplicationController; import io.fabric8.kubernetes.api.model.ReplicationControllerSpec; import io.fabric8.kubernetes.api.model.apps.DaemonSet; @@ -84,6 +87,7 @@ import io.fabric8.openshift.api.model.DeploymentConfigSpec; import io.fabric8.openshift.api.model.Template; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.eclipse.jkube.kit.common.KitLogger; import org.eclipse.jkube.kit.common.ResourceFileType; @@ -94,6 +98,10 @@ */ public class KubernetesHelper { protected static final String DATE_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ssX"; + private static final String FILENAME_PATTERN_REGEX = "^(?.*?)(-(?[^-]+))?\\.(?yaml|yml|json)$"; + private static final String PROFILES_PATTERN_REGEX = "^profiles?\\.ya?ml$"; + private static final Pattern FILENAME_PATTERN = Pattern.compile(FILENAME_PATTERN_REGEX); + private static final Pattern EXCLUDE_PATTERN = Pattern.compile(PROFILES_PATTERN_REGEX); /** * Validates that the given value is valid according to the kubernetes ID parsing rules, throwing an exception if not. @@ -828,5 +836,74 @@ public static boolean removeEnvVar(List envVarList, String name) { } return removed; } + + + /** + * Get a specific resource fragment ending with some suffix + * + * @param resourceDirFinal resource directory + * @param remotes list remote fragments if provided + * @param resourceNameSuffix resource name suffix + * @param log log object + * @return file if present or null + */ + public static File getResourceFragmentFromSource(File resourceDirFinal, List remotes, String resourceNameSuffix, KitLogger log) { + if (resourceDirFinal != null) { + File[] resourceFiles = listResourceFragments(resourceDirFinal, remotes, log); + + if (resourceFiles != null) { + for (File file : resourceFiles) { + if (file.getName().endsWith(resourceNameSuffix)) { + return file; + } + } + } + } + return null; + } + + /** + * Get requests or limit objects from string hashmaps + * + * @param quantity hashmap of strings + * @return hashmap of string to quantity + */ + public static Map getQuantityFromString(Map quantity) { + Map stringQuantityMap = new HashMap<>(); + if (quantity != null && !quantity.isEmpty()) { + for (Map.Entry entry : quantity.entrySet()) { + stringQuantityMap.put(entry.getKey(), new Quantity(entry.getValue())); + } + } + return stringQuantityMap; + } + + public static File[] listResourceFragments(File localResourceDir, List remotes, KitLogger log) { + File[] resourceFiles = listResourceFragments(localResourceDir); + + if(remotes != null) { + File[] remoteResourceFiles = listRemoteResourceFragments(remotes, log); + if (remoteResourceFiles.length > 0) { + resourceFiles = ArrayUtils.addAll(resourceFiles, remoteResourceFiles); + } + } + return resourceFiles; + } + + private static File[] listResourceFragments(File resourceDir) { + return resourceDir.listFiles((File dir, String name) -> FILENAME_PATTERN.matcher(name).matches() && !EXCLUDE_PATTERN.matcher(name).matches()); + } + + private static File[] listRemoteResourceFragments(List remotes, KitLogger log) { + if (remotes != null && !remotes.isEmpty()) { + final File remoteResources = FileUtil.createTempDirectory(); + FileUtil.downloadRemotes(remoteResources, remotes, log); + + if (remoteResources.isDirectory()) { + return remoteResources.listFiles(); + } + } + return new File[0]; + } } diff --git a/jkube-kit/common/src/test/java/org/eclipse/jkube/kit/common/util/KubernetesHelperTest.java b/jkube-kit/common/src/test/java/org/eclipse/jkube/kit/common/util/KubernetesHelperTest.java new file mode 100644 index 0000000000..4a3ab374cb --- /dev/null +++ b/jkube-kit/common/src/test/java/org/eclipse/jkube/kit/common/util/KubernetesHelperTest.java @@ -0,0 +1,56 @@ +/** + * Copyright (c) 2019 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at: + * + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.jkube.kit.common.util; + +import mockit.Mocked; +import org.eclipse.jkube.kit.common.KitLogger; +import org.junit.Test; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class KubernetesHelperTest { + @Mocked + KitLogger logger; + + @Test + public void testListResourceFragments() { + File localResourceDir = new File(getClass().getResource("/util/fragments").getPath()); + + assertLocalFragments(KubernetesHelper.listResourceFragments(localResourceDir, null, logger), 2); + } + + @Test + public void testResourceFragmentsWithRemotes() { + List remoteStrList = new ArrayList<>(); + + remoteStrList.add("https://gist.githubusercontent.com/lordofthejars/ac2823cec7831697d09444bbaa76cd50/raw/e4b43f1b6494766dfc635b5959af7730c1a58a93/deployment.yaml"); + remoteStrList.add("https://gist.githubusercontent.com/rohanKanojia/c4ac4ae5533f0bf0dd77d13c905face7/raw/8a7de1e27c1f437c1ccbd186ed247efd967953ee/sa.yml"); + File localResourceDir = new File(getClass().getResource("/util/fragments").getPath()); + + File[] fragments = KubernetesHelper.listResourceFragments(localResourceDir, remoteStrList, logger); + assertLocalFragments(fragments, 4); + assertEquals("sa.yml", fragments[2].getName()); + assertEquals("deployment.yaml", fragments[3].getName()); + } + + private void assertLocalFragments(File[] fragments, int expectedSize) { + assertEquals(expectedSize, fragments.length); + assertEquals("deployment.yml", fragments[0].getName()); + assertEquals("service.yml", fragments[1].getName()); + } +} diff --git a/jkube-kit/common/src/test/resources/util/fragments/deployment.yml b/jkube-kit/common/src/test/resources/util/fragments/deployment.yml new file mode 100644 index 0000000000..ac2559135b --- /dev/null +++ b/jkube-kit/common/src/test/resources/util/fragments/deployment.yml @@ -0,0 +1,35 @@ +# +# Copyright (c) 2019 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at: +# +# https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +# Contributors: +# Red Hat, Inc. - initial API and implementation +# + +spec: + replicas: 1 + template: + spec: + volumes: + - name: config + gitRepo: + repository: 'https://github.com/jstrachan/sample-springboot-config.git' + revision: 667ee4db6bc842b127825351e5c9bae5a4fb2147 + directory: . + containers: + - volumeMounts: + - name: config + mountPath: /app/config + env: + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + serviceAccount: ribbon diff --git a/jkube-kit/common/src/test/resources/util/fragments/service.yml b/jkube-kit/common/src/test/resources/util/fragments/service.yml new file mode 100644 index 0000000000..7f33538400 --- /dev/null +++ b/jkube-kit/common/src/test/resources/util/fragments/service.yml @@ -0,0 +1,19 @@ +# +# Copyright (c) 2019 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at: +# +# https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +# Contributors: +# Red Hat, Inc. - initial API and implementation +# + +metadata: + annotations: + api.service.kubernetes.io/path: /hello +spec: + type: LoadBalancer \ No newline at end of file diff --git a/jkube-kit/config/resource/src/main/java/org/eclipse/jkube/kit/config/resource/OpenshiftBuildConfig.java b/jkube-kit/config/resource/src/main/java/org/eclipse/jkube/kit/config/resource/OpenshiftBuildConfig.java new file mode 100644 index 0000000000..33bb9326d3 --- /dev/null +++ b/jkube-kit/config/resource/src/main/java/org/eclipse/jkube/kit/config/resource/OpenshiftBuildConfig.java @@ -0,0 +1,66 @@ +/** + * Copyright (c) 2019 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at: + * + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.jkube.kit.config.resource; + +import java.util.Map; + +public class OpenshiftBuildConfig { + private Map limits; + private Map requests; + + public Map getRequests() { + return requests; + } + + public void setRequests(Map requests) { + this.requests = requests; + } + + public Map getLimits() { + return limits; + } + + public void setLimits(Map resourceLimits) { + this.limits = resourceLimits; + } + + public static class Builder { + private OpenshiftBuildConfig openshiftBuildConfig; + + public Builder() { + this.openshiftBuildConfig = new OpenshiftBuildConfig(); + } + + public Builder(OpenshiftBuildConfig openshiftBuildConfig) { + if (openshiftBuildConfig != null) { + this.openshiftBuildConfig.limits = openshiftBuildConfig.limits; + this.openshiftBuildConfig.requests = openshiftBuildConfig.requests; + } + } + + public Builder limits(Map limits) { + this.openshiftBuildConfig.limits = limits; + return this; + } + + public Builder requests(Map requests) { + this.openshiftBuildConfig.requests = requests; + return this; + } + + public OpenshiftBuildConfig build() { + return this.openshiftBuildConfig; + } + } +} diff --git a/jkube-kit/config/resource/src/main/java/org/eclipse/jkube/kit/config/resource/ResourceConfig.java b/jkube-kit/config/resource/src/main/java/org/eclipse/jkube/kit/config/resource/ResourceConfig.java index 2a64dbf10f..e17da74fc7 100644 --- a/jkube-kit/config/resource/src/main/java/org/eclipse/jkube/kit/config/resource/ResourceConfig.java +++ b/jkube-kit/config/resource/src/main/java/org/eclipse/jkube/kit/config/resource/ResourceConfig.java @@ -71,6 +71,8 @@ public class ResourceConfig { private List ingressRules; + private OpenshiftBuildConfig openshiftBuildConfig; + public Optional> getEnv() { return Optional.ofNullable(env); } @@ -149,6 +151,10 @@ public List getRemotes() { public List getIngressRules() { return ingressRules; } + public OpenshiftBuildConfig getOpenshiftBuildConfig() { + return openshiftBuildConfig; + } + // ============================================================================================= public static class Builder { @@ -250,6 +256,11 @@ public Builder withCustomResourceDefinitions(List customResourceDefiniti return this; } + public Builder withOpenshiftBuildConfig(OpenshiftBuildConfig openshiftBuildConfig) { + config.openshiftBuildConfig = openshiftBuildConfig; + return this; + } + public ResourceConfig build() { return config; } diff --git a/jkube-kit/resource/service/src/main/java/org/eclipse/jkube/kit/config/service/BuildService.java b/jkube-kit/resource/service/src/main/java/org/eclipse/jkube/kit/config/service/BuildService.java index caa71e2a02..5534c9e339 100644 --- a/jkube-kit/resource/service/src/main/java/org/eclipse/jkube/kit/config/service/BuildService.java +++ b/jkube-kit/resource/service/src/main/java/org/eclipse/jkube/kit/config/service/BuildService.java @@ -20,6 +20,7 @@ import org.eclipse.jkube.kit.build.service.docker.helper.Task; import org.eclipse.jkube.kit.config.image.build.OpenShiftBuildStrategy; import org.eclipse.jkube.kit.config.resource.BuildRecreateMode; +import org.eclipse.jkube.kit.config.resource.ResourceConfig; import java.io.File; @@ -71,6 +72,10 @@ class BuildServiceConfig { private boolean s2iImageStreamLookupPolicyLocal; + private ResourceConfig resourceConfig; + + private File resourceDir; + public BuildServiceConfig() { } @@ -126,6 +131,14 @@ public void attachArtifact(String classifier, File destFile) { } } + public ResourceConfig getResourceConfig() { + return resourceConfig; + } + + public File getResourceDir() { + return resourceDir; + } + public static class Builder { private BuildServiceConfig config; @@ -197,6 +210,16 @@ public Builder imagePullManager(ImagePullManager imagePullManager) { return this; } + public Builder resourceConfig(ResourceConfig resourceConfig) { + config.resourceConfig = resourceConfig; + return this; + } + + public Builder resourceDir(File resourceDir) { + config.resourceDir = resourceDir; + return this; + } + public BuildServiceConfig build() { return config; } diff --git a/jkube-kit/resource/service/src/main/java/org/eclipse/jkube/kit/config/service/openshift/OpenshiftBuildService.java b/jkube-kit/resource/service/src/main/java/org/eclipse/jkube/kit/config/service/openshift/OpenshiftBuildService.java index 2db3fe2f31..b4c24ad33a 100644 --- a/jkube-kit/resource/service/src/main/java/org/eclipse/jkube/kit/config/service/openshift/OpenshiftBuildService.java +++ b/jkube-kit/resource/service/src/main/java/org/eclipse/jkube/kit/config/service/openshift/OpenshiftBuildService.java @@ -21,6 +21,7 @@ import io.fabric8.kubernetes.api.model.LocalObjectReferenceBuilder; import io.fabric8.kubernetes.api.model.ObjectReference; import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.Quantity; import io.fabric8.kubernetes.api.model.Status; import io.fabric8.kubernetes.client.KubernetesClientException; import io.fabric8.kubernetes.client.Watch; @@ -29,6 +30,7 @@ import io.fabric8.openshift.api.model.Build; import io.fabric8.openshift.api.model.BuildConfig; import io.fabric8.openshift.api.model.BuildConfigSpec; +import io.fabric8.openshift.api.model.BuildConfigSpecBuilder; import io.fabric8.openshift.api.model.BuildOutput; import io.fabric8.openshift.api.model.BuildOutputBuilder; import io.fabric8.openshift.api.model.BuildSource; @@ -78,6 +80,7 @@ public class OpenshiftBuildService implements BuildService { private static final String DEFAULT_S2I_BUILD_SUFFIX = "-s2i"; + public static final String DEFAULT_S2I_SOURCE_TYPE = "Binary"; private final OpenShiftClient client; private final KitLogger log; @@ -193,7 +196,7 @@ public void postProcess(BuildServiceConfig config) { config.attachArtifact("is", getImageStreamFile(config)); } - private String updateOrCreateBuildConfig(BuildServiceConfig config, OpenShiftClient client, KubernetesListBuilder builder, ImageConfiguration imageConfig, String openshiftPullSecret) { + protected String updateOrCreateBuildConfig(BuildServiceConfig config, OpenShiftClient client, KubernetesListBuilder builder, ImageConfiguration imageConfig, String openshiftPullSecret) { ImageName imageName = new ImageName(imageConfig.getName()); String buildName = getS2IBuildName(config, imageName); String imageStreamName = getImageStreamName(imageName); @@ -245,19 +248,49 @@ private BuildConfigSpec getBuildConfigSpec(BuildConfig buildConfig) { return spec; } + private BuildConfigSpec getBuildConfigSpec(BuildStrategy buildStrategyResource, BuildOutput buildOutput) { + BuildConfigSpecBuilder specBuilder = null; + + // Check for BuildConfig resource fragment + File buildConfigResourceFragment = KubernetesHelper.getResourceFragmentFromSource(config.getResourceDir(), config.getResourceConfig() != null ? config.getResourceConfig().getRemotes() : null, "buildconfig.yml", log); + if (buildConfigResourceFragment != null) { + BuildConfig buildConfigFragment = client.buildConfigs().load(buildConfigResourceFragment).get(); + specBuilder = new BuildConfigSpecBuilder(buildConfigFragment.getSpec()); + } else { + specBuilder = new BuildConfigSpecBuilder(); + } + + if (specBuilder.buildSource() == null) { + specBuilder.withNewSource() + .withType(DEFAULT_S2I_SOURCE_TYPE) + .endSource(); + } + + if (specBuilder.buildStrategy() == null) { + specBuilder.withStrategy(buildStrategyResource); + } + + if (specBuilder.buildOutput() == null) { + specBuilder.withOutput(buildOutput); + } + + Map> requestsLimitsMap = getRequestsAndLimits(); + if (requestsLimitsMap.containsKey("requests")) { + specBuilder.editOrNewResources().addToRequests(requestsLimitsMap.get("requests")).endResources(); + } + if (requestsLimitsMap.containsKey("limits")) { + specBuilder.editOrNewResources().addToLimits(requestsLimitsMap.get("limits")).endResources(); + } + return specBuilder.build(); + } + private String createBuildConfig(KubernetesListBuilder builder, String buildName, BuildStrategy buildStrategyResource, BuildOutput buildOutput) { log.info("Creating BuildServiceConfig %s for %s build", buildName, buildStrategyResource.getType()); builder.addNewBuildConfigItem() .withNewMetadata() .withName(buildName) .endMetadata() - .withNewSpec() - .withNewSource() - .withType("Binary") - .endSource() - .withStrategy(buildStrategyResource) - .withOutput(buildOutput) - .endSpec() + .withSpec(getBuildConfigSpec(buildStrategyResource, buildOutput)) .endBuildConfigItem(); return buildName; } @@ -719,4 +752,19 @@ private String getMapValueWithDefault(Map map, String field, Str return value != null ? value : defaultValue; } + private Map> getRequestsAndLimits() { + Map> keyToQuantityMap = new HashMap<>(); + if (config.getResourceConfig() != null && config.getResourceConfig().getOpenshiftBuildConfig() != null) { + Map limits = KubernetesHelper.getQuantityFromString(config.getResourceConfig().getOpenshiftBuildConfig().getLimits()); + if (!limits.isEmpty()) { + keyToQuantityMap.put("limits", limits); + } + Map requests = KubernetesHelper.getQuantityFromString(config.getResourceConfig().getOpenshiftBuildConfig().getRequests()); + if (!requests.isEmpty()) { + keyToQuantityMap.put("requests", requests); + } + } + return keyToQuantityMap; + } + } diff --git a/jkube-kit/resource/service/src/test/java/org/eclipse/jkube/kit/config/service/openshift/OpenshiftBuildServiceTest.java b/jkube-kit/resource/service/src/test/java/org/eclipse/jkube/kit/config/service/openshift/OpenshiftBuildServiceTest.java index 00e127fbb7..28642b3688 100644 --- a/jkube-kit/resource/service/src/test/java/org/eclipse/jkube/kit/config/service/openshift/OpenshiftBuildServiceTest.java +++ b/jkube-kit/resource/service/src/test/java/org/eclipse/jkube/kit/config/service/openshift/OpenshiftBuildServiceTest.java @@ -41,6 +41,8 @@ import org.eclipse.jkube.kit.common.KitLogger; import org.eclipse.jkube.kit.config.image.build.OpenShiftBuildStrategy; import org.eclipse.jkube.kit.config.resource.BuildRecreateMode; +import org.eclipse.jkube.kit.config.resource.OpenshiftBuildConfig; +import org.eclipse.jkube.kit.config.resource.ResourceConfig; import org.eclipse.jkube.kit.config.service.BuildService; import org.eclipse.jkube.kit.config.service.JKubeServiceException; import mockit.Expectations; @@ -56,15 +58,16 @@ import java.io.FileReader; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; - public class OpenshiftBuildServiceTest { private static final Logger LOG = LoggerFactory.getLogger(OpenshiftBuildServiceTest.class); @@ -452,6 +455,40 @@ public void checkTarPackageSecret() throws Exception { }); } + @Test + public void testBuildConfigResourceConfig() throws Exception { + retryInMockServer(() -> { + Map limitsMap = new HashMap<>(); + limitsMap.put("cpu", "100m"); + limitsMap.put("memory", "256Mi"); + + BuildService.BuildServiceConfig config = defaultConfig + .resourceConfig(new ResourceConfig.Builder() + .withOpenshiftBuildConfig(new OpenshiftBuildConfig.Builder().limits(limitsMap).build()).build()).build(); + OpenShiftMockServer mockServer = new OpenShiftMockServer(); + + OpenShiftClient client = mockServer.createOpenShiftClient(); + final OpenshiftBuildService service = new OpenshiftBuildService(client, logger, dockerServiceHub, config); + + ImageConfiguration imageWithEnv = new ImageConfiguration.Builder(image) + .buildConfig(new JKubeBuildConfiguration.Builder(image.getBuildConfiguration()) + .env(Collections.singletonMap("FOO", "BAR")) + .build() + ).build(); + + KubernetesListBuilder builder = new KubernetesListBuilder(); + service.createBuildArchive(imageWithEnv); + service.updateOrCreateBuildConfig(config, client, builder, imageWithEnv, null); + BuildConfig buildConfig = (BuildConfig) builder.buildFirstItem(); + assertNotNull(buildConfig); + assertNotNull(buildConfig.getSpec().getResources()); + assertEquals("256", buildConfig.getSpec().getResources().getLimits().get("memory").getAmount()); + assertEquals("Mi", buildConfig.getSpec().getResources().getLimits().get("memory").getFormat()); + assertEquals("100", buildConfig.getSpec().getResources().getLimits().get("cpu").getAmount()); + assertEquals("m", buildConfig.getSpec().getResources().getLimits().get("cpu").getFormat()); + }); + } + @FunctionalInterface private interface MockServerRetryable { void run() throws JKubeServiceException, IOException;