Skip to content

Commit

Permalink
Merge pull request #1927 from sahibamittal/Issue-1903-osv-ecosystem
Browse files Browse the repository at this point in the history
Issue 1903 : removed ecosystem hardcoding
  • Loading branch information
nscuro committed Sep 7, 2022
2 parents 7685a29 + 984f29a commit e9304da
Show file tree
Hide file tree
Showing 8 changed files with 211 additions and 70 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public enum ConfigPropertyConstants {
VULNERABILITY_SOURCE_NVD_FEEDS_URL("vuln-source", "nvd.feeds.url", "https://nvd.nist.gov/feeds", PropertyType.URL, "A base URL pointing to the hostname and path of the NVD feeds"),
VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ENABLED("vuln-source", "github.advisories.enabled", "false", PropertyType.BOOLEAN, "Flag to enable/disable GitHub Advisories"),
VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ACCESS_TOKEN("vuln-source", "github.advisories.access.token", null, PropertyType.STRING, "The access token used for GitHub API authentication"),
VULNERABILITY_SOURCE_GOOGLE_OSV_ENABLED("vuln-source", "google.osv.enabled", "false", PropertyType.BOOLEAN, "Flag to enable/disable Google OSV"),
VULNERABILITY_SOURCE_GOOGLE_OSV_ENABLED("vuln-source", "google.osv.enabled", null, PropertyType.STRING, "List of enabled ecosystems to mirror OSV"),
VULNERABILITY_SOURCE_EPSS_ENABLED("vuln-source", "epss.enabled", "true", PropertyType.BOOLEAN, "Flag to enable/disable Exploit Prediction Scoring System"),
VULNERABILITY_SOURCE_EPSS_FEEDS_URL("vuln-source", "epss.feeds.url", "https://epss.cyentia.com", PropertyType.URL, "A base URL pointing to the hostname and path of the EPSS feeds"),
ACCEPT_ARTIFACT_CYCLONEDX("artifact", "cyclonedx.enabled", "true", PropertyType.BOOLEAN, "Flag to enable/disable the systems ability to accept CycloneDX uploads"),
Expand Down
33 changes: 0 additions & 33 deletions src/main/java/org/dependencytrack/parser/osv/model/Ecosystem.java

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,16 @@

import alpine.common.logging.Logger;
import alpine.event.framework.Event;
import alpine.model.ConfigProperty;
import alpine.model.ManagedUser;
import alpine.model.Permission;
import alpine.model.Team;
import alpine.server.auth.PasswordService;
import org.apache.commons.io.FileUtils;
import org.dependencytrack.RequirementsVerifier;
import org.dependencytrack.auth.Permissions;
import org.dependencytrack.event.IndexEvent;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.ConfigPropertyConstants;
import org.dependencytrack.model.License;
import org.dependencytrack.model.NotificationPublisher;
import org.dependencytrack.model.Project;
import org.dependencytrack.model.RepositoryType;
import org.dependencytrack.model.Vulnerability;
Expand All @@ -45,15 +42,10 @@

import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import java.io.File;
import java.io.IOException;
import java.net.URLDecoder;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;

import static java.nio.charset.StandardCharsets.UTF_8;

/**
* Creates default objects on an empty database.
*
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/*
* This file is part of Dependency-Track.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* Copyright (c) Steve Springett. All Rights Reserved.
*/
package org.dependencytrack.resources.v1;

import alpine.server.auth.PermissionRequired;
import alpine.server.resources.AlpineResource;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
import io.swagger.annotations.Authorization;
import org.dependencytrack.auth.Permissions;
import org.dependencytrack.tasks.OsvDownloadTask;

import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.util.List;

@Path("/v1/integration/osv/ecosystem")
@Api(value = "ecosystem", authorizations = @Authorization(value = "X-Api-Key"))
public class OsvEcosytemResource extends AlpineResource {

@GET
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(
value = "Returns a list of all ecosystems in OSV",
response = String.class,
responseContainer = "List"
)
@ApiResponses(value = {
@ApiResponse(code = 401, message = "Unauthorized")
})
@PermissionRequired(Permissions.Constants.SYSTEM_CONFIGURATION)
public Response getAllEcosystems() {
final List<String> ecosystems = OsvDownloadTask.getEcosystems();
return Response.ok(ecosystems).build();
}
}
92 changes: 65 additions & 27 deletions src/main/java/org/dependencytrack/tasks/OsvDownloadTask.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.dependencytrack.model.VulnerableSoftware;
import org.dependencytrack.parser.common.resolver.CweResolver;
import org.dependencytrack.parser.osv.OsvAdvisoryParser;
import org.dependencytrack.parser.osv.model.Ecosystem;
import org.dependencytrack.parser.osv.model.OsvAdvisory;
import org.dependencytrack.parser.osv.model.OsvAffectedPackage;
import org.dependencytrack.persistence.QueryManager;
Expand All @@ -39,56 +38,70 @@
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Scanner;
import java.util.Arrays;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GOOGLE_OSV_ENABLED;
import static org.dependencytrack.model.Severity.getSeverityByLevel;
import static org.dependencytrack.util.VulnerabilityUtil.*;
import static org.dependencytrack.util.VulnerabilityUtil.normalizedCvssV3Score;
import static org.dependencytrack.util.VulnerabilityUtil.normalizedCvssV2Score;

public class OsvDownloadTask implements LoggableSubscriber {

private static final Logger LOGGER = Logger.getLogger(OsvDownloadTask.class);

private static final String OSV_BASE_URL = "https://osv-vulnerabilities.storage.googleapis.com/";
private final boolean isEnabled;
private HttpUriRequest request;
private String ecosystemConfig;
private List<String> ecosystems;

public List<String> getEnabledEcosystems() {
return this.ecosystems;
}

public OsvDownloadTask() {
try (final QueryManager qm = new QueryManager()) {
final ConfigProperty enabled = qm.getConfigProperty(VULNERABILITY_SOURCE_GOOGLE_OSV_ENABLED.getGroupName(), VULNERABILITY_SOURCE_GOOGLE_OSV_ENABLED.getPropertyName());
this.isEnabled = enabled != null && Boolean.valueOf(enabled.getPropertyValue());
if (enabled != null) {
this.ecosystemConfig = enabled.getPropertyValue();
if (this.ecosystemConfig != null) {
ecosystems = Arrays.stream(this.ecosystemConfig.split(";")).map(String::trim).toList();
}
}
}
}

@Override
public void inform(Event e) {

if (e instanceof OsvMirrorEvent && this.isEnabled) {

for (Ecosystem ecosystem : Ecosystem.values()) {
LOGGER.info("Updating datasource with Google OSV advisories for ecosystem " + ecosystem.getValue());
try {
String url = "https://osv-vulnerabilities.storage.googleapis.com/"
+ URLEncoder.encode(ecosystem.getValue(), StandardCharsets.UTF_8.toString()).replace("+", "%20")
+ "/all.zip";
request = new HttpGet(url);
try (final CloseableHttpResponse response = HttpClientPool.getClient().execute(request)) {
final StatusLine status = response.getStatusLine();
if (status.getStatusCode() == 200) {
try (InputStream in = response.getEntity().getContent();
ZipInputStream zipInput = new ZipInputStream(in)) {
unzipFolder(zipInput);
if (e instanceof OsvMirrorEvent) {

if(this.ecosystems != null && !this.ecosystems.isEmpty()) {
for (String ecosystem : this.ecosystems) {
LOGGER.info("Updating datasource with Google OSV advisories for ecosystem " + ecosystem);
try {
String url = OSV_BASE_URL + URLEncoder.encode(ecosystem, StandardCharsets.UTF_8.toString()).replace("+", "%20")
+ "/all.zip";
HttpUriRequest request = new HttpGet(url);
try (final CloseableHttpResponse response = HttpClientPool.getClient().execute(request)) {
final StatusLine status = response.getStatusLine();
if (status.getStatusCode() == 200) {
try (InputStream in = response.getEntity().getContent();
ZipInputStream zipInput = new ZipInputStream(in)) {
unzipFolder(zipInput);
}
} else {
LOGGER.error("Download failed : " + status.getStatusCode() + ": " + status.getReasonPhrase());
}
} else {
LOGGER.error("Download failed " + status.getStatusCode() + ": " + status.getReasonPhrase() + url);
} catch (Exception ex) {
LOGGER.error("Exception while executing Http client request", ex);
}
} catch (Exception ex) {
LOGGER.error("Exception while executing Http client request", ex);
} catch (UnsupportedEncodingException ex) {
LOGGER.error("Exception while encoding URL for ecosystem " + ecosystem);
}
} catch (UnsupportedEncodingException ex) {
LOGGER.error("Exception while encoding URL for ecosystem " + ecosystem.getValue());
}
} else {
LOGGER.info("Google OSV mirroring is disabled. No ecosystem selected.");
}
}
}
Expand Down Expand Up @@ -297,4 +310,29 @@ private boolean isVulnerabilitySourceClashingWithGithubOrNvd(String source) {
return Vulnerability.Source.GITHUB.toString().equals(source)
|| Vulnerability.Source.NVD.toString().equals(source);
}

public static List<String> getEcosystems() {
ArrayList<String> ecosystems = new ArrayList<>();
String url = OSV_BASE_URL + "ecosystems.txt";
HttpUriRequest request = new HttpGet(url);
try (final CloseableHttpResponse response = HttpClientPool.getClient().execute(request)) {
final StatusLine status = response.getStatusLine();
if (status.getStatusCode() == 200) {
try (InputStream in = response.getEntity().getContent();
Scanner scanner = new Scanner(in, StandardCharsets.UTF_8.name())) {
while (scanner.hasNextLine()) {
final String line = scanner.nextLine();
if(!line.isBlank()) {
ecosystems.add(line.trim());
}
}
}
} else {
LOGGER.error("Ecosystem download failed : " + status.getStatusCode() + ": " + status.getReasonPhrase());
}
} catch (Exception ex) {
LOGGER.error("Exception while executing Http request for ecosystems", ex);
}
return ecosystems;
}
}
1 change: 1 addition & 0 deletions src/test/java/org/dependencytrack/ResourceTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ public abstract class ResourceTest extends JerseyTest {
protected final String V1_NOTIFICATION_RULE = "/v1/notification/rule";
protected final String V1_OIDC = "/v1/oidc";
protected final String V1_PERMISSION = "/v1/permission";
protected final String V1_OSV_ECOSYSTEM = "/v1/integration/osv/ecosystem";
protected final String V1_POLICY = "/v1/policy";
protected final String V1_POLICY_VIOLATION = "/v1/violation";
protected final String V1_PROJECT = "/v1/project";
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
* This file is part of Dependency-Track.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* Copyright (c) Steve Springett. All Rights Reserved.
*/
package org.dependencytrack.resources.v1;

import alpine.server.filters.ApiFilter;
import alpine.server.filters.AuthenticationFilter;
import org.dependencytrack.ResourceTest;
import org.glassfish.jersey.server.ResourceConfig;
import org.glassfish.jersey.servlet.ServletContainer;
import org.glassfish.jersey.test.DeploymentContext;
import org.glassfish.jersey.test.ServletDeploymentContext;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

import javax.json.JsonArray;
import javax.ws.rs.core.Response;

public class OsvEcosystemResourceTest extends ResourceTest {

@Override
protected DeploymentContext configureDeployment() {
return ServletDeploymentContext.forServlet(new ServletContainer(
new ResourceConfig(OsvEcosytemResource.class)
.register(ApiFilter.class)
.register(AuthenticationFilter.class)))
.build();
}

@Before
public void before() throws Exception {
super.before();
}

@Test
public void getAllEcosystemsTest() {
Response response = target(V1_OSV_ECOSYSTEM).request()
.header(X_API_KEY, apiKey)
.get(Response.class);
Assert.assertEquals(200, response.getStatus(), 0);
Assert.assertNull(response.getHeaderString(TOTAL_COUNT_HEADER));
JsonArray json = parseJsonArray(response);
Assert.assertNotNull(json);
Assert.assertFalse(json.isEmpty());
}
}
Loading

0 comments on commit e9304da

Please sign in to comment.