Skip to content

Commit

Permalink
improve maintainability
Browse files Browse the repository at this point in the history
  • Loading branch information
@Jonathan-Oliveira
Jonathan-Oliveira committed May 7, 2022
1 parent 7d44480 commit 8564f5e
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 175 deletions.
138 changes: 39 additions & 99 deletions src/main/java/org/jabref/logic/importer/fetcher/BiodiversityLibrary.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.stream.IntStream;

import org.jabref.logic.importer.FetcherException;
import org.jabref.logic.importer.ParseException;
Expand All @@ -16,6 +17,8 @@
import org.jabref.logic.importer.util.JsonReader;
import org.jabref.logic.net.URLDownload;
import org.jabref.logic.util.BuildInfo;
import org.jabref.model.entry.Author;
import org.jabref.model.entry.AuthorList;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.field.StandardField;
import org.jabref.model.entry.types.StandardEntryType;
Expand Down Expand Up @@ -80,46 +83,28 @@ public JSONObject getDetails(URL url) throws IOException {
return new JSONObject(response).getJSONArray("Result").getJSONObject(0);
}

public BibEntry getMostDetails(JSONObject item, BibEntry entry) throws IOException, URISyntaxException { // FixMe ???? Method name ist unfug
public BibEntry mountDetails(JSONObject item, BibEntry entry) throws IOException, URISyntaxException { // FixMe ???? Method name ist unfug
if (item.has("BHLType")) {
if (item.getString("BHLType").equals("Part")) {
URL url = getPartMetadataURL(item.getString("PartID"));
JSONObject itemsDetails = getDetails(url);
if (itemsDetails.has("Language")) {
entry.setField(StandardField.LANGUAGE, itemsDetails.getString("Language"));
}
if (itemsDetails.has("Doi")) {
entry.setField(StandardField.DOI, itemsDetails.getString("Doi"));
}
if (itemsDetails.has("PublisherName")) {
entry.setField(StandardField.PUBLISHER, itemsDetails.getString("PublisherName"));
}
if (itemsDetails.has("Volume") && !entry.hasField(StandardField.VOLUME)) {
entry.setField(StandardField.VOLUME, itemsDetails.getString("Volume"));
}
if (itemsDetails.has("Date") && !entry.hasField(StandardField.DATE) && !entry.hasField(StandardField.YEAR)) {
entry.setField(StandardField.DATE, itemsDetails.getString("Date"));
}
if (itemsDetails.has("PartUrl")) {
entry.setField(StandardField.URL, itemsDetails.getString("PartUrl"));
}
entry.setField(StandardField.LANGUAGE, itemsDetails.optString("Language", ""));

entry.setField(StandardField.DOI, itemsDetails.optString("Doi", ""));

entry.setField(StandardField.PUBLISHER, itemsDetails.optString("PublisherName", ""));
entry.setField(StandardField.DATE, itemsDetails.optString("Date", ""));
entry.setField(StandardField.VOLUME, itemsDetails.optString("Volume", ""));
entry.setField(StandardField.URL, itemsDetails.optString("PartUrl", ""));
}

if (item.getString("BHLType").equals("Item")) {
URL url = getItemMetadataURL(item.getString("ItemID"));
JSONObject itemsDetails = getDetails(url);
if (itemsDetails.has("Sponsor")) {
entry.setField(StandardField.EDITOR, itemsDetails.getString("Sponsor"));
}
if (itemsDetails.has("HoldingInstitution")) {
entry.setField(StandardField.PUBLISHER, itemsDetails.getString("HoldingInstitution"));
}
if (itemsDetails.has("Language")) {
entry.setField(StandardField.LANGUAGE, itemsDetails.getString("Language"));
}
if (itemsDetails.has("ItemUrl")) {
entry.setField(StandardField.URL, itemsDetails.getString("ItemUrl"));
}
entry.setField(StandardField.EDITOR, itemsDetails.optString("Sponsor", ""));
entry.setField(StandardField.PUBLISHER, itemsDetails.optString("HoldingInstitution", ""));
entry.setField(StandardField.LANGUAGE, itemsDetails.optString("Language", ""));
entry.setField(StandardField.URL, itemsDetails.optString("ItemUrl", ""));
if (itemsDetails.has("Date") && !entry.hasField(StandardField.DATE) && !entry.hasField(StandardField.YEAR)) {
entry.setField(StandardField.DATE, itemsDetails.getString("Date"));
}
Expand All @@ -137,65 +122,34 @@ public BibEntry jsonResultToBibEntry(JSONObject item) {
} else {
entry.setType(StandardEntryType.Article);
}
entry.setField(StandardField.TITLE, item.optString("Title", ""));

if (item.has("Title")) {
entry.setField(StandardField.TITLE, item.optString("Title"));
}
entry.setField(StandardField.AUTHOR, toAuthors(item.optJSONArray("Authors")));

if (item.has("Authors")) {
JSONArray authors = item.getJSONArray("Authors");
List<String> authorList = new ArrayList<>();
for (int i = 0; i < authors.length(); i++) {
if (authors.getJSONObject(i).has("Name")) {
authorList.add(authors.getJSONObject(i).getString("Name"));
} else {
LOGGER.debug("Empty author name.");
}
}
entry.setField(StandardField.AUTHOR, String.join(" and ", authorList));
} else {
LOGGER.debug("Empty author name");
}
entry.setField(StandardField.PAGES, item.optString("PageRange", ""));
entry.setField(StandardField.PUBSTATE, item.optString("PublisherPlace", ""));
entry.setField(StandardField.PUBLISHER, item.optString("PublisherName", ""));

if (item.has("PageRange")) {
entry.setField(StandardField.PAGES, item.getString("PageRange"));
} else {
LOGGER.debug("Empty pages number");
}
entry.setField(StandardField.DATE, item.optString("Date", ""));
entry.setField(StandardField.YEAR, item.optString("PublicationDate", ""));
entry.setField(StandardField.JOURNALTITLE, item.optString("ContainerTitle", ""));
entry.setField(StandardField.VOLUME, item.optString("Volume", ""));

if (item.has("PublisherPlace")) {
entry.setField(StandardField.PUBSTATE, item.getString("PublisherPlace"));
} else {
LOGGER.debug("Empty Publisher Place");
}

if (item.has("PublisherName")) {
entry.setField(StandardField.PUBLISHER, item.getString("PublisherName"));
} else {
LOGGER.debug("Empty Publisher Name");
}

if (item.has("Date")) {
entry.setField(StandardField.DATE, item.getString("Date"));
} else if (item.has("PublicationDate")) {
entry.setField(StandardField.YEAR, item.getString("PublicationDate"));
} else {
LOGGER.debug("Empty date");
}

if (item.has("ContainerTitle")) {
entry.setField(StandardField.JOURNALTITLE, item.getString("ContainerTitle"));
} else {
LOGGER.debug("Empty journal name");
}
return entry;
}

if (item.has("Volume")) {
entry.setField(StandardField.VOLUME, item.getString("Volume"));
} else {
LOGGER.debug("Empty volume number");
private String toAuthors(JSONArray authors) {
if (authors == null) {
return "";
}

return entry;
// input: list of { "Name": "Author name,"}
return IntStream.range(0, authors.length())
.mapToObj(authors::getJSONObject)
.map((author) -> new Author(
author.optString("Name", ""), "", "", "", ""))
.collect(AuthorList.collect())
.getAsFirstLastNamesWithAnd();
}

@Override
Expand All @@ -217,7 +171,7 @@ public Parser getParser() {
JSONObject item = items.getJSONObject(i);
BibEntry entry = jsonResultToBibEntry(item);
try {
entry = getMostDetails(item, entry);
entry = mountDetails(item, entry);
} catch (JSONException | IOException | URISyntaxException exception) {
throw new ParseException("Error when parsing entry", exception);
}
Expand All @@ -232,22 +186,8 @@ public Parser getParser() {
public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException {
URIBuilder uriBuilder = new URIBuilder(getBaseURL().toURI());
BiodiversityLibraryTransformer transformer = new BiodiversityLibraryTransformer();
transformer.transformLuceneQuery(luceneQuery).orElse(""); // FixMe: ????? result ignored
uriBuilder.addParameter("op", "PublicationSearchAdvanced");

if (transformer.getAuthor().isPresent()) {
uriBuilder.addParameter("authorname", transformer.getAuthor().get());
}

if (transformer.getTitle().isPresent()) {
uriBuilder.addParameter("title", transformer.getTitle().get());
uriBuilder.addParameter("titleop", "all");
}

if (transformer.getTitle().isEmpty() && transformer.getAuthor().isEmpty()) {
throw new FetcherException("Must add author or title");
}

uriBuilder.addParameter("authorname", transformer.transformLuceneQuery(luceneQuery).orElse(""));
return uriBuilder.build().toURL();
}
}
66 changes: 6 additions & 60 deletions ...n/java/org/jabref/logic/importer/fetcher/transformers/BiodiversityLibraryTransformer.java
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,6 @@
package org.jabref.logic.importer.fetcher.transformers;

import java.util.List;
import java.util.Objects;
import java.util.Optional;

import org.jabref.model.strings.StringUtil;

public class BiodiversityLibraryTransformer extends YearRangeByFilteringQueryTransformer {

private static final List<String> STOP_WORDS = List.of("a", "and", "for", "or", "with");

private String journal;
private String articleNumber;
private String author;
private String title;
public class BiodiversityLibraryTransformer extends AbstractQueryTransformer {

@Override
protected String getLogicalAndOperator() {
Expand All @@ -32,63 +19,22 @@ protected String getLogicalNotOperator() {

@Override
protected String handleAuthor(String author) {
this.author = author;
return StringUtil.quoteStringIfSpaceIsContained(author);
return author;
}

@Override
protected String handleTitle(String title) {
this.title = title;
return StringUtil.quoteStringIfSpaceIsContained(title);
return null;
}

@Override
protected String handleJournal(String journal) {
this.journal = journal;
return StringUtil.quoteStringIfSpaceIsContained(journal);
protected String handleJournal(String journalTitle) {
return null;
}

@Override
protected String handleYear(String year) {
startYear = Math.min(startYear, Integer.parseInt(year));
endYear = Math.max(endYear, Integer.parseInt(year));
return "";
}

@Override
protected Optional<String> handleOtherField(String fieldAsString, String term) {
return switch (fieldAsString) {
case "article_number" -> handleArticleNumber(term);
default -> super.handleOtherField(fieldAsString, term);
};
}

@Override
protected Optional<String> handleUnFieldedTerm(String term) {
if (STOP_WORDS.contains(term)) {
return Optional.empty();
}
return super.handleUnFieldedTerm(term);
return null;
}

private Optional<String> handleArticleNumber(String term) {
articleNumber = term;
return Optional.empty();
}

public Optional<String> getJournal() {
return Objects.isNull(journal) ? Optional.empty() : Optional.of(journal);
}

public Optional<String> getArticleNumber() {
return Objects.isNull(articleNumber) ? Optional.empty() : Optional.of(articleNumber);
}

public Optional<String> getTitle() {
return Objects.isNull(title) ? Optional.empty() : Optional.of(title);
}

public Optional<String> getAuthor() {
return Objects.isNull(author) ? Optional.empty() : Optional.of(author);
}
}
26 changes: 10 additions & 16 deletions src/test/java/org/jabref/logic/importer/fetcher/BiodiversityLibraryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@

import kong.unirest.json.JSONObject;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
Expand Down Expand Up @@ -49,27 +51,19 @@ public void baseURLConstruction() throws MalformedURLException, URISyntaxExcepti

}

@Test
public void getPartMetadaUrl() throws MalformedURLException, URISyntaxException {
String id = "1234";
@ParameterizedTest
@ValueSource(strings = {"1234", "331", "121"})
public void getPartMetadaUrl(String id) throws MalformedURLException, URISyntaxException {

String expected_base = (BASE_URL
.concat("apikey=")
.concat(buildInfo.biodiversityHeritageApiKey)
.concat(RESPONSE_FORMAT)
.concat("&op=GetPartMetadata&pages=f&names=f")
.concat("&id=")
);
String expected = expected_base.concat(id);

assertEquals(expected, fetcher.getPartMetadataURL(id).toString());

id = "4321";
expected = expected_base.concat(id);
assertEquals(expected, fetcher.getPartMetadataURL(id).toString());

id = "331";
expected = expected_base.concat(id);
assertEquals(expected, fetcher.getPartMetadataURL(id).toString());
assertEquals(expected_base.concat(id), fetcher.getPartMetadataURL(id).toString());

}

Expand Down Expand Up @@ -102,7 +96,7 @@ public void jsonResultToBibEntry() {
JSONObject input = new JSONObject("{\n\"BHLType\": \"Part\",\n\"FoundIn\": \"Metadata\",\n\"Volume\": \"3\",\n\"Authors\": [\n{\n\"Name\": \"Dimmock, George,\"\n}\n],\n\"PartUrl\": \"https://www.biodiversitylibrary.org/part/181199\",\n\"PartID\": \"181199\",\n\"Genre\": \"Article\",\n\"Title\": \"The Cocoons of Cionus Scrophulariae\",\n\"ContainerTitle\": \"Psyche.\",\n\"Date\": \"1882\",\n\"PageRange\": \"411--413\"\n}");
BibEntry expect = new BibEntry(StandardEntryType.Article)
.withField(StandardField.TITLE, "The Cocoons of Cionus Scrophulariae")
.withField(StandardField.AUTHOR, "Dimmock, George,")
.withField(StandardField.AUTHOR, "Dimmock, George, ")
.withField(StandardField.PAGES, "411--413")
.withField(StandardField.DATE, "1882")
.withField(StandardField.JOURNALTITLE, "Psyche.")
Expand Down Expand Up @@ -131,7 +125,7 @@ public void jsonResultToBibEntry() {
" }");
expect = new BibEntry(StandardEntryType.Book)
.withField(StandardField.TITLE, "Potatoes : the poor man's own crop : illustrated with plates, showing the decay and disease of the potatoe [sic] : with hints to improve the land and life of the poor man : published to aid the Industrial Marlborough Exhibition")
.withField(StandardField.AUTHOR, "George, George")
.withField(StandardField.AUTHOR, "George, George ")
.withField(StandardField.YEAR, "1861")
.withField(StandardField.PUBSTATE, "Salisbury")
.withField(StandardField.PUBLISHER, "Frederick A. Blake,");
Expand All @@ -158,7 +152,7 @@ public void jsonResultToBibEntry() {
" }");
expect = new BibEntry(StandardEntryType.Book)
.withField(StandardField.TITLE, "The extra cost of producing clean milk.")
.withField(StandardField.AUTHOR, "Whitaker, George M. (George Mason)")
.withField(StandardField.AUTHOR, "Whitaker, George M. (George Mason) ")
.withField(StandardField.YEAR, "1911")
.withField(StandardField.PUBSTATE, "Washington")
.withField(StandardField.PUBLISHER, "Government Prining Office,");
Expand Down

0 comments on commit 8564f5e

Please sign in to comment.