From cd8a19d1080875672e1922c999c1899c78048523 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Tue, 22 Oct 2019 20:04:32 +0200 Subject: [PATCH 01/15] Changed performSearch to default implementation Removes method as it is essentially the same as the default implementation. --- CHANGELOG.md | 1 + .../fetcher/AstrophysicsDataSystem.java | 47 ++----------------- 2 files changed, 6 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 796c8e28ff5..4169fb01b44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `# - We added an option to show the preview as an extra tab in the entry editor (instead of in a split view). [#5244](https://github.com/JabRef/jabref/issues/5244) - A custom Open/LibreOffice jstyle file now requires a layout line for the entry type `default` [#5452](https://github.com/JabRef/jabref/issues/5452) - The entry editor is now open by default when JabRef starts up. [#5460](https://github.com/JabRef/jabref/issues/5460) +- We changed the ADS fetcher to use the new ADS API [#4949](https://github.com/JabRef/jabref/issues/4949) ### Fixed diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index 708039914e3..2ac10652bc9 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -1,13 +1,8 @@ package org.jabref.logic.importer.fetcher; -import java.io.IOException; -import java.io.InputStream; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; -import java.net.URLConnection; -import java.util.Collections; -import java.util.List; import java.util.Objects; import java.util.Optional; @@ -21,29 +16,24 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; -import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; import org.jabref.logic.importer.fileformat.BibtexParser; -import org.jabref.logic.l10n.Localization; -import org.jabref.logic.net.URLDownload; import org.jabref.model.cleanup.FieldFormatterCleanup; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; -import org.jabref.model.strings.StringUtil; import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; /** * Fetches data from the SAO/NASA Astrophysics Data System (http://www.adsabs.harvard.edu/) - * - * Search query-based: http://adsabs.harvard.edu/basic_search.html - * Entry -based: http://adsabs.harvard.edu/abstract_service.html - * - * There is also a new API (https://github.com/adsabs/adsabs-dev-api) but it returns JSON - * (or at least needs multiple calls to get BibTeX, status: September 2016) + *

+ * Search query-based: http://adsabs.harvard.edu/basic_search.html Entry -based: http://adsabs.harvard.edu/abstract_service.html + *

+ * There is also a new API (https://github.com/adsabs/adsabs-dev-api) but it returns JSON (or at least needs multiple + * calls to get BibTeX, status: September 2016) */ public class AstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher { @@ -126,33 +116,6 @@ public Parser getParser() { return new BibtexParser(preferences, new DummyFileUpdateMonitor()); } - @Override - public List performSearch(String query) throws FetcherException { - if (StringUtil.isBlank(query)) { - return Collections.emptyList(); - } - - try { - URLConnection connection = getURLForQuery(query).openConnection(); - connection.setRequestProperty("User-Agent", URLDownload.USER_AGENT); - try (InputStream stream = connection.getInputStream()) { - List fetchedEntries = getParser().parseEntries(stream); - - // Post-cleanup - fetchedEntries.forEach(this::doPostCleanup); - return fetchedEntries; - } catch (IOException e) { - throw new FetcherException("An I/O exception occurred", e); - } - } catch (URISyntaxException | MalformedURLException e) { - throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("An I/O exception occurred", e); - } catch (ParseException e) { - throw new FetcherException("Error occurred when parsing entry", Localization.lang("Error occurred when parsing entry"), e); - } - } - @Override public void doPostCleanup(BibEntry entry) { new FieldFormatterCleanup(StandardField.ABSTRACT, new RemoveBracesFormatter()).cleanup(entry); From bbe261828b0f4556d26324eb308f15a2d772f0ff Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Tue, 22 Oct 2019 23:25:30 +0200 Subject: [PATCH 02/15] Added new ADS fetcher Added a new ADS fetcher that uses the new ADS API, ref #4949 Preserve the old fetcher until the new fetcher is stable/ well tested --- CHANGELOG.md | 2 +- src/main/java/module-info.java | 1 + .../jabref/logic/importer/WebFetchers.java | 4 + .../fetcher/NewAstrophysicsDataSystem.java | 180 ++++++++++++++++++ 4 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 4169fb01b44..ed9808660c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `# - We added an option to show the preview as an extra tab in the entry editor (instead of in a split view). [#5244](https://github.com/JabRef/jabref/issues/5244) - A custom Open/LibreOffice jstyle file now requires a layout line for the entry type `default` [#5452](https://github.com/JabRef/jabref/issues/5452) - The entry editor is now open by default when JabRef starts up. [#5460](https://github.com/JabRef/jabref/issues/5460) -- We changed the ADS fetcher to use the new ADS API [#4949](https://github.com/JabRef/jabref/issues/4949) +- We add a new ADS fetcher to use the new ADS API [#4949](https://github.com/JabRef/jabref/issues/4949) ### Fixed diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index 791e2e50698..bfb3a41fdcc 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -65,4 +65,5 @@ requires de.saxsys.mvvmfx.validation; requires richtextfx; requires unirest.java; + requires json; } diff --git a/src/main/java/org/jabref/logic/importer/WebFetchers.java b/src/main/java/org/jabref/logic/importer/WebFetchers.java index 66319f5e6b4..4161786d0c4 100644 --- a/src/main/java/org/jabref/logic/importer/WebFetchers.java +++ b/src/main/java/org/jabref/logic/importer/WebFetchers.java @@ -27,6 +27,7 @@ import org.jabref.logic.importer.fetcher.LibraryOfCongress; import org.jabref.logic.importer.fetcher.MathSciNet; import org.jabref.logic.importer.fetcher.MedlineFetcher; +import org.jabref.logic.importer.fetcher.NewAstrophysicsDataSystem; import org.jabref.logic.importer.fetcher.OpenAccessDoi; import org.jabref.logic.importer.fetcher.RfcFetcher; import org.jabref.logic.importer.fetcher.ScienceDirect; @@ -88,6 +89,7 @@ public static SortedSet getSearchBasedFetchers(ImportFormatP set.add(new GvkFetcher()); set.add(new MedlineFetcher()); set.add(new AstrophysicsDataSystem(importFormatPreferences)); + set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new MathSciNet(importFormatPreferences)); set.add(new ZbMATH(importFormatPreferences)); set.add(new ACMPortalFetcher(importFormatPreferences)); @@ -108,6 +110,7 @@ public static SortedSet getIdBasedFetchers(ImportFormatPreferenc SortedSet set = new TreeSet<>(Comparator.comparing(WebFetcher::getName)); set.add(new ArXiv(importFormatPreferences)); set.add(new AstrophysicsDataSystem(importFormatPreferences)); + set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new IsbnFetcher(importFormatPreferences)); set.add(new DiVA(importFormatPreferences)); set.add(new DoiFetcher(importFormatPreferences)); @@ -127,6 +130,7 @@ public static SortedSet getIdBasedFetchers(ImportFormatPreferenc public static SortedSet getEntryBasedFetchers(ImportFormatPreferences importFormatPreferences) { SortedSet set = new TreeSet<>(Comparator.comparing(WebFetcher::getName)); set.add(new AstrophysicsDataSystem(importFormatPreferences)); + set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new DoiFetcher(importFormatPreferences)); set.add(new IsbnFetcher(importFormatPreferences)); set.add(new MathSciNet(importFormatPreferences)); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java new file mode 100644 index 00000000000..8d93f6ee4aa --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -0,0 +1,180 @@ +package org.jabref.logic.importer.fetcher; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import org.jabref.logic.help.HelpFile; +import org.jabref.logic.importer.EntryBasedParserFetcher; +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.IdBasedParserFetcher; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.ParseException; +import org.jabref.logic.importer.Parser; +import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fileformat.BibtexParser; +import org.jabref.logic.net.URLDownload; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.strings.StringUtil; +import org.jabref.model.util.DummyFileUpdateMonitor; + +import org.apache.http.client.utils.URIBuilder; +import org.json.JSONObject; + +//TODO Replace Old ADS after the new one is mature + +/** + * Fetches data from the SAO/NASA Astrophysics Data System (http://www.adsabs.harvard.edu/) + *

+ * Search query-based: http://adsabs.harvard.edu/basic_search.html Entry -based: http://adsabs.harvard.edu/abstract_service.html + *

+ * There is also a new API (https://github.com/adsabs/adsabs-dev-api) but it returns JSON (or at least needs multiple + * calls to get BibTeX, status: September 2016) + */ +public class NewAstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher { + + private static String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; + private static String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; + + private static String API_KEY = ""; //TODO Add API Token + + private final ImportFormatPreferences preferences; + + public NewAstrophysicsDataSystem(ImportFormatPreferences preferences) { + this.preferences = Objects.requireNonNull(preferences); + } + + private String buildPostData(String... bibcodes) { + JSONObject obj = new JSONObject(); + obj.put("bibcode", bibcodes); + return obj.toString(); + } + + @Override + public String getName() { + return "New SAO/NASA Astrophysics Data System"; + } + + @Override + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + URIBuilder builder = new URIBuilder(API_SEARCH_URL); + builder.addParameter("q", query); + builder.addParameter("fl", "bibcode"); + return builder.build().toURL(); + } + + @Override + public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedURLException, FetcherException { + StringBuilder stringBuilder = new StringBuilder(); + + Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:" + t); + Optional author = entry.getFieldOrAlias(StandardField.TITLE).map(a -> "author" + a); + + if (title.isPresent()) { + stringBuilder.append(title.get()) + .append(author.map(s -> " OR " + s) + .orElse("")); + } else { + stringBuilder.append(author.orElse("")); + } + String query = stringBuilder.toString().trim(); + + URIBuilder builder = new URIBuilder(API_SEARCH_URL); + builder.addParameter("q", query); + builder.addParameter("fl", "bibcode"); + return builder.build().toURL(); + } + + @Override + public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { + return new URIBuilder(API_EXPORT_URL).build().toURL(); + } + + @Override + public Optional getHelpPage() { + return Optional.of(HelpFile.FETCHER_ADS); + } + + @Override + public Parser getParser() { + return new BibtexParser(preferences, new DummyFileUpdateMonitor()); + } + + @Override + public void doPostCleanup(BibEntry entry) { + + } + + @Override + public List performSearch(BibEntry entry) throws FetcherException { + return Collections.emptyList(); + } + + @Override + public Optional performSearchById(String identifier) throws FetcherException { + + Optional> results = performSearchByIds(identifier); + if (results.isEmpty()) { + return Optional.empty(); + } + + List fetchedEntries = results.get(); + + if (fetchedEntries.size() > 1) { + LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier + + ". We will use the first entry."); + } + + BibEntry entry = fetchedEntries.get(0); + return Optional.of(entry); + } + + private Optional> performSearchByIds(String... identifiers) throws FetcherException { + + long idCount = Arrays.stream(identifiers).filter(identifier -> !StringUtil.isBlank(identifier)).count(); + if (idCount == 0) { + return Optional.empty(); + } + + try { + + String postData = buildPostData(identifiers); + URLDownload download = new URLDownload(getURLForID("")); + download.addHeader("Authorization", "Bearer " + API_KEY); + download.addHeader("ContentType", "application/json"); + download.setPostData(postData); + download.asString(); + String content = download.asString(); + JSONObject obj = new JSONObject(content); + + List fetchedEntries = getParser().parseEntries(obj.getString("export")); + + if (fetchedEntries.isEmpty()) { + return Optional.empty(); + } + // Post-cleanup + fetchedEntries.forEach(this::doPostCleanup); + + return Optional.of(fetchedEntries); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } catch (ParseException e) { + throw new FetcherException("An internal parser error occurred", e); + } + } + + @Override + public List performSearch(String query) throws FetcherException { + + return Collections.emptyList(); + } +} From 167a617828eab8dea6c9fcef94b71b949fd0d87c Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 20:02:29 +0200 Subject: [PATCH 03/15] Added new ADS fetcher Added a new ADS fetcher that uses the new ADS API, ref #4949 Preserve the old fetcher until the new fetcher is stable/ well tested --- .../fetcher/NewAstrophysicsDataSystem.java | 103 +++++++++++++----- 1 file changed, 77 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 8d93f6ee4aa..0ca438eff28 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -4,6 +4,7 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -26,6 +27,8 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; //TODO Replace Old ADS after the new one is mature @@ -40,8 +43,8 @@ */ public class NewAstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher { - private static String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; - private static String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; + private static final String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; + private static final String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; private static String API_KEY = ""; //TODO Add API Token @@ -75,7 +78,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR StringBuilder stringBuilder = new StringBuilder(); Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:" + t); - Optional author = entry.getFieldOrAlias(StandardField.TITLE).map(a -> "author" + a); + Optional author = entry.getFieldOrAlias(StandardField.TITLE).map(a -> "author:" + a); if (title.isPresent()) { stringBuilder.append(title.get()) @@ -89,6 +92,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR URIBuilder builder = new URIBuilder(API_SEARCH_URL); builder.addParameter("q", query); builder.addParameter("fl", "bibcode"); + builder.addParameter("rows", "20"); return builder.build().toURL(); } @@ -114,18 +118,69 @@ public void doPostCleanup(BibEntry entry) { @Override public List performSearch(BibEntry entry) throws FetcherException { - return Collections.emptyList(); + + if (entry.getFieldOrAlias(StandardField.TITLE).isEmpty() && entry.getFieldOrAlias(StandardField.AUTHOR).isEmpty()) { + return Collections.emptyList(); + } + + try { + List bibcodes = fetchBibcodes(getURLForEntry(entry)); + String[] bibcodeArray = new String[bibcodes.size()]; + return performSearchByIds(bibcodes.toArray(bibcodeArray)); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } } @Override - public Optional performSearchById(String identifier) throws FetcherException { + public List performSearch(String query) throws FetcherException { - Optional> results = performSearchByIds(identifier); - if (results.isEmpty()) { - return Optional.empty(); + if (StringUtil.isBlank(query)) { + return Collections.emptyList(); + } + + try { + List bibcodes = fetchBibcodes(getURLForQuery(query)); + String[] bibcodeArray = new String[bibcodes.size()]; + return performSearchByIds(bibcodes.toArray(bibcodeArray)); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } + } + + private List fetchBibcodes(URL url) throws FetcherException { + + try { + URLDownload download = new URLDownload(url); + download.addHeader("Authorization", "Bearer " + API_KEY); + String content = download.asString(); + JSONObject obj = new JSONObject(content); + + try { + JSONArray codes = obj.getJSONObject("response").getJSONArray("docs"); + List bibcodes = new ArrayList<>(); + for (int i = 0; i < codes.length(); i++) { + bibcodes.add(codes.getJSONObject(i).getString("bibcode")); + } + return bibcodes; + } catch (JSONException e) { + return Collections.emptyList(); + } + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); } + } - List fetchedEntries = results.get(); + @Override + public Optional performSearchById(String identifier) throws FetcherException { + List fetchedEntries = performSearchByIds(identifier); + if (fetchedEntries.isEmpty()) { + return Optional.empty(); + } if (fetchedEntries.size() > 1) { LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier @@ -136,33 +191,33 @@ public Optional performSearchById(String identifier) throws FetcherExc return Optional.of(entry); } - private Optional> performSearchByIds(String... identifiers) throws FetcherException { + private List performSearchByIds(String... identifiers) throws FetcherException { long idCount = Arrays.stream(identifiers).filter(identifier -> !StringUtil.isBlank(identifier)).count(); if (idCount == 0) { - return Optional.empty(); + return Collections.emptyList(); } - try { - String postData = buildPostData(identifiers); URLDownload download = new URLDownload(getURLForID("")); download.addHeader("Authorization", "Bearer " + API_KEY); download.addHeader("ContentType", "application/json"); download.setPostData(postData); - download.asString(); String content = download.asString(); JSONObject obj = new JSONObject(content); - List fetchedEntries = getParser().parseEntries(obj.getString("export")); - - if (fetchedEntries.isEmpty()) { - return Optional.empty(); + try { + List fetchedEntries = getParser().parseEntries(obj.optString("export")); + if (fetchedEntries.isEmpty()) { + return Collections.emptyList(); + } + // Post-cleanup + fetchedEntries.forEach(this::doPostCleanup); + + return fetchedEntries; + } catch (JSONException e) { + return Collections.emptyList(); } - // Post-cleanup - fetchedEntries.forEach(this::doPostCleanup); - - return Optional.of(fetchedEntries); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { @@ -172,9 +227,5 @@ private Optional> performSearchByIds(String... identifiers) throw } } - @Override - public List performSearch(String query) throws FetcherException { - return Collections.emptyList(); - } } From 7279e1b24777777c3c982746bb7c3195ea91e82b Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 20:36:04 +0200 Subject: [PATCH 04/15] Added new ADS fetcher Added a new ADS fetcher that uses the new ADS API, ref #4949 Preserve the old fetcher until the new fetcher is stable/ well tested --- .../logic/importer/fetcher/NewAstrophysicsDataSystem.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 0ca438eff28..824cf89c6e2 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -226,6 +226,4 @@ private List performSearchByIds(String... identifiers) throws FetcherE throw new FetcherException("An internal parser error occurred", e); } } - - } From 329d1dd533c70a49f6eea07c1d49527f627318ec Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 21:10:29 +0200 Subject: [PATCH 05/15] added cleanup --- .../fetcher/NewAstrophysicsDataSystem.java | 19 +- .../NewAstrophysicsDataSystemTest.java | 206 ++++++++++++++++++ 2 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 824cf89c6e2..9481aace5bc 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -11,6 +11,11 @@ import java.util.Objects; import java.util.Optional; +import org.jabref.logic.cleanup.MoveFieldCleanup; +import org.jabref.logic.formatter.bibtexfields.ClearFormatter; +import org.jabref.logic.formatter.bibtexfields.NormalizeNamesFormatter; +import org.jabref.logic.formatter.bibtexfields.RemoveBracesFormatter; +import org.jabref.logic.formatter.bibtexfields.RemoveNewlinesFormatter; import org.jabref.logic.help.HelpFile; import org.jabref.logic.importer.EntryBasedParserFetcher; import org.jabref.logic.importer.FetcherException; @@ -21,8 +26,10 @@ import org.jabref.logic.importer.SearchBasedParserFetcher; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.net.URLDownload; +import org.jabref.model.cleanup.FieldFormatterCleanup; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; import org.jabref.model.strings.StringUtil; import org.jabref.model.util.DummyFileUpdateMonitor; @@ -113,7 +120,17 @@ public Parser getParser() { @Override public void doPostCleanup(BibEntry entry) { - + new FieldFormatterCleanup(StandardField.ABSTRACT, new RemoveBracesFormatter()).cleanup(entry); + new FieldFormatterCleanup(StandardField.ABSTRACT, new RemoveNewlinesFormatter()).cleanup(entry); + new FieldFormatterCleanup(StandardField.TITLE, new RemoveBracesFormatter()).cleanup(entry); + new FieldFormatterCleanup(StandardField.AUTHOR, new NormalizeNamesFormatter()).cleanup(entry); + + // Remove ADS note + new FieldFormatterCleanup(new UnknownField("adsnote"), new ClearFormatter()).cleanup(entry); + // Move adsurl to url field + new MoveFieldCleanup(new UnknownField("adsurl"), StandardField.URL).cleanup(entry); + // The fetcher adds some garbage (number of found entries etc before) + entry.setCommentsBeforeEntry(""); } @Override diff --git a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java new file mode 100644 index 00000000000..649a39a9054 --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java @@ -0,0 +1,206 @@ +package org.jabref.logic.importer.fetcher; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import org.jabref.logic.bibtex.FieldContentParserPreferences; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.testutils.category.FetcherTest; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@FetcherTest +public class NewAstrophysicsDataSystemTest { + + private NewAstrophysicsDataSystem fetcher; + private BibEntry diezSliceTheoremEntry, famaeyMcGaughEntry, sunWelchEntry, xiongSunEntry, ingersollPollardEntry, luceyPaulEntry; + + @BeforeEach + public void setUp() throws Exception { + ImportFormatPreferences importFormatPreferences = mock(ImportFormatPreferences.class); + when(importFormatPreferences.getFieldContentParserPreferences()).thenReturn( + mock(FieldContentParserPreferences.class)); + fetcher = new NewAstrophysicsDataSystem(importFormatPreferences); + + diezSliceTheoremEntry = new BibEntry(); + diezSliceTheoremEntry.setType(StandardEntryType.Article); + diezSliceTheoremEntry.setCiteKey("2014arXiv1405.2249D"); + diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, T."); + diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem for Fr$\\backslash$'echet group actions and covariant symplectic field theory"); + diezSliceTheoremEntry.setField(StandardField.YEAR, "2014"); + diezSliceTheoremEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); + diezSliceTheoremEntry.setField(StandardField.EPRINT, "1405.2249"); + diezSliceTheoremEntry.setField(StandardField.JOURNAL, "ArXiv e-prints"); + diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematical Physics, Mathematics - Differential Geometry, Mathematics - Symplectic Geometry, 58B99, 58Z05, 58B25, 22E65, 58D19, 53D20, 53D42"); + diezSliceTheoremEntry.setField(StandardField.MONTH, "#may#"); + diezSliceTheoremEntry.setField(new UnknownField("primaryclass"), "math-ph"); + diezSliceTheoremEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2014arXiv1405.2249D"); + diezSliceTheoremEntry.setField(StandardField.ABSTRACT, + "A general slice theorem for the action of a Fr$\\backslash$'echet Lie group on a " + + "Fr$\\backslash$'echet manifolds is established. The Nash-Moser theorem provides the " + + "fundamental tool to generalize the result of Palais to this " + + "infinite-dimensional setting. The presented slice theorem is illustrated " + + "by its application to gauge theories: the action of the gauge " + + "transformation group admits smooth slices at every point and thus the " + + "gauge orbit space is stratified by Fr$\\backslash$'echet manifolds. Furthermore, a " + + "covariant and symplectic formulation of classical field theory is " + + "proposed and extensively discussed. At the root of this novel framework " + + "is the incorporation of field degrees of freedom F and spacetime M into " + + "the product manifold F * M. The induced bigrading of differential forms " + + "is used in order to carry over the usual symplectic theory to this new " + + "setting. The examples of the Klein-Gordon field and general Yang-Mills " + + "theory illustrate that the presented approach conveniently handles the " + + "occurring symmetries."); + + famaeyMcGaughEntry = new BibEntry(); + famaeyMcGaughEntry.setType(StandardEntryType.Article); + famaeyMcGaughEntry.setCiteKey("2012LRR....15...10F"); + famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, B. and McGaugh, S. S."); + famaeyMcGaughEntry.setField(StandardField.TITLE, "Modified Newtonian Dynamics (MOND): Observational Phenomenology and Relativistic Extensions"); + famaeyMcGaughEntry.setField(StandardField.JOURNAL, "Living Reviews in Relativity"); + famaeyMcGaughEntry.setField(StandardField.YEAR, "2012"); + famaeyMcGaughEntry.setField(StandardField.VOLUME, "15"); + famaeyMcGaughEntry.setField(StandardField.MONTH, "#sep#"); + famaeyMcGaughEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); + famaeyMcGaughEntry.setField(StandardField.DOI, "10.12942/lrr-2012-10"); + famaeyMcGaughEntry.setField(new UnknownField("eid"), "10"); + famaeyMcGaughEntry.setField(StandardField.EPRINT, "1112.3960"); + famaeyMcGaughEntry.setField(StandardField.PAGES, "10"); + famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics"); + famaeyMcGaughEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012LRR....15...10F"); + + sunWelchEntry = new BibEntry(); + sunWelchEntry.setType(StandardEntryType.Article); + sunWelchEntry.setCiteKey("2012NatMa..11...44S"); + sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Y. and Welch, G. C. and Leong, W. L. and Takacs, C. J. and Bazan, G. C. and Heeger, A. J."); + sunWelchEntry.setField(StandardField.DOI, "10.1038/nmat3160"); + sunWelchEntry.setField(StandardField.JOURNAL, "Nature Materials"); + sunWelchEntry.setField(StandardField.MONTH, "#jan#"); + sunWelchEntry.setField(StandardField.PAGES, "44-48"); + sunWelchEntry.setField(StandardField.TITLE, "Solution-processed small-molecule solar cells with 6.7\\% efficiency"); + sunWelchEntry.setField(StandardField.VOLUME, "11"); + sunWelchEntry.setField(StandardField.YEAR, "2012"); + sunWelchEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012NatMa..11...44S"); + + xiongSunEntry = new BibEntry(); + xiongSunEntry.setType(StandardEntryType.Article); + xiongSunEntry.setCiteKey("2007ITGRS..45..879X"); + xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, X. and Sun, J. and Barnes, W. and Salomonson, V. and Esposito, J. and Erives, H. and Guenther, B."); + xiongSunEntry.setField(StandardField.DOI, "10.1109/TGRS.2006.890567"); + xiongSunEntry.setField(StandardField.JOURNAL, "IEEE Transactions on Geoscience and Remote Sensing"); + xiongSunEntry.setField(StandardField.MONTH, "#apr#"); + xiongSunEntry.setField(StandardField.PAGES, "879-889"); + xiongSunEntry.setField(StandardField.TITLE, "Multiyear On-Orbit Calibration and Performance of Terra MODIS Reflective Solar Bands"); + xiongSunEntry.setField(StandardField.VOLUME, "45"); + xiongSunEntry.setField(StandardField.YEAR, "2007"); + xiongSunEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2007ITGRS..45..879X"); + + ingersollPollardEntry = new BibEntry(); + ingersollPollardEntry.setType(StandardEntryType.Article); + ingersollPollardEntry.setCiteKey("1982Icar...52...62I"); + ingersollPollardEntry.setField(StandardField.AUTHOR, "Ingersoll, A. P. and Pollard, D."); + ingersollPollardEntry.setField(StandardField.DOI, "10.1016/0019-1035(82)90169-5"); + ingersollPollardEntry.setField(StandardField.JOURNAL, "\\icarus"); + ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS"); + ingersollPollardEntry.setField(StandardField.MONTH, "#oct#"); + ingersollPollardEntry.setField(StandardField.PAGES, "62-80"); + ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn - Scale analysis, anelastic equations, barotropic stability criterion"); + ingersollPollardEntry.setField(StandardField.VOLUME, "52"); + ingersollPollardEntry.setField(StandardField.YEAR, "1982"); + ingersollPollardEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/1982Icar...52...62I"); + + luceyPaulEntry = new BibEntry(); + luceyPaulEntry.setType(StandardEntryType.Article); + luceyPaulEntry.setCiteKey("2000JGR...10520297L"); + luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, P. G. and Blewett, D. T. and Jolliff, B. L."); + luceyPaulEntry.setField(StandardField.DOI, "10.1029/1999JE001117"); + luceyPaulEntry.setField(StandardField.JOURNAL, "\\jgr"); + luceyPaulEntry.setField(StandardField.KEYWORDS, "Planetology: Solid Surface Planets: Composition, Planetology: Solid Surface Planets: Remote sensing, Planetology: Solid Surface Planets: Surface materials and properties, Planetology: Solar System Objects: Moon (1221)"); + luceyPaulEntry.setField(StandardField.PAGES, "20297-20306"); + luceyPaulEntry.setField(StandardField.TITLE, "Lunar iron and titanium abundance algorithms based on final processing of Clementine ultraviolet-visible images"); + luceyPaulEntry.setField(StandardField.VOLUME, "105"); + luceyPaulEntry.setField(StandardField.YEAR, "2000"); + luceyPaulEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2000JGR...10520297L"); + } + + @Test + public void testHelpPage() { + assertEquals("ADS", fetcher.getHelpPage().get().getPageName()); + } + + @Test + public void testGetName() { + assertEquals("New SAO/NASA Astrophysics Data System", fetcher.getName()); + } + + @Test + public void searchByQueryFindsEntry() throws Exception { + List fetchedEntries = fetcher.performSearch("Diez slice theorem Lie"); + assertEquals(Collections.singletonList(diezSliceTheoremEntry), fetchedEntries); + } + + @Test + public void searchByEntryFindsEntry() throws Exception { + BibEntry searchEntry = new BibEntry(); + searchEntry.setField(StandardField.TITLE, "slice theorem"); + searchEntry.setField(StandardField.AUTHOR, "Diez"); + + List fetchedEntries = fetcher.performSearch(searchEntry); + assertFalse(fetchedEntries.isEmpty()); + assertEquals(diezSliceTheoremEntry, fetchedEntries.get(0)); + } + + @Test + public void testPerformSearchByFamaeyMcGaughEntry() throws Exception { + Optional fetchedEntry = fetcher.performSearchById("10.12942/lrr-2012-10"); + fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT));//Remove abstract due to copyright + assertEquals(Optional.of(famaeyMcGaughEntry), fetchedEntry); + } + + @Test + public void testPerformSearchByIdEmptyDOI() throws Exception { + Optional fetchedEntry = fetcher.performSearchById(""); + assertEquals(Optional.empty(), fetchedEntry); + } + + @Test + public void testPerformSearchByIdInvalidDoi() throws Exception { + assertEquals(Optional.empty(), fetcher.performSearchById("this.doi.will.fail")); + } + + @Test + public void testPerformSearchBySunWelchEntry() throws Exception { + Optional fetchedEntry = fetcher.performSearchById("10.1038/nmat3160"); + fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT)); //Remove abstract due to copyright + assertEquals(Optional.of(sunWelchEntry), fetchedEntry); + } + + @Test + public void testPerformSearchByXiongSunEntry() throws Exception { + Optional fetchedEntry = fetcher.performSearchById("10.1109/TGRS.2006.890567"); + assertEquals(Optional.of(xiongSunEntry), fetchedEntry); + } + + @Test + public void testPerformSearchByIngersollPollardEntry() throws Exception { + Optional fetchedEntry = fetcher.performSearchById("10.1016/0019-1035(82)90169-5"); + assertEquals(Optional.of(ingersollPollardEntry), fetchedEntry); + } + + @Test + public void testPerformSearchByLuceyPaulEntry() throws Exception { + Optional fetchedEntry = fetcher.performSearchById("2000JGR...10520297L"); + assertEquals(Optional.of(luceyPaulEntry), fetchedEntry); + } +} From 605519db040dae259ca9d563a9830ca37dc025b3 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 21:45:32 +0200 Subject: [PATCH 06/15] support doi and ads bibcodes --- .../fetcher/NewAstrophysicsDataSystem.java | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 9481aace5bc..5183a3d8e67 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -67,6 +67,10 @@ private String buildPostData(String... bibcodes) { return obj.toString(); } + private URL getURLforExport() throws URISyntaxException, MalformedURLException { + return new URIBuilder(API_EXPORT_URL).build().toURL(); + } + @Override public String getName() { return "New SAO/NASA Astrophysics Data System"; @@ -89,7 +93,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR if (title.isPresent()) { stringBuilder.append(title.get()) - .append(author.map(s -> " OR " + s) + .append(author.map(s -> " AND " + s) .orElse("")); } else { stringBuilder.append(author.orElse("")); @@ -105,7 +109,11 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR @Override public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { - return new URIBuilder(API_EXPORT_URL).build().toURL(); + String query = "doi:" + identifier + " OR " + "bibcode:" + identifier; + URIBuilder builder = new URIBuilder(API_SEARCH_URL); + builder.addParameter("q", query); + builder.addParameter("fl", "bibcode"); + return builder.build().toURL(); } @Override @@ -194,18 +202,29 @@ private List fetchBibcodes(URL url) throws FetcherException { @Override public Optional performSearchById(String identifier) throws FetcherException { - List fetchedEntries = performSearchByIds(identifier); - if (fetchedEntries.isEmpty()) { + if (StringUtil.isBlank(identifier)) { return Optional.empty(); } - if (fetchedEntries.size() > 1) { - LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier - + ". We will use the first entry."); - } + try { + List bibcodes = fetchBibcodes(getURLForID(identifier)); + String[] bibcodeArray = new String[bibcodes.size()]; + List fetchedEntries = performSearchByIds(bibcodes.toArray(bibcodeArray)); - BibEntry entry = fetchedEntries.get(0); - return Optional.of(entry); + if (fetchedEntries.isEmpty()) { + return Optional.empty(); + } + if (fetchedEntries.size() > 1) { + LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier + + ". We will use the first entry."); + } + BibEntry entry = fetchedEntries.get(0); + return Optional.of(entry); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } } private List performSearchByIds(String... identifiers) throws FetcherException { @@ -216,7 +235,7 @@ private List performSearchByIds(String... identifiers) throws FetcherE } try { String postData = buildPostData(identifiers); - URLDownload download = new URLDownload(getURLForID("")); + URLDownload download = new URLDownload(getURLforExport()); download.addHeader("Authorization", "Bearer " + API_KEY); download.addHeader("ContentType", "application/json"); download.setPostData(postData); From cba78290dee278deee0bdb6fced9fa570dd291f2 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 23:07:22 +0200 Subject: [PATCH 07/15] fixes wrong search syntax --- .../importer/fetcher/NewAstrophysicsDataSystem.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 5183a3d8e67..510e213ebe1 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -88,8 +88,8 @@ public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLE public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedURLException, FetcherException { StringBuilder stringBuilder = new StringBuilder(); - Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:" + t); - Optional author = entry.getFieldOrAlias(StandardField.TITLE).map(a -> "author:" + a); + Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:\"" + t + "\""); + Optional author = entry.getFieldOrAlias(StandardField.AUTHOR).map(a -> "author:\"" + a + "\""); if (title.isPresent()) { stringBuilder.append(title.get()) @@ -109,7 +109,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR @Override public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { - String query = "doi:" + identifier + " OR " + "bibcode:" + identifier; + String query = "doi:\"" + identifier + "\" OR " + "bibcode:\"" + identifier + "\""; URIBuilder builder = new URIBuilder(API_SEARCH_URL); builder.addParameter("q", query); builder.addParameter("fl", "bibcode"); @@ -151,7 +151,8 @@ public List performSearch(BibEntry entry) throws FetcherException { try { List bibcodes = fetchBibcodes(getURLForEntry(entry)); String[] bibcodeArray = new String[bibcodes.size()]; - return performSearchByIds(bibcodes.toArray(bibcodeArray)); + bibcodes.toArray(bibcodeArray); + return performSearchByIds(bibcodeArray); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { From bd6d84551450807762f11b13484e24a1f10de283 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 23:08:17 +0200 Subject: [PATCH 08/15] fixed test cases to API response --- .../NewAstrophysicsDataSystemTest.java | 80 +++++++++---------- 1 file changed, 38 insertions(+), 42 deletions(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java index 649a39a9054..561d316a0d0 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java @@ -35,95 +35,89 @@ public void setUp() throws Exception { diezSliceTheoremEntry = new BibEntry(); diezSliceTheoremEntry.setType(StandardEntryType.Article); - diezSliceTheoremEntry.setCiteKey("2014arXiv1405.2249D"); - diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, T."); - diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem for Fr$\\backslash$'echet group actions and covariant symplectic field theory"); - diezSliceTheoremEntry.setField(StandardField.YEAR, "2014"); + diezSliceTheoremEntry.setCiteKey("2018arXiv181204698D"); + diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, Tobias and Rudolph, Gerd"); + diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem and orbit type stratification in infinite dimensions"); + diezSliceTheoremEntry.setField(StandardField.YEAR, "2018"); diezSliceTheoremEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); - diezSliceTheoremEntry.setField(StandardField.EPRINT, "1405.2249"); - diezSliceTheoremEntry.setField(StandardField.JOURNAL, "ArXiv e-prints"); - diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematical Physics, Mathematics - Differential Geometry, Mathematics - Symplectic Geometry, 58B99, 58Z05, 58B25, 22E65, 58D19, 53D20, 53D42"); - diezSliceTheoremEntry.setField(StandardField.MONTH, "#may#"); - diezSliceTheoremEntry.setField(new UnknownField("primaryclass"), "math-ph"); - diezSliceTheoremEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2014arXiv1405.2249D"); + diezSliceTheoremEntry.setField(StandardField.EPRINT, "1812.04698"); + diezSliceTheoremEntry.setField(StandardField.JOURNAL, "arXiv e-prints"); + diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematics - Differential Geometry, Mathematical Physics, 58B25, (58D19, 58B20, 22E99, 58A35)"); + diezSliceTheoremEntry.setField(StandardField.MONTH, "Dec"); + diezSliceTheoremEntry.setField(StandardField.PAGES, "arXiv:1812.04698"); + diezSliceTheoremEntry.setField(new UnknownField("eid"), "arXiv:1812.04698"); + diezSliceTheoremEntry.setField(StandardField.PRIMARYCLASS, "math.DG"); + diezSliceTheoremEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2018arXiv181204698D"); diezSliceTheoremEntry.setField(StandardField.ABSTRACT, - "A general slice theorem for the action of a Fr$\\backslash$'echet Lie group on a " - + "Fr$\\backslash$'echet manifolds is established. The Nash-Moser theorem provides the " - + "fundamental tool to generalize the result of Palais to this " - + "infinite-dimensional setting. The presented slice theorem is illustrated " - + "by its application to gauge theories: the action of the gauge " - + "transformation group admits smooth slices at every point and thus the " - + "gauge orbit space is stratified by Fr$\\backslash$'echet manifolds. Furthermore, a " - + "covariant and symplectic formulation of classical field theory is " - + "proposed and extensively discussed. At the root of this novel framework " - + "is the incorporation of field degrees of freedom F and spacetime M into " - + "the product manifold F * M. The induced bigrading of differential forms " - + "is used in order to carry over the usual symplectic theory to this new " - + "setting. The examples of the Klein-Gordon field and general Yang-Mills " - + "theory illustrate that the presented approach conveniently handles the " - + "occurring symmetries."); + "We establish a general slice theorem for the action of a locally convex Lie group on a locally convex manifold, which generalizes the classical slice theorem of Palais to infinite dimensions. We discuss two important settings under which the assumptions of this theorem are fulfilled. First, using Gl{\\\"o}ckner's inverse function theorem, we show that the linear action of a compact Lie group on a Fr{\\'e}chet space admits a slice. Second, using the Nash--Moser theorem, we establish a slice theorem for the tame action of a tame Fr{\\'e}chet Lie group on a tame Fr{\\'e}chet manifold. For this purpose, we develop the concept of a graded Riemannian metric, which allows the construction of a path-length metric compatible with the manifold topology and of a local addition. Finally, generalizing a classical result in finite dimensions, we prove that the existence of a slice implies that the decomposition of the manifold into orbit types of the group action is a stratification."); famaeyMcGaughEntry = new BibEntry(); famaeyMcGaughEntry.setType(StandardEntryType.Article); famaeyMcGaughEntry.setCiteKey("2012LRR....15...10F"); - famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, B. and McGaugh, S. S."); + famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, Beno{\\^\\i}t and McGaugh, Stacy S."); famaeyMcGaughEntry.setField(StandardField.TITLE, "Modified Newtonian Dynamics (MOND): Observational Phenomenology and Relativistic Extensions"); famaeyMcGaughEntry.setField(StandardField.JOURNAL, "Living Reviews in Relativity"); famaeyMcGaughEntry.setField(StandardField.YEAR, "2012"); famaeyMcGaughEntry.setField(StandardField.VOLUME, "15"); - famaeyMcGaughEntry.setField(StandardField.MONTH, "#sep#"); + famaeyMcGaughEntry.setField(StandardField.MONTH, "Sep"); + famaeyMcGaughEntry.setField(StandardField.NUMBER, "1"); famaeyMcGaughEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); famaeyMcGaughEntry.setField(StandardField.DOI, "10.12942/lrr-2012-10"); + famaeyMcGaughEntry.setField(StandardField.PRIMARYCLASS, "astro-ph.CO"); famaeyMcGaughEntry.setField(new UnknownField("eid"), "10"); famaeyMcGaughEntry.setField(StandardField.EPRINT, "1112.3960"); famaeyMcGaughEntry.setField(StandardField.PAGES, "10"); - famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics"); - famaeyMcGaughEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012LRR....15...10F"); + famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics, Astrophysics - Cosmology and Nongalactic Astrophysics, Astrophysics - Astrophysics of Galaxies, General Relativity and Quantum Cosmology, High Energy Physics - Phenomenology, High Energy Physics - Theory"); + famaeyMcGaughEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012LRR....15...10F"); sunWelchEntry = new BibEntry(); sunWelchEntry.setType(StandardEntryType.Article); sunWelchEntry.setCiteKey("2012NatMa..11...44S"); - sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Y. and Welch, G. C. and Leong, W. L. and Takacs, C. J. and Bazan, G. C. and Heeger, A. J."); + sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Yanming and Welch, Gregory C. and Leong, Wei Lin and Takacs, Christopher J. and Bazan, Guillermo C. and Heeger, Alan J."); sunWelchEntry.setField(StandardField.DOI, "10.1038/nmat3160"); sunWelchEntry.setField(StandardField.JOURNAL, "Nature Materials"); - sunWelchEntry.setField(StandardField.MONTH, "#jan#"); + sunWelchEntry.setField(StandardField.MONTH, "Jan"); + sunWelchEntry.setField(StandardField.NUMBER, "1"); sunWelchEntry.setField(StandardField.PAGES, "44-48"); sunWelchEntry.setField(StandardField.TITLE, "Solution-processed small-molecule solar cells with 6.7\\% efficiency"); sunWelchEntry.setField(StandardField.VOLUME, "11"); sunWelchEntry.setField(StandardField.YEAR, "2012"); - sunWelchEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012NatMa..11...44S"); + sunWelchEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012NatMa..11...44S"); xiongSunEntry = new BibEntry(); xiongSunEntry.setType(StandardEntryType.Article); xiongSunEntry.setCiteKey("2007ITGRS..45..879X"); - xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, X. and Sun, J. and Barnes, W. and Salomonson, V. and Esposito, J. and Erives, H. and Guenther, B."); + xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, Xiaoxiong and Sun, Junqiang and Barnes, William and Salomonson, Vincent and Esposito, Joseph and Erives, Hector and Guenther, Bruce"); xiongSunEntry.setField(StandardField.DOI, "10.1109/TGRS.2006.890567"); xiongSunEntry.setField(StandardField.JOURNAL, "IEEE Transactions on Geoscience and Remote Sensing"); - xiongSunEntry.setField(StandardField.MONTH, "#apr#"); + xiongSunEntry.setField(StandardField.MONTH, "Apr"); + xiongSunEntry.setField(StandardField.NUMBER, "4"); xiongSunEntry.setField(StandardField.PAGES, "879-889"); xiongSunEntry.setField(StandardField.TITLE, "Multiyear On-Orbit Calibration and Performance of Terra MODIS Reflective Solar Bands"); xiongSunEntry.setField(StandardField.VOLUME, "45"); xiongSunEntry.setField(StandardField.YEAR, "2007"); - xiongSunEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2007ITGRS..45..879X"); + xiongSunEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2007ITGRS..45..879X"); ingersollPollardEntry = new BibEntry(); ingersollPollardEntry.setType(StandardEntryType.Article); ingersollPollardEntry.setCiteKey("1982Icar...52...62I"); + ingersollPollardEntry.setField(StandardField.ABSTRACT, "If Jupiter's and Saturn's fluid interiors were inviscid and adiabatic, any steady zonal motion would take the form of differentially rotating cylinders concentric about the planetary axis of rotation. B. A. Smith et al. [ Science215, 504-537 (1982)] showed that Saturn's observed zonal wind profile extends a significant distance below cloud base. Further extension into the interior occurs if the values of the eddy viscosity and superadiabaticity are small. We estimate these values using a scaling analysis of deep convection in the presence of differential rotation. The differential rotation inhibits the convection and reduces the effective eddy viscosity. Viscous dissipation of zonal mean kinetic energy is then within the bounds set by the internal heat source. The differential rotation increases the superadiabaticity, but not so much as to eliminate the cylindrical structure of the flow. Very large departures from adiabaticity, necessary for decoupling the atmosphere and interior, do not occur. Using our scaling analysis we develop the anelastic equations that describe motions in Jupiter's and Saturn's interiors. A simple problem is solved, that of an adiabatic fluid with a steady zonal wind varying as a function of cylindrical radius. Low zonal wavenumber perturbations are two dimensional (independent of the axial coordinate) and obey a modified barotropic stability equation. The parameter analogous to {\\ensuremath{\\beta}} is negative and is three to four times larger than the {\\ensuremath{\\beta}} for thin atmospheres. Jupiter's and Saturn's observed zonal wind profiles are close to marginal stability according to this deep sphere criterion, but are several times supercritical according to the thin atmosphere criterion."); ingersollPollardEntry.setField(StandardField.AUTHOR, "Ingersoll, A. P. and Pollard, D."); ingersollPollardEntry.setField(StandardField.DOI, "10.1016/0019-1035(82)90169-5"); ingersollPollardEntry.setField(StandardField.JOURNAL, "\\icarus"); - ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS"); - ingersollPollardEntry.setField(StandardField.MONTH, "#oct#"); + ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS, Lunar and Planetary Exploration; Planets"); + ingersollPollardEntry.setField(StandardField.MONTH, "Oct"); + ingersollPollardEntry.setField(StandardField.NUMBER, "1"); ingersollPollardEntry.setField(StandardField.PAGES, "62-80"); - ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn - Scale analysis, anelastic equations, barotropic stability criterion"); + ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn: scale analysis, anelastic equations, barotropic stability criterion"); ingersollPollardEntry.setField(StandardField.VOLUME, "52"); ingersollPollardEntry.setField(StandardField.YEAR, "1982"); - ingersollPollardEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/1982Icar...52...62I"); + ingersollPollardEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/1982Icar...52...62I"); luceyPaulEntry = new BibEntry(); luceyPaulEntry.setType(StandardEntryType.Article); luceyPaulEntry.setCiteKey("2000JGR...10520297L"); - luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, P. G. and Blewett, D. T. and Jolliff, B. L."); + luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, Paul G. and Blewett, David T. and Jolliff, Bradley L."); luceyPaulEntry.setField(StandardField.DOI, "10.1029/1999JE001117"); luceyPaulEntry.setField(StandardField.JOURNAL, "\\jgr"); luceyPaulEntry.setField(StandardField.KEYWORDS, "Planetology: Solid Surface Planets: Composition, Planetology: Solid Surface Planets: Remote sensing, Planetology: Solid Surface Planets: Surface materials and properties, Planetology: Solar System Objects: Moon (1221)"); @@ -131,7 +125,9 @@ public void setUp() throws Exception { luceyPaulEntry.setField(StandardField.TITLE, "Lunar iron and titanium abundance algorithms based on final processing of Clementine ultraviolet-visible images"); luceyPaulEntry.setField(StandardField.VOLUME, "105"); luceyPaulEntry.setField(StandardField.YEAR, "2000"); - luceyPaulEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2000JGR...10520297L"); + luceyPaulEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2000JGR...10520297L"); + luceyPaulEntry.setField(StandardField.MONTH, "Jan"); + luceyPaulEntry.setField(StandardField.NUMBER, "E8"); } @Test @@ -165,7 +161,7 @@ public void searchByEntryFindsEntry() throws Exception { public void testPerformSearchByFamaeyMcGaughEntry() throws Exception { Optional fetchedEntry = fetcher.performSearchById("10.12942/lrr-2012-10"); fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT));//Remove abstract due to copyright - assertEquals(Optional.of(famaeyMcGaughEntry), fetchedEntry); + assertEquals(famaeyMcGaughEntry, fetchedEntry.get()); } @Test From 27243ed1b82f4e31f91e898ad8537c56b2a1d832 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 23:28:18 +0200 Subject: [PATCH 09/15] fixed bug where equals doesnt work with unknown field --- .../fetcher/NewAstrophysicsDataSystemTest.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java index 561d316a0d0..403bfaefa55 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java @@ -1,6 +1,5 @@ package org.jabref.logic.importer.fetcher; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -8,7 +7,6 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.field.UnknownField; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; @@ -17,6 +15,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -45,7 +44,7 @@ public void setUp() throws Exception { diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematics - Differential Geometry, Mathematical Physics, 58B25, (58D19, 58B20, 22E99, 58A35)"); diezSliceTheoremEntry.setField(StandardField.MONTH, "Dec"); diezSliceTheoremEntry.setField(StandardField.PAGES, "arXiv:1812.04698"); - diezSliceTheoremEntry.setField(new UnknownField("eid"), "arXiv:1812.04698"); + diezSliceTheoremEntry.setField(StandardField.EID, "arXiv:1812.04698"); diezSliceTheoremEntry.setField(StandardField.PRIMARYCLASS, "math.DG"); diezSliceTheoremEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2018arXiv181204698D"); diezSliceTheoremEntry.setField(StandardField.ABSTRACT, @@ -64,7 +63,7 @@ public void setUp() throws Exception { famaeyMcGaughEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); famaeyMcGaughEntry.setField(StandardField.DOI, "10.12942/lrr-2012-10"); famaeyMcGaughEntry.setField(StandardField.PRIMARYCLASS, "astro-ph.CO"); - famaeyMcGaughEntry.setField(new UnknownField("eid"), "10"); + famaeyMcGaughEntry.setField(StandardField.EID, "10"); famaeyMcGaughEntry.setField(StandardField.EPRINT, "1112.3960"); famaeyMcGaughEntry.setField(StandardField.PAGES, "10"); famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics, Astrophysics - Cosmology and Nongalactic Astrophysics, Astrophysics - Astrophysics of Galaxies, General Relativity and Quantum Cosmology, High Energy Physics - Phenomenology, High Energy Physics - Theory"); @@ -143,7 +142,7 @@ public void testGetName() { @Test public void searchByQueryFindsEntry() throws Exception { List fetchedEntries = fetcher.performSearch("Diez slice theorem Lie"); - assertEquals(Collections.singletonList(diezSliceTheoremEntry), fetchedEntries); + assertTrue(fetchedEntries.contains(diezSliceTheoremEntry)); } @Test @@ -161,7 +160,7 @@ public void searchByEntryFindsEntry() throws Exception { public void testPerformSearchByFamaeyMcGaughEntry() throws Exception { Optional fetchedEntry = fetcher.performSearchById("10.12942/lrr-2012-10"); fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT));//Remove abstract due to copyright - assertEquals(famaeyMcGaughEntry, fetchedEntry.get()); + assertEquals(Optional.of(famaeyMcGaughEntry), fetchedEntry); } @Test From 8910676fcc5f784d4760be10847f7a0f6ed7f5df Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 17:44:14 +0200 Subject: [PATCH 10/15] changed array method parameters to Collections --- .../fetcher/NewAstrophysicsDataSystem.java | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 510e213ebe1..37dedfb6ca9 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -5,11 +5,12 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import org.jabref.logic.cleanup.MoveFieldCleanup; import org.jabref.logic.formatter.bibtexfields.ClearFormatter; @@ -61,7 +62,7 @@ public NewAstrophysicsDataSystem(ImportFormatPreferences preferences) { this.preferences = Objects.requireNonNull(preferences); } - private String buildPostData(String... bibcodes) { + private String buildPostData(Collection bibcodes) { JSONObject obj = new JSONObject(); obj.put("bibcode", bibcodes); return obj.toString(); @@ -150,9 +151,7 @@ public List performSearch(BibEntry entry) throws FetcherException { try { List bibcodes = fetchBibcodes(getURLForEntry(entry)); - String[] bibcodeArray = new String[bibcodes.size()]; - bibcodes.toArray(bibcodeArray); - return performSearchByIds(bibcodeArray); + return performSearchByIds(bibcodes); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { @@ -169,8 +168,7 @@ public List performSearch(String query) throws FetcherException { try { List bibcodes = fetchBibcodes(getURLForQuery(query)); - String[] bibcodeArray = new String[bibcodes.size()]; - return performSearchByIds(bibcodes.toArray(bibcodeArray)); + return performSearchByIds(bibcodes); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { @@ -209,8 +207,7 @@ public Optional performSearchById(String identifier) throws FetcherExc try { List bibcodes = fetchBibcodes(getURLForID(identifier)); - String[] bibcodeArray = new String[bibcodes.size()]; - List fetchedEntries = performSearchByIds(bibcodes.toArray(bibcodeArray)); + List fetchedEntries = performSearchByIds(bibcodes); if (fetchedEntries.isEmpty()) { return Optional.empty(); @@ -228,14 +225,14 @@ public Optional performSearchById(String identifier) throws FetcherExc } } - private List performSearchByIds(String... identifiers) throws FetcherException { + private List performSearchByIds(Collection identifiers) throws FetcherException { - long idCount = Arrays.stream(identifiers).filter(identifier -> !StringUtil.isBlank(identifier)).count(); - if (idCount == 0) { + List ids = identifiers.stream().filter(identifier -> !StringUtil.isBlank(identifier)).collect(Collectors.toList()); + if (ids.isEmpty()) { return Collections.emptyList(); } try { - String postData = buildPostData(identifiers); + String postData = buildPostData(ids); URLDownload download = new URLDownload(getURLforExport()); download.addHeader("Authorization", "Bearer " + API_KEY); download.addHeader("ContentType", "application/json"); From b811c0bd7d8ff807bc261d199ba200e9da16b612 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 17:45:02 +0200 Subject: [PATCH 11/15] removes json module dependency --- src/main/java/module-info.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index bfb3a41fdcc..791e2e50698 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -65,5 +65,4 @@ requires de.saxsys.mvvmfx.validation; requires richtextfx; requires unirest.java; - requires json; } From 638dd947ca28e68f051c0e1dd45879579c49ed9b Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 17:52:28 +0200 Subject: [PATCH 12/15] replace old fetcher with the new one --- .../jabref/logic/importer/WebFetchers.java | 4 - .../fetcher/AstrophysicsDataSystem.java | 219 ++++++++++++--- .../fetcher/NewAstrophysicsDataSystem.java | 263 ------------------ .../fetcher/AstrophysicsDataSystemTest.java | 89 +++--- .../NewAstrophysicsDataSystemTest.java | 201 ------------- 5 files changed, 215 insertions(+), 561 deletions(-) delete mode 100644 src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java delete mode 100644 src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java diff --git a/src/main/java/org/jabref/logic/importer/WebFetchers.java b/src/main/java/org/jabref/logic/importer/WebFetchers.java index 4161786d0c4..66319f5e6b4 100644 --- a/src/main/java/org/jabref/logic/importer/WebFetchers.java +++ b/src/main/java/org/jabref/logic/importer/WebFetchers.java @@ -27,7 +27,6 @@ import org.jabref.logic.importer.fetcher.LibraryOfCongress; import org.jabref.logic.importer.fetcher.MathSciNet; import org.jabref.logic.importer.fetcher.MedlineFetcher; -import org.jabref.logic.importer.fetcher.NewAstrophysicsDataSystem; import org.jabref.logic.importer.fetcher.OpenAccessDoi; import org.jabref.logic.importer.fetcher.RfcFetcher; import org.jabref.logic.importer.fetcher.ScienceDirect; @@ -89,7 +88,6 @@ public static SortedSet getSearchBasedFetchers(ImportFormatP set.add(new GvkFetcher()); set.add(new MedlineFetcher()); set.add(new AstrophysicsDataSystem(importFormatPreferences)); - set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new MathSciNet(importFormatPreferences)); set.add(new ZbMATH(importFormatPreferences)); set.add(new ACMPortalFetcher(importFormatPreferences)); @@ -110,7 +108,6 @@ public static SortedSet getIdBasedFetchers(ImportFormatPreferenc SortedSet set = new TreeSet<>(Comparator.comparing(WebFetcher::getName)); set.add(new ArXiv(importFormatPreferences)); set.add(new AstrophysicsDataSystem(importFormatPreferences)); - set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new IsbnFetcher(importFormatPreferences)); set.add(new DiVA(importFormatPreferences)); set.add(new DoiFetcher(importFormatPreferences)); @@ -130,7 +127,6 @@ public static SortedSet getIdBasedFetchers(ImportFormatPreferenc public static SortedSet getEntryBasedFetchers(ImportFormatPreferences importFormatPreferences) { SortedSet set = new TreeSet<>(Comparator.comparing(WebFetcher::getName)); set.add(new AstrophysicsDataSystem(importFormatPreferences)); - set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new DoiFetcher(importFormatPreferences)); set.add(new IsbnFetcher(importFormatPreferences)); set.add(new MathSciNet(importFormatPreferences)); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index 2ac10652bc9..faa99419c0c 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -1,10 +1,16 @@ package org.jabref.logic.importer.fetcher; +import java.io.IOException; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import org.jabref.logic.cleanup.MoveFieldCleanup; import org.jabref.logic.formatter.bibtexfields.ClearFormatter; @@ -16,16 +22,22 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; import org.jabref.logic.importer.fileformat.BibtexParser; +import org.jabref.logic.net.URLDownload; import org.jabref.model.cleanup.FieldFormatterCleanup; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.strings.StringUtil; import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; /** * Fetches data from the SAO/NASA Astrophysics Data System (http://www.adsabs.harvard.edu/) @@ -37,73 +49,69 @@ */ public class AstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher { - private static String API_QUERY_URL = "http://adsabs.harvard.edu/cgi-bin/nph-basic_connect"; - private static String API_ENTRY_URL = "http://adsabs.harvard.edu/cgi-bin/nph-abs_connect"; - private static String API_DOI_URL = "http://adsabs.harvard.edu/doi/"; + private static final String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; + private static final String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; - private final String patternRemoveDOI = "^(doi:|DOI:)"; + private static final String API_KEY = ""; //TODO Add API Token private final ImportFormatPreferences preferences; public AstrophysicsDataSystem(ImportFormatPreferences preferences) { this.preferences = Objects.requireNonNull(preferences); } + private String buildPostData(Collection bibcodes) { + JSONObject obj = new JSONObject(); + obj.put("bibcode", bibcodes); + return obj.toString(); + } + + private URL getURLforExport() throws URISyntaxException, MalformedURLException { + return new URIBuilder(API_EXPORT_URL).build().toURL(); + } + @Override public String getName() { return "SAO/NASA Astrophysics Data System"; } - private URIBuilder getBaseUrl(String apiUrl) throws URISyntaxException { - URIBuilder uriBuilder = new URIBuilder(apiUrl); - uriBuilder.addParameter("data_type", "BIBTEXPLUS"); - uriBuilder.addParameter("start_nr", String.valueOf(1)); - uriBuilder.addParameter("nr_to_return", String.valueOf(200)); - return uriBuilder; - } - @Override public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { - URIBuilder uriBuilder = getBaseUrl(API_QUERY_URL); - uriBuilder.addParameter("qsearch", query); - return uriBuilder.build().toURL(); + URIBuilder builder = new URIBuilder(API_SEARCH_URL); + builder.addParameter("q", query); + builder.addParameter("fl", "bibcode"); + return builder.build().toURL(); } @Override public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedURLException, FetcherException { - URIBuilder uriBuilder = getBaseUrl(API_ENTRY_URL); - - // Search astronomy + physics + arXiv db - uriBuilder.addParameter("db_key", "AST"); - uriBuilder.addParameter("db_key", "PHY"); - uriBuilder.addParameter("db_key", "PRE"); - - // Add title search - entry.getFieldOrAlias(StandardField.TITLE).ifPresent(title -> { - uriBuilder.addParameter("ttl_logic", "OR"); - uriBuilder.addParameter("title", title); - uriBuilder.addParameter("ttl_syn", "YES"); // Synonym replacement - uriBuilder.addParameter("ttl_wt", "0.3"); // Weight - uriBuilder.addParameter("ttl_wgt", "YES"); // Consider Weight - }); - - // Add author search - entry.getFieldOrAlias(StandardField.AUTHOR).ifPresent(author -> { - uriBuilder.addParameter("aut_logic", "OR"); - uriBuilder.addParameter("author", author); - uriBuilder.addParameter("aut_syn", "YES"); // Synonym replacement - uriBuilder.addParameter("aut_wt", "1.0"); // Weight - uriBuilder.addParameter("aut_wgt", "YES"); // Consider weight - }); - - return uriBuilder.build().toURL(); + StringBuilder stringBuilder = new StringBuilder(); + + Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:\"" + t + "\""); + Optional author = entry.getFieldOrAlias(StandardField.AUTHOR).map(a -> "author:\"" + a + "\""); + + if (title.isPresent()) { + stringBuilder.append(title.get()) + .append(author.map(s -> " AND " + s) + .orElse("")); + } else { + stringBuilder.append(author.orElse("")); + } + String query = stringBuilder.toString().trim(); + + URIBuilder builder = new URIBuilder(API_SEARCH_URL); + builder.addParameter("q", query); + builder.addParameter("fl", "bibcode"); + builder.addParameter("rows", "20"); + return builder.build().toURL(); } @Override - public URL getURLForID(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { - String key = identifier.replaceAll(patternRemoveDOI, ""); - URIBuilder uriBuilder = new URIBuilder(API_DOI_URL + key); - uriBuilder.addParameter("data_type", "BIBTEXPLUS"); - return uriBuilder.build().toURL(); + public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { + String query = "doi:\"" + identifier + "\" OR " + "bibcode:\"" + identifier + "\""; + URIBuilder builder = new URIBuilder(API_SEARCH_URL); + builder.addParameter("q", query); + builder.addParameter("fl", "bibcode"); + return builder.build().toURL(); } @Override @@ -130,4 +138,123 @@ public void doPostCleanup(BibEntry entry) { // The fetcher adds some garbage (number of found entries etc before) entry.setCommentsBeforeEntry(""); } + + @Override + public List performSearch(BibEntry entry) throws FetcherException { + + if (entry.getFieldOrAlias(StandardField.TITLE).isEmpty() && entry.getFieldOrAlias(StandardField.AUTHOR).isEmpty()) { + return Collections.emptyList(); + } + + try { + List bibcodes = fetchBibcodes(getURLForEntry(entry)); + return performSearchByIds(bibcodes); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } + } + + @Override + public List performSearch(String query) throws FetcherException { + + if (StringUtil.isBlank(query)) { + return Collections.emptyList(); + } + + try { + List bibcodes = fetchBibcodes(getURLForQuery(query)); + return performSearchByIds(bibcodes); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } + } + + private List fetchBibcodes(URL url) throws FetcherException { + + try { + URLDownload download = new URLDownload(url); + download.addHeader("Authorization", "Bearer " + API_KEY); + String content = download.asString(); + JSONObject obj = new JSONObject(content); + + try { + JSONArray codes = obj.getJSONObject("response").getJSONArray("docs"); + List bibcodes = new ArrayList<>(); + for (int i = 0; i < codes.length(); i++) { + bibcodes.add(codes.getJSONObject(i).getString("bibcode")); + } + return bibcodes; + } catch (JSONException e) { + return Collections.emptyList(); + } + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } + } + + @Override + public Optional performSearchById(String identifier) throws FetcherException { + if (StringUtil.isBlank(identifier)) { + return Optional.empty(); + } + + try { + List bibcodes = fetchBibcodes(getURLForID(identifier)); + List fetchedEntries = performSearchByIds(bibcodes); + + if (fetchedEntries.isEmpty()) { + return Optional.empty(); + } + if (fetchedEntries.size() > 1) { + LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier + + ". We will use the first entry."); + } + BibEntry entry = fetchedEntries.get(0); + return Optional.of(entry); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } + } + + private List performSearchByIds(Collection identifiers) throws FetcherException { + + List ids = identifiers.stream().filter(identifier -> !StringUtil.isBlank(identifier)).collect(Collectors.toList()); + if (ids.isEmpty()) { + return Collections.emptyList(); + } + try { + String postData = buildPostData(ids); + URLDownload download = new URLDownload(getURLforExport()); + download.addHeader("Authorization", "Bearer " + API_KEY); + download.addHeader("ContentType", "application/json"); + download.setPostData(postData); + String content = download.asString(); + JSONObject obj = new JSONObject(content); + + try { + List fetchedEntries = getParser().parseEntries(obj.optString("export")); + if (fetchedEntries.isEmpty()) { + return Collections.emptyList(); + } + // Post-cleanup + fetchedEntries.forEach(this::doPostCleanup); + + return fetchedEntries; + } catch (JSONException e) { + return Collections.emptyList(); + } + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } catch (ParseException e) { + throw new FetcherException("An internal parser error occurred", e); + } + } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java deleted file mode 100644 index 37dedfb6ca9..00000000000 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ /dev/null @@ -1,263 +0,0 @@ -package org.jabref.logic.importer.fetcher; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; - -import org.jabref.logic.cleanup.MoveFieldCleanup; -import org.jabref.logic.formatter.bibtexfields.ClearFormatter; -import org.jabref.logic.formatter.bibtexfields.NormalizeNamesFormatter; -import org.jabref.logic.formatter.bibtexfields.RemoveBracesFormatter; -import org.jabref.logic.formatter.bibtexfields.RemoveNewlinesFormatter; -import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.EntryBasedParserFetcher; -import org.jabref.logic.importer.FetcherException; -import org.jabref.logic.importer.IdBasedParserFetcher; -import org.jabref.logic.importer.ImportFormatPreferences; -import org.jabref.logic.importer.ParseException; -import org.jabref.logic.importer.Parser; -import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fileformat.BibtexParser; -import org.jabref.logic.net.URLDownload; -import org.jabref.model.cleanup.FieldFormatterCleanup; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.field.UnknownField; -import org.jabref.model.strings.StringUtil; -import org.jabref.model.util.DummyFileUpdateMonitor; - -import org.apache.http.client.utils.URIBuilder; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -//TODO Replace Old ADS after the new one is mature - -/** - * Fetches data from the SAO/NASA Astrophysics Data System (http://www.adsabs.harvard.edu/) - *

- * Search query-based: http://adsabs.harvard.edu/basic_search.html Entry -based: http://adsabs.harvard.edu/abstract_service.html - *

- * There is also a new API (https://github.com/adsabs/adsabs-dev-api) but it returns JSON (or at least needs multiple - * calls to get BibTeX, status: September 2016) - */ -public class NewAstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher { - - private static final String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; - private static final String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; - - private static String API_KEY = ""; //TODO Add API Token - - private final ImportFormatPreferences preferences; - - public NewAstrophysicsDataSystem(ImportFormatPreferences preferences) { - this.preferences = Objects.requireNonNull(preferences); - } - - private String buildPostData(Collection bibcodes) { - JSONObject obj = new JSONObject(); - obj.put("bibcode", bibcodes); - return obj.toString(); - } - - private URL getURLforExport() throws URISyntaxException, MalformedURLException { - return new URIBuilder(API_EXPORT_URL).build().toURL(); - } - - @Override - public String getName() { - return "New SAO/NASA Astrophysics Data System"; - } - - @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { - URIBuilder builder = new URIBuilder(API_SEARCH_URL); - builder.addParameter("q", query); - builder.addParameter("fl", "bibcode"); - return builder.build().toURL(); - } - - @Override - public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedURLException, FetcherException { - StringBuilder stringBuilder = new StringBuilder(); - - Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:\"" + t + "\""); - Optional author = entry.getFieldOrAlias(StandardField.AUTHOR).map(a -> "author:\"" + a + "\""); - - if (title.isPresent()) { - stringBuilder.append(title.get()) - .append(author.map(s -> " AND " + s) - .orElse("")); - } else { - stringBuilder.append(author.orElse("")); - } - String query = stringBuilder.toString().trim(); - - URIBuilder builder = new URIBuilder(API_SEARCH_URL); - builder.addParameter("q", query); - builder.addParameter("fl", "bibcode"); - builder.addParameter("rows", "20"); - return builder.build().toURL(); - } - - @Override - public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { - String query = "doi:\"" + identifier + "\" OR " + "bibcode:\"" + identifier + "\""; - URIBuilder builder = new URIBuilder(API_SEARCH_URL); - builder.addParameter("q", query); - builder.addParameter("fl", "bibcode"); - return builder.build().toURL(); - } - - @Override - public Optional getHelpPage() { - return Optional.of(HelpFile.FETCHER_ADS); - } - - @Override - public Parser getParser() { - return new BibtexParser(preferences, new DummyFileUpdateMonitor()); - } - - @Override - public void doPostCleanup(BibEntry entry) { - new FieldFormatterCleanup(StandardField.ABSTRACT, new RemoveBracesFormatter()).cleanup(entry); - new FieldFormatterCleanup(StandardField.ABSTRACT, new RemoveNewlinesFormatter()).cleanup(entry); - new FieldFormatterCleanup(StandardField.TITLE, new RemoveBracesFormatter()).cleanup(entry); - new FieldFormatterCleanup(StandardField.AUTHOR, new NormalizeNamesFormatter()).cleanup(entry); - - // Remove ADS note - new FieldFormatterCleanup(new UnknownField("adsnote"), new ClearFormatter()).cleanup(entry); - // Move adsurl to url field - new MoveFieldCleanup(new UnknownField("adsurl"), StandardField.URL).cleanup(entry); - // The fetcher adds some garbage (number of found entries etc before) - entry.setCommentsBeforeEntry(""); - } - - @Override - public List performSearch(BibEntry entry) throws FetcherException { - - if (entry.getFieldOrAlias(StandardField.TITLE).isEmpty() && entry.getFieldOrAlias(StandardField.AUTHOR).isEmpty()) { - return Collections.emptyList(); - } - - try { - List bibcodes = fetchBibcodes(getURLForEntry(entry)); - return performSearchByIds(bibcodes); - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); - } - } - - @Override - public List performSearch(String query) throws FetcherException { - - if (StringUtil.isBlank(query)) { - return Collections.emptyList(); - } - - try { - List bibcodes = fetchBibcodes(getURLForQuery(query)); - return performSearchByIds(bibcodes); - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); - } - } - - private List fetchBibcodes(URL url) throws FetcherException { - - try { - URLDownload download = new URLDownload(url); - download.addHeader("Authorization", "Bearer " + API_KEY); - String content = download.asString(); - JSONObject obj = new JSONObject(content); - - try { - JSONArray codes = obj.getJSONObject("response").getJSONArray("docs"); - List bibcodes = new ArrayList<>(); - for (int i = 0; i < codes.length(); i++) { - bibcodes.add(codes.getJSONObject(i).getString("bibcode")); - } - return bibcodes; - } catch (JSONException e) { - return Collections.emptyList(); - } - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); - } - } - - @Override - public Optional performSearchById(String identifier) throws FetcherException { - if (StringUtil.isBlank(identifier)) { - return Optional.empty(); - } - - try { - List bibcodes = fetchBibcodes(getURLForID(identifier)); - List fetchedEntries = performSearchByIds(bibcodes); - - if (fetchedEntries.isEmpty()) { - return Optional.empty(); - } - if (fetchedEntries.size() > 1) { - LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier - + ". We will use the first entry."); - } - BibEntry entry = fetchedEntries.get(0); - return Optional.of(entry); - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); - } - } - - private List performSearchByIds(Collection identifiers) throws FetcherException { - - List ids = identifiers.stream().filter(identifier -> !StringUtil.isBlank(identifier)).collect(Collectors.toList()); - if (ids.isEmpty()) { - return Collections.emptyList(); - } - try { - String postData = buildPostData(ids); - URLDownload download = new URLDownload(getURLforExport()); - download.addHeader("Authorization", "Bearer " + API_KEY); - download.addHeader("ContentType", "application/json"); - download.setPostData(postData); - String content = download.asString(); - JSONObject obj = new JSONObject(content); - - try { - List fetchedEntries = getParser().parseEntries(obj.optString("export")); - if (fetchedEntries.isEmpty()) { - return Collections.emptyList(); - } - // Post-cleanup - fetchedEntries.forEach(this::doPostCleanup); - - return fetchedEntries; - } catch (JSONException e) { - return Collections.emptyList(); - } - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); - } catch (ParseException e) { - throw new FetcherException("An internal parser error occurred", e); - } - } -} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java index 4ece520646d..f7f393c49dd 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java @@ -1,6 +1,5 @@ package org.jabref.logic.importer.fetcher; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -8,7 +7,6 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.field.UnknownField; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; @@ -17,6 +15,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -35,95 +34,89 @@ public void setUp() throws Exception { diezSliceTheoremEntry = new BibEntry(); diezSliceTheoremEntry.setType(StandardEntryType.Article); - diezSliceTheoremEntry.setCiteKey("2014arXiv1405.2249D"); - diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, T."); - diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem for Fr$\\backslash$'echet group actions and covariant symplectic field theory"); - diezSliceTheoremEntry.setField(StandardField.YEAR, "2014"); + diezSliceTheoremEntry.setCiteKey("2018arXiv181204698D"); + diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, Tobias and Rudolph, Gerd"); + diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem and orbit type stratification in infinite dimensions"); + diezSliceTheoremEntry.setField(StandardField.YEAR, "2018"); diezSliceTheoremEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); - diezSliceTheoremEntry.setField(StandardField.EPRINT, "1405.2249"); - diezSliceTheoremEntry.setField(StandardField.JOURNAL, "ArXiv e-prints"); - diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematical Physics, Mathematics - Differential Geometry, Mathematics - Symplectic Geometry, 58B99, 58Z05, 58B25, 22E65, 58D19, 53D20, 53D42"); - diezSliceTheoremEntry.setField(StandardField.MONTH, "#may#"); - diezSliceTheoremEntry.setField(new UnknownField("primaryclass"), "math-ph"); - diezSliceTheoremEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2014arXiv1405.2249D"); + diezSliceTheoremEntry.setField(StandardField.EPRINT, "1812.04698"); + diezSliceTheoremEntry.setField(StandardField.JOURNAL, "arXiv e-prints"); + diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematics - Differential Geometry, Mathematical Physics, 58B25, (58D19, 58B20, 22E99, 58A35)"); + diezSliceTheoremEntry.setField(StandardField.MONTH, "Dec"); + diezSliceTheoremEntry.setField(StandardField.PAGES, "arXiv:1812.04698"); + diezSliceTheoremEntry.setField(StandardField.EID, "arXiv:1812.04698"); + diezSliceTheoremEntry.setField(StandardField.PRIMARYCLASS, "math.DG"); + diezSliceTheoremEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2018arXiv181204698D"); diezSliceTheoremEntry.setField(StandardField.ABSTRACT, - "A general slice theorem for the action of a Fr$\\backslash$'echet Lie group on a " - + "Fr$\\backslash$'echet manifolds is established. The Nash-Moser theorem provides the " - + "fundamental tool to generalize the result of Palais to this " - + "infinite-dimensional setting. The presented slice theorem is illustrated " - + "by its application to gauge theories: the action of the gauge " - + "transformation group admits smooth slices at every point and thus the " - + "gauge orbit space is stratified by Fr$\\backslash$'echet manifolds. Furthermore, a " - + "covariant and symplectic formulation of classical field theory is " - + "proposed and extensively discussed. At the root of this novel framework " - + "is the incorporation of field degrees of freedom F and spacetime M into " - + "the product manifold F * M. The induced bigrading of differential forms " - + "is used in order to carry over the usual symplectic theory to this new " - + "setting. The examples of the Klein-Gordon field and general Yang-Mills " - + "theory illustrate that the presented approach conveniently handles the " - + "occurring symmetries."); + "We establish a general slice theorem for the action of a locally convex Lie group on a locally convex manifold, which generalizes the classical slice theorem of Palais to infinite dimensions. We discuss two important settings under which the assumptions of this theorem are fulfilled. First, using Gl{\\\"o}ckner's inverse function theorem, we show that the linear action of a compact Lie group on a Fr{\\'e}chet space admits a slice. Second, using the Nash--Moser theorem, we establish a slice theorem for the tame action of a tame Fr{\\'e}chet Lie group on a tame Fr{\\'e}chet manifold. For this purpose, we develop the concept of a graded Riemannian metric, which allows the construction of a path-length metric compatible with the manifold topology and of a local addition. Finally, generalizing a classical result in finite dimensions, we prove that the existence of a slice implies that the decomposition of the manifold into orbit types of the group action is a stratification."); famaeyMcGaughEntry = new BibEntry(); famaeyMcGaughEntry.setType(StandardEntryType.Article); famaeyMcGaughEntry.setCiteKey("2012LRR....15...10F"); - famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, B. and McGaugh, S. S."); + famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, Beno{\\^\\i}t and McGaugh, Stacy S."); famaeyMcGaughEntry.setField(StandardField.TITLE, "Modified Newtonian Dynamics (MOND): Observational Phenomenology and Relativistic Extensions"); famaeyMcGaughEntry.setField(StandardField.JOURNAL, "Living Reviews in Relativity"); famaeyMcGaughEntry.setField(StandardField.YEAR, "2012"); famaeyMcGaughEntry.setField(StandardField.VOLUME, "15"); - famaeyMcGaughEntry.setField(StandardField.MONTH, "#sep#"); + famaeyMcGaughEntry.setField(StandardField.MONTH, "Sep"); + famaeyMcGaughEntry.setField(StandardField.NUMBER, "1"); famaeyMcGaughEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); famaeyMcGaughEntry.setField(StandardField.DOI, "10.12942/lrr-2012-10"); - famaeyMcGaughEntry.setField(new UnknownField("eid"), "10"); + famaeyMcGaughEntry.setField(StandardField.PRIMARYCLASS, "astro-ph.CO"); + famaeyMcGaughEntry.setField(StandardField.EID, "10"); famaeyMcGaughEntry.setField(StandardField.EPRINT, "1112.3960"); famaeyMcGaughEntry.setField(StandardField.PAGES, "10"); - famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics"); - famaeyMcGaughEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012LRR....15...10F"); + famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics, Astrophysics - Cosmology and Nongalactic Astrophysics, Astrophysics - Astrophysics of Galaxies, General Relativity and Quantum Cosmology, High Energy Physics - Phenomenology, High Energy Physics - Theory"); + famaeyMcGaughEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012LRR....15...10F"); sunWelchEntry = new BibEntry(); sunWelchEntry.setType(StandardEntryType.Article); sunWelchEntry.setCiteKey("2012NatMa..11...44S"); - sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Y. and Welch, G. C. and Leong, W. L. and Takacs, C. J. and Bazan, G. C. and Heeger, A. J."); + sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Yanming and Welch, Gregory C. and Leong, Wei Lin and Takacs, Christopher J. and Bazan, Guillermo C. and Heeger, Alan J."); sunWelchEntry.setField(StandardField.DOI, "10.1038/nmat3160"); sunWelchEntry.setField(StandardField.JOURNAL, "Nature Materials"); - sunWelchEntry.setField(StandardField.MONTH, "#jan#"); + sunWelchEntry.setField(StandardField.MONTH, "Jan"); + sunWelchEntry.setField(StandardField.NUMBER, "1"); sunWelchEntry.setField(StandardField.PAGES, "44-48"); sunWelchEntry.setField(StandardField.TITLE, "Solution-processed small-molecule solar cells with 6.7\\% efficiency"); sunWelchEntry.setField(StandardField.VOLUME, "11"); sunWelchEntry.setField(StandardField.YEAR, "2012"); - sunWelchEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012NatMa..11...44S"); + sunWelchEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012NatMa..11...44S"); xiongSunEntry = new BibEntry(); xiongSunEntry.setType(StandardEntryType.Article); xiongSunEntry.setCiteKey("2007ITGRS..45..879X"); - xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, X. and Sun, J. and Barnes, W. and Salomonson, V. and Esposito, J. and Erives, H. and Guenther, B."); + xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, Xiaoxiong and Sun, Junqiang and Barnes, William and Salomonson, Vincent and Esposito, Joseph and Erives, Hector and Guenther, Bruce"); xiongSunEntry.setField(StandardField.DOI, "10.1109/TGRS.2006.890567"); xiongSunEntry.setField(StandardField.JOURNAL, "IEEE Transactions on Geoscience and Remote Sensing"); - xiongSunEntry.setField(StandardField.MONTH, "#apr#"); + xiongSunEntry.setField(StandardField.MONTH, "Apr"); + xiongSunEntry.setField(StandardField.NUMBER, "4"); xiongSunEntry.setField(StandardField.PAGES, "879-889"); xiongSunEntry.setField(StandardField.TITLE, "Multiyear On-Orbit Calibration and Performance of Terra MODIS Reflective Solar Bands"); xiongSunEntry.setField(StandardField.VOLUME, "45"); xiongSunEntry.setField(StandardField.YEAR, "2007"); - xiongSunEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2007ITGRS..45..879X"); + xiongSunEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2007ITGRS..45..879X"); ingersollPollardEntry = new BibEntry(); ingersollPollardEntry.setType(StandardEntryType.Article); ingersollPollardEntry.setCiteKey("1982Icar...52...62I"); + ingersollPollardEntry.setField(StandardField.ABSTRACT, "If Jupiter's and Saturn's fluid interiors were inviscid and adiabatic, any steady zonal motion would take the form of differentially rotating cylinders concentric about the planetary axis of rotation. B. A. Smith et al. [ Science215, 504-537 (1982)] showed that Saturn's observed zonal wind profile extends a significant distance below cloud base. Further extension into the interior occurs if the values of the eddy viscosity and superadiabaticity are small. We estimate these values using a scaling analysis of deep convection in the presence of differential rotation. The differential rotation inhibits the convection and reduces the effective eddy viscosity. Viscous dissipation of zonal mean kinetic energy is then within the bounds set by the internal heat source. The differential rotation increases the superadiabaticity, but not so much as to eliminate the cylindrical structure of the flow. Very large departures from adiabaticity, necessary for decoupling the atmosphere and interior, do not occur. Using our scaling analysis we develop the anelastic equations that describe motions in Jupiter's and Saturn's interiors. A simple problem is solved, that of an adiabatic fluid with a steady zonal wind varying as a function of cylindrical radius. Low zonal wavenumber perturbations are two dimensional (independent of the axial coordinate) and obey a modified barotropic stability equation. The parameter analogous to {\\ensuremath{\\beta}} is negative and is three to four times larger than the {\\ensuremath{\\beta}} for thin atmospheres. Jupiter's and Saturn's observed zonal wind profiles are close to marginal stability according to this deep sphere criterion, but are several times supercritical according to the thin atmosphere criterion."); ingersollPollardEntry.setField(StandardField.AUTHOR, "Ingersoll, A. P. and Pollard, D."); ingersollPollardEntry.setField(StandardField.DOI, "10.1016/0019-1035(82)90169-5"); ingersollPollardEntry.setField(StandardField.JOURNAL, "\\icarus"); - ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS"); - ingersollPollardEntry.setField(StandardField.MONTH, "#oct#"); + ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS, Lunar and Planetary Exploration; Planets"); + ingersollPollardEntry.setField(StandardField.MONTH, "Oct"); + ingersollPollardEntry.setField(StandardField.NUMBER, "1"); ingersollPollardEntry.setField(StandardField.PAGES, "62-80"); - ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn - Scale analysis, anelastic equations, barotropic stability criterion"); + ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn: scale analysis, anelastic equations, barotropic stability criterion"); ingersollPollardEntry.setField(StandardField.VOLUME, "52"); ingersollPollardEntry.setField(StandardField.YEAR, "1982"); - ingersollPollardEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/1982Icar...52...62I"); + ingersollPollardEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/1982Icar...52...62I"); luceyPaulEntry = new BibEntry(); luceyPaulEntry.setType(StandardEntryType.Article); luceyPaulEntry.setCiteKey("2000JGR...10520297L"); - luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, P. G. and Blewett, D. T. and Jolliff, B. L."); + luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, Paul G. and Blewett, David T. and Jolliff, Bradley L."); luceyPaulEntry.setField(StandardField.DOI, "10.1029/1999JE001117"); luceyPaulEntry.setField(StandardField.JOURNAL, "\\jgr"); luceyPaulEntry.setField(StandardField.KEYWORDS, "Planetology: Solid Surface Planets: Composition, Planetology: Solid Surface Planets: Remote sensing, Planetology: Solid Surface Planets: Surface materials and properties, Planetology: Solar System Objects: Moon (1221)"); @@ -131,7 +124,9 @@ public void setUp() throws Exception { luceyPaulEntry.setField(StandardField.TITLE, "Lunar iron and titanium abundance algorithms based on final processing of Clementine ultraviolet-visible images"); luceyPaulEntry.setField(StandardField.VOLUME, "105"); luceyPaulEntry.setField(StandardField.YEAR, "2000"); - luceyPaulEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2000JGR...10520297L"); + luceyPaulEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2000JGR...10520297L"); + luceyPaulEntry.setField(StandardField.MONTH, "Jan"); + luceyPaulEntry.setField(StandardField.NUMBER, "E8"); } @Test @@ -141,13 +136,13 @@ public void testHelpPage() { @Test public void testGetName() { - assertEquals("SAO/NASA Astrophysics Data System", fetcher.getName()); + assertEquals("New SAO/NASA Astrophysics Data System", fetcher.getName()); } @Test public void searchByQueryFindsEntry() throws Exception { List fetchedEntries = fetcher.performSearch("Diez slice theorem Lie"); - assertEquals(Collections.singletonList(diezSliceTheoremEntry), fetchedEntries); + assertTrue(fetchedEntries.contains(diezSliceTheoremEntry)); } @Test @@ -200,7 +195,7 @@ public void testPerformSearchByIngersollPollardEntry() throws Exception { @Test public void testPerformSearchByLuceyPaulEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("10.1029/1999JE001117"); + Optional fetchedEntry = fetcher.performSearchById("2000JGR...10520297L"); assertEquals(Optional.of(luceyPaulEntry), fetchedEntry); } } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java deleted file mode 100644 index 403bfaefa55..00000000000 --- a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java +++ /dev/null @@ -1,201 +0,0 @@ -package org.jabref.logic.importer.fetcher; - -import java.util.List; -import java.util.Optional; - -import org.jabref.logic.bibtex.FieldContentParserPreferences; -import org.jabref.logic.importer.ImportFormatPreferences; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.types.StandardEntryType; -import org.jabref.testutils.category.FetcherTest; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@FetcherTest -public class NewAstrophysicsDataSystemTest { - - private NewAstrophysicsDataSystem fetcher; - private BibEntry diezSliceTheoremEntry, famaeyMcGaughEntry, sunWelchEntry, xiongSunEntry, ingersollPollardEntry, luceyPaulEntry; - - @BeforeEach - public void setUp() throws Exception { - ImportFormatPreferences importFormatPreferences = mock(ImportFormatPreferences.class); - when(importFormatPreferences.getFieldContentParserPreferences()).thenReturn( - mock(FieldContentParserPreferences.class)); - fetcher = new NewAstrophysicsDataSystem(importFormatPreferences); - - diezSliceTheoremEntry = new BibEntry(); - diezSliceTheoremEntry.setType(StandardEntryType.Article); - diezSliceTheoremEntry.setCiteKey("2018arXiv181204698D"); - diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, Tobias and Rudolph, Gerd"); - diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem and orbit type stratification in infinite dimensions"); - diezSliceTheoremEntry.setField(StandardField.YEAR, "2018"); - diezSliceTheoremEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); - diezSliceTheoremEntry.setField(StandardField.EPRINT, "1812.04698"); - diezSliceTheoremEntry.setField(StandardField.JOURNAL, "arXiv e-prints"); - diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematics - Differential Geometry, Mathematical Physics, 58B25, (58D19, 58B20, 22E99, 58A35)"); - diezSliceTheoremEntry.setField(StandardField.MONTH, "Dec"); - diezSliceTheoremEntry.setField(StandardField.PAGES, "arXiv:1812.04698"); - diezSliceTheoremEntry.setField(StandardField.EID, "arXiv:1812.04698"); - diezSliceTheoremEntry.setField(StandardField.PRIMARYCLASS, "math.DG"); - diezSliceTheoremEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2018arXiv181204698D"); - diezSliceTheoremEntry.setField(StandardField.ABSTRACT, - "We establish a general slice theorem for the action of a locally convex Lie group on a locally convex manifold, which generalizes the classical slice theorem of Palais to infinite dimensions. We discuss two important settings under which the assumptions of this theorem are fulfilled. First, using Gl{\\\"o}ckner's inverse function theorem, we show that the linear action of a compact Lie group on a Fr{\\'e}chet space admits a slice. Second, using the Nash--Moser theorem, we establish a slice theorem for the tame action of a tame Fr{\\'e}chet Lie group on a tame Fr{\\'e}chet manifold. For this purpose, we develop the concept of a graded Riemannian metric, which allows the construction of a path-length metric compatible with the manifold topology and of a local addition. Finally, generalizing a classical result in finite dimensions, we prove that the existence of a slice implies that the decomposition of the manifold into orbit types of the group action is a stratification."); - - famaeyMcGaughEntry = new BibEntry(); - famaeyMcGaughEntry.setType(StandardEntryType.Article); - famaeyMcGaughEntry.setCiteKey("2012LRR....15...10F"); - famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, Beno{\\^\\i}t and McGaugh, Stacy S."); - famaeyMcGaughEntry.setField(StandardField.TITLE, "Modified Newtonian Dynamics (MOND): Observational Phenomenology and Relativistic Extensions"); - famaeyMcGaughEntry.setField(StandardField.JOURNAL, "Living Reviews in Relativity"); - famaeyMcGaughEntry.setField(StandardField.YEAR, "2012"); - famaeyMcGaughEntry.setField(StandardField.VOLUME, "15"); - famaeyMcGaughEntry.setField(StandardField.MONTH, "Sep"); - famaeyMcGaughEntry.setField(StandardField.NUMBER, "1"); - famaeyMcGaughEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); - famaeyMcGaughEntry.setField(StandardField.DOI, "10.12942/lrr-2012-10"); - famaeyMcGaughEntry.setField(StandardField.PRIMARYCLASS, "astro-ph.CO"); - famaeyMcGaughEntry.setField(StandardField.EID, "10"); - famaeyMcGaughEntry.setField(StandardField.EPRINT, "1112.3960"); - famaeyMcGaughEntry.setField(StandardField.PAGES, "10"); - famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics, Astrophysics - Cosmology and Nongalactic Astrophysics, Astrophysics - Astrophysics of Galaxies, General Relativity and Quantum Cosmology, High Energy Physics - Phenomenology, High Energy Physics - Theory"); - famaeyMcGaughEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012LRR....15...10F"); - - sunWelchEntry = new BibEntry(); - sunWelchEntry.setType(StandardEntryType.Article); - sunWelchEntry.setCiteKey("2012NatMa..11...44S"); - sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Yanming and Welch, Gregory C. and Leong, Wei Lin and Takacs, Christopher J. and Bazan, Guillermo C. and Heeger, Alan J."); - sunWelchEntry.setField(StandardField.DOI, "10.1038/nmat3160"); - sunWelchEntry.setField(StandardField.JOURNAL, "Nature Materials"); - sunWelchEntry.setField(StandardField.MONTH, "Jan"); - sunWelchEntry.setField(StandardField.NUMBER, "1"); - sunWelchEntry.setField(StandardField.PAGES, "44-48"); - sunWelchEntry.setField(StandardField.TITLE, "Solution-processed small-molecule solar cells with 6.7\\% efficiency"); - sunWelchEntry.setField(StandardField.VOLUME, "11"); - sunWelchEntry.setField(StandardField.YEAR, "2012"); - sunWelchEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012NatMa..11...44S"); - - xiongSunEntry = new BibEntry(); - xiongSunEntry.setType(StandardEntryType.Article); - xiongSunEntry.setCiteKey("2007ITGRS..45..879X"); - xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, Xiaoxiong and Sun, Junqiang and Barnes, William and Salomonson, Vincent and Esposito, Joseph and Erives, Hector and Guenther, Bruce"); - xiongSunEntry.setField(StandardField.DOI, "10.1109/TGRS.2006.890567"); - xiongSunEntry.setField(StandardField.JOURNAL, "IEEE Transactions on Geoscience and Remote Sensing"); - xiongSunEntry.setField(StandardField.MONTH, "Apr"); - xiongSunEntry.setField(StandardField.NUMBER, "4"); - xiongSunEntry.setField(StandardField.PAGES, "879-889"); - xiongSunEntry.setField(StandardField.TITLE, "Multiyear On-Orbit Calibration and Performance of Terra MODIS Reflective Solar Bands"); - xiongSunEntry.setField(StandardField.VOLUME, "45"); - xiongSunEntry.setField(StandardField.YEAR, "2007"); - xiongSunEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2007ITGRS..45..879X"); - - ingersollPollardEntry = new BibEntry(); - ingersollPollardEntry.setType(StandardEntryType.Article); - ingersollPollardEntry.setCiteKey("1982Icar...52...62I"); - ingersollPollardEntry.setField(StandardField.ABSTRACT, "If Jupiter's and Saturn's fluid interiors were inviscid and adiabatic, any steady zonal motion would take the form of differentially rotating cylinders concentric about the planetary axis of rotation. B. A. Smith et al. [ Science215, 504-537 (1982)] showed that Saturn's observed zonal wind profile extends a significant distance below cloud base. Further extension into the interior occurs if the values of the eddy viscosity and superadiabaticity are small. We estimate these values using a scaling analysis of deep convection in the presence of differential rotation. The differential rotation inhibits the convection and reduces the effective eddy viscosity. Viscous dissipation of zonal mean kinetic energy is then within the bounds set by the internal heat source. The differential rotation increases the superadiabaticity, but not so much as to eliminate the cylindrical structure of the flow. Very large departures from adiabaticity, necessary for decoupling the atmosphere and interior, do not occur. Using our scaling analysis we develop the anelastic equations that describe motions in Jupiter's and Saturn's interiors. A simple problem is solved, that of an adiabatic fluid with a steady zonal wind varying as a function of cylindrical radius. Low zonal wavenumber perturbations are two dimensional (independent of the axial coordinate) and obey a modified barotropic stability equation. The parameter analogous to {\\ensuremath{\\beta}} is negative and is three to four times larger than the {\\ensuremath{\\beta}} for thin atmospheres. Jupiter's and Saturn's observed zonal wind profiles are close to marginal stability according to this deep sphere criterion, but are several times supercritical according to the thin atmosphere criterion."); - ingersollPollardEntry.setField(StandardField.AUTHOR, "Ingersoll, A. P. and Pollard, D."); - ingersollPollardEntry.setField(StandardField.DOI, "10.1016/0019-1035(82)90169-5"); - ingersollPollardEntry.setField(StandardField.JOURNAL, "\\icarus"); - ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS, Lunar and Planetary Exploration; Planets"); - ingersollPollardEntry.setField(StandardField.MONTH, "Oct"); - ingersollPollardEntry.setField(StandardField.NUMBER, "1"); - ingersollPollardEntry.setField(StandardField.PAGES, "62-80"); - ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn: scale analysis, anelastic equations, barotropic stability criterion"); - ingersollPollardEntry.setField(StandardField.VOLUME, "52"); - ingersollPollardEntry.setField(StandardField.YEAR, "1982"); - ingersollPollardEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/1982Icar...52...62I"); - - luceyPaulEntry = new BibEntry(); - luceyPaulEntry.setType(StandardEntryType.Article); - luceyPaulEntry.setCiteKey("2000JGR...10520297L"); - luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, Paul G. and Blewett, David T. and Jolliff, Bradley L."); - luceyPaulEntry.setField(StandardField.DOI, "10.1029/1999JE001117"); - luceyPaulEntry.setField(StandardField.JOURNAL, "\\jgr"); - luceyPaulEntry.setField(StandardField.KEYWORDS, "Planetology: Solid Surface Planets: Composition, Planetology: Solid Surface Planets: Remote sensing, Planetology: Solid Surface Planets: Surface materials and properties, Planetology: Solar System Objects: Moon (1221)"); - luceyPaulEntry.setField(StandardField.PAGES, "20297-20306"); - luceyPaulEntry.setField(StandardField.TITLE, "Lunar iron and titanium abundance algorithms based on final processing of Clementine ultraviolet-visible images"); - luceyPaulEntry.setField(StandardField.VOLUME, "105"); - luceyPaulEntry.setField(StandardField.YEAR, "2000"); - luceyPaulEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2000JGR...10520297L"); - luceyPaulEntry.setField(StandardField.MONTH, "Jan"); - luceyPaulEntry.setField(StandardField.NUMBER, "E8"); - } - - @Test - public void testHelpPage() { - assertEquals("ADS", fetcher.getHelpPage().get().getPageName()); - } - - @Test - public void testGetName() { - assertEquals("New SAO/NASA Astrophysics Data System", fetcher.getName()); - } - - @Test - public void searchByQueryFindsEntry() throws Exception { - List fetchedEntries = fetcher.performSearch("Diez slice theorem Lie"); - assertTrue(fetchedEntries.contains(diezSliceTheoremEntry)); - } - - @Test - public void searchByEntryFindsEntry() throws Exception { - BibEntry searchEntry = new BibEntry(); - searchEntry.setField(StandardField.TITLE, "slice theorem"); - searchEntry.setField(StandardField.AUTHOR, "Diez"); - - List fetchedEntries = fetcher.performSearch(searchEntry); - assertFalse(fetchedEntries.isEmpty()); - assertEquals(diezSliceTheoremEntry, fetchedEntries.get(0)); - } - - @Test - public void testPerformSearchByFamaeyMcGaughEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("10.12942/lrr-2012-10"); - fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT));//Remove abstract due to copyright - assertEquals(Optional.of(famaeyMcGaughEntry), fetchedEntry); - } - - @Test - public void testPerformSearchByIdEmptyDOI() throws Exception { - Optional fetchedEntry = fetcher.performSearchById(""); - assertEquals(Optional.empty(), fetchedEntry); - } - - @Test - public void testPerformSearchByIdInvalidDoi() throws Exception { - assertEquals(Optional.empty(), fetcher.performSearchById("this.doi.will.fail")); - } - - @Test - public void testPerformSearchBySunWelchEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("10.1038/nmat3160"); - fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT)); //Remove abstract due to copyright - assertEquals(Optional.of(sunWelchEntry), fetchedEntry); - } - - @Test - public void testPerformSearchByXiongSunEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("10.1109/TGRS.2006.890567"); - assertEquals(Optional.of(xiongSunEntry), fetchedEntry); - } - - @Test - public void testPerformSearchByIngersollPollardEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("10.1016/0019-1035(82)90169-5"); - assertEquals(Optional.of(ingersollPollardEntry), fetchedEntry); - } - - @Test - public void testPerformSearchByLuceyPaulEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("2000JGR...10520297L"); - assertEquals(Optional.of(luceyPaulEntry), fetchedEntry); - } -} From c9f46df5ec648ac82c96b2bb42c2420a40d3dc7f Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 17:57:47 +0200 Subject: [PATCH 13/15] replace old fetcher with the new one --- .../logic/importer/fetcher/AstrophysicsDataSystemTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java index f7f393c49dd..f3141019524 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java @@ -136,7 +136,7 @@ public void testHelpPage() { @Test public void testGetName() { - assertEquals("New SAO/NASA Astrophysics Data System", fetcher.getName()); + assertEquals("SAO/NASA Astrophysics Data System", fetcher.getName()); } @Test From 9319bb139f354f1522f59c8b360b1ff3d00a1343 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 19:15:32 +0200 Subject: [PATCH 14/15] added Javadoc --- .../fetcher/AstrophysicsDataSystem.java | 55 +++++++++++++------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index faa99419c0c..f7d8c7f9c82 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -40,12 +40,7 @@ import org.json.JSONObject; /** - * Fetches data from the SAO/NASA Astrophysics Data System (http://www.adsabs.harvard.edu/) - *

- * Search query-based: http://adsabs.harvard.edu/basic_search.html Entry -based: http://adsabs.harvard.edu/abstract_service.html - *

- * There is also a new API (https://github.com/adsabs/adsabs-dev-api) but it returns JSON (or at least needs multiple - * calls to get BibTeX, status: September 2016) + * Fetches data from the SAO/NASA Astrophysics Data System (https://ui.adsabs.harvard.edu/) */ public class AstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher { @@ -59,13 +54,19 @@ public AstrophysicsDataSystem(ImportFormatPreferences preferences) { this.preferences = Objects.requireNonNull(preferences); } - private String buildPostData(Collection bibcodes) { + /** + * @param bibcodes collection of bibcodes for which a JSON object should be created + */ + private static String buildPostData(Collection bibcodes) { JSONObject obj = new JSONObject(); obj.put("bibcode", bibcodes); return obj.toString(); } - private URL getURLforExport() throws URISyntaxException, MalformedURLException { + /** + * @return export URL endpoint + */ + private static URL getURLforExport() throws URISyntaxException, MalformedURLException { return new URIBuilder(API_EXPORT_URL).build().toURL(); } @@ -74,6 +75,10 @@ public String getName() { return "SAO/NASA Astrophysics Data System"; } + /** + * @param query query string, matching the apache solr format + * @return URL which points to a search request for given query + */ @Override public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder builder = new URIBuilder(API_SEARCH_URL); @@ -82,6 +87,10 @@ public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLE return builder.build().toURL(); } + /** + * @param entry BibEntry for which a search URL is created + * @return URL which points to a search request for given entry + */ @Override public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedURLException, FetcherException { StringBuilder stringBuilder = new StringBuilder(); @@ -105,6 +114,10 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR return builder.build().toURL(); } + /** + * @param identifier bibcode or doi for which a search URL is created + * @return URL which points to a search URL for given identifier + */ @Override public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { String query = "doi:\"" + identifier + "\" OR " + "bibcode:\"" + identifier + "\""; @@ -173,6 +186,10 @@ public List performSearch(String query) throws FetcherException { } } + /** + * @param url search ul for which bibcode will be returned + * @return list of bibcodes matching the search request. May be empty + */ private List fetchBibcodes(URL url) throws FetcherException { try { @@ -180,19 +197,16 @@ private List fetchBibcodes(URL url) throws FetcherException { download.addHeader("Authorization", "Bearer " + API_KEY); String content = download.asString(); JSONObject obj = new JSONObject(content); - - try { - JSONArray codes = obj.getJSONObject("response").getJSONArray("docs"); - List bibcodes = new ArrayList<>(); - for (int i = 0; i < codes.length(); i++) { - bibcodes.add(codes.getJSONObject(i).getString("bibcode")); - } - return bibcodes; - } catch (JSONException e) { - return Collections.emptyList(); + JSONArray codes = obj.getJSONObject("response").getJSONArray("docs"); + List bibcodes = new ArrayList<>(); + for (int i = 0; i < codes.length(); i++) { + bibcodes.add(codes.getJSONObject(i).getString("bibcode")); } + return bibcodes; } catch (IOException e) { throw new FetcherException("A network error occurred", e); + } catch (JSONException e) { + return Collections.emptyList(); } } @@ -222,6 +236,11 @@ public Optional performSearchById(String identifier) throws FetcherExc } } + /** + * @param identifiers bibcodes for which bibentries ahould be fetched + * @return list of bibentries matching the bibcodes. Can be empty and differ in size to the size of requested + * bibcodes + */ private List performSearchByIds(Collection identifiers) throws FetcherException { List ids = identifiers.stream().filter(identifier -> !StringUtil.isBlank(identifier)).collect(Collectors.toList()); From 6e81ed07ea9e86232c783f48adfecd75e187d4d2 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 24 Oct 2019 20:20:48 +0200 Subject: [PATCH 15/15] Add token --- .../jabref/logic/importer/fetcher/AstrophysicsDataSystem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index f7d8c7f9c82..70f81f095dd 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -47,7 +47,7 @@ public class AstrophysicsDataSystem implements IdBasedParserFetcher, SearchBased private static final String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; private static final String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; - private static final String API_KEY = ""; //TODO Add API Token + private static final String API_KEY = "tDueGIu6zl96OqkcCS5LOHboWbTgEEx8yAR7Etta"; private final ImportFormatPreferences preferences; public AstrophysicsDataSystem(ImportFormatPreferences preferences) {