Fix some fetcher tests

.github/workflows/deployment.yml

@@ -17,6 +17,8 @@ on:
 
 env:
   SpringerNatureAPIKey: ${{ secrets.SpringerNatureAPIKey }}
+  AstrophysicsDataSystemAPIKey: ${{ secrets.AstrophysicsDataSystemAPIKey }}
+  IEEEAPIKey: ${{ secrets.IEEEAPIKey }}
 
 jobs:
   build:

.github/workflows/tests-fetchers.yml

@@ -23,6 +23,8 @@ on:
 
 env:
   SpringerNatureAPIKey: ${{ secrets.SpringerNatureAPIKey }}
+  AstrophysicsDataSystemAPIKey: ${{ secrets.AstrophysicsDataSystemAPIKey }}
+  IEEEAPIKey: ${{ secrets.IEEEAPIKey }}
 
 jobs:
   fetchertests:

.github/workflows/tests.yml

@@ -9,6 +9,8 @@ on:
 
 env:
   SpringerNatureAPIKey: ${{ secrets.SpringerNatureAPIKey }}
+  AstrophysicsDataSystemAPIKey: ${{ secrets.AstrophysicsDataSystemAPIKey }}
+  IEEEAPIKey: ${{ secrets.IEEEAPIKey }}
 
 jobs:
   checkstyle:

build.gradle

@@ -276,6 +276,8 @@ processResources {
                 "developers": new File('DEVELOPERS').readLines().findAll { !it.startsWith("#") }.join(", "),
                 "azureInstrumentationKey": System.getenv('AzureInstrumentationKey'),
                 "springerNatureAPIKey": System.getenv('SpringerNatureAPIKey'),
+                "astrophysicsDataSystemAPIKey": System.getenv('AstrophysicsDataSystemAPIKey'),
+                "ieeeAPIKey": System.getenv('IEEEAPIKey'),
                 "minRequiredJavaVersion": minRequiredJavaVersion,
                 "allowJava9": allowJava9
 

src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java

@@ -28,6 +28,7 @@
 import org.jabref.logic.importer.SearchBasedParserFetcher;
 import org.jabref.logic.importer.fileformat.BibtexParser;
 import org.jabref.logic.net.URLDownload;
+import org.jabref.logic.util.BuildInfo;
 import org.jabref.model.cleanup.FieldFormatterCleanup;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.entry.field.StandardField;
@@ -48,7 +49,7 @@ public class AstrophysicsDataSystem implements IdBasedParserFetcher, SearchBased
     private static final String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query";
     private static final String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs";
 
-    private static final String API_KEY = "tDueGIu6zl96OqkcCS5LOHboWbTgEEx8yAR7Etta";
+    private static final String API_KEY = new BuildInfo().astrophysicsDataSystemAPIKey;
     private final ImportFormatPreferences preferences;
 
     public AstrophysicsDataSystem(ImportFormatPreferences preferences) {

src/main/java/org/jabref/logic/importer/fetcher/IEEE.java

@@ -6,6 +6,7 @@
 import java.net.MalformedURLException;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
@@ -15,12 +16,12 @@
 import java.util.stream.Collectors;
 
 import org.jabref.logic.help.HelpFile;
-import org.jabref.logic.importer.FetcherException;
 import org.jabref.logic.importer.FulltextFetcher;
 import org.jabref.logic.importer.ImportFormatPreferences;
 import org.jabref.logic.importer.Parser;
 import org.jabref.logic.importer.SearchBasedParserFetcher;
 import org.jabref.logic.net.URLDownload;
+import org.jabref.logic.util.BuildInfo;
 import org.jabref.logic.util.OS;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.entry.LinkedFile;
@@ -51,6 +52,7 @@ public class IEEE implements FulltextFetcher, SearchBasedParserFetcher {
     private static final Pattern PDF_PATTERN = Pattern.compile("\"(https://ieeexplore.ieee.org/ielx[0-9/]+\\.pdf[^\"]+)\"");
     private static final String IEEE_DOI = "10.1109";
     private static final String BASE_URL = "https://ieeexplore.ieee.org";
+    private static final String API_KEY = new BuildInfo().ieeeAPIKey;
 
     private final ImportFormatPreferences preferences;
 
@@ -88,7 +90,7 @@ private static BibEntry parseJsonRespone(JSONObject jsonEntry, Character keyword
             JSONObject author = (JSONObject) authorPure;
             authors.add(author.optString("full_name"));
         });
-        entry.setField(StandardField.AUTHOR, authors.stream().collect(Collectors.joining(" and ")));
+        entry.setField(StandardField.AUTHOR, String.join(" and ", authors));
         entry.setField(StandardField.LOCATION, jsonEntry.optString("conference_location"));
         entry.setField(StandardField.DOI, jsonEntry.optString("doi"));
         entry.setField(StandardField.YEAR, jsonEntry.optString("publication_year"));
@@ -113,7 +115,7 @@ private static BibEntry parseJsonRespone(JSONObject jsonEntry, Character keyword
         entry.setField(StandardField.ISBN, jsonEntry.optString("isbn"));
         entry.setField(StandardField.ISSN, jsonEntry.optString("issn"));
         entry.setField(StandardField.ISSUE, jsonEntry.optString("issue"));
-        entry.addFile(new LinkedFile("", jsonEntry.optString("pdf_url"), "PDF"));
+        entry.addFile(new LinkedFile("", Path.of(jsonEntry.optString("pdf_url")), "PDF"));
         entry.setField(StandardField.JOURNALTITLE, jsonEntry.optString("publication_title"));
         entry.setField(StandardField.DATE, jsonEntry.optString("publication_date"));
         entry.setField(StandardField.EVENTTITLEADDON, jsonEntry.optString("conference_location"));
@@ -195,9 +197,9 @@ public TrustLevel getTrustLevel() {
     }
 
     @Override
-    public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException {
+    public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException {
         URIBuilder uriBuilder = new URIBuilder("https://ieeexploreapi.ieee.org/api/v1/search/articles");
-        uriBuilder.addParameter("apikey", "86wnawtvtc986d3wtnqynm8c");
+        uriBuilder.addParameter("apikey", API_KEY);
         uriBuilder.addParameter("querytext", query);
 
         URLDownload.bypassSSLVerification();

src/main/java/org/jabref/logic/importer/util/GrobidService.java

@@ -51,7 +51,7 @@ public String processCitation(String rawCitation, ConsolidateCitations consolida
         urlDownload.setPostData("citations=" + rawCitation + "&consolidateCitations=" + consolidateCitations);
         String httpResponse = urlDownload.asString();
 
-        if (httpResponse == null || httpResponse.equals("@misc{-1,\n\n}\n")) { // This filters empty BibTeX entries
+        if (httpResponse == null || httpResponse.equals("@misc{-1,\n  author = {}\n}\n")) { // This filters empty BibTeX entries
             throw new IOException("The GROBID server response does not contain anything.");
         }
 

src/main/java/org/jabref/logic/util/BuildInfo.java

@@ -22,6 +22,8 @@ public final class BuildInfo {
     public final String year;
     public final String azureInstrumentationKey;
     public final String springerNatureAPIKey;
+    public final String astrophysicsDataSystemAPIKey;
+    public final String ieeeAPIKey;
     public final String minRequiredJavaVersion;
     public final boolean allowJava9;
 
@@ -48,6 +50,8 @@ public BuildInfo(String path) {
         developers = properties.getProperty("developers", "");
         azureInstrumentationKey = properties.getProperty("azureInstrumentationKey", "");
         springerNatureAPIKey = properties.getProperty("springerNatureAPIKey", "");
+        astrophysicsDataSystemAPIKey = properties.getProperty("astrophysicsDataSystemAPIKey", "");
+        ieeeAPIKey = properties.getProperty("ieeeAPIKey", "");
         minRequiredJavaVersion = properties.getProperty("minRequiredJavaVersion", "1.8");
         allowJava9 = "true".equals(properties.getProperty("allowJava9", ""));
     }

src/main/resources/build.properties

@@ -4,5 +4,7 @@ authors=${authors}
 developers=${developers}
 azureInstrumentationKey=${azureInstrumentationKey}
 springerNatureAPIKey=${springerNatureAPIKey}
+astrophysicsDataSystemAPIKey=${astrophysicsDataSystemAPIKey}
+ieeeAPIKey=${ieeeAPIKey}
 minRequiredJavaVersion = ${minRequiredJavaVersion}
 allowJava9 = ${allowJava9}

src/test/java/org/jabref/logic/importer/fetcher/AbstractIsbnFetcherTest.java

@@ -19,8 +19,6 @@ public abstract class AbstractIsbnFetcherTest {
 
     public abstract void testName();
 
-    public abstract void testHelpPage();
-
     public abstract void authorsAreCorrectlyFormatted() throws Exception;
 
     public abstract void searchByIdSuccessfulWithShortISBN() throws FetcherException;

src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java

@@ -134,11 +134,6 @@ public void setUp() throws Exception {
         luceyPaulEntry.setField(StandardField.NUMBER, "E8");
     }
 
-    @Test
-    public void testHelpPage() {
-        assertEquals("import-using-online-bibliographic-database/ads", fetcher.getHelpPage().get().getPageName());
-    }
-
     @Test
     public void testGetName() {
         assertEquals("SAO/NASA Astrophysics Data System", fetcher.getName());

src/test/java/org/jabref/logic/importer/fetcher/DiVATest.java

@@ -2,7 +2,6 @@
 
 import java.util.Optional;
 
-import org.jabref.logic.help.HelpFile;
 import org.jabref.logic.importer.ImportFormatPreferences;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.entry.field.StandardField;
@@ -33,11 +32,6 @@ public void testGetName() {
         assertEquals("DiVA", fetcher.getName());
     }
 
-    @Test
-    public void testGetHelpPage() {
-        assertEquals(HelpFile.FETCHER_DIVA, HelpFile.FETCHER_DIVA);
-    }
-
     @Test
     public void testPerformSearchById() throws Exception {
         BibEntry entry = new BibEntry();

src/test/java/org/jabref/logic/importer/fetcher/DoiFetcherTest.java

@@ -55,11 +55,6 @@ public void testGetName() {
         assertEquals("DOI", fetcher.getName());
     }
 
-    @Test
-    public void testGetHelpPage() {
-        assertEquals("import-using-publication-identifiers/doitobibtex", fetcher.getHelpPage().get().getPageName());
-    }
-
     @Test
     public void testPerformSearchBurd2011() throws FetcherException {
         Optional<BibEntry> fetchedEntry = fetcher.performSearchById("10.1002/9781118257517");

src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java

@@ -23,7 +23,7 @@ public class GrobidCitationFetcherTest {
 
     static String example1 = "Derwing, T. M., Rossiter, M. J., & Munro, M. J. (2002). Teaching native speakers to listen to foreign-accented speech. Journal of Multilingual and Multicultural Development, 23(4), 245-259.";
     static BibEntry example1AsBibEntry = new BibEntry(StandardEntryType.Article).withCiteKey("-1")
-                .withField(StandardField.AUTHOR, "T Derwing and M Rossiter and M Munro")
+                .withField(StandardField.AUTHOR, "Derwing, T and Rossiter, M and Munro, M")
                 .withField(StandardField.TITLE, "Teaching native speakers to listen to foreign-accented speech")
                 .withField(StandardField.JOURNAL, "Journal of Multilingual and Multicultural Development")
                 .withField(StandardField.YEAR, "2002")
@@ -32,20 +32,20 @@ public class GrobidCitationFetcherTest {
                 .withField(StandardField.NUMBER, "4");
     static String example2 = "Thomas, H. K. (2004). Training strategies for improving listeners' comprehension of foreign-accented speech (Doctoral dissertation). University of Colorado, Boulder.";
     static BibEntry example2AsBibEntry = new BibEntry(BibEntry.DEFAULT_TYPE).withCiteKey("-1")
-            .withField(StandardField.AUTHOR, "H Thomas")
+            .withField(StandardField.AUTHOR, "Thomas, H")
             .withField(StandardField.TITLE, "Training strategies for improving listeners' comprehension of foreign-accented speech (Doctoral dissertation)")
             .withField(StandardField.YEAR, "2004")
             .withField(StandardField.ADDRESS, "Boulder");
     static String example3 = "Turk, J., Graham, P., & Verhulst, F. (2007). Child and adolescent psychiatry : A developmental approach. Oxford, England: Oxford University Press.";
     static BibEntry example3AsBibEntry = new BibEntry(BibEntry.DEFAULT_TYPE).withCiteKey("-1")
-            .withField(StandardField.AUTHOR, "J Turk and P Graham and F Verhulst")
+            .withField(StandardField.AUTHOR, "Turk, J and Graham, P and Verhulst, F")
             .withField(StandardField.TITLE, "Child and adolescent psychiatry : A developmental approach")
             .withField(StandardField.PUBLISHER, "Oxford University Press")
             .withField(StandardField.YEAR, "2007")
             .withField(StandardField.ADDRESS, "Oxford, England");
     static String example4 = "Carr, I., & Kidner, R. (2003). Statutes and conventions on international trade law (4th ed.). London, England: Cavendish.";
-    static BibEntry example4AsBibEntry = new BibEntry(StandardEntryType.Article).withCiteKey("-1")
-            .withField(StandardField.AUTHOR, "I Carr and R Kidner")
+    static BibEntry example4AsBibEntry = new BibEntry(StandardEntryType.InBook).withCiteKey("-1")
+            .withField(StandardField.AUTHOR, "Carr, I and Kidner, R")
             .withField(StandardField.BOOKTITLE, "Statutes and conventions on international trade law")
             .withField(StandardField.PUBLISHER, "Cavendish")
             .withField(StandardField.YEAR, "2003")

src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java

@@ -63,11 +63,6 @@ public void testGetName() {
         assertEquals("GVK", fetcher.getName());
     }
 
-    @Test
-    public void testGetHelpPage() {
-        assertEquals("import-using-online-bibliographic-database/gvk", fetcher.getHelpPage().get().getPageName());
-    }
-
     @Test
     public void simpleSearchQueryStringCorrect() {
         String query = "java jdk";

src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java

@@ -55,7 +55,7 @@ public void searchByIdentifierFindsEntry() throws Exception {
         article.setField(StandardField.AUTHOR, "Melnikov, Kirill and Yelkhovsky, Alexander");
         article.setField(StandardField.TITLE, "Top quark production at threshold with O(alpha-s**2) accuracy");
         article.setField(StandardField.DOI, "10.1016/S0550-3213(98)00348-4");
-        article.setField(StandardField.JOURNAL, "Nucl.\\ Phys.\\ B");
+        article.setField(StandardField.JOURNAL, "Nucl. Phys. B");
         article.setField(StandardField.PAGES, "59--72");
         article.setField(StandardField.VOLUME, "528");
         article.setField(StandardField.YEAR, "1998");

src/test/java/org/jabref/logic/importer/fetcher/IsbnFetcherTest.java

@@ -49,11 +49,6 @@ void testName() {
         assertEquals("ISBN", fetcher.getName());
     }
 
-    @Test
-    void testHelpPage() {
-        assertEquals("import-using-publication-identifiers/isbntobibtex", fetcher.getHelpPage().get().getPageName());
-    }
-
     @Test
     void searchByIdSuccessfulWithShortISBN() throws FetcherException {
         Optional<BibEntry> fetchedEntry = fetcher.performSearchById("0134685997");

src/test/java/org/jabref/logic/importer/fetcher/IsbnViaEbookDeFetcherTest.java

@@ -43,12 +43,6 @@ public void testName() {
         assertEquals("ISBN (ebook.de)", fetcher.getName());
     }
 
-    @Test
-    @Override
-    public void testHelpPage() {
-        assertEquals("import-using-publication-identifiers/isbntobibtex", fetcher.getHelpPage().get().getPageName());
-    }
-
     @Test
     @Override
     public void searchByIdSuccessfulWithShortISBN() throws FetcherException {

src/test/java/org/jabref/logic/importer/fetcher/IsbnViaOttoBibFetcherTest.java

@@ -40,12 +40,6 @@ public void testName() {
         assertEquals("ISBN (OttoBib)", fetcher.getName());
     }
 
-    @Test
-    @Override
-    public void testHelpPage() {
-        assertEquals("import-using-publication-identifiers/isbntobibtex", fetcher.getHelpPage().get().getPageName());
-    }
-
     @Test
     @Override
     public void searchByIdSuccessfulWithShortISBN() throws FetcherException {

src/test/java/org/jabref/logic/importer/fetcher/MedlineFetcherTest.java

@@ -133,35 +133,38 @@ public void testGetName() {
         assertEquals("Medline/PubMed", fetcher.getName());
     }
 
-    @Test
-    public void testGetHelpPage() {
-        assertEquals("import-using-online-bibliographic-database/medline", fetcher.getHelpPage().get().getPageName());
-    }
-
     @Test
     public void testSearchByIDWijedasa() throws Exception {
         Optional<BibEntry> fetchedEntry = fetcher.performSearchById("27670948");
+        assertTrue(fetchedEntry.isPresent());
+
         fetchedEntry.get().clearField(StandardField.ABSTRACT); // Remove abstract due to copyright
         assertEquals(Optional.of(entryWijedasa), fetchedEntry);
     }
 
     @Test
     public void testSearchByIDEndharti() throws Exception {
         Optional<BibEntry> fetchedEntry = fetcher.performSearchById("27670445");
+        assertTrue(fetchedEntry.isPresent());
+
         fetchedEntry.get().clearField(StandardField.ABSTRACT); // Remove abstract due to copyright
         assertEquals(Optional.of(entryEndharti), fetchedEntry);
     }
 
     @Test
     public void testSearchByIDIchikawa() throws Exception {
         Optional<BibEntry> fetchedEntry = fetcher.performSearchById("26197440");
+        assertTrue(fetchedEntry.isPresent());
+
         fetchedEntry.get().clearField(StandardField.ABSTRACT); // Remove abstract due to copyright
         assertEquals(Optional.of(bibEntryIchikawa), fetchedEntry);
     }
 
     @Test
     public void testSearchByIDSari() throws Exception {
         Optional<BibEntry> fetchedEntry = fetcher.performSearchById("26867355");
+        assertTrue(fetchedEntry.isPresent());
+
         fetchedEntry.get().clearField(StandardField.ABSTRACT); // Remove abstract due to copyright
         assertEquals(Optional.of(bibEntrySari), fetchedEntry);
     }

src/test/java/org/jabref/logic/importer/fetcher/RfcFetcherTest.java

@@ -33,7 +33,7 @@ public void setUp() {
         bibEntry.setField(StandardField.PUBLISHER, "RFC Editor");
         bibEntry.setField(StandardField.DOI, "10.17487/RFC1945");
         bibEntry.setField(StandardField.URL, "https://rfc-editor.org/rfc/rfc1945.txt");
-        bibEntry.setField(StandardField.AUTHOR, "Henrik Frystyk Nielsen and Roy T. Fielding and Tim Berners-Lee");
+        bibEntry.setField(StandardField.AUTHOR, "Henrik Nielsen and Roy T. Fielding and Tim Berners-Lee");
         bibEntry.setField(StandardField.TITLE, "{Hypertext Transfer Protocol -- HTTP/1.0}");
         bibEntry.setField(StandardField.PAGETOTAL, "60");
         bibEntry.setField(StandardField.YEAR, "1996");
@@ -47,11 +47,6 @@ public void getNameReturnsEqualIdName() {
         assertEquals("RFC", fetcher.getName());
     }
 
-    @Test
-    public void getHelpPageReturnsEqualHelpPage() {
-        assertEquals("import-using-publication-identifiers/rfctobibtex", fetcher.getHelpPage().get().getPageName());
-    }
-
     @Test
     public void performSearchByIdFindsEntryWithRfcPrefix() throws Exception {
         assertEquals(Optional.of(bibEntry), fetcher.performSearchById("RFC1945"));

src/test/java/org/jabref/logic/importer/fetcher/TitleFetcherTest.java

@@ -42,11 +42,6 @@ public void testGetName() {
         assertEquals("Title", fetcher.getName());
     }
 
-    @Test
-    public void testGetHelpPage() {
-        assertEquals("import-using-publication-identifiers/titletobibtex", fetcher.getHelpPage().get().getPageName());
-    }
-
     @Test
     public void testPerformSearchKopp2007() throws FetcherException {
         Optional<BibEntry> fetchedEntry = fetcher.performSearchById("BPELscript: A simplified script syntax for WS-BPEL 2.0");

src/test/java/org/jabref/logic/importer/util/GrobidServiceTest.java

@@ -19,7 +19,7 @@ public class GrobidServiceTest {
 
     @BeforeAll
     public static void setup() {
-        grobidService = new GrobidService("http://grobid.cm.in.tum.de:8070");
+        grobidService = new GrobidService("http://grobid.jabref.org:8070");
     }
 
     @Test
@@ -30,7 +30,7 @@ public void processValidCitationTest() throws IOException {
         String[] responseRows = response.split("\n");
         assertNotNull(response);
         assertEquals('@', response.charAt(0));
-        assertTrue(responseRows[1].contains("author") && responseRows[1].contains("Derwing and M Rossiter"));
+        assertTrue(responseRows[1].contains("author") && responseRows[1].contains("Derwing, T and Rossiter, M"));
         assertTrue(responseRows[2].contains("title") && responseRows[2].contains("Teaching native speakers"));
         assertTrue(responseRows[3].contains("journal") && responseRows[3].contains("Journal of Multilingual and Multicultural"));
         assertTrue(responseRows[4].contains("year") && responseRows[4].contains("2002"));