Merge pull request #287 from oscargus/doajsearch

DOAJ search

CHANGELOG

@@ -1,4 +1,5 @@
 [master]
+    - Feature: Search DOAJ, Directory of Open Access Journals
     - Removes options to set PDF and PS directories per .bib database as the general options have also been deleted.
     - Removes option to disable renaming in FileChooser dialogs.
     - Removes option to hide the BibTeX Code tab in the entry editor.

src/main/java/net/sf/jabref/importer/fetcher/DOAJFetcher.java

@@ -0,0 +1,146 @@
+/*  Copyright (C) 2015 Oscar Gustafsson.
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+package net.sf.jabref.importer.fetcher;
+
+import javax.swing.JOptionPane;
+import javax.swing.JPanel;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import com.mashape.unirest.http.HttpResponse;
+import com.mashape.unirest.http.JsonNode;
+import com.mashape.unirest.http.Unirest;
+import com.mashape.unirest.http.exceptions.UnirestException;
+
+import net.sf.jabref.importer.ImportInspector;
+import net.sf.jabref.importer.OutputPrinter;
+import net.sf.jabref.importer.fileformat.JSONEntryParser;
+import net.sf.jabref.logic.l10n.Localization;
+import net.sf.jabref.model.entry.BibtexEntry;
+
+public class DOAJFetcher implements EntryFetcher {
+
+    private final String searchURL = "https://doaj.org/api/v1/search/articles/";
+    private static final Log LOGGER = LogFactory.getLog(DOAJFetcher.class);
+    private final int maxPerPage = 100;
+    private boolean shouldContinue;
+
+
+    private final JSONEntryParser jsonConverter = new JSONEntryParser();
+
+    public DOAJFetcher() {
+        super();
+    }
+
+    @Override
+    public void stopFetching() {
+        shouldContinue = false;
+    }
+
+    @Override
+    public boolean processQuery(String query, ImportInspector inspector, OutputPrinter status) {
+        shouldContinue = true;
+        try {
+            status.setStatus(Localization.lang("Searching DOAJ..."));
+            HttpResponse<JsonNode> jsonResponse;
+            jsonResponse = Unirest.get(searchURL + query + "?pageSize=1").header("accept", "application/json").asJson();
+            JSONObject jo = jsonResponse.getBody().getObject();
+            int hits = jo.getInt("total");
+            int numberToFetch = 0;
+            if (hits > 0) {
+                if (hits > maxPerPage) {
+                    while (true) {
+                        String strCount = JOptionPane
+                                .showInputDialog(
+                                        Localization.lang("References found") + ": " + hits + "  "
+                                                + Localization.lang("Number of references to fetch?"),
+                                        Integer.toString(hits));
+
+                        if (strCount == null) {
+                            status.setStatus(Localization.lang("DOAJ search canceled"));
+                            return false;
+                        }
+
+                        try {
+                            numberToFetch = Integer.parseInt(strCount.trim());
+                            break;
+                        } catch (RuntimeException ex) {
+                            status.showMessage(Localization.lang("Please enter a valid number"));
+                        }
+                    }
+                } else {
+                    numberToFetch = hits;
+                }
+
+                int fetched = 0; // Keep track of number of items fetched for the progress bar
+                for (int page = 1; ((page - 1) * maxPerPage) <= numberToFetch; page++) {
+                    if (!shouldContinue) {
+                        break;
+                    }
+
+                    int noToFetch = Math.min(maxPerPage, numberToFetch - ((page - 1) * maxPerPage));
+                    jsonResponse = Unirest.get(searchURL + query + "?page=" + page + "&pageSize=" + noToFetch)
+                            .header("accept", "application/json").asJson();
+                    jo = jsonResponse.getBody().getObject();
+                    if (jo.has("results")) {
+                        JSONArray results = jo.getJSONArray("results");
+                        for (int i = 0; i < results.length(); i++) {
+                            JSONObject bibJsonEntry = results.getJSONObject(i).getJSONObject("bibjson");
+                            BibtexEntry entry = jsonConverter.BibJSONtoBibtex(bibJsonEntry);
+                            inspector.addEntry(entry);
+                            fetched++;
+                            inspector.setProgress(fetched, numberToFetch);
+                        }
+                    }
+                }
+                return true;
+            } else {
+                status.showMessage(Localization.lang("No entries found for the search string '%0'", query),
+                        Localization.lang("Search DOAJ"), JOptionPane.INFORMATION_MESSAGE);
+                return false;
+            }
+        } catch (UnirestException e) {
+            LOGGER.warn("Problem searching DOAJ", e);
+            return false;
+        }
+
+    }
+
+    @Override
+    public String getTitle() {
+        return "DOAJ";
+    }
+
+    @Override
+    public String getKeyName() {
+        return "DOAJ";
+    }
+
+    @Override
+    public String getHelpPage() {
+        return "DOAJHelp.html";
+    }
+
+    @Override
+    public JPanel getOptionsPanel() {
+        // No additional options available
+        return null;
+    }
+
+}

src/main/java/net/sf/jabref/importer/fetcher/EntryFetchers.java

@@ -42,6 +42,7 @@ public EntryFetchers() {
         entryFetchers.add(new SPIRESFetcher());
         entryFetchers.add(new ACMPortalFetcher());
         entryFetchers.add(new GoogleScholarFetcher());
+        entryFetchers.add(new DOAJFetcher());
     }
 
     public List<EntryFetcher> getEntryFetchers() {

src/main/java/net/sf/jabref/importer/fileformat/JSONEntryParser.java

@@ -0,0 +1,150 @@
+/*  Copyright (C) 2015 Oscar Gustafsson.
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+package net.sf.jabref.importer.fileformat;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import net.sf.jabref.model.entry.BibtexEntry;
+import net.sf.jabref.model.entry.BibtexEntryTypes;
+
+public class JSONEntryParser {
+
+    private static final Log LOGGER = LogFactory.getLog(JSONEntryParser.class);
+
+
+    public JSONEntryParser() {
+
+    }
+
+    public BibtexEntry BibJSONtoBibtex(JSONObject bibJsonEntry) {
+        // Fields that are directly accessible at the top level BibJson object
+        String[] singleFieldStrings = {"year", "title", "abstract", "month"};
+
+        // Fields that are accessible in the journal part of the BibJson object
+        String[] journalSingleFieldStrings = {"publisher", "number", "volume"};
+
+
+
+        BibtexEntry entry = new BibtexEntry();
+        entry.setType(BibtexEntryTypes.ARTICLE);
+
+        // Authors
+        if (bibJsonEntry.has("author")) {
+            JSONArray authors = bibJsonEntry.getJSONArray("author");
+            StringBuffer sb = new StringBuffer();
+            for (int i = 0; i < authors.length(); i++) {
+                if (authors.getJSONObject(i).has("name")) {
+                    sb.append(authors.getJSONObject(i).getString("name"));
+                    if (i < (authors.length() - 1)) {
+                        sb.append(" and ");
+                    }
+                } else {
+                    LOGGER.info("Empty author name.");
+                }
+            }
+            entry.setField("author", sb.toString());
+        } else {
+            LOGGER.info("No author found.");
+        }
+
+        // Direct accessible fields
+        for (String field : singleFieldStrings) {
+            if (bibJsonEntry.has(field)) {
+                entry.setField(field, bibJsonEntry.getString(field));
+            }
+        }
+
+        // Page numbers
+        if (bibJsonEntry.has("start_page")) {
+            if (bibJsonEntry.has("end_page")) {
+                entry.setField("pages",
+                        bibJsonEntry.getString("start_page") + "--" + bibJsonEntry.getString("end_page"));
+            } else {
+                entry.setField("pages", bibJsonEntry.getString("start_page"));
+            }
+        }
+
+        // Journal
+        if (bibJsonEntry.has("journal")) {
+            JSONObject journal = bibJsonEntry.getJSONObject("journal");
+            // Journal title
+            if (journal.has("title")) {
+                entry.setField("journal", journal.getString("title"));
+            } else {
+                LOGGER.info("No journal title found.");
+            }
+            // Other journal related fields
+            for (String field : journalSingleFieldStrings) {
+                if (journal.has(field)) {
+                    entry.setField(field, journal.getString(field));
+                }
+            }
+        } else {
+            LOGGER.info("No journal information found.");
+        }
+
+        // Keywords
+        if (bibJsonEntry.has("keywords")) {
+            JSONArray keywords = bibJsonEntry.getJSONArray("keywords");
+            StringBuffer sb = new StringBuffer();
+            for (int i = 0; i < keywords.length(); i++) {
+                if (!keywords.isNull(i)) {
+                    sb.append(keywords.getString(i));
+                    if (i < (keywords.length() - 1)) {
+                        sb.append(", ");
+                    }
+                }
+            }
+            entry.setField("keywords", sb.toString());
+        }
+
+        // Identifiers
+        if (bibJsonEntry.has("identifier")) {
+            JSONArray identifiers = bibJsonEntry.getJSONArray("identifier");
+            for (int i = 0; i < identifiers.length(); i++) {
+                String type = identifiers.getJSONObject(i).getString("type");
+                if (type.equals("doi")) {
+                    entry.setField("doi", identifiers.getJSONObject(i).getString("id"));
+                } else if (type.equals("pissn")) {
+                    entry.setField("issn", identifiers.getJSONObject(i).getString("id"));
+                } else if (type.equals("eissn")) {
+                    entry.setField("issn", identifiers.getJSONObject(i).getString("id"));
+                }
+            }
+        }
+
+        // Links
+        if (bibJsonEntry.has("link")) {
+            JSONArray links = bibJsonEntry.getJSONArray("link");
+            for (int i = 0; i < links.length(); i++) {
+                if (links.getJSONObject(i).has("type")) {
+                    String type = links.getJSONObject(i).getString("type");
+                    if (type.equals("fulltext")) {
+                        if (links.getJSONObject(i).has("url")) {
+                            entry.setField("url", links.getJSONObject(i).getString("url"));
+                        }
+                    }
+                }
+            }
+        }
+
+        return entry;
+    }
+}

src/main/resources/help/en/Contents.html

@@ -57,6 +57,7 @@ <h2>Import/Export</h2>
       <li><a href="ACMPortalHelp.html">Fetching entries from <em>ACM</em> Portal</a></li>
       <li><a href="CiteSeerHelp.html">Fetching entries from <em>CiteSeerX</em></a></li>
       <li><a href="DiVAtoBibTeXHelp.html">Fetching entries from <em>DiVA</em></a></li>
+      <li><a href="DOAJHelp.html">Fetching entries from <em>DOAJ</em></a></li>
       <li><a href="DOItoBibTeXHelp.html">Fetching entries using the <em>DOI to BibTeX Converter</em></a></li>
       <li><a href="GoogleScholarHelp.html">Fetching entries from <em>Google Scholar</em></a></li>
       <li><a href="IEEEXploreHelp.html">Fetching entries from <em>IEEExplore</em></a></li>

src/main/resources/help/en/DOAJHelp.html

@@ -0,0 +1,27 @@
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<link rel="stylesheet" type="text/css" href="jabref_help.css"/>
+</head>
+
+<body>
+    <h1>Fetching entries from DOAJ</h1>
+
+    <p><a href="http://doaj.org/">DOAJ (Directory of Open Access Journals)</a> is a database
+    covering open access journals.
+
+    <p>To use this feature, choose <b>Search -&gt; Web search</b>, and the
+    search interface will appear in the side pane. Select <b>DOAJ (Directory of Open Access Journals)</b> in the dropdown menu.</p>
+
+    <p>Enter the text you want to search for in the search field
+        and press <b>Enter</b> or the <b>Fetch</b> button. 
+
+	<p>It is possible to limit the search by adding a field name to the search, as field:text. The supported fields area
+	<ul>
+	<li> title: The title of the article
+    <li> doi: The DOI of the article
+    <li> issn: The ISSN of the journal
+    <li> publisher: The publisher of the journal
+    <li> abstract: The abstract of the article
+	</ul>
+</body>
+</html>