Merge remote-tracking branch 'upstream/master' into ooPanel

* upstream/master:
  Refactor BibEntry deprecated method (#4554)
  Added extra stats to be sent with MrDLib recommendations (#4452)

src/main/java/org/jabref/gui/entryeditor/RelatedArticlesTab.java

@@ -4,6 +4,7 @@
 import java.util.List;
 
 import javafx.scene.control.Button;
+import javafx.scene.control.CheckBox;
 import javafx.scene.control.Hyperlink;
 import javafx.scene.control.ProgressIndicator;
 import javafx.scene.control.ScrollPane;
@@ -125,10 +126,12 @@ private ScrollPane getPrivacyDialog(BibEntry entry) {
 
         Button button = new Button(Localization.lang("I Agree"));
         button.setDefaultButton(true);
-        Text line1 = new Text(Localization.lang("Mr. DLib is an external service which provides article recommendations based on the currently selected entry. Data about the selected entry must be sent to Mr. DLib in order to provide these recommendations. Do you agree that this data may be sent?"));
 
+        Text line1 = new Text(Localization.lang("JabRef requests recommendations from Mr. DLib, which is an external service. To enable Mr. DLib to calculate recommendations, some of your data must be shared with Mr. DLib. Generally, the more data is shared the better recommendations can be calculated. However, we understand that some of your data in JabRef is sensitive, and you may not want to share it. Therefore, Mr. DLib offers a choice of which data you would like to share."));
         line1.setWrappingWidth(1300.0);
-        Text line2 = new Text(Localization.lang("This setting may be changed in preferences at any time."));
+        Text line2 = new Text(Localization.lang("Whatever option you choose, Mr. DLib may share its data with research partners to further improve recommendation quality as part of a 'living lab'. Mr. DLib may also release public datasets that may contain anonymized information about you and the recommendations (sensitive information such as metadata of your articles will be anonymised through e.g. hashing). Research partners are obliged to adhere to the same strict data protection policy as Mr. DLib."));
+        line2.setWrappingWidth(1300.0);
+        Text line3 = new Text(Localization.lang("This setting may be changed in preferences at any time."));
         Hyperlink mdlLink = new Hyperlink(Localization.lang("Further information about Mr DLib. for JabRef users."));
         mdlLink.setOnAction(event -> {
             try {
@@ -138,15 +141,30 @@ private ScrollPane getPrivacyDialog(BibEntry entry) {
                 dialogService.showErrorDialogAndWait(e);
             }
         });
+        VBox vb = new VBox();
+        CheckBox cbTitle = new CheckBox(Localization.lang("Entry Title (Required to deliver recommendations.)"));
+        cbTitle.setSelected(true);
+        cbTitle.setDisable(true);
+        CheckBox cbVersion = new CheckBox(Localization.lang("JabRef Version (Required to ensure backwards compatibility with Mr. DLib's Web Service)"));
+        cbVersion.setSelected(true);
+        cbVersion.setDisable(true);
+        CheckBox cbLanguage = new CheckBox(Localization.lang("JabRef Language (Provides for better recommendations by giving an indication of user's preferred language.)"));
+        CheckBox cbOS = new CheckBox(Localization.lang("Operating System (Provides for better recommendations by giving an indication of user's system set-up.)"));
+        CheckBox cbTimezone = new CheckBox(Localization.lang("Timezone (Provides for better recommendations by indicating the time of day the request is being made.)"));
+        vb.getChildren().addAll(cbTitle, cbVersion, cbLanguage, cbOS, cbTimezone);
+        vb.setSpacing(10);
 
         button.setOnAction(event -> {
             JabRefPreferences prefs = JabRefPreferences.getInstance();
             prefs.putBoolean(JabRefPreferences.ACCEPT_RECOMMENDATIONS, true);
+            prefs.putBoolean(JabRefPreferences.SEND_LANGUAGE_DATA, cbLanguage.isSelected());
+            prefs.putBoolean(JabRefPreferences.SEND_OS_DATA, cbOS.isSelected());
+            prefs.putBoolean(JabRefPreferences.SEND_TIMEZONE_DATA, cbTimezone.isSelected());
             dialogService.showWarningDialogAndWait(Localization.lang("Restart"), Localization.lang("Please restart JabRef for preferences to take effect."));
             setContent(getRelatedArticlesPane(entry));
         });
 
-        vbox.getChildren().addAll(line1, mdlLink, line2, button);
+        vbox.getChildren().addAll(line1, line2, mdlLink, line3, vb, button);
         root.setContent(vbox);
 
         return root;

src/main/java/org/jabref/logic/importer/fetcher/MrDLibFetcher.java

@@ -4,6 +4,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.Calendar;
 import java.util.List;
 import java.util.Optional;
 
@@ -16,6 +17,7 @@
 import org.jabref.model.database.BibDatabase;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.entry.FieldName;
+import org.jabref.preferences.JabRefPreferences;
 
 import org.apache.http.client.utils.URIBuilder;
 import org.slf4j.Logger;
@@ -27,6 +29,7 @@
 public class MrDLibFetcher implements EntryBasedFetcher {
     private static final Logger LOGGER = LoggerFactory.getLogger(MrDLibFetcher.class);
     private static final String NAME = "MDL_FETCHER";
+    private static final String MDL_JABREF_PARTNER_ID = "1";
     private final String LANGUAGE;
     private final Version VERSION;
 
@@ -104,11 +107,22 @@ private String constructQuery(String queryWithTitle) {
         URIBuilder builder = new URIBuilder();
         builder.setScheme("http");
         builder.setHost(getMdlUrl());
-        builder.setPath("/v2/items/" + queryWithTitle + "/related_items");
-        builder.addParameter("partner_id", "jabref");
+        builder.setPath("/v2/documents/" + queryWithTitle + "/related_documents");
+        builder.addParameter("partner_id", MDL_JABREF_PARTNER_ID);
         builder.addParameter("app_id", "jabref_desktop");
         builder.addParameter("app_version", VERSION.getFullVersion());
-        builder.addParameter("app_lang", LANGUAGE);
+
+        JabRefPreferences prefs = JabRefPreferences.getInstance();
+        if (prefs.getBoolean(JabRefPreferences.SEND_LANGUAGE_DATA)) {
+            builder.addParameter("app_lang", LANGUAGE);
+        }
+        if (prefs.getBoolean(JabRefPreferences.SEND_OS_DATA)) {
+            builder.addParameter("os", System.getProperty("os.name"));
+        }
+        if (prefs.getBoolean(JabRefPreferences.SEND_TIMEZONE_DATA)) {
+            builder.addParameter("timezone", Calendar.getInstance().getTimeZone().getID());
+        }
+
         try {
             URI uri = builder.build();
             LOGGER.trace("Request: " + uri.toString());
@@ -120,6 +134,6 @@ private String constructQuery(String queryWithTitle) {
     }
 
     private String getMdlUrl() {
-        return VERSION.isDevelopmentVersion() ? "api-dev.darwingoliath.com" : "api.darwingoliath.com";
+        return VERSION.isDevelopmentVersion() ? "api-dev.darwingoliath.com" : "api.mr-dlib.org";
     }
 }

src/main/java/org/jabref/logic/importer/fileformat/BiblioscapeImporter.java

@@ -13,6 +13,7 @@
 import org.jabref.logic.importer.ParserResult;
 import org.jabref.logic.util.StandardFileType;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibtexEntryTypes;
 import org.jabref.model.entry.FieldName;
 
 /**
@@ -255,7 +256,7 @@ public ParserResult importDatabase(BufferedReader reader) throws IOException {
                 if (!comments.isEmpty()) { // set comment if present
                     hm.put(FieldName.COMMENT, String.join(";", comments));
                 }
-                BibEntry b = new BibEntry(bibtexType);
+                BibEntry b = new BibEntry(BibtexEntryTypes.getTypeOrDefault(bibtexType));
                 b.setField(hm);
                 bibItems.add(b);
 

src/main/java/org/jabref/logic/importer/fileformat/BibtexParser.java

@@ -29,6 +29,7 @@
 import org.jabref.model.database.BibDatabase;
 import org.jabref.model.database.KeyCollisionException;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibtexEntryTypes;
 import org.jabref.model.entry.BibtexString;
 import org.jabref.model.entry.CustomEntryType;
 import org.jabref.model.entry.EntryType;
@@ -504,7 +505,8 @@ private String parsePreamble() throws IOException {
     }
 
     private BibEntry parseEntry(String entryType) throws IOException {
-        BibEntry result = new BibEntry(entryType);
+        BibEntry result = new BibEntry(BibtexEntryTypes.getTypeOrDefault(entryType));
+
         skipWhitespace();
         consume('{', '(');
         int character = peek();

src/main/java/org/jabref/logic/importer/fileformat/CopacImporter.java

@@ -11,6 +11,7 @@
 import org.jabref.logic.importer.ParserResult;
 import org.jabref.logic.util.StandardFileType;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibtexEntryTypes;
 import org.jabref.model.entry.FieldName;
 
 /**
@@ -98,7 +99,7 @@ public ParserResult importDatabase(BufferedReader reader) throws IOException {
 
             // Copac does not contain enough information on the type of the
             // document. A book is assumed.
-            BibEntry b = new BibEntry("book");
+            BibEntry b = new BibEntry(BibtexEntryTypes.BOOK);
 
             String[] lines = entry.split("\n");
 

src/main/java/org/jabref/logic/importer/fileformat/EndnoteImporter.java

@@ -15,6 +15,7 @@
 import org.jabref.logic.util.StandardFileType;
 import org.jabref.model.entry.AuthorList;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibtexEntryTypes;
 import org.jabref.model.entry.FieldName;
 
 /**
@@ -253,7 +254,7 @@ else if ("P".equals(prefix)) {
                 hm.put(FieldName.PAGES, artnum);
             }
 
-            BibEntry b = new BibEntry(type);
+            BibEntry b = new BibEntry(BibtexEntryTypes.getTypeOrDefault(type));
             b.setField(hm);
             if (!b.getFieldNames().isEmpty()) {
                 bibitems.add(b);

src/main/java/org/jabref/logic/importer/fileformat/GvkParser.java

@@ -12,6 +12,7 @@
 import org.jabref.logic.importer.ParseException;
 import org.jabref.logic.importer.Parser;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibtexEntryTypes;
 import org.jabref.model.entry.FieldName;
 
 import com.google.common.base.Strings;
@@ -360,7 +361,7 @@ private BibEntry parseEntry(Element e) {
          * dann @incollection annehmen, wenn weder ISBN noch
          * ZDB-ID vorhanden sind.
          */
-        BibEntry result = new BibEntry(entryType);
+        BibEntry result = new BibEntry(BibtexEntryTypes.getTypeOrDefault(entryType));
 
         // Zuordnung der Felder in Abhängigkeit vom Dokumenttyp
         if (author != null) {

src/main/java/org/jabref/logic/importer/fileformat/InspecImporter.java

@@ -13,6 +13,7 @@
 import org.jabref.logic.util.StandardFileType;
 import org.jabref.model.entry.AuthorList;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibtexEntryTypes;
 import org.jabref.model.entry.FieldName;
 
 /**
@@ -120,7 +121,7 @@ public ParserResult importDatabase(BufferedReader reader) throws IOException {
                     }
                 }
             }
-            BibEntry b = new BibEntry(type);
+            BibEntry b = new BibEntry(BibtexEntryTypes.getTypeOrDefault(type));
             b.setField(h);
 
             bibitems.add(b);

src/main/java/org/jabref/logic/importer/fileformat/IsiImporter.java

@@ -17,6 +17,7 @@
 import org.jabref.logic.importer.ParserResult;
 import org.jabref.logic.util.StandardFileType;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibtexEntryTypes;
 import org.jabref.model.entry.FieldName;
 import org.jabref.model.entry.Month;
 
@@ -301,7 +302,7 @@ public ParserResult importDatabase(BufferedReader reader) throws IOException {
                 continue;
             }
 
-            BibEntry b = new BibEntry(Type);
+            BibEntry b = new BibEntry(BibtexEntryTypes.getTypeOrDefault(Type));
             // id assumes an existing database so don't
 
             // Remove empty fields:

src/main/java/org/jabref/logic/importer/fileformat/MedlineImporter.java

@@ -74,6 +74,7 @@
 import org.jabref.logic.importer.fileformat.medline.Text;
 import org.jabref.logic.util.StandardFileType;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibtexEntryTypes;
 import org.jabref.model.entry.FieldName;
 import org.jabref.model.strings.StringUtil;
 
@@ -262,7 +263,7 @@ private void parseBookArticle(PubmedBookArticle currentArticle, List<BibEntry> b
             putIfValueNotNull(fields, "pubstatus", bookData.getPublicationStatus());
         }
 
-        BibEntry entry = new BibEntry("article");
+        BibEntry entry = new BibEntry(BibtexEntryTypes.ARTICLE);
         entry.setField(fields);
 
         bibItems.add(entry);
@@ -410,7 +411,7 @@ private void parseArticle(PubmedArticle article, List<BibEntry> bibItems) {
             }
         }
 
-        BibEntry entry = new BibEntry("article");
+        BibEntry entry = new BibEntry(BibtexEntryTypes.ARTICLE);
         entry.setField(fields);
 
         bibItems.add(entry);

src/main/java/org/jabref/logic/importer/fileformat/MedlinePlainImporter.java

@@ -16,6 +16,7 @@
 import org.jabref.logic.util.StandardFileType;
 import org.jabref.model.entry.AuthorList;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibtexEntryTypes;
 import org.jabref.model.entry.FieldName;
 
 /**
@@ -208,7 +209,7 @@ public ParserResult importDatabase(BufferedReader reader) throws IOException {
                 fields.put(FieldName.COMMENT, comment);
             }
 
-            BibEntry b = new BibEntry(type);
+            BibEntry b = new BibEntry(BibtexEntryTypes.getTypeOrDefault(type));
 
             // Remove empty fields:
             fields.entrySet().stream().filter(n -> n.getValue().trim().isEmpty()).forEach(fields::remove);

src/main/java/org/jabref/logic/importer/fileformat/MrDLibImporter.java

@@ -146,10 +146,10 @@ private RankedBibEntry populateBibEntry(JSONObject recommendation) {
         // parse each of the relevant fields into variables
         String authors = isRecommendationFieldPresent(recommendation, "authors") ? getAuthorsString(recommendation) : "";
         String title = isRecommendationFieldPresent(recommendation, "title") ? recommendation.getString("title") : "";
-        String year = isRecommendationFieldPresent(recommendation, "year_published") ? Integer.toString(recommendation.getInt("year_published")) : "";
+        String year = isRecommendationFieldPresent(recommendation, "published_year") ? Integer.toString(recommendation.getInt("published_year")) : "";
         String journal = isRecommendationFieldPresent(recommendation, "published_in") ? recommendation.getString("published_in") : "";
         String url = isRecommendationFieldPresent(recommendation, "url") ? recommendation.getString("url") : "";
-        Integer rank = isRecommendationFieldPresent(recommendation, "url") ? recommendation.getInt("recommendation_id") : 100;
+        Integer rank = isRecommendationFieldPresent(recommendation, "recommendation_id") ? recommendation.getInt("recommendation_id") : 100;
 
         // Populate bib entry with relevant data
         current.setField(FieldName.AUTHOR, authors);

src/main/java/org/jabref/logic/importer/fileformat/OvidImporter.java

@@ -14,6 +14,7 @@
 import org.jabref.logic.util.StandardFileType;
 import org.jabref.model.entry.AuthorList;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibtexEntryTypes;
 import org.jabref.model.entry.FieldName;
 
 /**
@@ -207,7 +208,7 @@ public ParserResult importDatabase(BufferedReader reader) throws IOException {
                 // Move the "chaptertitle" to just "title":
                 h.put(FieldName.TITLE, h.remove("chaptertitle"));
             }
-            BibEntry b = new BibEntry(entryType);
+            BibEntry b = new BibEntry(BibtexEntryTypes.getTypeOrDefault(entryType));
             b.setField(h);
 
             bibitems.add(b);

src/main/java/org/jabref/logic/importer/fileformat/RisImporter.java

@@ -16,6 +16,7 @@
 import org.jabref.logic.util.StandardFileType;
 import org.jabref.model.entry.AuthorList;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibtexEntryTypes;
 import org.jabref.model.entry.FieldName;
 import org.jabref.model.entry.Month;
 
@@ -265,7 +266,7 @@ else if ("AV".equals(tag)) {
 
             // create one here
             // type is set in the loop above
-            BibEntry entry = new BibEntry(type);
+            BibEntry entry = new BibEntry(BibtexEntryTypes.getTypeOrDefault(type));
             entry.setField(fields);
             // month has a special treatment as we use the separate method "setMonth" of BibEntry instead of directly setting the value
             month.ifPresent(entry::setMonth);

src/main/java/org/jabref/logic/importer/fileformat/SilverPlatterImporter.java

@@ -14,6 +14,7 @@
 import org.jabref.logic.util.StandardFileType;
 import org.jabref.model.entry.AuthorList;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibtexEntryTypes;
 import org.jabref.model.entry.FieldName;
 
 /**
@@ -180,7 +181,7 @@ public ParserResult importDatabase(BufferedReader reader) throws IOException {
 
             }
 
-            BibEntry b = new BibEntry(type);
+            BibEntry b = new BibEntry(BibtexEntryTypes.getTypeOrDefault(type));
             // create one here
             b.setField(h);
 

src/main/java/org/jabref/logic/msbib/BibTeXConverter.java

@@ -9,6 +9,7 @@
 import java.util.stream.Collectors;
 
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibtexEntryTypes;
 import org.jabref.model.entry.FieldName;
 import org.jabref.model.entry.Month;
 
@@ -29,7 +30,7 @@ public static BibEntry convert(MSBibEntry entry) {
         Map<String, String> fieldValues = new HashMap<>();
 
         String bibTexEntryType = MSBibMapping.getBiblatexEntryType(entry.getType());
-        result = new BibEntry(bibTexEntryType);
+        result = new BibEntry(BibtexEntryTypes.getTypeOrDefault(bibTexEntryType));
 
         // add String fields
         for (Map.Entry<String, String> field : entry.fields.entrySet()) {

src/main/java/org/jabref/logic/util/TestEntry.java

@@ -11,7 +11,7 @@ private TestEntry() {
 
     public static BibEntry getTestEntry() {
 
-        BibEntry entry = new BibEntry("article");
+        BibEntry entry = new BibEntry(BibtexEntryTypes.ARTICLE);
         entry.setCiteKey("Smith2016");
         entry.setField(FieldName.AUTHOR, "Smith, Bill and Jones, Bob and Williams, Jeff");
         entry.setField(FieldName.EDITOR, "Taylor, Phil");

src/main/java/org/jabref/model/entry/BibEntry.java

@@ -111,7 +111,7 @@ private BibEntry(String id, String type) {
      * Constructs a new BibEntry. The internal ID is set to IdGenerator.next()
      */
     public BibEntry(EntryType type) {
-        this(type.getName());
+        this(IdGenerator.next(),type.getName());
     }
 
     public Optional<FieldChange> setMonth(Month parsedMonth) {
@@ -544,7 +544,7 @@ private boolean atLeastOnePresent(String[] fieldsToCheck, BibDatabase database)
      */
     @Override
     public Object clone() {
-        BibEntry clone = new BibEntry(type.getValue());
+        BibEntry clone = new BibEntry(IdGenerator.next(),type.getValue());
         clone.fields = FXCollections.observableMap(new ConcurrentHashMap<>(fields));
         return clone;
     }

src/main/java/org/jabref/model/entry/BibtexEntryTypes.java

@@ -287,4 +287,8 @@ private BibtexEntryTypes() {
     public static Optional<EntryType> getType(String name) {
         return ALL.stream().filter(e -> e.getName().equalsIgnoreCase(name)).findFirst();
     }
+
+    public static EntryType getTypeOrDefault(String name) {
+        return getType(name).orElseGet(() -> new CustomEntryType(name, "required", "optional"));
+    }
 }