Fix for issue 5850: Journal abbreviations in UTF-8 not recognized

CHANGELOG.md

@@ -76,6 +76,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve
 - We fixed an issue where opening BibTex file (doubleclick) from Folder with spaces not working. [#6487](https://github.com/JabRef/jabref/issues/6487)
 - We fixed an issue with saving large `.bib` files [#7265](https://github.com/JabRef/jabref/issues/7265)
 - We fixed an issue with very large page numbers [#7590](https://github.com/JabRef/jabref/issues/7590)
+- We fixed an issue where journal abbreviations in UTF-8 not recognized [#5850](https://github.com/JabRef/jabref/issues/5850)
 
 ### Removed
 

build.gradle

@@ -758,4 +758,3 @@ task downloadDependencies {
     }
 }
 
-

src/main/java/org/jabref/logic/integrity/IntegrityCheck.java

@@ -44,7 +44,7 @@ public IntegrityCheck(BibDatabaseContext bibDatabaseContext,
         } else {
             entryCheckers.addAll(List.of(
                     new JournalInAbbreviationListChecker(StandardField.JOURNAL, journalAbbreviationRepository),
-                    new ASCIICharacterChecker(),
+                    new UTF8Checker(),
                     new NoBibtexFieldChecker(),
                     new BibTeXEntryTypeChecker())
             );
@@ -59,7 +59,6 @@ List<IntegrityMessage> check() {
         for (BibEntry entry : database.getEntries()) {
             result.addAll(checkEntry(entry));
         }
-
         result.addAll(checkDatabase(database));
 
         return result;

src/main/java/org/jabref/logic/integrity/UTF8Checker.java

@@ -0,0 +1,44 @@
+package org.jabref.logic.integrity;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.jabref.logic.l10n.Localization;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.field.Field;
+
+public class UTF8Checker implements EntryChecker {
+
+    /**
+     * Detect any non UTF-8 encoded field
+     */
+    @Override
+    public List<IntegrityMessage> check(BibEntry entry) {
+        List<IntegrityMessage> results = new ArrayList<>();
+        for (Map.Entry<Field, String> field : entry.getFieldMap().entrySet()) {
+            Charset charset = Charset.forName(System.getProperty("file.encoding"));
+            boolean utfOnly = UTF8EncodingChecker(field.getValue().getBytes(charset));
+            if (!utfOnly) {
+                results.add(new IntegrityMessage(Localization.lang("Non-UTF-8 encoded found"), entry,
+                        field.getKey()));
+            }
+        }
+        return results;
+    }
+
+    public static boolean UTF8EncodingChecker(byte[] data) {
+        CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder();
+        try {
+            decoder.decode(ByteBuffer.wrap(data));
+        } catch (CharacterCodingException ex) {
+            return false;
+        }
+        return true;
+    }
+}

src/test/java/org/jabref/logic/integrity/UTF8CheckerTest.java

@@ -0,0 +1,39 @@
+package org.jabref.logic.integrity;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.field.StandardField;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class UTF8CheckerTest {
+
+    private final BibEntry entry = new BibEntry();
+
+    @Test
+    void fieldAcceptsUTF8() {
+        UTF8Checker checker = new UTF8Checker();
+        entry.setField(StandardField.TITLE, "Only ascii characters!'@12");
+        assertEquals(Collections.emptyList(), checker.check(entry));
+    }
+
+    @Test
+    void fieldDoesNotAcceptUmlauts() {
+        System.getProperties().put("file.encoding", "GBK");
+        UTF8Checker checker = new UTF8Checker();
+        String NonUTF8 = "";
+        try {
+            NonUTF8 = new String("你好，这条语句使用GBK字符集".getBytes(), "GBK");
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+        entry.setField(StandardField.MONTH, NonUTF8);
+        assertEquals(List.of(new IntegrityMessage("Non-UTF-8 encoded found", entry, StandardField.MONTH)), checker.check(entry));
+    }
+
+
+}