fix: update fstring handling to better handle unicode

convert.py

@@ -78,9 +78,7 @@ def convert_sav_to_json(filename, output_path, minify):
     print(f"Writing JSON to {output_path}")
     with open(output_path, "w", encoding="utf8") as f:
         indent = None if minify else "\t"
-        json.dump(
-            gvas_file.dump(), f, indent=indent, cls=CustomEncoder, ensure_ascii=False
-        )
+        json.dump(gvas_file.dump(), f, indent=indent, cls=CustomEncoder)
 
 
 def convert_json_to_sav(filename, output_path):

lib/archive.py

@@ -96,18 +96,20 @@ def read_fstring(self):
         if size == 0:
             return ""
 
+        data: bytes
+        encoding: str
         if LoadUCS2Char:
-            data = []
-            for i in range(size):
-                if i == size - 1:
-                    self.read_uint16()
-                else:
-                    data.append(self.read_uint16())
-            string = "".join([chr(v) for v in data])
-            return string
+            data = self.read(size * 2)[:-2]
+            encoding = "utf-16-le"
         else:
-            byte = self.data.read(size)[:-1]
-            return byte.decode("utf-8")
+            data = self.read(size)[:-1]
+            encoding = "ascii"
+        try:
+            return data.decode(encoding)
+        except Exception as e:
+            raise Exception(
+                f"Error decoding {encoding} string of length {size}: {bytes(data)}"
+            ) from e
 
     def read_int16(self):
         return struct.unpack("h", self.data.read(2))[0]

tests/test_cli_scripts.py

@@ -99,6 +99,63 @@ def test_level_roundtrip(self):
             with contextlib.suppress(FileNotFoundError):
                 os.remove("tests/testdata/Level-3.sav.json")
 
+    def test_level_tricky_unicode_player_name_roundtrip(self):
+        try:
+            run = subprocess.run(
+                [
+                    "python3",
+                    "convert.py",
+                    "tests/testdata/Level-tricky-unicode-player-name.sav",
+                ]
+            )
+            self.assertEqual(run.returncode, 0)
+            self.assertTrue(
+                os.path.exists(
+                    "tests/testdata/Level-tricky-unicode-player-name.sav.json"
+                )
+            )
+            os.rename(
+                "tests/testdata/Level-tricky-unicode-player-name.sav.json",
+                "tests/testdata/Level-tricky-unicode-player-name-2.sav.json",
+            )
+            run = subprocess.run(
+                [
+                    "python3",
+                    "convert.py",
+                    "tests/testdata/Level-tricky-unicode-player-name-2.sav.json",
+                ]
+            )
+            self.assertEqual(run.returncode, 0)
+            self.assertTrue(
+                os.path.exists("tests/testdata/Level-tricky-unicode-player-name-2.sav")
+            )
+            os.rename(
+                "tests/testdata/Level-tricky-unicode-player-name-2.sav",
+                "tests/testdata/Level-tricky-unicode-player-name-3.sav",
+            )
+            run = subprocess.run(
+                [
+                    "python3",
+                    "convert.py",
+                    "tests/testdata/Level-tricky-unicode-player-name-3.sav",
+                ]
+            )
+            self.assertEqual(run.returncode, 0)
+            self.assertTrue(
+                os.path.exists(
+                    "tests/testdata/Level-tricky-unicode-player-name-3.sav.json"
+                )
+            )
+        finally:
+            with contextlib.suppress(FileNotFoundError):
+                os.remove("tests/testdata/Level-tricky-unicode-player-name.sav.json")
+            with contextlib.suppress(FileNotFoundError):
+                os.remove("tests/testdata/Level-tricky-unicode-player-name-2.sav.json")
+            with contextlib.suppress(FileNotFoundError):
+                os.remove("tests/testdata/Level-tricky-unicode-player-name-3.sav")
+            with contextlib.suppress(FileNotFoundError):
+                os.remove("tests/testdata/Level-tricky-unicode-player-name-3.sav.json")
+
     def test_levelmeta_roundtrip(self):
         try:
             run = subprocess.run(

tests/test_gvas.py

@@ -133,6 +133,29 @@ def test_level_sav(self):
             "sav does not match expected after roundtrip",
         )
 
+    def test_level_tricky_unicode_sav(self):
+        with open("tests/testdata/Level-tricky-unicode-player-name.sav", "rb") as f:
+            data = f.read()
+        gvas_data, _ = decompress_sav_to_gvas(data)
+        gvas_file = GvasFile.read(
+            gvas_data, PALWORLD_TYPE_HINTS, PALWORLD_CUSTOM_PROPERTIES
+        )
+        self.assertEqual(
+            gvas_file.header.dump()["save_game_class_name"],
+            "/Script/Pal.PalWorldSaveGame",
+            "sav save_game_class_name does not match expected",
+        )
+        dump = gvas_file.dump()
+        js = json.dumps(dump, cls=CustomEncoder)
+        new_js = json.loads(js)
+        new_gvas_file = GvasFile.load(new_js)
+        new_gvas_data = new_gvas_file.write(PALWORLD_CUSTOM_PROPERTIES)
+        self.assertEqual(
+            gvas_data,
+            new_gvas_data,
+            "sav does not match expected after roundtrip",
+        )
+
     def test_levelmeta_sav(self):
         with open("tests/testdata/LevelMeta.sav", "rb") as f:
             data = f.read()