Added support for utf8 characters as tileset ids

Turns out the game supports this, as long as the tiles string is not run length encoded

src/helpers/fake_tiles.lua

@@ -5,6 +5,7 @@ local drawableSprite = require("structs.drawable_sprite")
 local drawableRectangle = require("structs.drawable_rectangle")
 local colors = require("consts.colors")
 local brushes = require("brushes")
+local utf8 = require("utf8")
 
 local fakeTilesHelper = {}
 
@@ -148,10 +149,10 @@ local function getEntityMaterialFromKey(entity, materialKey)
         fromKey = tostring(fromKey)
     end
 
-    if fromKeyType == "string" and #fromKey == 1 then
+    if fromKeyType == "string" and utf8.len(fromKey) == 1 then
         return fakeTilesHelper.getMaterialMatrix(entity, fromKey)
 
-    elseif materialKeyType == "string" and #materialKey == 1 then
+    elseif materialKeyType == "string" and utf8.len(materialKey) == 1 then
         return fakeTilesHelper.getMaterialMatrix(entity, materialKey)
 
     elseif materialKeyType == "matrix" then

src/mapcoder.lua

@@ -1,6 +1,7 @@
 local binfile = require("utils.binfile")
 local binaryReader = require("utils.binary_reader")
 local binaryWriter = require("utils.binary_writer")
+local utf8 = require("utf8")
 
 local mapcoder = {}
 
@@ -180,8 +181,13 @@ function mapcoder.encodeString(writer, s, lookup)
     else
         local encodedString = binfile.encodeRunLength(s)
         local encodedLength = #encodedString
+        local utf8Length = utf8.len(s)
+        local length = #s
 
-        if encodedLength < #s and encodedLength < 2^15 then
+        -- Only allow run length encoding if the string contains only 1 byte characters
+        -- Celeste does not read it as expected otherwise, this is mainly a tile issue
+        -- Run length encoding is also hardcoded max length, make sure we don't exceed the limit
+        if length == utf8Length and encodedLength < utf8Length and encodedLength < 2^15 then
             writer:writeByte(7)
             writer:writeSignedShort(encodedLength)
             writer:write(encodedString)

src/structs/tiles.lua

@@ -1,4 +1,5 @@
 local matrix = require("utils.matrix")
+local utils = require("utils")
 
 local tilesStruct = {}
 
@@ -78,8 +79,8 @@ function tilesStruct.matrixToTileStringMinimized(matrix, seperator, empty)
     return table.concat(lines, "\n")
 end
 
-function tilesStruct.tileStringToMatrix(tiles, seperator, empty)
-    seperator = seperator or 1
+function tilesStruct.tileStringToMatrix(tiles, separator, empty)
+    separator = separator or 1
     empty = empty or "0"
     tiles = tiles:gsub("\r\n", "\n")
 
@@ -95,9 +96,7 @@ function tilesStruct.tileStringToMatrix(tiles, seperator, empty)
     local res = matrix.filled(empty, cols, rows)
 
     for y, line in ipairs(lines) do
-        local chars = line:split(seperator)()
-
-        for x, char in ipairs(chars) do
+        for x, char in ipairs(utils.splitUTF8(line)) do
             res:setInbounds(x, y, char)
         end
     end

src/utils/utils.lua

@@ -4,6 +4,7 @@ local requireUtils = require("utils.require")
 local xnaColors = require("consts.xna_colors")
 local bit = require("bit")
 local ffi = require("ffi")
+local utf8 = require("utf8")
 
 local rectangles = require("structs.rectangle")
 
@@ -227,6 +228,26 @@ function utils.findCharacter(string, character)
     end
 end
 
+function utils.splitUTF8(s, separator)
+    separator = separator or 1
+
+    local res = {}
+    local separatorType = type(separator)
+
+    if separatorType == "string" then
+        res = s:split(separator)()
+
+    elseif separatorType == "number" then
+        local length = utf8.len(s)
+
+        for i = 1, length, separator do
+            table.insert(res, utf8.char(utf8.codepoint(s, utf8.offset(s, i), utf8.offset(s, math.min(i + separator - 1, length)))))
+        end
+    end
+
+    return res
+end
+
 function utils.titleCase(name)
     return name:gsub("(%a)(%a*)", function(a, b) return string.upper(a) .. b end)
 end