[add] Enable kuromoji for Japanease language search

highemerly · Nov 17, 2023 · 89becdd · 89becdd
1 parent d1ea05f
commit 89becdd
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 14 deletions.
diff --git a/app/chewy/public_statuses_index.rb b/app/chewy/public_statuses_index.rb
@@ -2,6 +2,13 @@
 
 class PublicStatusesIndex < Chewy::Index
   settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: {
+    tokenizer: {
+      kuromoji_user_dict: {
+        type: 'kuromoji_tokenizer',
+        user_dictionary: 'userdic.txt',
+      },
+    },
+
     filter: {
       english_stop: {
         type: 'stop',
@@ -26,15 +33,18 @@ class PublicStatusesIndex < Chewy::Index
       },
 
       content: {
-        tokenizer: 'standard',
+        tokenizer: 'kuromoji_user_dict',
         filter: %w(
-          lowercase
-          asciifolding
-          cjk_width
-          elision
-          english_possessive_stemmer
+          kuromoji_baseform
+          kuromoji_stemmer
+          kuromoji_part_of_speech
+          ja_stop
+          kuromoji_number
           english_stop
           english_stemmer
+          english_possessive_stemmer
+          cjk_width
+          lowercase
         ),
       },
 
@@ -58,7 +68,7 @@ class PublicStatusesIndex < Chewy::Index
   root date_detection: false do
     field(:id, type: 'long')
     field(:account_id, type: 'long')
-    field(:text, type: 'text', analyzer: 'verbatim', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
+    field(:text, type: 'text', analyzer: 'content', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
     field(:tags, type: 'text', analyzer: 'hashtag', value: ->(status) { status.tags.map(&:display_name) })
     field(:language, type: 'keyword')
     field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties })

diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb
@@ -2,6 +2,13 @@
 
 class StatusesIndex < Chewy::Index
   settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: {
+    tokenizer: {
+      kuromoji_user_dict: {
+        type: 'kuromoji_tokenizer',
+        user_dictionary: 'userdic.txt',
+      },
+    },
+
     filter: {
       english_stop: {
         type: 'stop',
@@ -26,15 +33,19 @@ class StatusesIndex < Chewy::Index
       },
 
       content: {
-        tokenizer: 'standard',
+        type: 'custom',
+        tokenizer: 'kuromoji_user_dict',
         filter: %w(
-          lowercase
-          asciifolding
-          cjk_width
-          elision
-          english_possessive_stemmer
+          kuromoji_baseform
+          kuromoji_stemmer
+          kuromoji_part_of_speech
+          ja_stop
+          kuromoji_number
           english_stop
           english_stemmer
+          english_possessive_stemmer
+          cjk_width
+          lowercase
         ),
       },
 
@@ -55,7 +66,7 @@ class StatusesIndex < Chewy::Index
   root date_detection: false do
     field(:id, type: 'long')
     field(:account_id, type: 'long')
-    field(:text, type: 'text', analyzer: 'verbatim', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
+    field(:text, type: 'text', analyzer: 'content', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
     field(:tags, type: 'text', analyzer: 'hashtag',  value: ->(status) { status.tags.map(&:display_name) })
     field(:searchable_by, type: 'long', value: ->(status) { status.searchable_by })
     field(:language, type: 'keyword')