Fix/cjk unicode lookup (#5421)

* fix: revert to use unicodeBlockLookup

* expose some blocks

* test: cjk label rendering

* rmtesthtmlforlocal

* fix: cover Ideo

* test: add cases cjk symbols

* revert: unexpose ideographs extension

* test: add example to show cjk symbols

* test: add render test for cjk-symbols

* #5421

* test: add macos, linux expected image

* add and rm expected image

* add  expected image

* chore: comment

* fix: revert rm comment

* chore: rm unneeded bracket

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### ✨ Features and improvements
 
+- Fix rendering Japanese symbols which are accidentally ignored. ([#5421](https://github.com/maplibre/maplibre-gl-js/pull/5421)
 - _...Add new stuff here..._
 
 ### 🐞 Bug fixes

src/render/glyph_manager.test.ts

@@ -130,6 +130,12 @@ describe('GlyphManager', () => {
         expect(manager._doesCharSupportLocalGlyph(0x3066)).toBe(true);
         // Hangul letter a 아
         expect(manager._doesCharSupportLocalGlyph(0xC544)).toBe(true);
+        // Japanese full-width dash ー
+        expect(manager._doesCharSupportLocalGlyph(0x30FC)).toBe(true);
+        // Halfwidth and Fullwidth Forms: full-width exclamation ！
+        expect(manager._doesCharSupportLocalGlyph(0xFF01)).toBe(true);
+        // CJK Symbols and Punctuation: Japanese Post mark 〒
+        expect(manager._doesCharSupportLocalGlyph(0x3012)).toBe(true);
     });
 
     test('GlyphManager caches locally generated glyphs', async () => {

src/render/glyph_manager.ts

@@ -1,6 +1,7 @@
 import {loadGlyphRange} from '../style/load_glyph_range';
 
 import TinySDF from '@mapbox/tiny-sdf';
+import {unicodeBlockLookup} from '../util/is_char_in_unicode_block';
 import {AlphaImage} from '../util/image';
 
 import type {StyleGlyph} from '../style/style_glyph';
@@ -126,7 +127,17 @@ export class GlyphManager {
         // text, also include any other CJKV or siniform ideograph or hangul,
         // hiragana, or katakana character.
         return !!this.localIdeographFontFamily &&
-            /\p{Ideo}|\p{sc=Hang}|\p{sc=Hira}|\p{sc=Kana}/u.test(String.fromCodePoint(id));
+        (/\p{Ideo}|\p{sc=Hang}|\p{sc=Hira}|\p{sc=Kana}/u.test(String.fromCodePoint(id)) ||
+        // fallback: RegExp can't cover all cases. refer Issue #5420
+        unicodeBlockLookup['CJK Unified Ideographs'](id) ||
+        unicodeBlockLookup['Hangul Syllables'](id) ||
+        unicodeBlockLookup['Hiragana'](id) ||
+        unicodeBlockLookup['Katakana'](id) || // includes "ー"
+        // memo: these symbols are not all. others could be added if needed.
+        unicodeBlockLookup['CJK Symbols and Punctuation'](id) || // 、。〃〄々〆〇〈〉《》「...
+        unicodeBlockLookup['Halfwidth and Fullwidth Forms'](id) // ！？＂＃＄％＆...
+        );
+
     }
 
     _tinySDF(entry: Entry, stack: string, id: number): StyleGlyph {

src/util/is_char_in_unicode_block.ts

@@ -112,7 +112,7 @@ export const unicodeBlockLookup: UnicodeBlockLookup = {
     // 'Kangxi Radicals': (char) => char >= 0x2F00 && char <= 0x2FDF,
     'Ideographic Description Characters': (char) => char >= 0x2FF0 && char <= 0x2FFF,
     'CJK Symbols and Punctuation': (char) => char >= 0x3000 && char <= 0x303F,
-    // 'Hiragana': (char) => char >= 0x3040 && char <= 0x309F,
+    'Hiragana': (char) => char >= 0x3040 && char <= 0x309F,
     'Katakana': (char) => char >= 0x30A0 && char <= 0x30FF,
     // 'Bopomofo': (char) => char >= 0x3100 && char <= 0x312F,
     // 'Hangul Compatibility Jamo': (char) => char >= 0x3130 && char <= 0x318F,
@@ -124,7 +124,7 @@ export const unicodeBlockLookup: UnicodeBlockLookup = {
     'CJK Compatibility': (char) => char >= 0x3300 && char <= 0x33FF,
     // 'CJK Unified Ideographs Extension A': (char) => char >= 0x3400 && char <= 0x4DBF,
     'Yijing Hexagram Symbols': (char) => char >= 0x4DC0 && char <= 0x4DFF,
-    // 'CJK Unified Ideographs': (char) => char >= 0x4E00 && char <= 0x9FFF,
+    'CJK Unified Ideographs': (char) => char >= 0x4E00 && char <= 0x9FFF,
     // 'Yi Syllables': (char) => char >= 0xA000 && char <= 0xA48F,
     // 'Yi Radicals': (char) => char >= 0xA490 && char <= 0xA4CF,
     // 'Lisu': (char) => char >= 0xA4D0 && char <= 0xA4FF,
@@ -151,7 +151,7 @@ export const unicodeBlockLookup: UnicodeBlockLookup = {
     // 'Latin Extended-E': (char) => char >= 0xAB30 && char <= 0xAB6F,
     // 'Cherokee Supplement': (char) => char >= 0xAB70 && char <= 0xABBF,
     // 'Meetei Mayek': (char) => char >= 0xABC0 && char <= 0xABFF,
-    // 'Hangul Syllables': (char) => char >= 0xAC00 && char <= 0xD7AF,
+    'Hangul Syllables': (char) => char >= 0xAC00 && char <= 0xD7AF,
     // 'Hangul Jamo Extended-B': (char) => char >= 0xD7B0 && char <= 0xD7FF,
     // 'High Surrogates': (char) => char >= 0xD800 && char <= 0xDB7F,
     // 'High Private Use Surrogates': (char) => char >= 0xDB80 && char <= 0xDBFF,

test/integration/render/tests/text-local-ideographs/cjk-symbols/style.json

@@ -0,0 +1,43 @@
+{
+    "version": 8,
+    "metadata": {
+        "test": {
+            "pixelRatio": 2,
+            "localIdeographFontFamily": "sans-serif",
+            "width": 800,
+            "height": 600
+        }
+    },
+    "zoom": 8,
+    "sources": {
+        "sample": {
+            "type": "geojson",
+            "data": {
+                "type": "Feature",
+                "geometry": {
+                    "type": "Point",
+                    "coordinates": [0, 0]
+                },
+                "properties": {
+                    "name_en": "a-b,c.",
+                    "name_ja": "あーい、う。"
+                }
+            }
+        }
+    },
+    "glyphs": "local://glyphs/{fontstack}/{range}.pbf",
+    "layers": [
+        {
+            "id": "sample-text-left",
+            "type": "symbol",
+            "source": "sample",
+            "layout": {
+                "text-anchor": "top",
+                "text-field": "{name_ja}{name_en}",
+                "text-font": ["Open Sans Semibold", "Arial Unicode MS Bold"],
+                "text-size": 30,
+                "text-offset": [0, -2]
+            }
+        }
+    ]
+}