Ticket #2457: multiLanguage.diff

source/gui/GUItext.cpp

@@ -30 +30 @@
 
 static const wchar_t TagStart = '[';
 static const wchar_t TagEnd   = ']';
+// List of word demlimitor bounds
+// The list contains ranges of word delimitors. The odd indexed chars are the start
+// of a range, the even are the end of a range. The list must be sorted in INCREASING ORDER
+static const int NUM_WORD_DELIMITORS = 4*2;
+static const u16 WordDelimitors[NUM_WORD_DELIMITORS] = {
+    ' '   , ' ',    // spaces
+    '-'   , '-',    // hyphens
+    0x3000, 0x3002, // ideographic symbols (maybe this range should be wider)
+    0x4E00, 0x9FFF  // characters in the Chinese unicode block
+// TODO add unicode blocks of other languages that don't use spaces
+};
 
 void CGUIString::SFeedback::Reset()
 {
@@ -513 +524 @@
     // Add a delimiter at start and at end, it helps when
     //  processing later, because we don't have make exceptions for
     //  those cases.
-    // We'll sort later.
     m_Words.push_back(0);
-    m_Words.push_back((int)m_RawString.length());
-
-    // Space: ' '
-    for (position=0, curpos=0;;position = curpos+1)
-    {
-        // Find the next word-delimiter.
-        long dl = m_RawString.Find(position, ' ');
-
-        if (dl == -1)
-            break;
-
-        curpos = dl;
-        m_Words.push_back((int)dl+1);
-    }
-
-    // Dash: '-'
-    for (position=0, curpos=0;;position = curpos+1)
-    {
-        // Find the next word-delimiter.
-        long dl = m_RawString.Find(position, '-');
-
-        if (dl == -1)
-            break;
-
-        curpos = dl;
-        m_Words.push_back((int)dl+1);
-    }
 
-    // New Line: '\n'
-    for (position=0, curpos=0;;position = curpos+1)
+    // Add word boundaries in increasing order
+    for (u32 i = 0; i < m_RawString.length(); ++i)
     {
-        // Find the next word-delimiter.
-        long dl = m_RawString.Find(position, '\n');
-
-        if (dl == -1)
-            break;
-
-        curpos = dl;
-
-        // Add before and
-        m_Words.push_back((int)dl);
-        m_Words.push_back((int)dl+1);
+        wchar_t c = m_RawString[i];
+        if (c == '\n')
+        {
+            m_Words.push_back((int)i);
+            m_Words.push_back((int)i+1);
+            continue;
+        }
+        for (int n = 0; n < NUM_WORD_DELIMITORS; n += 2)
+        {
+            if (c <= WordDelimitors[n+1])
+            {
+                if (c >= WordDelimitors[n])
+                    m_Words.push_back((int)i+1);
+                // assume the WordDelimitors list is stored in increasing order
+                break;
+            }
+        }
     }
 
-    sort(m_Words.begin(), m_Words.end());
+    m_Words.push_back((int)m_RawString.length());
 
     // Remove duplicates (only if larger than 2)
     if (m_Words.size() > 2)

source/tools/fontbuilder2/fontbuilder.py

@@ -7 +7 @@
 
 # Representation of a rendered glyph
 class Glyph(object):
-    def __init__(self, ctx, renderstyle, char, idx):
+    def __init__(self, ctx, renderstyle, char, idx, face, size):
         self.renderstyle = renderstyle
         self.char = char
         self.idx = idx
+        self.face = face
+        self.size = size
         self.glyph = (idx, 0, 0)
 
+        if not ctx.get_font_face() == self.face:
+            ctx.set_font_face(self.face)
+            ctx.set_font_size(self.size)
         extents = ctx.glyph_extents([self.glyph])
 
         self.xadvance = round(extents[4])
@@ -52 +57 @@
         self.pos = packer.Pack(self.w, self.h)
 
     def render(self, ctx):
+        if not ctx.get_font_face() == self.face:
+            ctx.set_font_face(self.face)
+            ctx.set_font_size(self.size)
         ctx.save()
         ctx.translate(self.x0, self.y0)
         ctx.translate(self.pos.x, self.pos.y)
@@ -81 +89 @@
     return set(chars)
 
 # Construct a Cairo context and surface for rendering text with the given parameters
-def setup_context(width, height, face, size, renderstyle):
+def setup_context(width, height, renderstyle):
     format = (cairo.FORMAT_ARGB32 if "colour" in renderstyle else cairo.FORMAT_A8)
     surface = cairo.ImageSurface(format, width, height)
     ctx = cairo.Context(surface)
-    ctx.set_font_face(face)
-    ctx.set_font_size(size)
     ctx.set_line_join(cairo.LINE_JOIN_ROUND)
     return ctx, surface
 
-def generate_font(chars, outname, ttf, loadopts, size, renderstyle):
+def generate_font(outname, ttfNames, loadopts, size, renderstyle, dsizes):
 
-    (face, indexes) = FontLoader.create_cairo_font_face_for_file(ttf, 0, loadopts)
+    faceList = []
+    indexList = []
+    for i in range(len(ttfNames)):
+        (face, indices) = FontLoader.create_cairo_font_face_for_file("../../../binaries/data/tools/fontbuilder/fonts/%s" % ttfNames[i], 0, loadopts)
+        faceList.append(face)
+        if not ttfNames[i] in dsizes:
+            dsizes[ttfNames[i]] = 0
+        indexList.append(indices)
 
-    (ctx, _) = setup_context(1, 1, face, size, renderstyle)
+    (ctx, _) = setup_context(1, 1, renderstyle)
 
-    (ascent, descent, linespacing, _, _) = ctx.font_extents()
+    # TODO this gets the line height from the default font
+    # while entire texts can be in the fallback font
+    ctx.set_font_face(faceList[0]);
+    ctx.set_font_size(size + dsizes[ttfNames[0]])
+    (_, _, linespacing, _, _) = ctx.font_extents()
 
     # Estimate the 'average' height of text, for vertical center alignment
-    charheight = round(ctx.glyph_extents([(indexes("I"), 0.0, 0.0)])[3])
+    charheight = round(ctx.glyph_extents([(indexList[0]("I"), 0.0, 0.0)])[3])
 
     # Translate all the characters into glyphs
     # (This is inefficient if multiple characters have the same glyph)
     glyphs = []
-    for c in chars:
-        idx = indexes(c)
-        if ord(c) == 0xFFFD and idx == 0: # use "?" if the missing-glyph glyph is missing
-            idx = indexes("?")
-        if idx:
-            glyphs.append(Glyph(ctx, renderstyle, c, idx))
+    #for c in chars:
+    for c in range(0x20, 0xFFFD):
+        for i in range(len(indexList)):
+            idx = indexList[i](unichr(c))
+            if c == 0xFFFD and idx == 0: # use "?" if the missing-glyph glyph is missing
+                idx = indexList[i]("?")
+            if idx:
+                glyphs.append(Glyph(ctx, renderstyle, unichr(c), idx, faceList[i], size + dsizes[ttfNames[i]]))
+                break
 
     # Sort by decreasing height (tie-break on decreasing width)
     glyphs.sort(key = lambda g: (-g.h, -g.w))
 
     # Try various sizes to pack the glyphs into
     sizes = []
-    for h in [32, 64, 128, 256, 512, 1024]:
-        for w in [32, 64, 128, 256, 512, 1024]:
-            sizes.append((w, h))
+    for h in [32, 64, 128, 256, 512, 1024, 2048, 4096]:
+        sizes.append((h, h))
+        sizes.append((h*2, h))
     sizes.sort(key = lambda (w, h): (w*h, max(w, h))) # prefer smaller and squarer
 
     for w, h in sizes:
         try:
-            #packer = Packer.DumbRectanglePacker(w, h)
-            packer = Packer.CygonRectanglePacker(w, h)
+            packer = Packer.DumbRectanglePacker(w, h)
+            #packer = Packer.CygonRectanglePacker(w, h)
             for g in glyphs:
                 g.pack(packer)
         except Packer.OutOfSpaceError:
             continue
 
-        ctx, surface = setup_context(w, h, face, size, renderstyle)
+        ctx, surface = setup_context(w, h, renderstyle)
         for g in glyphs:
-            g.render(ctx)
+             g.render(ctx)
         surface.write_to_png("%s.png" % outname)
 
         # Output the .fnt file with all the glyph positions etc
@@ -143 +163 @@
         fnt.write("%d\n" % len(glyphs))
         fnt.write("%d\n" % linespacing)
         fnt.write("%d\n" % charheight)
-        glyphs.sort(key = lambda g: ord(g.char))
+        # sorting unneeded, as glyphs are added in increasing order
+        #glyphs.sort(key = lambda g: ord(g.char))
         for g in glyphs:
             x0 = g.x0
             y0 = g.y0
@@ -168 +189 @@
 stroked2 = { "colour": True, "stroke": [((0, 0, 0, 1), 2.0)], "fill": [(1, 1, 1, 1), (1, 1, 1, 1)] }
 stroked3 = { "colour": True, "stroke": [((0, 0, 0, 1), 2.5)], "fill": [(1, 1, 1, 1), (1, 1, 1, 1)] }
 
-chars = load_char_list("charset.txt")
+DejaVuSansMono = (["DejaVuSansMono.ttf","FreeMono.ttf", "HanaMinA.ttf"], FontLoader.FT_LOAD_DEFAULT)
+DejaVuSans = (["DejaVuSans.ttf","FreeSans.ttf", "HanaMinA.ttf"], FontLoader.FT_LOAD_DEFAULT)
+PagellaRegular = (["texgyrepagella-regular.otf","FreeSerif.ttf", "HanaMinA.ttf"], FontLoader.FT_LOAD_NO_HINTING)
+PagellaBold = (["texgyrepagella-bold.otf","FreeSerifBold.ttf", "HanaMinA.ttf"], FontLoader.FT_LOAD_NO_HINTING)
 
-DejaVuSansMono = ("DejaVuSansMono.ttf", FontLoader.FT_LOAD_DEFAULT)
-DejaVuSans = ("DejaVuSans.ttf", FontLoader.FT_LOAD_DEFAULT)
-PagellaRegular = ("texgyrepagella-regular.otf", FontLoader.FT_LOAD_NO_HINTING)
-PagellaBold = ("texgyrepagella-bold.otf", FontLoader.FT_LOAD_NO_HINTING)
+# Define the size differences used to render different fallback fonts
+dsizes = {'HanaMinA.ttf': 2}
 
 fonts = (
     ("mono-10", DejaVuSansMono, 10, filled),
@@ -201 +223 @@
     ("serif-stroke-16", PagellaRegular, 16, stroked2),
 )
 
-for (name, (fontname, loadopts), size, style) in fonts:
+for (name, (fontnames, loadopts), size, style) in fonts:
     print "%s..." % name
-    generate_font(chars, "../../../binaries/data/mods/public/fonts/%s" % name, "../../../binaries/data/tools/fontbuilder/fonts/%s" % fontname, loadopts, size, style)
+    generate_font("../../../binaries/data/mods/public/fonts/%s" % name, fontnames, loadopts, size, style, dsizes)