libpagemaker 0.0.4 drops every text byte from 0x80 up: the control-character test compares a signed char with 0x20, so accented letters, typographic quotes and the glyphs of legacy Indic fonts (Kruti Dev, Akruti Oriya), which sit in those bytes, vanish. PageMaker for Windows stores text in the ANSI code page, so the bytes are now read as Windows-1252 and passed on as UTF-8 (written for johnmcm.com, MPL-2.0 like libpagemaker). --- a/src/lib/PMDCollector.cpp +++ b/src/lib/PMDCollector.cpp @@ -34,6 +34,35 @@ } } +/* Windows-1252 bytes 0x80-0x9F as Unicode; the five bytes it leaves undefined stay as the C1 controls. */ +const unsigned short CP1252_HIGH[32] = +{ + 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, + 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178 +}; + +/* A text byte as UTF-8: PageMaker for Windows stores text in the ANSI code page (Windows-1252). */ +void appendByte(std::string &out, const unsigned char c) +{ + if (c < 0x80) + { + out.push_back(char(c)); + return; + } + const unsigned u = c < 0xA0 ? CP1252_HIGH[c - 0x80] : c; + if (u < 0x800) + { + out.push_back(char(0xC0 | (u >> 6))); + out.push_back(char(0x80 | (u & 0x3F))); + } + else + { + out.push_back(char(0xE0 | (u >> 12))); + out.push_back(char(0x80 | ((u >> 6) & 0x3F))); + out.push_back(char(0x80 | (u & 0x3F))); + } +} + void writeTextSpan(const std::string &text, const std::size_t charStart, std::size_t charEnd, librevenge::RVNGDrawingInterface *const painter) { ++charEnd; @@ -70,14 +99,14 @@ default: // Ignore control characters that do not have known use in PageMaker. // Specific control characters are handled in the switch already. - if (c < 0x20) + if ((unsigned char)c < 0x20) { PMD_DEBUG_MSG(("skipping control character %#x\n", c)); break; } else { - currentText.push_back(c); + appendByte(currentText, (unsigned char)c); } }