path: root/src/silfont/ftml_builder.py

#!/usr/bin/env python3
"""classes and functions for building ftml tests from glyph_data.csv and UFO"""
__url__ = 'https://github.com/silnrsi/pysilfont'
__copyright__ = 'Copyright (c) 2018 SIL International  (https://www.sil.org)'
__license__ = 'Released under the MIT License (https://opensource.org/licenses/MIT)'
__author__ = 'Bob Hallissy'

from silfont.ftml import Fxml, Ftestgroup, Ftest, Ffontsrc
from palaso.unicode.ucd import get_ucd
from itertools import product
import re
import collections.abc

# This module comprises two related functionalities:
#  1. The FTML object which acts as a staging object for ftml test data. The methods of this class
#     permit a gradual build-up of an ftml file, e.g.,
#
#       startTestGroup(...)
#       setFeatures(...)
#       addToTest(...)
#       addToTest(...)
#       clearFeatures(...)
#       setLang(...)
#       addToTest(...)
#       closeTestGroup(...)
#       ...
#       writeFile(...)
#
#     The module is clever enough, for example, to automatically close a test when changing features, languages or direction.
#
#  2. The FTMLBuilder object which reads and processes glyph_data.csv and provides assistance in iterating over
#     the characters, features, and languages that should be supported by the font, e.g.:
#
#       ftml.startTestGroup('Encoded characters')
#       for uid in sorted(builder.uids()):
#           if uid < 32: continue
#           c = builder.char(uid)
#           for featlist in builder.permuteFeatures(uids=[uid]):
#               ftml.setFeatures(featlist)
#               builder.render([uid], ftml)
#           ftml.clearFeatures()
#           for langID in sorted(c.langs):
#               ftml.setLang(langID)
#               builder.render([uid], ftml)
#           ftml.clearLang()
#
# See examples/psfgenftml.py for ideas

class FTML(object):
    """a staging class for collecting ftml content and finally writing the xml"""

    # Assumes no nesting of test groups

    def __init__(self, title, logger, comment = None, fontsrc = None, fontlabel = None, fontscale = None,
                 widths = None, rendercheck = True, xslfn = None, defaultrtl = False):
        self.logger = logger
        # Initialize an Fxml object
        fxml = Fxml(testgrouplabel = "dummy")
        fxml.stylesheet = {'type': 'text/xsl', 'href': xslfn if xslfn is not None else 'ftml.xsl'}
        fxml.head.title = title
        fxml.head.comment = comment
        if isinstance(fontsrc, (tuple, list)):
            # Allow multiple fontsrc
            fxml.head.fontsrc = [Ffontsrc(fxml.head, text=fontsrc,
                                          label=fontlabel[i] if fontlabel is not None and i < len(fontlabel) else None)
                                 for i, fontsrc in enumerate(fontsrc)]
        elif fontsrc:
            fxml.head.fontsrc = Ffontsrc(fxml.head, text=fontsrc, label=fontlabel)

        if fontscale: fxml.head.fontscale = int(fontscale)
        if widths: fxml.head.widths = widths
        fxml.testgroups.pop() # Remove dummy test group
        # Save object
        self._fxml = fxml
        # Initialize state
        self._curTest = None
        self.closeTestGroup()
        self.defaultRTL = defaultrtl
        # Add first testgroup if requested
        if rendercheck:
            self.startTestGroup("Rendering Check", background="#F0F0F0")
            self.addToTest(None, "RenderingUnknown", "check", rtl = False)
            self.closeTest()
            self.closeTestGroup()

    _colorMap = {
        'aqua':    '#00ffff',
        'black':   '#000000',
        'blue':    '#0000ff',
        'fuchsia': '#ff00ff',
        'green':   '#008000',
        'grey':    '#808080',
        'lime':    '#00ff00',
        'maroon':  '#800000',
        'navy':    '#000080',
        'olive':   '#808000',
        'purple':  '#800080',
        'red':     '#ff0000',
        'silver':  '#c0c0c0',
        'teal':    '#008080',
        'white':   '#ffffff',
        'yellow':  '#ffff00',
        'orange':  '#ffa500'
    }

    @staticmethod
    def _getColor(color):
        if color is None or len(color) == 0:
            return None
        color = color.lower()
        if color in FTML._colorMap:
            return FTML._colorMap[color]
        if re.match(r'#[0-9a-f]{6}$', color):
            return color
        self.logger.log(f'Color "{color}" not understood; ignored', 'W')
        return None

    def closeTest(self, comment = None):
        if self._curTest:
            if comment is not None:
                self._curTest.comment = comment
            if self._curColor:
                self._curTest.background = self._curColor
        self._curTest = None
        self._lastUID = None
        self._lastRTL = None

    def addToTest(self, uid, s = "", label = None, comment = None, rtl = None):
        if rtl is None: rtl = self.defaultRTL
        if (self._lastUID and uid and uid not in range(self._lastUID, self._lastUID + 2))\
                or (self._lastRTL is not None and rtl != self._lastRTL):
            self.closeTest()
        self._lastUID = uid
        self._lastRTL = rtl
        if self._curTestGroup is None:
            # Create a new Ftestgroup
            self.startTestGroup("Group")
        if self._curTest is None:
            # Create a new Ftest
            if label is None:
                label = "U+{0:04X}".format(uid) if uid is not None else "test"
            test = Ftest(self._curTestGroup, label = label, string = '')
            if comment:
                test.comment = comment
            if rtl: test.rtl = "True"
            # Construct stylename and add style if needed:
            x = ['{}_{}'.format(t,v) for t,v in self._curFeatures.items()] if self._curFeatures else []
            if self._curLang:
                x.insert(0,self._curLang)
            if len(x):
                test.stylename = '_'.join(x)
                self._fxml.head.addstyle(test.stylename, feats = self._curFeatures, lang = self._curLang)
            # Append to current test group
            self._curTestGroup.tests.append(test)
            self._curTest = test
        if len(self._curTest.string): self._curTest.string += ' '
        # Special hack until we get to python3 with full unicode support
        self._curTest.string += ''.join([ c if ord(c) < 128 else '\\u{0:06X}'.format(ord(c)) for c in s ])
        # self._curTest.string += s

    def setFeatures(self, features):
        # features can be None or a list; list elements can be:
        #   None
        #   a feature setting in the form [tag,value]
        if features is None:
            return self.clearFeatures()
        features = [x for x in features if x]
        if len(features) == 0:
            return self.clearFeatures()
        features = dict(features)   # Convert to a dictionary -- this is what we'll keep.
        if features != self._curFeatures:
            self.closeTest()
        self._curFeatures = features

    def clearFeatures(self):
        if self._curFeatures is not None:
            self.closeTest()
        self._curFeatures = None

    def setLang(self, langID):
        if langID != self._curLang:
            self.closeTest();
        self._curLang = langID

    def clearLang(self):
        if self._curLang:
            self.closeTest()
        self._curLang = None

    def setBackground(self, color):
        color = self._getColor(color)
        if color != self._curColor:
            self.closeTest()
        self._curColor = color

    def clearBackground(self):
        if self._curColor is not None:
            self.closeTest()
        self._curColor = None

    def closeTestGroup(self):
        self.closeTest()
        self._curTestGroup = None
        self._curFeatures = None
        self._curLang = None
        self._curColor = None

    def startTestGroup(self, label, background = None):
        if self._curTestGroup is not None:
            if label == self._curTestGroup.label:
                return
            self.closeTestGroup()
        # Add new test group
        self._curTestGroup = Ftestgroup(self._fxml, label = label)
        background = self._getColor(background)
        if background is not None:
            self._curTestGroup.background = background

        # append to root test groups
        self._fxml.testgroups.append(self._curTestGroup)

    def writeFile(self, output):
        self.closeTestGroup()
        self._fxml.save(output)


class Feature(object):
    """abstraction of a feature"""

    def __init__(self, tag):
        self.tag = tag
        self.default = 0
        self.maxval = 1
        self._tvlist = None

    def __getattr__(self,name):
        if name == "tvlist":
            # tvlist is a list of all possible tag,value pairs (except the default but including None) for this feature
            # This attribute shouldn't be needed until all the possible feature value are known,
            # therefore we'll generate this the first time we need it and save it
            if self._tvlist is None:
                self._tvlist = [ None ]
                for v in range (0, self.maxval+1):
                    if v != self.default:
                        self._tvlist.append( [self.tag, str(v)])
            return self._tvlist


class FChar(object):
    """abstraction of an encoded glyph in the font"""

    def __init__(self, uids, basename, logger):
        self.logger = logger
        # uids can be a singleton integer or, for multiple-encoded glyphs, some kind of sequence of integers
        if isinstance(uids,collections.abc.Sequence):
            uids1 = uids
        else:
            uids1 = (uids,)
        # test each uid to make sure valid; remove if not.
        uids2=[]
        self.general = "unknown"
        for uid in uids1:
            try:
                gc = get_ucd(uid,'gc')
                if self.general == "unknown":
                    self.general = gc
                uids2.append(uid)
            except (TypeError, IndexError):
                self.logger.log(f'Invalid USV "{uid}" -- ignored.', 'E')
                continue
            except KeyError:
                self.logger.log('USV %04X not defined; no properties known' % uid, 'W')
        # make sure there's at least one left
        assert len(uids2) > 0, f'No valid USVs found in {repr(uids)}'
        self._uids = tuple(uids2)
        self.basename = basename
        self.feats = set()  # feat tags that affect this char
        self.langs = set()  # lang tags that affect this char
        self.aps = set()
        self.altnames = {}  # alternate glyph names.
            # the above is a dict keyed by either:
            #   lang tag e.g., 'ur', or
            #   feat tag and value, e.g., 'cv24=3'
            # and returns a the glyphname for that alternate.
        # Additional info from UFO:
        self.takesMarks = self.isMark = self.isBase = self.notInUFO = False

    # Most callers don't need to support or or care about multiple-encoded glyphs, so we
    # support the old .uid attribute by returning the first (I guess we consider it primary) uid.
    def __getattr__(self,name):
        if name == 'uids':
            return self._uids
        elif name == 'uid':
            return self._uids[0]
        else:
            raise AttributeError

    # the static method FTMLBuilder.checkGlyph is likely preferred
    #   but leave this instance method for backwards compatibility
    def checkGlyph(self, gname, font, apRE):
        # glean info from UFO if glyph is present
        if gname in font.deflayer:
            self.notInUFO = False
            for a in font.deflayer[gname]['anchor'] :
                name = a.element.get('name')
                if apRE.match(name) is None:
                    continue
                self.aps.add(name)
                if name.startswith("_") :
                    self.isMark = True
                else:
                    self.takesMarks = True
            self.isBase = self.takesMarks and not self.isMark
        else:
            self.notInUFO = True


class FSpecial(object):
    """abstraction of a ligature or other interesting sequence"""

    # Similar to FChar but takes a uid list rather than a single uid
    def __init__(self, uids, basename, logger):
        self.logger = logger
        self.uids = uids
        self.basename = basename
        # a couple of properties based on the first uid:
        try:
            self.general = get_ucd(uids[0],'gc')
        except KeyError:
            self.logger.log('USV %04X not defined; no properties known' % uids[0], 'W')
        self.feats = set()  # feat tags that affect this char
        self.aps = set()
        self.langs = set()  # lang tags that affect this char
        self.altnames = {}  # alternate glyph names.
        self.takesMarks = self.isMark = self.isBase = self.notInUFO = False

class FTMLBuilder(object):
    """glyph_data and UFO processing for building FTML"""

    def __init__(self, logger, incsv = None, fontcode = None, font = None, langs = None, rtlenable = False, ap = None ):
        self.logger = logger
        self.rtlEnable = rtlenable

        # Default diacritic base:
        self.diacBase = 0x25CC

        # Default joinBefore and joinAfter sequence
        self.joinBefore = '\u200D'  # put before a sequence to force joining shape; def = zwj
        self.joinAfter = '\u200D'   # put after a sequence to force joining shape; def = zwj

        # Dict mapping tag to Feature
        self.features = {}

        # Set of all languages seen
        if langs is not None:
            # Use a list so we keep the order (assuming caller wouldn't give us dups
            self.allLangs = list(re.split(r'\s*[\s,]\s*', langs)) # Allow comma- or space-separated tags
            self._langsComplete = True  # We have all the lang tags desired
        else:
            # use a set because the langtags are going to dribble in and be repeated.
            self.allLangs = set()
            self._langsComplete = False # Add lang_tags from glyph_data

        # Be able to find chars and specials:
        self._charFromUID = {}
        self._charFromBasename = {}
        self._specialFromUIDs = {}
        self._specialFromBasename = {}

        # list of USVs that are in the CSV but whose glyphs are not in the UFO
        self.uidsMissingFromUFO = set()

        # DummyUSV  (see charAuto())
        self.curDummyUSV = 0x100000 # Supplemental Private Use Area B

        # Compile --ap parameter
        if ap is None:
            ap = "."
        try:
            self.apRE = re.compile(ap)
        except re.error as e:
            logger.log("--ap parameter '{}' doesn't compile as regular expression: {}".format(ap, e), "S")

        if incsv is not None:
            self.readGlyphData(incsv, fontcode, font)

    def addChar(self, uids, basename):
        # Add an FChar
        # assume parameters are OK:
        c = FChar(uids, basename, self.logger)
        # fatal error if the basename or any of uids have already been seen
        fatal = False
        for uid in c.uids:
            if uid in self._charFromUID:
                self.logger.log('Attempt to add duplicate USV %04X' % uid, 'E')
                fatal = True
            self._charFromUID[uid] = c
        if basename in self._charFromBasename:
            self.logger.log('Attempt to add duplicate basename %s' % basename, 'E')
            fatal = True
        self._charFromBasename[basename] = c
        if fatal:
            self.logger.log('Cannot continue due to previous errors', 'S')
        return c

    def uids(self):
        """ returns list of uids in glyph_data """
        return self._charFromUID.keys()

    def char(self, x):
        """ finds an FChar based either basename or uid;
            generates KeyError if not found."""
        return self._charFromBasename[x] if isinstance(x, str) else self._charFromUID[x]

    def charAuto(self, x):
        """ Like char() but will issue a warning and add a dummy """
        try:
            return self._charFromBasename[x] if isinstance(x, str) else self._charFromUID[x]
        except KeyError:
            # Issue error message and create dummy Char object for this character
            if isinstance(x, str):
                self.logger.log(f'Glyph "{x}" isn\'t in glyph_data.csv - adding dummy', 'E')
                while self.curDummyUSV in self._charFromUID:
                    self.curDummyUSV += 1
                c = self.addChar(self.curDummyUSV, x)
            else:
                self.logger.log(f'Char U+{x:04x} isn\'t in glyph_data.csv - adding dummy', 'E')
                c = self.addChar(x, f'U+{x:04x}')
            return c

    def addSpecial(self, uids, basename):
        # Add an FSpecial:
        # fatal error if basename has already been seen:
        if basename in self._specialFromBasename:
            self.logger.log('Attempt to add duplicate basename %s' % basename, 'S')
        c = FSpecial(uids, basename, self.logger)
        # remember it:
        self._specialFromUIDs[tuple(uids)] = c
        self._specialFromBasename[basename] = c
        return c

    def specials(self):
        """returns a list of the basenames of specials"""
        return self._specialFromBasename.keys()

    def special(self, x):
        """ finds an FSpecial based either basename or uid sequence;
            generates KeyError if not found."""
        return self._specialFromBasename[x] if isinstance(x, str) else self._specialFromUIDs[tuple(x)]

    def _csvWarning(self, msg, exception = None):
        m = "glyph_data line {1}: {0}".format(msg, self.incsv.line_num)
        if exception is not None:
            m += '; ' + str(exception)
        self.logger.log(m, 'W')

    def readGlyphData(self, incsv, fontcode = None, font = None):
        # Remember csv file for other methods:
        self.incsv = incsv

        # Validate fontcode, if provided
        if fontcode is not None:
            whichfont = fontcode.strip().lower()
            if len(whichfont) != 1:
                self.logger.log('fontcode must be a single letter', 'S')
        else:
            whichfont = None

        # Get headings from csvfile:
        fl = incsv.firstline
        if fl is None: self.logger.log("Empty input file", "S")
        # required columns:
        try:
            nameCol = fl.index('glyph_name');
            usvCol = fl.index('USV')
        except ValueError as e:
            self.logger.log('Missing csv input field: ' + str(e), 'S')
        except Exception as e:
            self.logger.log('Error reading csv input field: ' + str(e), 'S')
        # optional columns:
        # If -f specified, make sure we have the fonts column
        if whichfont is not None:
            if 'Fonts' not in fl: self.logger.log('-f requires "Fonts" column in glyph_data', 'S')
            fontsCol = fl.index('Fonts')
        # Allow for projects that use only production glyph names (ps_name same as glyph_name)
        psCol = fl.index('ps_name') if 'ps_name' in fl else nameCol
        # Allow for projects that have no feature and/or lang-specific behaviors
        featCol = fl.index('Feat') if 'Feat' in fl else None
        bcp47Col = fl.index('bcp47tags') if 'bcp47tags' in fl else None

        next(incsv.reader, None)  # Skip first line with headers

        # RE that matches names of glyphs we don't care about
        namesToSkipRE = re.compile('^(?:[._].*|null|cr|nonmarkingreturn|tab|glyph_name)$',re.IGNORECASE)

        # RE that matches things like 'cv23' or 'cv23=4' or 'cv23=2,3'
        featRE = re.compile('^(\w{2,4})(?:=([\d,]+))?$')

        # RE that matches USV sequences for ligatures
        ligatureRE = re.compile('^[0-9A-Fa-f]{4,6}(?:_[0-9A-Fa-f]{4,6})+$')
        
        # RE that matches space-separated USV sequences
        USVsRE = re.compile('^[0-9A-Fa-f]{4,6}(?:\s+[0-9A-Fa-f]{4,6})*$')

        # keep track of glyph names we've seen to detect duplicates
        namesSeen = set()
        psnamesSeen = set()

        # OK, process all records in glyph_data
        for line in incsv:
            gname = line[nameCol].strip()

            # things to ignore:
            if namesToSkipRE.match(gname):
                continue
            if whichfont is not None and line[fontsCol] != '*' and line[fontsCol].lower().find(whichfont) < 0:
                continue
            if len(gname) == 0:
                self._csvWarning('empty glyph name in glyph_data; ignored')
                continue
            if gname.startswith('#'):
                continue
            if gname in namesSeen:
                self._csvWarning('glyph name %s previously seen in glyph_data; ignored' % gname)
                continue

            psname = line[psCol].strip() or gname   # If psname absent, working name will be production name
            if psname in psnamesSeen:
                self._csvWarning('psname %s previously seen; ignored' % psname)
                continue
            namesSeen.add(gname)
            psnamesSeen.add(psname)

            # compute basename-- the glyph name without extensions:
            basename = gname.split('.',1)[0]

            # Process USV(s)
            # could be empty string, a single USV, space-separated list of USVs for multiple encoding,
            # or underscore-connected USVs indicating ligatures.

            usvs = line[usvCol].strip()
            if len(usvs) == 0:
                # Empty USV field, unencoded glyph
                usvs = ()
            elif USVsRE.match(usvs):
                # space-separated hex values:
                usvs = usvs.split()
                isLigature = False
            elif ligatureRE.match(usvs):
                # '_' separated hex values (ligatures)
                usvs = usvs.split('_')
                isLigature = True
            else:
                self._csvWarning(f"invalid USV field '{usvs}'; ignored")
                usvs = ()
            uids = [int(x, 16) for x in usvs]

            if len(uids) == 0:
                # Handle unencoded glyphs
                uids = None # Prevents using this record to set default feature values
                if basename in self._charFromBasename:
                    c = self._charFromBasename[basename]
                    # Check for additional AP info
                    c.checkGlyph(gname, font, self.apRE)
                elif basename in self._specialFromBasename:
                    c = self._specialFromBasename[basename]
                else:
                    self._csvWarning('unencoded variant %s found before encoded glyph' % gname)
                    c = None
            elif isLigature:
                # Handle ligatures
                c = self.addSpecial(uids, basename)
                uids = None  # Prevents using this record to set default feature values  (TODO: Research this)
            else:
                # Handle simple encoded glyphs (could be multiple uids!)
                # Create character object
                c = self.addChar(uids, basename)
                if font is not None:
                    # Examine APs to determine if this character takes marks:
                    c.checkGlyph(gname, font, self.apRE)
                    if c.notInUFO:
                        self.uidsMissingFromUFO.update(uids)

            if featCol is not None:
                feats = line[featCol].strip()
                if len(feats) > 0 and not(feats.startswith('#')):
                    feats = feats.split(';')
                    for feat in feats:
                        m = featRE.match(feat)
                        if m is None:
                            self._csvWarning('incorrectly formed feature specification "%s"; ignored' % feat)
                        else:
                            # find/create structure for this feature:
                            tag = m.group(1)
                            try:
                                feature = self.features[tag]
                            except KeyError:
                                feature = Feature(tag)
                                self.features[tag] = feature
                            # if values supplied, collect default and maximum values for this feature:
                            if m.group(2) is not None:
                                vals = [int(i) for i in m.group(2).split(',')]
                                if len(vals) > 0:
                                    if uids is not None:
                                        feature.default = vals[0]
                                    elif len(feats) == 1:  # TODO: This seems like wrong test.
                                        for v in vals:
                                            # remember the glyph name for this feature/value combination:
                                            feat = '{}={}'.format(tag,v)
                                            if c is not None and feat not in c.altnames:
                                                c.altnames[feat] = gname
                                    vals.append(feature.maxval)
                                    feature.maxval = max(vals)
                            if c is not None:
                                # Record that this feature affects this character:
                                c.feats.add(tag)
                            else:
                                self._csvWarning('untestable feature "%s" : no known USV' % tag)

            if bcp47Col is not None:
                bcp47 = line[bcp47Col].strip()
                if len(bcp47) > 0 and not(bcp47.startswith('#')):
                    if c is not None:
                        for tag in re.split(r'\s*[\s,]\s*', bcp47): # Allow comma- or space-separated tags
                            c.langs.add(tag)        # lang-tags mentioned for this character
                            if not self._langsComplete:
                                self.allLangs.add(tag)  # keep track of all possible lang-tags
                    else:
                        self._csvWarning('untestable langs: no known USV')

        # We're finally done, but if allLangs is a set, let's order it (for lack of anything better) and make a list:
        if not self._langsComplete:
            self.allLangs = list(sorted(self.allLangs))

    def permuteFeatures(self, uids = None, feats = None):
        """ returns an iterator that provides all combinations of feature/value pairs, for a list of uids and/or a specific list of feature tags"""
        feats = set(feats) if feats is not None else set()
        if uids is not None:
            for uid in uids:
                if uid in self._charFromUID:
                    feats.update(self._charFromUID[uid].feats)
        l = [self.features[tag].tvlist for tag in sorted(feats)]
        return product(*l)

    @staticmethod
    def checkGlyph(obj, gname, font, apRE):
        # glean info from UFO if glyph is present
        if gname in font.deflayer:
            obj.notInUFO = False
            for a in font.deflayer[gname]['anchor']:
                name = a.element.get('name')
                if apRE.match(name) is None:
                    continue
                obj.aps.add(name)
                if name.startswith("_"):
                    obj.isMark = True
                else:
                    obj.takesMarks = True
            obj.isBase = obj.takesMarks and not obj.isMark
        else:
            obj.notInUFO = True

    @staticmethod
    def matchMarkBase(c_mark, c_base):
        """ test whether an _AP on c_mark matches an AP on c_base """
        for apM in c_mark.aps:
            if apM.startswith("_"):
                ap = apM[1:]
                for apB in c_base.aps:
                    if apB == ap:
                        return True
        return False

    def render(self, uids, ftml, keyUID = 0, addBreaks = True, rtl = None, dualJoinMode = 3, label = None, comment = None):
        """ general purpose (but not required) function to generate ftml for a character sequence """
        if len(uids) == 0:
            return
        # Make a copy so we don't affect caller
        uids = list(uids)
        # Remember first uid and original length for later
        startUID = uids[0]
        uidLen = len(uids)
        # if keyUID wasn't supplied, use startUID
        if keyUID == 0: keyUID = startUID
        if label is None:
            # Construct label from uids:
            label = '\n'.join(['U+{0:04X}'.format(u) for u in uids])
        if comment is None:
            # Construct comment from glyph names:
            comment = ' '.join([self._charFromUID[u].basename for u in uids])
        # see if uid list includes a mirrored char
        hasMirrored = bool(len([x for x in uids if get_ucd(x,'Bidi_M')]))
        # Analyze first and last joining char
        joiningChars = [x for x in uids if get_ucd(x, 'jt') != 'T']
        if len(joiningChars):
            # If first or last non-TRANSPARENT char is a joining char, then we need to emit examples with zwj
            # Assumes any non-TRANSPARENT char that is bc != L must be a rtl character of some sort
            uid = joiningChars[0]
            zwjBefore = (get_ucd(uid,'jt') == 'D'
                         or (get_ucd(uid,'bc') == 'L' and get_ucd(uid,'jt') == 'L')
                         or (get_ucd(uid,'bc') != 'L' and get_ucd(uid,'jt') == 'R'))
            uid = joiningChars[-1]
            zwjAfter = (get_ucd(uid,'jt') == 'D'
                         or (get_ucd(uid,'bc') == 'L' and get_ucd(uid,'jt') == 'R')
                         or (get_ucd(uid,'bc') != 'L' and get_ucd(uid,'jt') == 'L'))
        else:
            zwjBefore = zwjAfter = False
        if get_ucd(startUID,'gc') == 'Mn':
            # First char is a NSM... prefix a suitable base
            uids.insert(0, self.diacBase)
            zwjBefore = False   # No longer any need to put zwj before
        elif get_ucd(startUID, 'WSpace'):
            # First char is whitespace -- prefix with baseline brackets:
            uids.insert(0, 0xF130)
        lastNonMark = [x for x in uids if get_ucd(x,'gc') != 'Mn'][-1]
        if get_ucd(lastNonMark, 'WSpace'):
            # Last non-mark is whitespace -- append baseline brackets:
            uids.append(0xF131)
        s = ''.join([chr(uid) for uid in uids])
        if zwjBefore or zwjAfter:
            # Show contextual forms:
            # Start with isolate
            t = u'{0} '.format(s)
            if zwjBefore and zwjAfter:
                # For sequences that show dual-joining behavior, what we show depends on dualJoinMode:
                if dualJoinMode & 1:
                    # show initial, medial, final separated by space:
                    t += u'{0}{2} {1}{0}{2} {1}{0} '.format(s, self.joinBefore, self.joinAfter)
                if dualJoinMode & 2:
                    # show 3 joined forms in sequence:
                    t += u'{0}{0}{0} '.format(s)
            elif zwjAfter:
                t += u'{0}{1} '.format(s, self.joinAfter)
            elif zwjBefore:
                t += u'{1}{0} '.format(s, self.joinBefore)
            if addBreaks: ftml.closeTest()
            ftml.addToTest(keyUID, t, label = label, comment = comment, rtl = rtl)
            if addBreaks: ftml.closeTest()
        elif hasMirrored and self.rtlEnable:
            # Contains mirrored and rtl enabled:
            if addBreaks: ftml.closeTest()
            ftml.addToTest(keyUID, u'{0} LTR: \u202A{0}\u202C RTL: \u202B{0}\u202C'.format(s), label = label, comment = comment, rtl = rtl)
            if addBreaks: ftml.closeTest()
        # elif is LRE, RLE, PDF
        # elif is LRI, RLI, FSI, PDI
        elif uidLen > 1:
            ftml.addToTest(keyUID, s , label = label, comment = comment, rtl = rtl)
        else:
            ftml.addToTest(keyUID, s , comment = comment, rtl = rtl)