#!/usr/bin/env python3 __doc__ = '''General classes and functions for use with SIL's github fonts repository, github.com/silnrsi/fonts''' __url__ = 'https://github.com/silnrsi/pysilfont' __copyright__ = 'Copyright (c) 2022 SIL International (https://www.sil.org)' __license__ = 'Released under the MIT License (https://opensource.org/licenses/MIT)' __author__ = 'David Raymond' import os, json, io import urllib.request as urllib2 from zipfile import ZipFile from silfont.util import prettyjson from silfont.core import splitfn, loggerobj from collections import OrderedDict from fontTools.ttLib import TTFont familyfields = OrderedDict([ ("familyid", {"opt": True, "manifest": False}), # req for families.json but not for base files; handled in code ("fallback", {"opt": True, "manifest": False}), ("family", {"opt": False, "manifest": True}), ("altfamily", {"opt": True, "manifest": False}), ("siteurl", {"opt": True, "manifest": False}), ("packageurl", {"opt": True, "manifest": False}), ("ziproot", {"opt": True, "manifest": False}), ("files", {"opt": True, "manifest": True}), ("defaults", {"opt": True, "manifest": True}), ("version", {"opt": True, "manifest": True}), ("status", {"opt": True, "manifest": False}), ("license", {"opt": True, "manifest": False}), ("distributable", {"opt": False, "manifest": False}), ("source", {"opt": True, "manifest": False}), ("googlefonts", {"opt": True, "manifest": False}), ("features", {"opt": True, "manifest": False}) ]) filefields = OrderedDict([ ("altfamily", {"opt": True, "manifest": True, "mopt": True}), ("url", {"opt": True, "manifest": False}), ("flourl", {"opt": True, "manifest": False}), ("packagepath", {"opt": True, "manifest": True}), ("zippath", {"opt": True, "manifest": False}), ("axes", {"opt": False, "manifest": True}) ]) defaultsfields = OrderedDict([ ("ttf", {"opt": True, "manifest": True}), ("woff", {"opt": True, "manifest": True, "mopt": True}), ("woff2", {"opt": True, "manifest": True, "mopt": True}) ]) class _familydata(object): """Family data key for use with families.json, font manifests and base files """ def __init__(self, id=None, data=None, filename = None, type="f", logger=None): # Initial input can be a dictionary (data) in which case id nneds to be set # or it can be read from a file (containing just one family record), in which case id is taken from the file # Type can be f, b or m for families, base or manifest # With f, this would be for just a single entry from a families.json file self.id = id self.data = data if data else {} self.filename = filename self.type = type self.logger = logger if logger else loggerobj() def fieldscheck(self, data, validfields, reqfields, logprefix, valid, logs): for key in data: # Check all keys have valid names if key not in validfields: logs.append((f'{logprefix}: Invalid field "{key}"', 'W')) valid = False continue # Are required fields present for key in reqfields: if key not in data: logs.append((f'{logprefix}: Required field "{key}" missing', 'W')) valid = False continue return (valid, logs) def validate(self): global familyfields, filefields, defaultsfields logs = [] valid = True if self.type == "m": validfields = reqfields = [key for key in familyfields if familyfields[key]["manifest"]] else: validfields = list(familyfields) reqfields = [key for key in familyfields if not familyfields[key]["opt"]] if self.type == "f": reqfields = reqfields + ["familyid"] else: # Must be b validfields = validfields + ["hosturl", "filesroot"] (valid, logs) = self.fieldscheck(self.data, validfields, reqfields, "Main", valid, logs) # Now check sub-fields if "files" in self.data: fdata = self.data["files"] if self.type == "m": validfields = [key for key in filefields if filefields[key]["manifest"]] reqfields = [key for key in filefields if filefields[key]["manifest"] and not ("mopt" in filefields[key] and filefields[key]["mopt"])] else: validfields = list(filefields) reqfields = [key for key in filefields if not filefields[key]["opt"]] # Now need to check values for each record in files for filen in fdata: frecord = fdata[filen] (valid, logs) = self.fieldscheck(frecord, validfields, reqfields, "Files: " + filen, valid, logs) if "axes" in frecord: # (Will already have been reported above if axes is missing!) adata = frecord["axes"] avalidfields = [key for key in adata if len(key) == 4] areqfields = ["wght", "ital"] if self.type == "m" else [] (valid, logs) = self.fieldscheck(adata, avalidfields, areqfields, "Files, axes: " + filen, valid, logs) if "defaults" in self.data: ddata = self.data["defaults"] if self.type == "m": validfields = [key for key in defaultsfields if defaultsfields[key]["manifest"]] reqfields = [key for key in defaultsfields if defaultsfields[key]["manifest"] and not ("mopt" in defaultsfields[key] and defaultsfields[key]["mopt"])] else: validfields = list(defaultsfields) reqfields = [key for key in defaultsfields if not defaultsfields[key]["opt"]] (valid, logs) = self.fieldscheck(ddata, validfields, reqfields, "Defaults:", valid, logs) return (valid, logs) def read(self, filename=None): # Read data from file (not for families.json) if filename: self.filename = filename with open(self.filename) as infile: try: filedata = json.load(infile) except Exception as e: self.logger.log(f'Error opening {infile}: {e}', 'S') if len(filedata) != 1: self.logger.log(f'Files must contain just one record; {self.filename} has {len(filedata)}') self.id = list(filedata.keys())[0] self.data = filedata[self.id] def write(self, filename=None): # Write data to a file (not for families.json) if filename is None: filename = self.filename self.logger.log(f'Writing to {filename}', 'P') filedata = {self.id: self.data} with open(filename, "w", encoding="utf-8") as outf: outf.write(prettyjson(filedata, oneliners=["files"])) class gfr_manifest(_familydata): # def __init__(self, id=None, data=None, filename = None, logger=None): super(gfr_manifest, self).__init__(id=id, data=data, filename=filename, type="m", logger=logger) def validate(self, version=None, filename=None, checkfiles=True): # Validate the manifest. # If version is supplied, check that that matches the version in the manifest # If self.filename not already set, the filename of the manifest must be supplied (valid, logs) = super(gfr_manifest, self).validate() # Field name validation based on _familydata validation if filename is None: filename = self.filename data = self.data if "files" in data and checkfiles: files = data["files"] mfilelist = {x: files[x]["packagepath"] for x in files} # Check files that are on disk match the manifest files (path, base, ext) = splitfn(filename) fontexts = ['.ttf', '.woff', '.woff2'] dfilelist = {} for dirname, subdirs, filenames in os.walk(path): for filen in filenames: (base, ext) = os.path.splitext(filen) if ext in fontexts: dfilelist[filen] = (os.path.relpath(os.path.join(dirname, filen), start=path).replace('\\', '/')) if mfilelist == dfilelist: logs.append(('Files OK', 'I')) else: valid = False logs.append(('Files on disk and in manifest do not match.', 'W')) logs.append(('Files on disk:', 'I')) for filen in sorted(dfilelist): logs.append((f' {dfilelist[filen]}', 'I')) logs.append(('Files in manifest:', 'I')) for filen in sorted(mfilelist): logs.append((f' {mfilelist[filen]}', 'I')) if "defaults" in data: defaults = data["defaults"] # Check defaults exist allthere = True for default in defaults: if defaults[default] not in mfilelist: allthere = False if allthere: logs.append(('Defaults OK', 'I')) else: valid = False logs.append(('At least one default missing', 'W')) if version: if "version" in data: mversion = data["version"] if version == mversion: logs.append(('Versions OK', 'I')) else: valid = False logs.append((f'Version mismatch: {version} supplied and {mversion} in manifest', "W")) return (valid, logs) class gfr_base(_familydata): # def __init__(self, id=None, data=None, filename = None, logger=None): super(gfr_base, self).__init__(id=id, data=data, filename=filename, type="b", logger=logger) class gfr_family(object): # For families.json. # def __init__(self, data=None, filename=None, logger=None): self.filename = filename self.logger = logger if logger else loggerobj() self.familyrecords = {} if data is not None: self.familyrecords = data def validate(self, familyid=None): allvalid = True alllogs = [] if familyid: record = self.familyrecords[familyid] (allvalid, alllogs) = record.validate() else: for familyid in self.familyrecords: record = self.familyrecords[familyid] (valid, logs) = record.validate() if not valid: allvalid = False alllogs.append(logs) return allvalid, alllogs def write(self, filename=None): # Write data to a file if filename is None: filename = self.filename self.logger.log(f'Writing to {filename}', "P") with open(filename, "w", encoding="utf-8") as outf: outf.write(prettyjson(self.familyrecords, oneliners=["files"])) def setpaths(logger): # Check that the script is being run from the root of the repository and set standard paths repopath = os.path.abspath(os.path.curdir) # Do cursory checks that this is the root of the fonts repo if repopath[-5:] != "fonts" or not os.path.isdir(os.path.join(repopath, "fonts/sil")): logger.log("GFR scripts must be run from the root of the fonts repo", "S") # Set up standard paths for scripts to use silpath = os.path.join(repopath, "fonts/sil") otherpath = os.path.join(repopath, "fonts/other") basespath = os.path.join(repopath, "basefiles") if not os.path.isdir(basespath): os.makedirs(basespath) return repopath, silpath, otherpath, basespath def getttfdata(ttf, logger): # Extract data from a ttf try: font = TTFont(ttf) except Exception as e: logger.log(f'Error opening {ttf}: {e}', 'S') name = font['name'] os2 = font['OS/2'] post = font['post'] values = {} name16 = name.getName(nameID=16, platformID=3, platEncID=1, langID=0x409) values["family"] = str(name16) if name16 else str(name.getName(nameID=1, platformID=3, platEncID=1, langID=0x409)) values["subfamily"] = str(name.getName(nameID=2, platformID=3, platEncID=1, langID=0x409)) values["version"] = str(name.getName(nameID=5, platformID=3, platEncID=1, langID=0x409))[8:] # Remove "Version " from the front values["wght"] = os2.usWeightClass values["ital"] = 0 if getattr(post, "italicAngle") == 0 else 1 return values def getziproot(url, ttfpath): req = urllib2.Request(url=url, headers={'User-Agent': 'Mozilla/4.0 (compatible; httpget)'}) try: reqdat = urllib2.urlopen(req) except Exception as e: return (None, f'{url} not valid: {str(e)}') zipdat = reqdat.read() zipinfile = io.BytesIO(initial_bytes=zipdat) try: zipf = ZipFile(zipinfile) except Exception as e: return (None, f'{url} is not a valid zip file') for zf in zipf.namelist(): if zf.endswith(ttfpath): # found a font, assume we want it ziproot = zf[:-len(ttfpath) - 1] # strip trailing / return (ziproot, "") else: return (None, f"Can't find {ttfpath} in {url}")