diff options
Diffstat (limited to 'Tools')
| -rwxr-xr-x | Tools/scripts/parse_html5_entities.py | 27 | ||||
| -rwxr-xr-x | Tools/scripts/parseentities.py | 64 |
2 files changed, 18 insertions, 73 deletions
diff --git a/Tools/scripts/parse_html5_entities.py b/Tools/scripts/parse_html5_entities.py index c011328b010..1e5bdad2165 100755 --- a/Tools/scripts/parse_html5_entities.py +++ b/Tools/scripts/parse_html5_entities.py @@ -2,10 +2,14 @@ """ Utility for parsing HTML5 entity definitions available from: - http://dev.w3.org/html5/spec/entities.json + https://html.spec.whatwg.org/entities.json + https://html.spec.whatwg.org/multipage/named-characters.html -Written by Ezio Melotti and Iuliia Proskurnia. +The page now contains the following note: + + "This list is static and will not be expanded or changed in the future." +Written by Ezio Melotti and Iuliia Proskurnia. """ import os @@ -14,7 +18,9 @@ import json from urllib.request import urlopen from html.entities import html5 -entities_url = 'http://dev.w3.org/html5/spec/entities.json' +PAGE_URL = 'https://html.spec.whatwg.org/multipage/named-characters.html' +ENTITIES_URL = 'https://html.spec.whatwg.org/entities.json' +HTML5_SECTION_START = '# HTML5 named character references' def get_json(url): """Download the json file from the url and returns a decoded object.""" @@ -62,9 +68,15 @@ def write_items(entities, file=sys.stdout): # be before their equivalent lowercase version. keys = sorted(entities.keys()) keys = sorted(keys, key=str.lower) + print(HTML5_SECTION_START, file=file) + print(f'# Generated by {sys.argv[0]!r}\n' + f'# from {ENTITIES_URL} and\n' + f'# {PAGE_URL}.\n' + f'# Map HTML5 named character references to the ' + f'equivalent Unicode character(s).', file=file) print('html5 = {', file=file) for name in keys: - print(' {!r}: {!a},'.format(name, entities[name]), file=file) + print(f' {name!r}: {entities[name]!a},', file=file) print('}', file=file) @@ -72,11 +84,8 @@ if __name__ == '__main__': # without args print a diff between html.entities.html5 and new_html5 # with --create print the new html5 dict # with --patch patch the Lib/html/entities.py file - new_html5 = create_dict(get_json(entities_url)) + new_html5 = create_dict(get_json(ENTITIES_URL)) if '--create' in sys.argv: - print('# map the HTML5 named character references to the ' - 'equivalent Unicode character(s)') - print('# Generated by {}. Do not edit manually.'.format(__file__)) write_items(new_html5) elif '--patch' in sys.argv: fname = 'Lib/html/entities.py' @@ -84,7 +93,7 @@ if __name__ == '__main__': with open(fname) as f1, open(temp_fname, 'w') as f2: skip = False for line in f1: - if line.startswith('html5 = {'): + if line.startswith(HTML5_SECTION_START): write_items(new_html5, file=f2) skip = True continue diff --git a/Tools/scripts/parseentities.py b/Tools/scripts/parseentities.py deleted file mode 100755 index 0229d3af86b..00000000000 --- a/Tools/scripts/parseentities.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 -""" Utility for parsing HTML entity definitions available from: - - http://www.w3.org/ as e.g. - http://www.w3.org/TR/REC-html40/HTMLlat1.ent - - Input is read from stdin, output is written to stdout in form of a - Python snippet defining a dictionary "entitydefs" mapping literal - entity name to character or numeric entity. - - Marc-Andre Lemburg, mal@lemburg.com, 1999. - Use as you like. NO WARRANTIES. - -""" -import re,sys - -entityRE = re.compile(r'<!ENTITY +(\w+) +CDATA +"([^"]+)" +-- +((?:.|\n)+?) *-->') - -def parse(text,pos=0,endpos=None): - - pos = 0 - if endpos is None: - endpos = len(text) - d = {} - while 1: - m = entityRE.search(text,pos,endpos) - if not m: - break - name,charcode,comment = m.groups() - d[name] = charcode,comment - pos = m.end() - return d - -def writefile(f,defs): - - f.write("entitydefs = {\n") - items = sorted(defs.items()) - for name, (charcode,comment) in items: - if charcode[:2] == '&#': - code = int(charcode[2:-1]) - if code < 256: - charcode = r"'\%o'" % code - else: - charcode = repr(charcode) - else: - charcode = repr(charcode) - comment = ' '.join(comment.split()) - f.write(" '%s':\t%s, \t# %s\n" % (name,charcode,comment)) - f.write('\n}\n') - -if __name__ == '__main__': - if len(sys.argv) > 1: - with open(sys.argv[1]) as infile: - text = infile.read() - else: - text = sys.stdin.read() - - defs = parse(text) - - if len(sys.argv) > 2: - with open(sys.argv[2],'w') as outfile: - writefile(outfile, defs) - else: - writefile(sys.stdout, defs) |