diff options
| -rw-r--r-- | stmhal/make-stmconst.py | 7 |
1 files changed, 6 insertions, 1 deletions
diff --git a/stmhal/make-stmconst.py b/stmhal/make-stmconst.py index 05c0c08c96..7afafe7206 100644 --- a/stmhal/make-stmconst.py +++ b/stmhal/make-stmconst.py @@ -42,12 +42,17 @@ class Lexer: ) def __init__(self, filename): - self.file = open(filename, 'rt') + self.file = open(filename, 'rb') self.line_number = 0 def next_match(self, strictly_next=False): while True: line = self.file.readline() + try: + line = str(line, 'utf8') + except ValueError: + # some files have invalid utf8 bytes, so filter them out + line = ''.join(chr(l) for l in line if l <= 126) self.line_number += 1 if len(line) == 0: return ('EOF', None) |