summaryrefslogtreecommitdiffstatshomepage
path: root/tests/bytecode/mp-tests/ptex.py
diff options
context:
space:
mode:
authorDamien George <damien.p.george@gmail.com>2015-08-14 12:24:11 +0100
committerDamien George <damien.p.george@gmail.com>2015-08-17 12:51:26 +0100
commit65dc960e3b22a8426e369607e47c19b380ce30ea (patch)
tree5e55ec2861df54e14fdb0eac1d030b34f684743b /tests/bytecode/mp-tests/ptex.py
parent0e978349a5e7696aa44a0faf5d046081a0616ca5 (diff)
downloadmicropython-65dc960e3b22a8426e369607e47c19b380ce30ea.tar.gz
micropython-65dc960e3b22a8426e369607e47c19b380ce30ea.zip
unix-cpy: Remove unix-cpy. It's no longer needed.
unix-cpy was originally written to get semantic equivalent with CPython without writing functional tests. When writing the initial implementation of uPy it was a long way between lexer and functional tests, so the half-way test was to make sure that the bytecode was correct. The idea was that if the uPy bytecode matched CPython 1-1 then uPy would be proper Python if the bytecodes acted correctly. And having matching bytecode meant that it was less likely to miss some deep subtlety in the Python semantics that would require an architectural change later on. But that is all history and it no longer makes sense to retain the ability to output CPython bytecode, because: 1. It outputs CPython 3.3 compatible bytecode. CPython's bytecode changes from version to version, and seems to have changed quite a bit in 3.5. There's no point in changing the bytecode output to match CPython anymore. 2. uPy and CPy do different optimisations to the bytecode which makes it harder to match. 3. The bytecode tests are not run. They were never part of Travis and are not run locally anymore. 4. The EMIT_CPYTHON option needs a lot of extra source code which adds heaps of noise, especially in compile.c. 5. Now that there is an extensive test suite (which tests functionality) there is no need to match the bytecode. Some very subtle behaviour is tested with the test suite and passing these tests is a much better way to stay Python-language compliant, rather than trying to match CPy bytecode.
Diffstat (limited to 'tests/bytecode/mp-tests/ptex.py')
-rw-r--r--tests/bytecode/mp-tests/ptex.py269
1 files changed, 0 insertions, 269 deletions
diff --git a/tests/bytecode/mp-tests/ptex.py b/tests/bytecode/mp-tests/ptex.py
deleted file mode 100644
index 8f23d78009..0000000000
--- a/tests/bytecode/mp-tests/ptex.py
+++ /dev/null
@@ -1,269 +0,0 @@
-import sys
-import os
-import os.path
-import datetime
-import argparse
-from xml.etree.ElementTree import Element, SubElement, tostring
-
-from log import Log
-from texparser import TexParser
-from latexparser import LatexParser
-from gettexfile import file_has_suffix
-from gettexfile import get_tex_file
-
-from xiwi.common.misc import buildFileList
-from xiwi.common import arxivid
-from xiwi.common.stats import Statistics
-
-def str_contains(s1, s2):
- return s1.find(s2) != -1
-
-def str_contains_one_of(st, st_list):
- for st2 in st_list:
- if str_contains(st, st2):
- return True
- return False
-
-def detect_file_kind(file_obj):
- """Simple detection of kind of source file."""
- kind = 'unknown'
- firstline = file_obj.readline()
- while firstline.isspace():
- firstline = file_obj.readline()
- if firstline.startswith('%!PS'):
- kind = 'PS'
- elif firstline.startswith('%auto-ignore'):
- kind = 'auto-ignore'
- else:
- file_obj.seek(0)
- for line in file_obj:
- if str_contains(line, '\\def'):
- # might be tex, if we don't find anything else
- kind = 'tex'
- if str_contains(line, '\\input'):
- # might be tex, if we don't find anything else
- kind = 'tex'
- if str_contains(line, 'amstex') or str_contains(line, 'harvmac'):
- # definitely tex
- kind = 'tex'
- break
- if str_contains(line, '\\documentclass'):
- # definitely latex
- kind = 'latex'
- break
- if str_contains(line, '\\documentstyle'):
- # could be tex or latex
- if str_contains(line, 'amsppt'):
- kind = 'tex'
- break
- else:
- kind = 'latex'
- break
- file_obj.seek(0)
- return kind
-
-class WithdrawnPaper(object):
- def __init__(self):
- pass
-
- def __getitem__(self, item):
- if item == 'refs':
- return []
- elif item == 'success':
- return True
-
- def parse(self):
- pass
-
-def process_article(filename):
- """Returns TexParserBase derived object on success, None on failure."""
-
- # get the tex file
- filename, file_obj, tarfile_obj = get_tex_file(filename)
- if file_obj is None:
- return None
-
- # detect the type of file
- kind = detect_file_kind(file_obj)
-
- # act on the type of file
- parser = None
- if kind == 'PS':
- print('skipping postscript file')
- elif kind == 'auto-ignore':
- print('asked to ignore file, most likely it was withdrawn')
- parser = WithdrawnPaper()
- if kind == 'tex':
- print('parsing as TeX')
- parser = TexParser(filename, file_obj, tarfile_obj)
- elif kind == 'latex':
- print('parsing as LaTeX')
- parser = LatexParser(filename, file_obj, tarfile_obj)
- else:
- print('cannot determine kind of file')
-
- # attempt to parse the document
- try:
- if parser is not None:
- parser.parse()
- except Exception as e:
- print('exception while trying to parse file:')
- print(str(e))
- parser = None
-
- # close the files
- file_obj.close()
- if tarfile_obj is not None:
- tarfile_obj.close()
-
- # return the parsed document
- return parser
-
-arxiv_classes = [
- 'acc-phys', 'adap-org', 'alg-geom', 'ao-sci', 'astro-ph', 'atom-ph',
- 'bayes-an', 'chao-dyn', 'chem-ph', 'cmp-lg', 'comp-gas', 'cond-mat',
- 'cs', 'dg-ga', 'funct-an', 'gr-qc', 'hep-ex', 'hep-lat',
- 'hep-ph', 'hep-th', 'math', 'math-ph', 'mtrl-th', 'nlin',
- 'nucl-ex', 'nucl-th', 'patt-sol', 'physics', 'plasm-ph', 'q-alg',
- 'q-bio', 'quant-ph', 'solv-int', 'supr-con'
-]
-
-def do_single_file(file_name, print_xml, write_xml_dir):
- arxiv_id, arxiv_version = arxivid.filenameToArxivAndVersion(file_name)
- if arxiv_id is None:
- print('WARN: could not determine arXiv identifier for', file_name)
- arxiv_id = '<unknown>'
- arxiv_version = 0
-
- Log.reset()
- Statistics.begin_item(arxiv_id)
-
- if file_has_suffix(file_name, '.pdf'):
- Statistics.count('1) pdf')
- succ = True
- else:
- Statistics.count('2) processed')
-
- parser = process_article(file_name)
-
- if parser is not None :
- succ = parser['success']
- bib_refs = parser['refs']
- else :
- succ = False
- bib_refs = []
-
- if str_contains_one_of(arxiv_id, ['gr-qc', 'hep-']):
- Statistics.count('hep-processed')
- if succ:
- Statistics.count('hep-success')
- if succ:
- print('-success--------')
- Statistics.count('3) success')
- else:
- print('-fail-----------')
- Statistics.count('4) fail')
-
- show_ref = False
-
- if succ and show_ref:
- for bib_ref in bib_refs:
- print(bib_ref.key, 'with', bib_ref.cite_count, 'citations in paper')
- if len(bib_ref.bib_info) == 0:
- print('no reference')
- else:
- print(bib_ref.bib_info_as_str(keep_comments=True))
-
- if succ and (print_xml or write_xml_dir):
- xml = Element('article')
- SubElement(xml, 'id').text = arxiv_id
- if arxiv_version > 0:
- SubElement(xml, 'version').text = str(arxiv_version)
- refs = SubElement(xml, 'refs')
- for bib_ref in bib_refs:
- bib_text = bib_ref.bib_info_as_str(keep_comments=True)
- if len(bib_text) != 0:
- ncites = bib_ref.cite_count
- if ncites < 1:
- ncites = 1
- ref = SubElement(refs, 'ref', order=str(bib_ref.ref_order_num), freq=str(ncites))
- ref.text = bib_text
- if print_xml:
- print(tostring(xml))
- if isinstance(write_xml_dir, str):
- if arxiv_id != '<unknown>':
- xml_file_name = os.path.join(write_xml_dir, arxiv_id.replace('/', '') + '.xml')
- else:
- fname = os.path.split(file_name)[1]
- if fname.rfind('.') > 0:
- fname = fname[:fname.rfind('.')]
- xml_file_name = write_xml_dir + '/' + fname + '.xml'
- file_obj = open(xml_file_name, 'wb')
- file_obj.write(tostring(xml, encoding='utf-8'))
- file_obj.close()
-
- Statistics.end_item()
-
- return succ
-
-summaryStrs = []
-
-if __name__ == "__main__":
- cmd_parser = argparse.ArgumentParser(description='Parse TeX/LaTeX to find references.')
- cmd_parser.add_argument('--filelist', action='store_true', help='file names on the command line each contain a list of files to process')
- cmd_parser.add_argument('--print-xml', action='store_true', help='print XML output to stdout')
- cmd_parser.add_argument('--write-xml', metavar='<dir>', help='destination directory to write XML output files')
- cmd_parser.add_argument('--failed', metavar='<file>', help='output file to write list of failed files')
- cmd_parser.add_argument('files', nargs='+', help='input files')
- args = cmd_parser.parse_args()
-
- # print date stamp
- timeStart = datetime.datetime.now()
- print('[ptex] started processing at', str(timeStart))
-
- print('given', len(args.files), 'files, first file:', args.files[0])
- print('================')
-
- Statistics.clear('article')
-
- # build list of files to process
- file_list = buildFileList(args.filelist, args.files)
-
- # ensure the destination directory exists
- if args.write_xml is not None and os.path.exists(args.write_xml):
- try:
- os.makedirs(args.write_xml)
- except:
- pass
-
- # process the files
- failed_files = []
- for file_name in file_list:
- success = do_single_file(file_name, args.print_xml, args.write_xml)
- if not success:
- failed_files.append(file_name)
-
- # write the failed files to an output file, if requested
- if args.failed is not None:
- file_obj = open(args.failed, 'w')
- file_obj.writelines(f + '\n' for f in failed_files)
- file_obj.close()
-
- print('================')
- Statistics.show()
- Statistics.show_detail('fail')
- #Statistics.show_detail('cite-range')
- #Statistics.show_detail('bad-ascii')
- #Statistics.show_detail('non-ascii')
-
- print('================')
-
- # print date stamp
- timeEnd = datetime.datetime.now()
- print('[ptex] finished processing at', str(timeEnd))
-
- # print summary for email
- summaryStrs.extend(Statistics.get_summary())
- summaryStrs.insert(0, 'started processing at %s, took %.1f minutes' % (timeStart.strftime('%H:%M'), (timeEnd - timeStart).total_seconds() / 60))
- for s in summaryStrs:
- print('**SUMMARY** [ptex]', s)