Add tests to test compiler and emitted byte code.

76 files changed, 979 insertions, 0 deletions

diff --git a/tests/bytecode/mp-tests/assert1.py b/tests/bytecode/mp-tests/assert1.py
new file mode 100644
index 0000000000..077defc970
--- /dev/null
+++ b/tests/bytecode/mp-tests/assert1.py
@@ -0,0 +1,2 @@
+assert x
+assert x, 'test'
diff --git a/tests/bytecode/mp-tests/assign1.py b/tests/bytecode/mp-tests/assign1.py
new file mode 100644
index 0000000000..64ae4a0c59
--- /dev/null
+++ b/tests/bytecode/mp-tests/assign1.py
@@ -0,0 +1,10 @@
+[] = ()
+[] = []
+a = b
+(a) = b
+a, b = c, d
+a, b, c = d, e, f
+a, b, c, d = e, f, g, h
+#(a, b) = c, d
+#a, b = (c, d)
+#(a, b) = (c, d)
diff --git a/tests/bytecode/mp-tests/assign2.py b/tests/bytecode/mp-tests/assign2.py
new file mode 100644
index 0000000000..f55a3221d0
--- /dev/null
+++ b/tests/bytecode/mp-tests/assign2.py
@@ -0,0 +1,10 @@
+*a, = b
+a, *b = c
+a, *b, = c
+a, *b, c = d
+
+[*a] = b
+[*a,] = b
+[a, *b] = c
+#[a, *b,] = c
+#[a, *b, c] = d
diff --git a/tests/bytecode/mp-tests/augassign1.py b/tests/bytecode/mp-tests/augassign1.py
new file mode 100644
index 0000000000..38a376af46
--- /dev/null
+++ b/tests/bytecode/mp-tests/augassign1.py
@@ -0,0 +1,5 @@
+[] = ()
+x += 1
+x.y += 1
+x.f().y += 1
+x[1] += 2
diff --git a/tests/bytecode/mp-tests/call1.py b/tests/bytecode/mp-tests/call1.py
new file mode 100644
index 0000000000..eb8a8bf5f1
--- /dev/null
+++ b/tests/bytecode/mp-tests/call1.py
@@ -0,0 +1 @@
+f(a, b=c)
diff --git a/tests/bytecode/mp-tests/class1.py b/tests/bytecode/mp-tests/class1.py
new file mode 100644
index 0000000000..bc87666806
--- /dev/null
+++ b/tests/bytecode/mp-tests/class1.py
@@ -0,0 +1,3 @@
+class C:
+    pass
+C()
diff --git a/tests/bytecode/mp-tests/class2.py b/tests/bytecode/mp-tests/class2.py
new file mode 100644
index 0000000000..1a3e89849d
--- /dev/null
+++ b/tests/bytecode/mp-tests/class2.py
@@ -0,0 +1,4 @@
+class A:
+    x = 1
+    y = x + z
+A()
diff --git a/tests/bytecode/mp-tests/class3.py b/tests/bytecode/mp-tests/class3.py
new file mode 100644
index 0000000000..f49e2e8114
--- /dev/null
+++ b/tests/bytecode/mp-tests/class3.py
@@ -0,0 +1,10 @@
+class A:
+    def f(x):
+        return x
+    def g(y):
+        def h(z):
+            return x + y + z
+        h(y)
+A()
+A.f(1)
+A.g(2)(3)
diff --git a/tests/bytecode/mp-tests/class4.py b/tests/bytecode/mp-tests/class4.py
new file mode 100644
index 0000000000..4cb6258093
--- /dev/null
+++ b/tests/bytecode/mp-tests/class4.py
@@ -0,0 +1,9 @@
+class A:
+    def __init__(self, x):
+        self.x = x
+        self.y = 0
+
+    def get(self):
+        return self.x + self.y
+A(1)
+A(2).get()
diff --git a/tests/bytecode/mp-tests/class5.py b/tests/bytecode/mp-tests/class5.py
new file mode 100644
index 0000000000..4bf96c8e2f
--- /dev/null
+++ b/tests/bytecode/mp-tests/class5.py
@@ -0,0 +1,8 @@
+class A(B):
+    pass
+class A(object):
+    pass
+class A(x.y()):
+    pass
+class A(B, C):
+    pass
diff --git a/tests/bytecode/mp-tests/closure1.py b/tests/bytecode/mp-tests/closure1.py
new file mode 100644
index 0000000000..fdfb4eaf27
--- /dev/null
+++ b/tests/bytecode/mp-tests/closure1.py
@@ -0,0 +1,2 @@
+# basic closure
+# to write!
diff --git a/tests/bytecode/mp-tests/closure2.py b/tests/bytecode/mp-tests/closure2.py
new file mode 100644
index 0000000000..08b4205810
--- /dev/null
+++ b/tests/bytecode/mp-tests/closure2.py
@@ -0,0 +1,7 @@
+# test closing over an argument
+
+def f(x):
+    y = 2 * x
+    def g(z):
+        return x + y + z
+    return g
diff --git a/tests/bytecode/mp-tests/closure3.py b/tests/bytecode/mp-tests/closure3.py
new file mode 100644
index 0000000000..905211317a
--- /dev/null
+++ b/tests/bytecode/mp-tests/closure3.py
@@ -0,0 +1,12 @@
+# test when different variables are closed over by different functions
+
+def f():
+    l1 = 1
+    l2 = 2
+    l3 = 3
+
+    def g():
+        return l1 + l2
+
+    def h():
+        return l2 + l3
diff --git a/tests/bytecode/mp-tests/closure4.py b/tests/bytecode/mp-tests/closure4.py
new file mode 100644
index 0000000000..6828f89008
--- /dev/null
+++ b/tests/bytecode/mp-tests/closure4.py
@@ -0,0 +1,13 @@
+# test when a function has cell and free vars
+
+def f():
+    f_local = 1
+    f_cell = 2
+
+    def g():
+        g_local = 3
+        g_cell = f_cell + 4
+
+        def h():
+            h1_local = 4
+            h2_local = f_cell + g_cell
diff --git a/tests/bytecode/mp-tests/compare1.py b/tests/bytecode/mp-tests/compare1.py
new file mode 100644
index 0000000000..32ba43e3bf
--- /dev/null
+++ b/tests/bytecode/mp-tests/compare1.py
@@ -0,0 +1,8 @@
+if 1 <= x <= 5:
+    f()
+
+if 1 <= x <= y <= 7:
+    f()
+
+if a < b > c in l != c is not d:
+    f()
diff --git a/tests/bytecode/mp-tests/const1.py b/tests/bytecode/mp-tests/const1.py
new file mode 100644
index 0000000000..545b334344
--- /dev/null
+++ b/tests/bytecode/mp-tests/const1.py
@@ -0,0 +1,9 @@
+x = 1
+#x = 1.2
+#x = 1.2e5
+#x = 1.2e+5
+#x = 1.2e-5
+x = ()
+x = (1,)
+x = (1,2)
+x = ('a',None,3)
diff --git a/tests/bytecode/mp-tests/continue1.py b/tests/bytecode/mp-tests/continue1.py
new file mode 100644
index 0000000000..3600691b1f
--- /dev/null
+++ b/tests/bytecode/mp-tests/continue1.py
@@ -0,0 +1,44 @@
+for a in b:
+    continue
+
+for a in b:
+    try:
+        f()
+    except:
+        continue
+    g()
+
+for a in b:
+    try:
+        f()
+        continue
+    except:
+        g()
+
+for a in b:
+    try:
+        f()
+    except:
+        try:
+            g()
+        except:
+            continue
+
+for a in b:
+    try:
+        f()
+    except:
+        try:
+            g()
+            continue
+        except:
+            h()
+
+for a in b:
+    try:
+        f()
+    except:
+        pass
+    else:
+        continue
+    g()
diff --git a/tests/bytecode/mp-tests/decorate1.py b/tests/bytecode/mp-tests/decorate1.py
new file mode 100644
index 0000000000..208aebc5bf
--- /dev/null
+++ b/tests/bytecode/mp-tests/decorate1.py
@@ -0,0 +1,20 @@
+@d
+def f():
+    pass
+
+@d
+@e
+def g():
+    pass
+
+@d.e.f
+def h():
+    pass
+
+@d(a + 1)
+def i():
+    pass
+
+@d(a + 1, b + 2)
+def i():
+    pass
diff --git a/tests/bytecode/mp-tests/del1.py b/tests/bytecode/mp-tests/del1.py
new file mode 100644
index 0000000000..0a259fac78
--- /dev/null
+++ b/tests/bytecode/mp-tests/del1.py
@@ -0,0 +1,16 @@
+del x
+del x.y
+del x().y
+del g
+del x[a]
+def f():
+    global g
+    del x
+    del g
+    local = 1
+    local2 = 2
+    local3 = 3
+    del local, local2, local3
+    def f2():
+        nonlocal local3
+        del local2, local3
diff --git a/tests/bytecode/mp-tests/del2.py b/tests/bytecode/mp-tests/del2.py
new file mode 100644
index 0000000000..1c63d15fcb
--- /dev/null
+++ b/tests/bytecode/mp-tests/del2.py
@@ -0,0 +1,11 @@
+del x
+del x,
+del x, y
+del x, y,
+del x, y, z
+del (x)
+del (x,)
+del (x, y)
+del (x, y,)
+del (x, y, z)
+del a, (b, c)
diff --git a/tests/bytecode/mp-tests/dict1.py b/tests/bytecode/mp-tests/dict1.py
new file mode 100644
index 0000000000..3243faa632
--- /dev/null
+++ b/tests/bytecode/mp-tests/dict1.py
@@ -0,0 +1,3 @@
+x = {}
+x = {'a':1}
+x = {'a':1, 'b':2}
diff --git a/tests/bytecode/mp-tests/dictcomp1.py b/tests/bytecode/mp-tests/dictcomp1.py
new file mode 100644
index 0000000000..9dca499c57
--- /dev/null
+++ b/tests/bytecode/mp-tests/dictcomp1.py
@@ -0,0 +1,2 @@
+x = {a:None for a in l}
+x = {b:c for c, b in l if c}
diff --git a/tests/bytecode/mp-tests/docstring1.py b/tests/bytecode/mp-tests/docstring1.py
new file mode 100644
index 0000000000..d1e0184547
--- /dev/null
+++ b/tests/bytecode/mp-tests/docstring1.py
@@ -0,0 +1,8 @@
+"""Module"""
+
+class A:
+    """Class"""
+    pass
+
+class B:
+    """Class B"""
diff --git a/tests/bytecode/mp-tests/docstring2.py b/tests/bytecode/mp-tests/docstring2.py
new file mode 100644
index 0000000000..5a2183aef9
--- /dev/null
+++ b/tests/bytecode/mp-tests/docstring2.py
@@ -0,0 +1,3 @@
+# comment before doc string
+
+"""Doc string"""
diff --git a/tests/bytecode/mp-tests/fun1.py b/tests/bytecode/mp-tests/fun1.py
new file mode 100644
index 0000000000..36e079c01e
--- /dev/null
+++ b/tests/bytecode/mp-tests/fun1.py
@@ -0,0 +1,2 @@
+def f(*args):
+    g(*args)
diff --git a/tests/bytecode/mp-tests/fun2.py b/tests/bytecode/mp-tests/fun2.py
new file mode 100644
index 0000000000..a6cba92aad
--- /dev/null
+++ b/tests/bytecode/mp-tests/fun2.py
@@ -0,0 +1,23 @@
+def f(*, b):
+    return b
+
+def f(a, *, b):
+    return a + b
+
+def f(a, *, b, c):
+    return a + b + c
+
+def f(a, *, b=c):
+    return a + b
+
+def f(a, *, b=c, c):
+    return a + b + c
+
+def f(a, *, b=c, c=d):
+    return a + b + c
+
+def f(a, *, b=c, c, d=e):
+    return a + b + c + d
+
+def f(a=None, *, b=None):
+    return a + b
diff --git a/tests/bytecode/mp-tests/fun3.py b/tests/bytecode/mp-tests/fun3.py
new file mode 100644
index 0000000000..5336a70797
--- /dev/null
+++ b/tests/bytecode/mp-tests/fun3.py
@@ -0,0 +1,3 @@
+def f(a, b):
+    def g(c, d=None, *, e=True):
+        return a + b + c + d + e
diff --git a/tests/bytecode/mp-tests/if1.py b/tests/bytecode/mp-tests/if1.py
new file mode 100644
index 0000000000..8c8a08ccdd
--- /dev/null
+++ b/tests/bytecode/mp-tests/if1.py
@@ -0,0 +1,24 @@
+if x:
+    x()
+if x:
+    x()
+elif y:
+    y()
+if x:
+    x()
+else:
+    zz()
+if x:
+    x()
+elif y:
+    y()
+else:
+    zz()
+if x:
+    x()
+elif y:
+    y()
+elif z:
+    z()
+else:
+    zz()
diff --git a/tests/bytecode/mp-tests/if2.py b/tests/bytecode/mp-tests/if2.py
new file mode 100644
index 0000000000..deb0cd5811
--- /dev/null
+++ b/tests/bytecode/mp-tests/if2.py
@@ -0,0 +1,26 @@
+def f(x):
+    if x:
+        return
+    if x:
+        return
+    elif y:
+        return
+    if x:
+        return
+    else:
+        return
+    if x:
+        return
+    elif y:
+        return
+    else:
+        return
+    if x:
+        return
+    elif y:
+        return
+    elif z:
+        return
+    else:
+        return
+    return None
diff --git a/tests/bytecode/mp-tests/if3.py b/tests/bytecode/mp-tests/if3.py
new file mode 100644
index 0000000000..bd01514d63
--- /dev/null
+++ b/tests/bytecode/mp-tests/if3.py
@@ -0,0 +1,6 @@
+if a and b:
+    f()
+if a or b:
+    f()
+if a and (b or c):
+    f()
diff --git a/tests/bytecode/mp-tests/if4.py b/tests/bytecode/mp-tests/if4.py
new file mode 100644
index 0000000000..4d5a86cd8b
--- /dev/null
+++ b/tests/bytecode/mp-tests/if4.py
@@ -0,0 +1,8 @@
+if not a:
+    f()
+if not a and b:
+    f()
+if not a and not b:
+    f()
+while not a:
+    f()
diff --git a/tests/bytecode/mp-tests/ifexpr1.py b/tests/bytecode/mp-tests/ifexpr1.py
new file mode 100644
index 0000000000..bdb2efc0a1
--- /dev/null
+++ b/tests/bytecode/mp-tests/ifexpr1.py
@@ -0,0 +1 @@
+x = 1 if a else 2
diff --git a/tests/bytecode/mp-tests/import1.py b/tests/bytecode/mp-tests/import1.py
new file mode 100644
index 0000000000..696f3a2708
--- /dev/null
+++ b/tests/bytecode/mp-tests/import1.py
@@ -0,0 +1,5 @@
+a = 1
+def f():
+    global a
+import a
+import b, c
diff --git a/tests/bytecode/mp-tests/import2.py b/tests/bytecode/mp-tests/import2.py
new file mode 100644
index 0000000000..2a89703d9d
--- /dev/null
+++ b/tests/bytecode/mp-tests/import2.py
@@ -0,0 +1 @@
+from a import b
diff --git a/tests/bytecode/mp-tests/import3.py b/tests/bytecode/mp-tests/import3.py
new file mode 100644
index 0000000000..7f365a51eb
--- /dev/null
+++ b/tests/bytecode/mp-tests/import3.py
@@ -0,0 +1,8 @@
+import a.b
+import a.b.c
+from a.b import d
+from a.b.c import d
+
+from a import *
+from a import d, e
+from a import (d, e)
diff --git a/tests/bytecode/mp-tests/import4.py b/tests/bytecode/mp-tests/import4.py
new file mode 100644
index 0000000000..ecc3786755
--- /dev/null
+++ b/tests/bytecode/mp-tests/import4.py
@@ -0,0 +1,3 @@
+import a as y
+import a.b as y
+import a.b.c as y
diff --git a/tests/bytecode/mp-tests/import5.py b/tests/bytecode/mp-tests/import5.py
new file mode 100644
index 0000000000..fb93862d19
--- /dev/null
+++ b/tests/bytecode/mp-tests/import5.py
@@ -0,0 +1,4 @@
+from a import b as c
+from a.b import c as d
+from a.b.c import d as e
+from a.b.c import d as e, f as h
diff --git a/tests/bytecode/mp-tests/lambda1.py b/tests/bytecode/mp-tests/lambda1.py
new file mode 100644
index 0000000000..559c7c20f5
--- /dev/null
+++ b/tests/bytecode/mp-tests/lambda1.py
@@ -0,0 +1,2 @@
+f = lambda: 0
+f = lambda x: 1 + x
diff --git a/tests/bytecode/mp-tests/lambda2.py b/tests/bytecode/mp-tests/lambda2.py
new file mode 100644
index 0000000000..1b4500c08f
--- /dev/null
+++ b/tests/bytecode/mp-tests/lambda2.py
@@ -0,0 +1 @@
+f = lambda *args: args
diff --git a/tests/bytecode/mp-tests/list1.py b/tests/bytecode/mp-tests/list1.py
new file mode 100644
index 0000000000..e2a1a3e9fa
--- /dev/null
+++ b/tests/bytecode/mp-tests/list1.py
@@ -0,0 +1,8 @@
+x = []
+x = [1]
+x = [1,] # not implemented
+x = [1, 2]
+x = [1, 2,]
+x = [1, 2, 3]
+x = [1, 2, 3, 4]
+x = [1, 2, 3, 4, 5]
diff --git a/tests/bytecode/mp-tests/list2.py b/tests/bytecode/mp-tests/list2.py
new file mode 100644
index 0000000000..90b21184da
--- /dev/null
+++ b/tests/bytecode/mp-tests/list2.py
@@ -0,0 +1,8 @@
+x = [()]
+x = [(a)]
+x = [(a,)]
+x = [(a)]
+x = [(a,)]
+x = [a, b]
+x = [(a, b)]
+x = [(a, b, c)]
diff --git a/tests/bytecode/mp-tests/listcomp1.py b/tests/bytecode/mp-tests/listcomp1.py
new file mode 100644
index 0000000000..3a0ef49791
--- /dev/null
+++ b/tests/bytecode/mp-tests/listcomp1.py
@@ -0,0 +1,4 @@
+x = (a for a in l)
+
+f(a for a in l)
+f(a + b for a, b in f())
diff --git a/tests/bytecode/mp-tests/listcomp2.py b/tests/bytecode/mp-tests/listcomp2.py
new file mode 100644
index 0000000000..5f52a5e6b0
--- /dev/null
+++ b/tests/bytecode/mp-tests/listcomp2.py
@@ -0,0 +1 @@
+[x.y for x in k.l]
diff --git a/tests/bytecode/mp-tests/listcomp3.py b/tests/bytecode/mp-tests/listcomp3.py
new file mode 100644
index 0000000000..77a8f2be20
--- /dev/null
+++ b/tests/bytecode/mp-tests/listcomp3.py
@@ -0,0 +1,3 @@
+x = (a + 1 for a in l if a.f())
+
+x = [a + 1 for a in l if a.f()]
diff --git a/tests/bytecode/mp-tests/listcomp4.py b/tests/bytecode/mp-tests/listcomp4.py
new file mode 100644
index 0000000000..6b29993097
--- /dev/null
+++ b/tests/bytecode/mp-tests/listcomp4.py
@@ -0,0 +1,4 @@
+# closing over a local variable in a list comprehension
+def f():
+    a = 1
+    x = [a + b for b in l]
diff --git a/tests/bytecode/mp-tests/listcomp5.py b/tests/bytecode/mp-tests/listcomp5.py
new file mode 100644
index 0000000000..a42d811b75
--- /dev/null
+++ b/tests/bytecode/mp-tests/listcomp5.py
@@ -0,0 +1,11 @@
+# nested ifs
+x = [a for a in l if a if a + 1]
+x = [a for a in l if a if a + 1 if a + 2]
+
+# nested for loops
+x = [a for a in l for l in ls]
+x = [a for ls in lss for l in ls for a in l]
+x = [a for a in l for l in ls for ls in lss]
+
+# nested ifs and for loops
+x = [a for a in l if a for l in ls if l if a for ls in lss if ls]
diff --git a/tests/bytecode/mp-tests/locals1.py b/tests/bytecode/mp-tests/locals1.py
new file mode 100644
index 0000000000..49c34da1ad
--- /dev/null
+++ b/tests/bytecode/mp-tests/locals1.py
@@ -0,0 +1,22 @@
+# to test the order of locals and arguments (LOAD_FAST, STORE_FAST)
+
+def f1():
+    b = 1
+    a = 2
+    return a + b
+
+def f2(b):
+    a = 2
+    return a + b
+
+def f3():
+    def f3f():
+        return True
+    a = 1
+    return f3f(a)
+
+def f4():
+    x = 1
+    def f3f():
+        return True
+    return f3f(x)
diff --git a/tests/bytecode/mp-tests/ptex.py b/tests/bytecode/mp-tests/ptex.py
new file mode 100644
index 0000000000..8f23d78009
--- /dev/null
+++ b/tests/bytecode/mp-tests/ptex.py
@@ -0,0 +1,269 @@
+import sys
+import os
+import os.path
+import datetime
+import argparse
+from xml.etree.ElementTree import Element, SubElement, tostring
+
+from log import Log
+from texparser import TexParser
+from latexparser import LatexParser
+from gettexfile import file_has_suffix
+from gettexfile import get_tex_file
+
+from xiwi.common.misc import buildFileList
+from xiwi.common import arxivid
+from xiwi.common.stats import Statistics
+
+def str_contains(s1, s2):
+    return s1.find(s2) != -1
+
+def str_contains_one_of(st, st_list):
+    for st2 in st_list:
+        if str_contains(st, st2):
+            return True
+    return False
+
+def detect_file_kind(file_obj):
+    """Simple detection of kind of source file."""
+    kind = 'unknown'
+    firstline = file_obj.readline()
+    while firstline.isspace():
+        firstline = file_obj.readline()
+    if firstline.startswith('%!PS'):
+        kind = 'PS'
+    elif firstline.startswith('%auto-ignore'):
+        kind = 'auto-ignore'
+    else:
+        file_obj.seek(0)
+        for line in file_obj:
+            if str_contains(line, '\\def'):
+                # might be tex, if we don't find anything else
+                kind = 'tex'
+            if str_contains(line, '\\input'):
+                # might be tex, if we don't find anything else
+                kind = 'tex'
+                if str_contains(line, 'amstex') or str_contains(line, 'harvmac'):
+                    # definitely tex
+                    kind = 'tex'
+                    break
+            if str_contains(line, '\\documentclass'):
+                # definitely latex
+                kind = 'latex'
+                break
+            if str_contains(line, '\\documentstyle'):
+                # could be tex or latex
+                if str_contains(line, 'amsppt'):
+                    kind = 'tex'
+                    break
+                else:
+                    kind = 'latex'
+                    break
+    file_obj.seek(0)
+    return kind
+
+class WithdrawnPaper(object):
+    def __init__(self):
+        pass
+
+    def __getitem__(self, item):
+        if item == 'refs':
+            return []
+        elif item == 'success':
+            return True
+
+    def parse(self):
+        pass
+
+def process_article(filename):
+    """Returns TexParserBase derived object on success, None on failure."""
+
+    # get the tex file
+    filename, file_obj, tarfile_obj = get_tex_file(filename)
+    if file_obj is None:
+        return None
+
+    # detect the type of file
+    kind = detect_file_kind(file_obj)
+
+    # act on the type of file
+    parser = None
+    if kind == 'PS':
+        print('skipping postscript file')
+    elif kind == 'auto-ignore':
+        print('asked to ignore file, most likely it was withdrawn')
+        parser = WithdrawnPaper()
+    if kind == 'tex':
+        print('parsing as TeX')
+        parser = TexParser(filename, file_obj, tarfile_obj)
+    elif kind == 'latex':
+        print('parsing as LaTeX')
+        parser = LatexParser(filename, file_obj, tarfile_obj)
+    else:
+        print('cannot determine kind of file')
+
+    # attempt to parse the document
+    try:
+        if parser is not None:
+            parser.parse()
+    except Exception as e:
+        print('exception while trying to parse file:')
+        print(str(e))
+        parser = None
+
+    # close the files
+    file_obj.close()
+    if tarfile_obj is not None:
+        tarfile_obj.close()
+
+    # return the parsed document
+    return parser
+
+arxiv_classes = [
+    'acc-phys', 'adap-org', 'alg-geom', 'ao-sci', 'astro-ph', 'atom-ph',
+    'bayes-an', 'chao-dyn', 'chem-ph', 'cmp-lg', 'comp-gas', 'cond-mat',
+    'cs', 'dg-ga', 'funct-an', 'gr-qc', 'hep-ex', 'hep-lat',
+    'hep-ph', 'hep-th', 'math', 'math-ph', 'mtrl-th', 'nlin',
+    'nucl-ex', 'nucl-th', 'patt-sol', 'physics', 'plasm-ph', 'q-alg',
+    'q-bio', 'quant-ph', 'solv-int', 'supr-con'
+]
+
+def do_single_file(file_name, print_xml, write_xml_dir):
+    arxiv_id, arxiv_version = arxivid.filenameToArxivAndVersion(file_name)
+    if arxiv_id is None:
+        print('WARN: could not determine arXiv identifier for', file_name)
+        arxiv_id = '<unknown>'
+        arxiv_version = 0
+
+    Log.reset()
+    Statistics.begin_item(arxiv_id)
+
+    if file_has_suffix(file_name, '.pdf'):
+        Statistics.count('1) pdf')
+        succ = True
+    else:
+        Statistics.count('2) processed')
+
+        parser = process_article(file_name)
+
+        if parser is not None :
+            succ = parser['success']
+            bib_refs = parser['refs']
+        else :
+            succ = False
+            bib_refs = []
+
+        if str_contains_one_of(arxiv_id, ['gr-qc', 'hep-']):
+            Statistics.count('hep-processed')
+            if succ:
+                Statistics.count('hep-success')
+        if succ:
+            print('-success--------')
+            Statistics.count('3) success')
+        else:
+            print('-fail-----------')
+            Statistics.count('4) fail')
+
+        show_ref = False
+
+        if succ and show_ref:
+            for bib_ref in bib_refs:
+                print(bib_ref.key, 'with', bib_ref.cite_count, 'citations in paper')
+                if len(bib_ref.bib_info) == 0:
+                    print('no reference')
+                else:
+                    print(bib_ref.bib_info_as_str(keep_comments=True))
+
+        if succ and (print_xml or write_xml_dir):
+            xml = Element('article')
+            SubElement(xml, 'id').text = arxiv_id
+            if arxiv_version > 0:
+                SubElement(xml, 'version').text = str(arxiv_version)
+            refs = SubElement(xml, 'refs')
+            for bib_ref in bib_refs:
+                bib_text = bib_ref.bib_info_as_str(keep_comments=True)
+                if len(bib_text) != 0:
+                    ncites = bib_ref.cite_count
+                    if ncites < 1:
+                        ncites = 1
+                    ref = SubElement(refs, 'ref', order=str(bib_ref.ref_order_num), freq=str(ncites))
+                    ref.text = bib_text
+            if print_xml:
+                print(tostring(xml))
+            if isinstance(write_xml_dir, str):
+                if arxiv_id != '<unknown>':
+                    xml_file_name = os.path.join(write_xml_dir, arxiv_id.replace('/', '') + '.xml')
+                else:
+                    fname = os.path.split(file_name)[1]
+                    if fname.rfind('.') > 0:
+                        fname = fname[:fname.rfind('.')]
+                    xml_file_name = write_xml_dir + '/' + fname + '.xml'
+                file_obj = open(xml_file_name, 'wb')
+                file_obj.write(tostring(xml, encoding='utf-8'))
+                file_obj.close()
+
+    Statistics.end_item()
+
+    return succ
+
+summaryStrs = []
+
+if __name__ == "__main__":
+    cmd_parser = argparse.ArgumentParser(description='Parse TeX/LaTeX to find references.')
+    cmd_parser.add_argument('--filelist', action='store_true', help='file names on the command line each contain a list of files to process')
+    cmd_parser.add_argument('--print-xml', action='store_true', help='print XML output to stdout')
+    cmd_parser.add_argument('--write-xml', metavar='<dir>', help='destination directory to write XML output files')
+    cmd_parser.add_argument('--failed', metavar='<file>', help='output file to write list of failed files')
+    cmd_parser.add_argument('files', nargs='+', help='input files')
+    args = cmd_parser.parse_args()
+
+    # print date stamp
+    timeStart = datetime.datetime.now()
+    print('[ptex] started processing at', str(timeStart))
+
+    print('given', len(args.files), 'files, first file:', args.files[0])
+    print('================')
+
+    Statistics.clear('article')
+
+    # build list of files to process
+    file_list = buildFileList(args.filelist, args.files)
+
+    # ensure the destination directory exists
+    if args.write_xml is not None and os.path.exists(args.write_xml):
+        try:
+            os.makedirs(args.write_xml)
+        except:
+            pass
+
+    # process the files
+    failed_files = []
+    for file_name in file_list:
+        success = do_single_file(file_name, args.print_xml, args.write_xml)
+        if not success:
+            failed_files.append(file_name)
+
+    # write the failed files to an output file, if requested
+    if args.failed is not None:
+        file_obj = open(args.failed, 'w')
+        file_obj.writelines(f + '\n' for f in failed_files)
+        file_obj.close()
+
+    print('================')
+    Statistics.show()
+    Statistics.show_detail('fail')
+    #Statistics.show_detail('cite-range')
+    #Statistics.show_detail('bad-ascii')
+    #Statistics.show_detail('non-ascii')
+
+    print('================')
+
+    # print date stamp
+    timeEnd = datetime.datetime.now()
+    print('[ptex] finished processing at', str(timeEnd))
+
+    # print summary for email
+    summaryStrs.extend(Statistics.get_summary())
+    summaryStrs.insert(0, 'started processing at %s, took %.1f minutes' % (timeStart.strftime('%H:%M'), (timeEnd - timeStart).total_seconds() / 60))
+    for s in summaryStrs:
+        print('**SUMMARY** [ptex]', s)
diff --git a/tests/bytecode/mp-tests/raise1.py b/tests/bytecode/mp-tests/raise1.py
new file mode 100644
index 0000000000..9cceed4944
--- /dev/null
+++ b/tests/bytecode/mp-tests/raise1.py
@@ -0,0 +1,11 @@
+def f():
+    raise
+def g():
+    raise 1
+def h():
+    raise 1 from 2
+def i():
+    try:
+        f()
+    except:
+        raise
diff --git a/tests/bytecode/mp-tests/scope0.py b/tests/bytecode/mp-tests/scope0.py
new file mode 100644
index 0000000000..5d81345ea4
--- /dev/null
+++ b/tests/bytecode/mp-tests/scope0.py
@@ -0,0 +1,7 @@
+x = 1
+print(x)
+
+# local store after load
+def f():
+    print(x)
+    x = 1
diff --git a/tests/bytecode/mp-tests/scope1.py b/tests/bytecode/mp-tests/scope1.py
new file mode 100644
index 0000000000..92a0f9fa8c
--- /dev/null
+++ b/tests/bytecode/mp-tests/scope1.py
@@ -0,0 +1,6 @@
+x = 1
+print(x)
+def f1():
+    print(x)
+def f2(x):
+    print(x)
diff --git a/tests/bytecode/mp-tests/scope2.py b/tests/bytecode/mp-tests/scope2.py
new file mode 100644
index 0000000000..af9e372318
--- /dev/null
+++ b/tests/bytecode/mp-tests/scope2.py
@@ -0,0 +1,18 @@
+# scope
+
+gl = 1
+
+def f(x):
+    global gl
+    gl += 2
+    lo1 = 3
+    lo2 = 4
+    lo3 = 5
+
+    def f2(x, y):
+        global gl
+        nonlocal lo3
+        lo3 = 5
+        lo4 = gl + lo2 + lo3
+
+    return f2
diff --git a/tests/bytecode/mp-tests/scope3.py b/tests/bytecode/mp-tests/scope3.py
new file mode 100644
index 0000000000..a5fc8d09fe
--- /dev/null
+++ b/tests/bytecode/mp-tests/scope3.py
@@ -0,0 +1,11 @@
+# test nested functions and scope
+
+def f(x):
+    def f2(y):
+        return y + x
+    print(f2(x))
+    return f2
+x=f(2)
+print(x, x(5))
+f=123
+print(f(f))
diff --git a/tests/bytecode/mp-tests/scope4.py b/tests/bytecode/mp-tests/scope4.py
new file mode 100644
index 0000000000..70968cdf30
--- /dev/null
+++ b/tests/bytecode/mp-tests/scope4.py
@@ -0,0 +1,14 @@
+# test scope
+
+def f(x):
+    global x42
+    print(x, x42)
+    x42 = x
+
+x42 = 123
+f(1)
+print(x42)
+
+x42 = 456
+f(2)
+print(x42)
diff --git a/tests/bytecode/mp-tests/scope5.py b/tests/bytecode/mp-tests/scope5.py
new file mode 100644
index 0000000000..a14de350ed
--- /dev/null
+++ b/tests/bytecode/mp-tests/scope5.py
@@ -0,0 +1,12 @@
+# test scope
+
+def f(x):
+    def f2(y):
+        print(y, x42, y42)
+    x42 = x = y42 = 123
+    myf2 = f2
+    x42 = 456
+    return myf2
+
+myf = f(1)
+myf(1)
diff --git a/tests/bytecode/mp-tests/scope6.py b/tests/bytecode/mp-tests/scope6.py
new file mode 100644
index 0000000000..4848378887
--- /dev/null
+++ b/tests/bytecode/mp-tests/scope6.py
@@ -0,0 +1,7 @@
+# closed over variable 2 deep
+
+def f():
+    x = 1
+    def g():
+        def h():
+            return 1 + x
diff --git a/tests/bytecode/mp-tests/scope7.py b/tests/bytecode/mp-tests/scope7.py
new file mode 100644
index 0000000000..699d12510c
--- /dev/null
+++ b/tests/bytecode/mp-tests/scope7.py
@@ -0,0 +1,15 @@
+# test order of closed over locals
+# not that CPython seems to sort closed over variables (but not fast locals)
+
+def f():
+    l1 = 1
+    l2 = 4
+    l3 = 3
+    l4 = 2
+    l5 = 5
+
+    def g():
+        return l1 + l4 + l3 + l2 + l5
+
+    def h():
+        return l1 + l2 + l3 + l4 + l5
diff --git a/tests/bytecode/mp-tests/set1.py b/tests/bytecode/mp-tests/set1.py
new file mode 100644
index 0000000000..f6de75606c
--- /dev/null
+++ b/tests/bytecode/mp-tests/set1.py
@@ -0,0 +1,6 @@
+x = set()
+x = {1}
+x = {1,}
+x = {1, 2}
+x = {1, 2,}
+x = {1, 2, 3}
diff --git a/tests/bytecode/mp-tests/setcomp1.py b/tests/bytecode/mp-tests/setcomp1.py
new file mode 100644
index 0000000000..82927f5d19
--- /dev/null
+++ b/tests/bytecode/mp-tests/setcomp1.py
@@ -0,0 +1,2 @@
+x = {a for a in l}
+x = {a + b for a, b in l if b}
diff --git a/tests/bytecode/mp-tests/slice1.py b/tests/bytecode/mp-tests/slice1.py
new file mode 100644
index 0000000000..008e57c182
--- /dev/null
+++ b/tests/bytecode/mp-tests/slice1.py
@@ -0,0 +1,16 @@
+x = x[:]
+x = x[::]
+x = x[::c]
+x = x[:b]
+x = x[:b:]
+x = x[:b:c]
+x = x[a]
+x = x[a:]
+x = x[a::]
+x = x[a::c]
+x = x[a:b]
+x = x[a:b:]
+x = x[a:b:c]
+
+x[0] = 1
+x[x] = x
diff --git a/tests/bytecode/mp-tests/slice2.py b/tests/bytecode/mp-tests/slice2.py
new file mode 100644
index 0000000000..e329156c3c
--- /dev/null
+++ b/tests/bytecode/mp-tests/slice2.py
@@ -0,0 +1,3 @@
+x = x[a, b]
+
+x[a, b] = x
diff --git a/tests/bytecode/mp-tests/string1.py b/tests/bytecode/mp-tests/string1.py
new file mode 100644
index 0000000000..d6ddc7ae43
--- /dev/null
+++ b/tests/bytecode/mp-tests/string1.py
@@ -0,0 +1,11 @@
+x = 'abc'
+x = "abc"
+x = r'abc'
+x = 'abc' \
+    'def'
+x = ('abc'
+    'def')
+
+x = 'ab"c'
+x = "ab'c"
+x = '''ab'c'''
diff --git a/tests/bytecode/mp-tests/string2.py b/tests/bytecode/mp-tests/string2.py
new file mode 100644
index 0000000000..70dc9924b0
--- /dev/null
+++ b/tests/bytecode/mp-tests/string2.py
@@ -0,0 +1,14 @@
+'abc'
+class f:
+    u"123"
+    pass
+x = 'abc'
+x = u"abc"
+x = u"ab\\c"
+x = r"ab\\c"
+x = b"abc"
+x = rb"abc"
+x = b"ab\\c"
+x = rb"ab\\c"
+x = """abc"""
+x = b"""abc"""
diff --git a/tests/bytecode/mp-tests/super1.py b/tests/bytecode/mp-tests/super1.py
new file mode 100644
index 0000000000..1512429939
--- /dev/null
+++ b/tests/bytecode/mp-tests/super1.py
@@ -0,0 +1,17 @@
+class A(B):
+    def f():
+        super.a()
+
+class B(C):
+    def g():
+        def h():
+            super.a()
+
+super.a()
+
+def i():
+    super.a()
+
+def j():
+    def k():
+        super.a()
diff --git a/tests/bytecode/mp-tests/try1.py b/tests/bytecode/mp-tests/try1.py
new file mode 100644
index 0000000000..10344c8ae3
--- /dev/null
+++ b/tests/bytecode/mp-tests/try1.py
@@ -0,0 +1,13 @@
+def f(x):
+    try:
+        f(x)
+    except:
+        f(x)
+    try:
+        f(x)
+    except Exception:
+        f(x)
+    try:
+        f(x)
+    except Exception as e:
+        f(x, e)
diff --git a/tests/bytecode/mp-tests/try2.py b/tests/bytecode/mp-tests/try2.py
new file mode 100644
index 0000000000..efdac04756
--- /dev/null
+++ b/tests/bytecode/mp-tests/try2.py
@@ -0,0 +1,5 @@
+def f():
+    try:
+        f()
+    finally:
+        g()
diff --git a/tests/bytecode/mp-tests/try3.py b/tests/bytecode/mp-tests/try3.py
new file mode 100644
index 0000000000..9741aaf681
--- /dev/null
+++ b/tests/bytecode/mp-tests/try3.py
@@ -0,0 +1,14 @@
+def f():
+    try:
+        f()
+    except:
+        g()
+    finally:
+        f()
+
+    try:
+        f()
+    except Exception:
+        g()
+    finally:
+        f()
diff --git a/tests/bytecode/mp-tests/try4.py b/tests/bytecode/mp-tests/try4.py
new file mode 100644
index 0000000000..412cb74ee5
--- /dev/null
+++ b/tests/bytecode/mp-tests/try4.py
@@ -0,0 +1,22 @@
+try:
+    f()
+except A:
+    g()
+except:
+    h()
+
+try:
+    f()
+except A:
+    g()
+except B as c:
+    h()
+
+try:
+    f()
+except A:
+    g()
+except B as c:
+    h()
+except:
+    i()
diff --git a/tests/bytecode/mp-tests/try5.py b/tests/bytecode/mp-tests/try5.py
new file mode 100644
index 0000000000..7ba7949125
--- /dev/null
+++ b/tests/bytecode/mp-tests/try5.py
@@ -0,0 +1,8 @@
+try:
+    f()
+except A:
+    g()
+except B as b:
+    h()
+finally:
+    i()
diff --git a/tests/bytecode/mp-tests/try6.py b/tests/bytecode/mp-tests/try6.py
new file mode 100644
index 0000000000..d5b68722e9
--- /dev/null
+++ b/tests/bytecode/mp-tests/try6.py
@@ -0,0 +1,15 @@
+try:
+    f()
+except:
+    g()
+else:
+    h()
+
+try:
+    f()
+except:
+    g()
+else:
+    h()
+finally:
+    i()
diff --git a/tests/bytecode/mp-tests/tuple1.py b/tests/bytecode/mp-tests/tuple1.py
new file mode 100644
index 0000000000..d70e4cf569
--- /dev/null
+++ b/tests/bytecode/mp-tests/tuple1.py
@@ -0,0 +1,17 @@
+x = ()
+x = a
+x = a,
+x = a, 2
+x = a, 2,
+x = a, 2, 3
+x = a, 2, 3, 4
+x = a, 2, 3, 4, 5
+
+x = ()
+x = (a)
+x = (a,)
+x = (a, 2)
+x = (a, 2,)
+x = (a, 2, 3)
+x = (a, 2, 3, 4)
+x = (a, 2, 3, 4, 5)
diff --git a/tests/bytecode/mp-tests/tuple2.py b/tests/bytecode/mp-tests/tuple2.py
new file mode 100644
index 0000000000..df11e74ce0
--- /dev/null
+++ b/tests/bytecode/mp-tests/tuple2.py
@@ -0,0 +1,15 @@
+x = t
+x, = t
+x, y = t
+x, y, = t
+x, y, z = t
+x, y, z, = t
+x, y, z, z = a, b, c, d
+
+(x) = t
+(x,) = t
+(x, y) = t
+(x, y,) = t
+(x, y, z) = t
+(x, y, z,) = t
+(x, y, z, z) = a, b, c, d
diff --git a/tests/bytecode/mp-tests/tuple3.py b/tests/bytecode/mp-tests/tuple3.py
new file mode 100644
index 0000000000..29ddd86d08
--- /dev/null
+++ b/tests/bytecode/mp-tests/tuple3.py
@@ -0,0 +1,4 @@
+def f(x):
+    return x, x + 1
+for a in b, c:
+    f(a)
diff --git a/tests/bytecode/mp-tests/with1.py b/tests/bytecode/mp-tests/with1.py
new file mode 100644
index 0000000000..897ec530fa
--- /dev/null
+++ b/tests/bytecode/mp-tests/with1.py
@@ -0,0 +1,8 @@
+with x:
+    f()
+with x():
+    f()
+with f() as x:
+    f(x)
+with f() as x, g() as y:
+    f(x, y)
diff --git a/tests/bytecode/mp-tests/yield1.py b/tests/bytecode/mp-tests/yield1.py
new file mode 100644
index 0000000000..114151e718
--- /dev/null
+++ b/tests/bytecode/mp-tests/yield1.py
@@ -0,0 +1,17 @@
+# generators and yield
+
+def main():
+    def f():
+        print(123)
+        yield
+        print(456)
+        yield 2
+        print(789)
+
+    a = f()
+    print(a)
+    print(a.__next__())
+    print(a.__next__())
+    #print(a.__next__())
+
+main()
diff --git a/tests/bytecode/mp-tests/yield2.py b/tests/bytecode/mp-tests/yield2.py
new file mode 100644
index 0000000000..140fe0795a
--- /dev/null
+++ b/tests/bytecode/mp-tests/yield2.py
@@ -0,0 +1,4 @@
+def f():
+    yield from a
+    yield from (a, b)
+    yield from f(a)