diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/check_code_size.sh | 23 | ||||
| -rw-r--r-- | tools/micropython-upip-1.1.3.tar.gz | bin | 4072 -> 0 bytes | |||
| -rwxr-xr-x | tools/pip-micropython | 93 | ||||
| -rwxr-xr-x | tools/tinytest-codegen.py | 9 | ||||
| -rw-r--r-- | tools/upip.py | 288 | ||||
| -rw-r--r-- | tools/upip_utarfile.py | 94 |
6 files changed, 409 insertions, 98 deletions
diff --git a/tools/check_code_size.sh b/tools/check_code_size.sh new file mode 100755 index 0000000000..c5f0c6ffdd --- /dev/null +++ b/tools/check_code_size.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# This script check that changes don't lead to code size regressions. +# (Size of the language core (== minimal port should not grow)). +# + +REFERENCE=$HOME/persist/firmware.bin +#REFERENCE=/tmp/micropython +#TRAVIS_PULL_REQUEST=false + +if [ -f $REFERENCE ]; then + size_old=$(stat -c%s $REFERENCE) + size_new=$(stat -c%s minimal/build/firmware.bin) + echo "Old size: $size_old new size: $size_new" + if [ $size_new -gt $size_old ]; then + echo "Validation failure: Core code size increased" + if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then + exit 1 + fi + fi +else + echo "Warning: reference file doesn't exist, code size check didn't run" +fi diff --git a/tools/micropython-upip-1.1.3.tar.gz b/tools/micropython-upip-1.1.3.tar.gz Binary files differdeleted file mode 100644 index 90f726d862..0000000000 --- a/tools/micropython-upip-1.1.3.tar.gz +++ /dev/null diff --git a/tools/pip-micropython b/tools/pip-micropython deleted file mode 100755 index c7b23f1b6d..0000000000 --- a/tools/pip-micropython +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/sh -# -# This tool can be used to install a new package into MicroPython -# library location (for unix port, default behavior), or produce -# complete library snapshot to be deployed on a device for baremetal -# ports (if PIP_MICROPY_DEST environment var is set). -# -# Note: this tool is deprecated in favor of "upip" native MicroPython -# package manager, which is bundled with MicroPython unix binary -# as a frozen module and can be run as "micropython -u pip" or installed -# from PyPI package "micropython-upip". This utility is left for -# reference, regression testing, debugging, etc. -# - -if [ "$1" != "install" ]; then - echo "Only install command is supported currently" - exit 1 -fi -shift - -if [ -z "$TMPDIR" ]; then - TMPDIR=/tmp -fi -TMPVENV="$TMPDIR/pip-micropy-venv" - -if [ -n "$PIP_MICROPY_DEST" ]; then - dest="$PIP_MICROPY_DEST" - echo "Destination snapshot directory: $dest" -elif [ -n "$MICROPYPATH" ]; then - libdest=$(echo "$MICROPYPATH" | awk -F: ' {print $1}') - echo "Destination library directory: $libdest" -else - echo "Warning: MICROPYPATH is not set, assuming default value" - libdest=~/.micropython/lib - echo "Destination library directory: $libdest" -fi - -# Due to bugs in pip, installation should happen with active virtualenv -# The issue (at least with pip 1.0 which is still what's shipped with many -# distros) is that even if --ignore-installed is used, package is not -# installed if it's already installed for main python distribution. -if [ ! -d "$TMPVENV" ]; then - virtualenv --no-site-packages "$TMPVENV" - # distutils, setuptools, pip are buggy and allow target packages affect - # their execution environment. For example, if distribution they install - # has re.py, they will import that instead of system re. So, we need - # to remove current dir from sys.path, but that appear to be quite uneasy - # with CPython, so we hook __import__ and exterminate it persistently. - # See also https://bitbucket.org/pypa/setuptools/issue/187/ - cat > $(ls -1d "$TMPVENV"/lib/python*/)/sitecustomize.py <<EOF -import sys -import __builtin__ -old_imp = __import__ -def new_imp(*a, **kw): - if not sys.path[0]: sys.path.pop(0) - return old_imp(*a, **kw) -__builtin__.__import__ = new_imp -EOF -fi -. "$TMPVENV"/bin/activate - -# We need to specify --record to override this switch as passed by pip -# pip will try to parse this file (at the location in specifies), and try to -# access files as specified in it. But paths there will be relative to --root -# we pass, so it won't find files and crash. However, if it won't find the -# file, it will just issue a warning and continue. -if [ -n "$dest" ]; then -pip install "$@" \ - --install-option="--install-base=." \ - --install-option="--install-purelib=lib" \ - --install-option="--install-platlib=lib" \ - --install-option="--install-scripts=." \ - --install-option="--install-headers=headers" \ - --install-option="--install-data=lib" \ - --install-option="--record=$TMPDIR/setuptools-record.txt" \ - --install-option="--no-compile" \ - --install-option="--root=$dest" -else -# Here we assume that base dir is lib dir, and install scripts a level -# higher. For default value of ~/.micropython/lib/ , this should give -# reasonable behavior, though better would make it overridable (or -# go bold and use ~/bin ?) -pip install "$@" \ - --install-option="--install-base=." \ - --install-option="--install-purelib=." \ - --install-option="--install-platlib=." \ - --install-option="--install-scripts=.." \ - --install-option="--install-headers=../headers" \ - --install-option="--install-data=." \ - --install-option="--record=$TMPDIR/setuptools-record.txt" \ - --install-option="--no-compile" \ - --install-option="--root=$libdest" -fi diff --git a/tools/tinytest-codegen.py b/tools/tinytest-codegen.py index bab937135f..8e505f21a1 100755 --- a/tools/tinytest-codegen.py +++ b/tools/tinytest-codegen.py @@ -46,14 +46,13 @@ testgroup_member = ( ## XXX: may be we could have `--without <groups>` argument... # currently these tests are selected because they pass on qemu-arm -test_dirs = ('basics', 'micropython', 'extmod', 'inlineasm') # 'float', 'import', 'io', 'misc') +test_dirs = ('basics', 'micropython', 'float', 'extmod', 'inlineasm') # 'import', 'io', 'misc') exclude_tests = ( + 'float/float2int_doubleprec.py', # requires double precision floating point to work 'inlineasm/asmfpaddsub.py', 'inlineasm/asmfpcmp.py', 'inlineasm/asmfpldrstr.py', 'inlineasm/asmfpmuldiv.py', 'inlineasm/asmfpsqrt.py', - 'extmod/time_ms_us.py', - 'extmod/ujson_dumps_float.py', 'extmod/ujson_loads_float.py', - 'extmod/uctypes_native_float.py', 'extmod/uctypes_le_float.py', + 'extmod/ticks_diff.py', 'extmod/time_ms_us.py', 'extmod/machine_pinbase.py', 'extmod/machine_pulse.py', - 'extmod/vfs_fat_ramdisk.py', + 'extmod/vfs_fat_ramdisk.py', 'extmod/vfs_fat_fileio.py', 'extmod/vfs_fat_fsusermount.py', 'extmod/vfs_fat_oldproto.py', ) output = [] diff --git a/tools/upip.py b/tools/upip.py new file mode 100644 index 0000000000..db18a7427e --- /dev/null +++ b/tools/upip.py @@ -0,0 +1,288 @@ +import sys +import gc +import uos as os +import uerrno as errno +import ujson as json +import uzlib +import upip_utarfile as tarfile +gc.collect() + + +debug = False +install_path = None +cleanup_files = [] +gzdict_sz = 16 + 15 + +file_buf = bytearray(512) + +class NotFoundError(Exception): + pass + +def op_split(path): + if path == "": + return ("", "") + r = path.rsplit("/", 1) + if len(r) == 1: + return ("", path) + head = r[0] + if not head: + head = "/" + return (head, r[1]) + +def op_basename(path): + return op_split(path)[1] + +# Expects *file* name +def _makedirs(name, mode=0o777): + ret = False + s = "" + comps = name.rstrip("/").split("/")[:-1] + if comps[0] == "": + s = "/" + for c in comps: + if s and s[-1] != "/": + s += "/" + s += c + try: + os.mkdir(s) + ret = True + except OSError as e: + if e.args[0] != errno.EEXIST and e.args[0] != errno.EISDIR: + raise + ret = False + return ret + + +def save_file(fname, subf): + global file_buf + with open(fname, "wb") as outf: + while True: + sz = subf.readinto(file_buf) + if not sz: + break + outf.write(file_buf, sz) + +def install_tar(f, prefix): + meta = {} + for info in f: + #print(info) + fname = info.name + try: + fname = fname[fname.index("/") + 1:] + except ValueError: + fname = "" + + save = True + for p in ("setup.", "PKG-INFO", "README"): + #print(fname, p) + if fname.startswith(p) or ".egg-info" in fname: + if fname.endswith("/requires.txt"): + meta["deps"] = f.extractfile(info).read() + save = False + if debug: + print("Skipping", fname) + break + + if save: + outfname = prefix + fname + if info.type != tarfile.DIRTYPE: + if debug: + print("Extracting " + outfname) + _makedirs(outfname) + subf = f.extractfile(info) + save_file(outfname, subf) + return meta + +def expandhome(s): + if "~/" in s: + h = os.getenv("HOME") + s = s.replace("~/", h + "/") + return s + +import ussl +import usocket +warn_ussl = True +def url_open(url): + global warn_ussl + proto, _, host, urlpath = url.split('/', 3) + ai = usocket.getaddrinfo(host, 443) + #print("Address infos:", ai) + addr = ai[0][4] + + s = usocket.socket(ai[0][0]) + #print("Connect address:", addr) + s.connect(addr) + + if proto == "https:": + s = ussl.wrap_socket(s) + if warn_ussl: + print("Warning: %s SSL certificate is not validated" % host) + warn_ussl = False + + # MicroPython rawsocket module supports file interface directly + s.write("GET /%s HTTP/1.0\r\nHost: %s\r\n\r\n" % (urlpath, host)) + l = s.readline() + protover, status, msg = l.split(None, 2) + if status != b"200": + if status == b"404": + print("Package not found") + raise ValueError(status) + while 1: + l = s.readline() + if not l: + raise ValueError("Unexpected EOF") + if l == b'\r\n': + break + + return s + + +def get_pkg_metadata(name): + f = url_open("https://pypi.python.org/pypi/%s/json" % name) + s = f.read() + f.close() + return json.loads(s) + + +def fatal(msg): + print(msg) + sys.exit(1) + +def install_pkg(pkg_spec, install_path): + data = get_pkg_metadata(pkg_spec) + + latest_ver = data["info"]["version"] + packages = data["releases"][latest_ver] + del data + gc.collect() + assert len(packages) == 1 + package_url = packages[0]["url"] + print("Installing %s %s from %s" % (pkg_spec, latest_ver, package_url)) + package_fname = op_basename(package_url) + f1 = url_open(package_url) + f2 = uzlib.DecompIO(f1, gzdict_sz) + f3 = tarfile.TarFile(fileobj=f2) + meta = install_tar(f3, install_path) + f1.close() + del f3 + del f2 + gc.collect() + return meta + +def install(to_install, install_path=None): + # Calculate gzip dictionary size to use + global gzdict_sz + sz = gc.mem_free() + gc.mem_alloc() + if sz <= 65536: + gzdict_sz = 16 + 12 + + if install_path is None: + install_path = get_install_path() + if install_path[-1] != "/": + install_path += "/" + if not isinstance(to_install, list): + to_install = [to_install] + print("Installing to: " + install_path) + # sets would be perfect here, but don't depend on them + installed = [] + try: + while to_install: + if debug: + print("Queue:", to_install) + pkg_spec = to_install.pop(0) + if pkg_spec in installed: + continue + meta = install_pkg(pkg_spec, install_path) + installed.append(pkg_spec) + if debug: + print(meta) + deps = meta.get("deps", "").rstrip() + if deps: + deps = deps.decode("utf-8").split("\n") + to_install.extend(deps) + except NotFoundError: + print("Error: cannot find '%s' package (or server error), packages may be partially installed" \ + % pkg_spec, file=sys.stderr) + +def get_install_path(): + global install_path + if install_path is None: + # sys.path[0] is current module's path + install_path = sys.path[1] + install_path = expandhome(install_path) + return install_path + +def cleanup(): + for fname in cleanup_files: + try: + os.unlink(fname) + except OSError: + print("Warning: Cannot delete " + fname) + +def help(): + print("""\ +upip - Simple PyPI package manager for MicroPython +Usage: micropython -m upip install [-p <path>] <package>... | -r <requirements.txt> +import upip; upip.install(package_or_list, [<path>]) + +If <path> is not given, packages will be installed into sys.path[1] +(can be set from MICROPYPATH environment variable, if current system +supports that).""") + print("Current value of sys.path[1]:", sys.path[1]) + print("""\ + +Note: only MicroPython packages (usually, named micropython-*) are supported +for installation, upip does not support arbitrary code in setup.py. +""") + +def main(): + global debug + global install_path + install_path = None + + if len(sys.argv) < 2 or sys.argv[1] == "-h" or sys.argv[1] == "--help": + help() + return + + if sys.argv[1] != "install": + fatal("Only 'install' command supported") + + to_install = [] + + i = 2 + while i < len(sys.argv) and sys.argv[i][0] == "-": + opt = sys.argv[i] + i += 1 + if opt == "-h" or opt == "--help": + help() + return + elif opt == "-p": + install_path = sys.argv[i] + i += 1 + elif opt == "-r": + list_file = sys.argv[i] + i += 1 + with open(list_file) as f: + while True: + l = f.readline() + if not l: + break + to_install.append(l.rstrip()) + elif opt == "--debug": + debug = True + else: + fatal("Unknown/unsupported option: " + opt) + + to_install.extend(sys.argv[i:]) + if not to_install: + help() + return + + install(to_install) + + if not debug: + cleanup() + + +if __name__ == "__main__": + main() diff --git a/tools/upip_utarfile.py b/tools/upip_utarfile.py new file mode 100644 index 0000000000..65ce0bdca8 --- /dev/null +++ b/tools/upip_utarfile.py @@ -0,0 +1,94 @@ +import uctypes + +# http://www.gnu.org/software/tar/manual/html_node/Standard.html +TAR_HEADER = { + "name": (uctypes.ARRAY | 0, uctypes.UINT8 | 100), + "size": (uctypes.ARRAY | 124, uctypes.UINT8 | 12), +} + +DIRTYPE = "dir" +REGTYPE = "file" + +def roundup(val, align): + return (val + align - 1) & ~(align - 1) + +class FileSection: + + def __init__(self, f, content_len, aligned_len): + self.f = f + self.content_len = content_len + self.align = aligned_len - content_len + + def read(self, sz=65536): + if self.content_len == 0: + return b"" + if sz > self.content_len: + sz = self.content_len + data = self.f.read(sz) + sz = len(data) + self.content_len -= sz + return data + + def readinto(self, buf): + if self.content_len == 0: + return 0 + if len(buf) > self.content_len: + buf = memoryview(buf)[:self.content_len] + sz = self.f.readinto(buf) + self.content_len -= sz + return sz + + def skip(self): + sz = self.content_len + self.align + if sz: + buf = bytearray(16) + while sz: + s = min(sz, 16) + self.f.readinto(buf, s) + sz -= s + +class TarInfo: + + def __str__(self): + return "TarInfo(%r, %s, %d)" % (self.name, self.type, self.size) + +class TarFile: + + def __init__(self, name=None, fileobj=None): + if fileobj: + self.f = fileobj + else: + self.f = open(name, "rb") + self.subf = None + + def next(self): + if self.subf: + self.subf.skip() + buf = self.f.read(512) + if not buf: + return None + + h = uctypes.struct(uctypes.addressof(buf), TAR_HEADER, uctypes.LITTLE_ENDIAN) + + # Empty block means end of archive + if h.name[0] == 0: + return None + + d = TarInfo() + d.name = str(h.name, "utf-8").rstrip() + d.size = int(bytes(h.size).rstrip(), 8) + d.type = [REGTYPE, DIRTYPE][d.name[-1] == "/"] + self.subf = d.subf = FileSection(self.f, d.size, roundup(d.size, 512)) + return d + + def __iter__(self): + return self + + def __next__(self): + v = self.next() + if v is None: + raise StopIteration + return v + + def extractfile(self, tarinfo): + return tarinfo.subf |