diff options
| -rw-r--r-- | tools/micropython-upip-1.1.4.tar.gz | bin | 4090 -> 0 bytes | |||
| -rw-r--r-- | tools/upip.py | 288 | ||||
| -rw-r--r-- | tools/upip_utarfile.py | 94 | ||||
| -rw-r--r-- | unix/Makefile | 31 |
4 files changed, 385 insertions, 28 deletions
diff --git a/tools/micropython-upip-1.1.4.tar.gz b/tools/micropython-upip-1.1.4.tar.gz Binary files differdeleted file mode 100644 index 588fc4d62c..0000000000 --- a/tools/micropython-upip-1.1.4.tar.gz +++ /dev/null diff --git a/tools/upip.py b/tools/upip.py new file mode 100644 index 0000000000..db18a7427e --- /dev/null +++ b/tools/upip.py @@ -0,0 +1,288 @@ +import sys +import gc +import uos as os +import uerrno as errno +import ujson as json +import uzlib +import upip_utarfile as tarfile +gc.collect() + + +debug = False +install_path = None +cleanup_files = [] +gzdict_sz = 16 + 15 + +file_buf = bytearray(512) + +class NotFoundError(Exception): + pass + +def op_split(path): + if path == "": + return ("", "") + r = path.rsplit("/", 1) + if len(r) == 1: + return ("", path) + head = r[0] + if not head: + head = "/" + return (head, r[1]) + +def op_basename(path): + return op_split(path)[1] + +# Expects *file* name +def _makedirs(name, mode=0o777): + ret = False + s = "" + comps = name.rstrip("/").split("/")[:-1] + if comps[0] == "": + s = "/" + for c in comps: + if s and s[-1] != "/": + s += "/" + s += c + try: + os.mkdir(s) + ret = True + except OSError as e: + if e.args[0] != errno.EEXIST and e.args[0] != errno.EISDIR: + raise + ret = False + return ret + + +def save_file(fname, subf): + global file_buf + with open(fname, "wb") as outf: + while True: + sz = subf.readinto(file_buf) + if not sz: + break + outf.write(file_buf, sz) + +def install_tar(f, prefix): + meta = {} + for info in f: + #print(info) + fname = info.name + try: + fname = fname[fname.index("/") + 1:] + except ValueError: + fname = "" + + save = True + for p in ("setup.", "PKG-INFO", "README"): + #print(fname, p) + if fname.startswith(p) or ".egg-info" in fname: + if fname.endswith("/requires.txt"): + meta["deps"] = f.extractfile(info).read() + save = False + if debug: + print("Skipping", fname) + break + + if save: + outfname = prefix + fname + if info.type != tarfile.DIRTYPE: + if debug: + print("Extracting " + outfname) + _makedirs(outfname) + subf = f.extractfile(info) + save_file(outfname, subf) + return meta + +def expandhome(s): + if "~/" in s: + h = os.getenv("HOME") + s = s.replace("~/", h + "/") + return s + +import ussl +import usocket +warn_ussl = True +def url_open(url): + global warn_ussl + proto, _, host, urlpath = url.split('/', 3) + ai = usocket.getaddrinfo(host, 443) + #print("Address infos:", ai) + addr = ai[0][4] + + s = usocket.socket(ai[0][0]) + #print("Connect address:", addr) + s.connect(addr) + + if proto == "https:": + s = ussl.wrap_socket(s) + if warn_ussl: + print("Warning: %s SSL certificate is not validated" % host) + warn_ussl = False + + # MicroPython rawsocket module supports file interface directly + s.write("GET /%s HTTP/1.0\r\nHost: %s\r\n\r\n" % (urlpath, host)) + l = s.readline() + protover, status, msg = l.split(None, 2) + if status != b"200": + if status == b"404": + print("Package not found") + raise ValueError(status) + while 1: + l = s.readline() + if not l: + raise ValueError("Unexpected EOF") + if l == b'\r\n': + break + + return s + + +def get_pkg_metadata(name): + f = url_open("https://pypi.python.org/pypi/%s/json" % name) + s = f.read() + f.close() + return json.loads(s) + + +def fatal(msg): + print(msg) + sys.exit(1) + +def install_pkg(pkg_spec, install_path): + data = get_pkg_metadata(pkg_spec) + + latest_ver = data["info"]["version"] + packages = data["releases"][latest_ver] + del data + gc.collect() + assert len(packages) == 1 + package_url = packages[0]["url"] + print("Installing %s %s from %s" % (pkg_spec, latest_ver, package_url)) + package_fname = op_basename(package_url) + f1 = url_open(package_url) + f2 = uzlib.DecompIO(f1, gzdict_sz) + f3 = tarfile.TarFile(fileobj=f2) + meta = install_tar(f3, install_path) + f1.close() + del f3 + del f2 + gc.collect() + return meta + +def install(to_install, install_path=None): + # Calculate gzip dictionary size to use + global gzdict_sz + sz = gc.mem_free() + gc.mem_alloc() + if sz <= 65536: + gzdict_sz = 16 + 12 + + if install_path is None: + install_path = get_install_path() + if install_path[-1] != "/": + install_path += "/" + if not isinstance(to_install, list): + to_install = [to_install] + print("Installing to: " + install_path) + # sets would be perfect here, but don't depend on them + installed = [] + try: + while to_install: + if debug: + print("Queue:", to_install) + pkg_spec = to_install.pop(0) + if pkg_spec in installed: + continue + meta = install_pkg(pkg_spec, install_path) + installed.append(pkg_spec) + if debug: + print(meta) + deps = meta.get("deps", "").rstrip() + if deps: + deps = deps.decode("utf-8").split("\n") + to_install.extend(deps) + except NotFoundError: + print("Error: cannot find '%s' package (or server error), packages may be partially installed" \ + % pkg_spec, file=sys.stderr) + +def get_install_path(): + global install_path + if install_path is None: + # sys.path[0] is current module's path + install_path = sys.path[1] + install_path = expandhome(install_path) + return install_path + +def cleanup(): + for fname in cleanup_files: + try: + os.unlink(fname) + except OSError: + print("Warning: Cannot delete " + fname) + +def help(): + print("""\ +upip - Simple PyPI package manager for MicroPython +Usage: micropython -m upip install [-p <path>] <package>... | -r <requirements.txt> +import upip; upip.install(package_or_list, [<path>]) + +If <path> is not given, packages will be installed into sys.path[1] +(can be set from MICROPYPATH environment variable, if current system +supports that).""") + print("Current value of sys.path[1]:", sys.path[1]) + print("""\ + +Note: only MicroPython packages (usually, named micropython-*) are supported +for installation, upip does not support arbitrary code in setup.py. +""") + +def main(): + global debug + global install_path + install_path = None + + if len(sys.argv) < 2 or sys.argv[1] == "-h" or sys.argv[1] == "--help": + help() + return + + if sys.argv[1] != "install": + fatal("Only 'install' command supported") + + to_install = [] + + i = 2 + while i < len(sys.argv) and sys.argv[i][0] == "-": + opt = sys.argv[i] + i += 1 + if opt == "-h" or opt == "--help": + help() + return + elif opt == "-p": + install_path = sys.argv[i] + i += 1 + elif opt == "-r": + list_file = sys.argv[i] + i += 1 + with open(list_file) as f: + while True: + l = f.readline() + if not l: + break + to_install.append(l.rstrip()) + elif opt == "--debug": + debug = True + else: + fatal("Unknown/unsupported option: " + opt) + + to_install.extend(sys.argv[i:]) + if not to_install: + help() + return + + install(to_install) + + if not debug: + cleanup() + + +if __name__ == "__main__": + main() diff --git a/tools/upip_utarfile.py b/tools/upip_utarfile.py new file mode 100644 index 0000000000..65ce0bdca8 --- /dev/null +++ b/tools/upip_utarfile.py @@ -0,0 +1,94 @@ +import uctypes + +# http://www.gnu.org/software/tar/manual/html_node/Standard.html +TAR_HEADER = { + "name": (uctypes.ARRAY | 0, uctypes.UINT8 | 100), + "size": (uctypes.ARRAY | 124, uctypes.UINT8 | 12), +} + +DIRTYPE = "dir" +REGTYPE = "file" + +def roundup(val, align): + return (val + align - 1) & ~(align - 1) + +class FileSection: + + def __init__(self, f, content_len, aligned_len): + self.f = f + self.content_len = content_len + self.align = aligned_len - content_len + + def read(self, sz=65536): + if self.content_len == 0: + return b"" + if sz > self.content_len: + sz = self.content_len + data = self.f.read(sz) + sz = len(data) + self.content_len -= sz + return data + + def readinto(self, buf): + if self.content_len == 0: + return 0 + if len(buf) > self.content_len: + buf = memoryview(buf)[:self.content_len] + sz = self.f.readinto(buf) + self.content_len -= sz + return sz + + def skip(self): + sz = self.content_len + self.align + if sz: + buf = bytearray(16) + while sz: + s = min(sz, 16) + self.f.readinto(buf, s) + sz -= s + +class TarInfo: + + def __str__(self): + return "TarInfo(%r, %s, %d)" % (self.name, self.type, self.size) + +class TarFile: + + def __init__(self, name=None, fileobj=None): + if fileobj: + self.f = fileobj + else: + self.f = open(name, "rb") + self.subf = None + + def next(self): + if self.subf: + self.subf.skip() + buf = self.f.read(512) + if not buf: + return None + + h = uctypes.struct(uctypes.addressof(buf), TAR_HEADER, uctypes.LITTLE_ENDIAN) + + # Empty block means end of archive + if h.name[0] == 0: + return None + + d = TarInfo() + d.name = str(h.name, "utf-8").rstrip() + d.size = int(bytes(h.size).rstrip(), 8) + d.type = [REGTYPE, DIRTYPE][d.name[-1] == "/"] + self.subf = d.subf = FileSection(self.f, d.size, roundup(d.size, 512)) + return d + + def __iter__(self): + return self + + def __next__(self): + v = self.next() + if v is None: + raise StopIteration + return v + + def extractfile(self, tarinfo): + return tarinfo.subf diff --git a/unix/Makefile b/unix/Makefile index fd98d2ced2..a14ed215c6 100644 --- a/unix/Makefile +++ b/unix/Makefile @@ -1,6 +1,8 @@ -include mpconfigport.mk include ../py/mkenv.mk +FROZEN_DIR = scripts + # define main target PROG = micropython @@ -148,17 +150,6 @@ SRC_C = \ fatfs_port.c \ $(SRC_MOD) -# Include builtin package manager in the standard build (and coverage) -ifeq ($(PROG),micropython) -SRC_C += $(BUILD)/_frozen_upip.c -else ifeq ($(PROG),micropython_coverage) -SRC_C += $(BUILD)/_frozen_upip.c -else ifeq ($(PROG), micropython_nanbox) -SRC_C += $(BUILD)/_frozen_upip.c -else ifeq ($(PROG), micropython_freedos) -SRC_C += $(BUILD)/_frozen_upip.c -endif - LIB_SRC_C = $(addprefix lib/,\ $(LIB_SRC_C_EXTRA) \ timeutils/timeutils.c \ @@ -235,7 +226,7 @@ fast: # build a minimal interpreter minimal: $(MAKE) COPT="-Os -DNDEBUG" CFLAGS_EXTRA='-DMP_CONFIGFILE="<mpconfigport_minimal.h>"' \ - BUILD=build-minimal PROG=micropython_minimal \ + BUILD=build-minimal PROG=micropython_minimal FROZEN_DIR= \ MICROPY_PY_BTREE=0 MICROPY_PY_FFI=0 MICROPY_PY_SOCKET=0 MICROPY_PY_THREAD=0 \ MICROPY_PY_TERMIOS=0 MICROPY_PY_USSL=0 \ MICROPY_USE_READLINE=0 MICROPY_FATFS=0 @@ -272,22 +263,6 @@ coverage_test: coverage gcov -o build-coverage/py ../py/*.c gcov -o build-coverage/extmod ../extmod/*.c -$(BUILD)/_frozen_upip.c: $(BUILD)/frozen_upip/upip.py - $(MAKE_FROZEN) $(dir $^) > $@ - -# Select latest upip version available -UPIP_TARBALL := $(shell ls -1 -v ../tools/micropython-upip-*.tar.gz | tail -n1) - -$(BUILD)/frozen_upip/upip.py: $(UPIP_TARBALL) - $(ECHO) "MISC Preparing upip as frozen module" - $(Q)mkdir -p $(BUILD) - $(Q)rm -rf $(BUILD)/micropython-upip-* - $(Q)tar -C $(BUILD) -xz -f $^ - $(Q)rm -rf $(dir $@) - $(Q)mkdir -p $(dir $@) - $(Q)cp $(BUILD)/micropython-upip-*/upip*.py $(dir $@) - - # Value of configure's --host= option (required for cross-compilation). # Deduce it from CROSS_COMPILE by default, but can be overriden. ifneq ($(CROSS_COMPILE),) |