diff options
Diffstat (limited to 'tools/mpremote')
| -rw-r--r-- | tools/mpremote/mpremote/commands.py | 49 | ||||
| -rw-r--r-- | tools/mpremote/mpremote/main.py | 62 | ||||
| -rw-r--r-- | tools/mpremote/mpremote/mip.py | 16 | ||||
| -rw-r--r-- | tools/mpremote/mpremote/repl.py | 107 | ||||
| -rwxr-xr-x | tools/mpremote/tests/test_filesystem.sh | 3 | ||||
| -rw-r--r-- | tools/mpremote/tests/test_filesystem.sh.exp | 2 | ||||
| -rwxr-xr-x | tools/mpremote/tests/test_fs_tree.sh | 114 | ||||
| -rw-r--r-- | tools/mpremote/tests/test_fs_tree.sh.exp | 225 |
8 files changed, 525 insertions, 53 deletions
diff --git a/tools/mpremote/mpremote/commands.py b/tools/mpremote/mpremote/commands.py index 452384728a..428600baf4 100644 --- a/tools/mpremote/mpremote/commands.py +++ b/tools/mpremote/mpremote/commands.py @@ -334,6 +334,49 @@ def do_filesystem_recursive_rm(state, path, args): print(f"removed: '{path}'") +def human_size(size, decimals=1): + for unit in ['B', 'K', 'M', 'G', 'T']: + if size < 1024.0 or unit == 'T': + break + size /= 1024.0 + return f"{size:.{decimals}f}{unit}" if unit != 'B' else f"{int(size)}" + + +def do_filesystem_tree(state, path, args): + """Print a tree of the device's filesystem starting at path.""" + connectors = ("├── ", "└── ") + + def _tree_recursive(path, prefix=""): + entries = state.transport.fs_listdir(path) + entries.sort(key=lambda e: e.name) + for i, entry in enumerate(entries): + connector = connectors[1] if i == len(entries) - 1 else connectors[0] + is_dir = entry.st_mode & 0x4000 # Directory + size_str = "" + # most MicroPython filesystems don't support st_size on directories, reduce clutter + if entry.st_size > 0 or not is_dir: + if args.size: + size_str = f"[{entry.st_size:>9}] " + elif args.human: + size_str = f"[{human_size(entry.st_size):>6}] " + print(f"{prefix}{connector}{size_str}{entry.name}") + if is_dir: + _tree_recursive( + _remote_path_join(path, entry.name), + prefix + (" " if i == len(entries) - 1 else "│ "), + ) + + if not path or path == ".": + path = state.transport.exec("import os;print(os.getcwd())").strip().decode("utf-8") + if not (path == "." or state.transport.fs_isdir(path)): + raise CommandError(f"tree: '{path}' is not a directory") + if args.verbose: + print(f":{path} on {state.transport.device_name}") + else: + print(f":{path}") + _tree_recursive(path) + + def do_filesystem(state, args): state.ensure_raw_repl() state.did_action() @@ -361,8 +404,8 @@ def do_filesystem(state, args): # leading ':' if the user included them. paths = [path[1:] if path.startswith(":") else path for path in paths] - # ls implicitly lists the cwd. - if command == "ls" and not paths: + # ls and tree implicitly lists the cwd. + if command in ("ls", "tree") and not paths: paths = [""] try: @@ -404,6 +447,8 @@ def do_filesystem(state, args): ) else: do_filesystem_cp(state, path, cp_dest, len(paths) > 1, not args.force) + elif command == "tree": + do_filesystem_tree(state, path, args) except OSError as er: raise CommandError("{}: {}: {}.".format(command, er.strerror, os.strerror(er.errno))) except TransportError as er: diff --git a/tools/mpremote/mpremote/main.py b/tools/mpremote/mpremote/main.py index b30a1a2135..0441857fab 100644 --- a/tools/mpremote/mpremote/main.py +++ b/tools/mpremote/mpremote/main.py @@ -181,7 +181,11 @@ def argparse_rtc(): def argparse_filesystem(): - cmd_parser = argparse.ArgumentParser(description="execute filesystem commands on the device") + cmd_parser = argparse.ArgumentParser( + description="execute filesystem commands on the device", + add_help=False, + ) + cmd_parser.add_argument("--help", action="help", help="show this help message and exit") _bool_flag(cmd_parser, "recursive", "r", False, "recursive (for cp and rm commands)") _bool_flag( cmd_parser, @@ -197,10 +201,26 @@ def argparse_filesystem(): None, "enable verbose output (defaults to True for all commands except cat)", ) + size_group = cmd_parser.add_mutually_exclusive_group() + size_group.add_argument( + "--size", + "-s", + default=False, + action="store_true", + help="show file size in bytes(tree command only)", + ) + size_group.add_argument( + "--human", + "-h", + default=False, + action="store_true", + help="show file size in a more human readable way (tree command only)", + ) + cmd_parser.add_argument( "command", nargs=1, - help="filesystem command (e.g. cat, cp, sha256sum, ls, rm, rmdir, touch)", + help="filesystem command (e.g. cat, cp, sha256sum, ls, rm, rmdir, touch, tree)", ) cmd_parser.add_argument("path", nargs="+", help="local and remote paths") return cmd_parser @@ -355,10 +375,31 @@ _BUILTIN_COMMAND_EXPANSIONS = { "rmdir": "fs rmdir", "sha256sum": "fs sha256sum", "touch": "fs touch", + "tree": "fs tree", # Disk used/free. "df": [ "exec", - "import os\nprint('mount \\tsize \\tused \\tavail \\tuse%')\nfor _m in [''] + os.listdir('/'):\n _s = os.stat('/' + _m)\n if not _s[0] & 1 << 14: continue\n _s = os.statvfs(_m)\n if _s[0]:\n _size = _s[0] * _s[2]; _free = _s[0] * _s[3]; print(_m, _size, _size - _free, _free, int(100 * (_size - _free) / _size), sep='\\t')", + """ +import os,vfs +_f = "{:<10}{:>9}{:>9}{:>9}{:>5} {}" +print(_f.format("filesystem", "size", "used", "avail", "use%", "mounted on")) +try: + _ms = vfs.mount() +except: + _ms = [] + for _m in [""] + os.listdir("/"): + _m = "/" + _m + _s = os.stat(_m) + if _s[0] & 1 << 14: + _ms.append(("<unknown>",_m)) +for _v,_p in _ms: + _s = os.statvfs(_p) + _sz = _s[0]*_s[2] + if _sz: + _av = _s[0]*_s[3] + _us = 100*(_sz-_av)//_sz + print(_f.format(str(_v), _sz, _sz-_av, _av, _us, _p)) +""", ], # Other shortcuts. "reset": { @@ -552,8 +593,13 @@ def main(): command_args = remaining_args extra_args = [] - # Special case: "fs ls" allowed have no path specified. - if cmd == "fs" and len(command_args) == 1 and command_args[0] == "ls": + # Special case: "fs ls" and "fs tree" can have only options and no path specified. + if ( + cmd == "fs" + and len(command_args) >= 1 + and command_args[0] in ("ls", "tree") + and sum(1 for a in command_args if not a.startswith('-')) == 1 + ): command_args.append("") # Use the command-specific argument parser. @@ -574,7 +620,11 @@ def main(): # If no commands were "actions" then implicitly finish with the REPL # using default args. if state.run_repl_on_completion(): - do_repl(state, argparse_repl().parse_args([])) + disconnected = do_repl(state, argparse_repl().parse_args([])) + + # Handle disconnection message + if disconnected: + print("\ndevice disconnected") return 0 except CommandError as e: diff --git a/tools/mpremote/mpremote/mip.py b/tools/mpremote/mpremote/mip.py index 26ae8bec5e..fa7974053f 100644 --- a/tools/mpremote/mpremote/mip.py +++ b/tools/mpremote/mpremote/mip.py @@ -34,6 +34,15 @@ def _ensure_path_exists(transport, path): prefix += "/" +# Check if the specified path exists and matches the hash. +def _check_exists(transport, path, short_hash): + try: + remote_hash = transport.fs_hashfile(path, "sha256") + except FileNotFoundError: + return False + return remote_hash.hex()[: len(short_hash)] == short_hash + + def _rewrite_url(url, branch=None): if not branch: branch = "HEAD" @@ -115,8 +124,11 @@ def _install_json(transport, package_json_url, index, target, version, mpy): raise CommandError(f"Invalid url for package: {package_json_url}") for target_path, short_hash in package_json.get("hashes", ()): fs_target_path = target + "/" + target_path - file_url = f"{index}/file/{short_hash[:2]}/{short_hash}" - _download_file(transport, file_url, fs_target_path) + if _check_exists(transport, fs_target_path, short_hash): + print("Exists:", fs_target_path) + else: + file_url = f"{index}/file/{short_hash[:2]}/{short_hash}" + _download_file(transport, file_url, fs_target_path) for target_path, url in package_json.get("urls", ()): fs_target_path = target + "/" + target_path if base_url and not url.startswith(allowed_mip_url_prefixes): diff --git a/tools/mpremote/mpremote/repl.py b/tools/mpremote/mpremote/repl.py index d24a7774ac..4fda04a2e2 100644 --- a/tools/mpremote/mpremote/repl.py +++ b/tools/mpremote/mpremote/repl.py @@ -7,51 +7,53 @@ def do_repl_main_loop( state, console_in, console_out_write, *, escape_non_printable, code_to_inject, file_to_inject ): while True: - console_in.waitchar(state.transport.serial) - c = console_in.readchar() - if c: - if c in (b"\x1d", b"\x18"): # ctrl-] or ctrl-x, quit - break - elif c == b"\x04": # ctrl-D - # special handling needed for ctrl-D if filesystem is mounted - state.transport.write_ctrl_d(console_out_write) - elif c == b"\x0a" and code_to_inject is not None: # ctrl-j, inject code - state.transport.serial.write(code_to_inject) - elif c == b"\x0b" and file_to_inject is not None: # ctrl-k, inject script - console_out_write(bytes("Injecting %s\r\n" % file_to_inject, "utf8")) - state.transport.enter_raw_repl(soft_reset=False) - with open(file_to_inject, "rb") as f: - pyfile = f.read() - try: - state.transport.exec_raw_no_follow(pyfile) - except TransportError as er: - console_out_write(b"Error:\r\n") - console_out_write(er) - state.transport.exit_raw_repl() - else: - state.transport.serial.write(c) - try: + console_in.waitchar(state.transport.serial) + c = console_in.readchar() + if c: + if c in (b"\x1d", b"\x18"): # ctrl-] or ctrl-x, quit + break + elif c == b"\x04": # ctrl-D + # special handling needed for ctrl-D if filesystem is mounted + state.transport.write_ctrl_d(console_out_write) + elif c == b"\x0a" and code_to_inject is not None: # ctrl-j, inject code + state.transport.serial.write(code_to_inject) + elif c == b"\x0b" and file_to_inject is not None: # ctrl-k, inject script + console_out_write(bytes("Injecting %s\r\n" % file_to_inject, "utf8")) + state.transport.enter_raw_repl(soft_reset=False) + with open(file_to_inject, "rb") as f: + pyfile = f.read() + try: + state.transport.exec_raw_no_follow(pyfile) + except TransportError as er: + console_out_write(b"Error:\r\n") + console_out_write(er) + state.transport.exit_raw_repl() + else: + state.transport.serial.write(c) + n = state.transport.serial.inWaiting() - except OSError as er: - if er.args[0] == 5: # IO error, device disappeared - print("device disconnected") - break + if n > 0: + dev_data_in = state.transport.serial.read(n) + if dev_data_in is not None: + if escape_non_printable: + # Pass data through to the console, with escaping of non-printables. + console_data_out = bytearray() + for c in dev_data_in: + if c in (8, 9, 10, 13, 27) or 32 <= c <= 126: + console_data_out.append(c) + else: + console_data_out.extend(b"[%02x]" % c) + else: + console_data_out = dev_data_in + console_out_write(console_data_out) - if n > 0: - dev_data_in = state.transport.serial.read(n) - if dev_data_in is not None: - if escape_non_printable: - # Pass data through to the console, with escaping of non-printables. - console_data_out = bytearray() - for c in dev_data_in: - if c in (8, 9, 10, 13, 27) or 32 <= c <= 126: - console_data_out.append(c) - else: - console_data_out.extend(b"[%02x]" % c) - else: - console_data_out = dev_data_in - console_out_write(console_data_out) + except OSError as er: + if _is_disconnect_exception(er): + return True + else: + raise + return False def do_repl(state, args): @@ -86,7 +88,7 @@ def do_repl(state, args): capture_file.flush() try: - do_repl_main_loop( + return do_repl_main_loop( state, console, console_out_write, @@ -98,3 +100,22 @@ def do_repl(state, args): console.exit() if capture_file is not None: capture_file.close() + + +def _is_disconnect_exception(exception): + """ + Check if an exception indicates device disconnect. + + Returns True if the exception indicates the device has disconnected, + False otherwise. + """ + if isinstance(exception, OSError): + if hasattr(exception, 'args') and len(exception.args) > 0: + # IO error, device disappeared + if exception.args[0] == 5: + return True + # Check for common disconnect messages in the exception string + exception_str = str(exception) + disconnect_indicators = ["Write timeout", "Device disconnected", "ClearCommError failed"] + return any(indicator in exception_str for indicator in disconnect_indicators) + return False diff --git a/tools/mpremote/tests/test_filesystem.sh b/tools/mpremote/tests/test_filesystem.sh index a29015e987..13c5394f71 100755 --- a/tools/mpremote/tests/test_filesystem.sh +++ b/tools/mpremote/tests/test_filesystem.sh @@ -237,3 +237,6 @@ echo ----- # try to delete existing folder in mounted filesystem $MPREMOTE mount "${TMP}" + rm -rv :package || echo "expect error" echo ----- +# fs without command should raise error +$MPREMOTE fs 2>/dev/null || echo "expect error: $?" +echo ----- diff --git a/tools/mpremote/tests/test_filesystem.sh.exp b/tools/mpremote/tests/test_filesystem.sh.exp index 3d9d0fe9ae..63411580b7 100644 --- a/tools/mpremote/tests/test_filesystem.sh.exp +++ b/tools/mpremote/tests/test_filesystem.sh.exp @@ -272,3 +272,5 @@ rm :package mpremote: rm -r not permitted on /remote directory expect error ----- +expect error: 2 +----- diff --git a/tools/mpremote/tests/test_fs_tree.sh b/tools/mpremote/tests/test_fs_tree.sh new file mode 100755 index 0000000000..d7fc433ae6 --- /dev/null +++ b/tools/mpremote/tests/test_fs_tree.sh @@ -0,0 +1,114 @@ +#!/bin/bash +set -e + +# Creates a RAM disk big enough to hold two copies of the test directory +# structure. +cat << EOF > "${TMP}/ramdisk.py" +class RAMBlockDev: + def __init__(self, block_size, num_blocks): + self.block_size = block_size + self.data = bytearray(block_size * num_blocks) + + def readblocks(self, block_num, buf): + for i in range(len(buf)): + buf[i] = self.data[block_num * self.block_size + i] + + def writeblocks(self, block_num, buf): + for i in range(len(buf)): + self.data[block_num * self.block_size + i] = buf[i] + + def ioctl(self, op, arg): + if op == 4: # get number of blocks + return len(self.data) // self.block_size + if op == 5: # get block size + return self.block_size + +import os + +bdev = RAMBlockDev(512, 50) +os.VfsFat.mkfs(bdev) +os.mount(bdev, '/ramdisk') +os.chdir('/ramdisk') +EOF + +# setup +echo ----- +$MPREMOTE run "${TMP}/ramdisk.py" +$MPREMOTE resume ls + +echo ----- +echo "empty tree" +$MPREMOTE resume tree : + +echo ----- +$MPREMOTE resume touch :a.py + touch :b.py +$MPREMOTE resume mkdir :foo + touch :foo/aa.py + touch :foo/ba.py + +echo "small tree - :" +$MPREMOTE resume tree : + +echo ----- +echo "no path" +$MPREMOTE resume tree + +echo ----- +echo "path = '.'" +$MPREMOTE resume tree . + +echo ----- +echo "path = ':.'" +$MPREMOTE resume tree :. + + +echo ----- +echo "multiple trees" +$MPREMOTE resume mkdir :bar + touch :bar/aaa.py + touch :bar/bbbb.py +$MPREMOTE resume mkdir :bar/baz + touch :bar/baz/aaa.py + touch :bar/baz/bbbb.py +$MPREMOTE resume mkdir :bar/baz/quux + touch :bar/baz/quux/aaa.py + touch :bar/baz/quux/bbbb.py +$MPREMOTE resume mkdir :bar/baz/quux/xen + touch :bar/baz/quux/xen/aaa.py + +$MPREMOTE resume tree + +echo ----- +echo single path +$MPREMOTE resume tree :foo + +echo ----- +echo "multiple paths" +$MPREMOTE resume tree :foo :bar + +echo ----- +echo "subtree" +$MPREMOTE resume tree bar/baz + +echo ----- +echo mountpoint +$MPREMOTE resume tree :/ramdisk + +echo ----- +echo non-existent folder : error +$MPREMOTE resume tree :not_there || echo "expect error: $?" + +echo ----- +echo file : error +$MPREMOTE resume tree :a.py || echo "expect error: $?" + +echo ----- +echo "tree -s :" +mkdir -p "${TMP}/data" +dd if=/dev/zero of="${TMP}/data/file1.txt" bs=1 count=20 > /dev/null 2>&1 +dd if=/dev/zero of="${TMP}/data/file2.txt" bs=1 count=204 > /dev/null 2>&1 +dd if=/dev/zero of="${TMP}/data/file3.txt" bs=1 count=1096 > /dev/null 2>&1 +dd if=/dev/zero of="${TMP}/data/file4.txt" bs=1 count=2192 > /dev/null 2>&1 + +$MPREMOTE resume cp -r "${TMP}/data" : +$MPREMOTE resume tree -s : +echo ----- +echo "tree -s" +$MPREMOTE resume tree -s +echo ----- +$MPREMOTE resume tree --human : +echo ----- +$MPREMOTE resume tree -s --human : || echo "expect error: $?" +echo ----- + diff --git a/tools/mpremote/tests/test_fs_tree.sh.exp b/tools/mpremote/tests/test_fs_tree.sh.exp new file mode 100644 index 0000000000..9a67883b1c --- /dev/null +++ b/tools/mpremote/tests/test_fs_tree.sh.exp @@ -0,0 +1,225 @@ +----- +ls : +----- +empty tree +tree : +:/ramdisk +----- +touch :a.py +touch :b.py +mkdir :foo +touch :foo/aa.py +touch :foo/ba.py +small tree - : +tree : +:/ramdisk +├── a.py +├── b.py +└── foo + ├── aa.py + └── ba.py +----- +no path +tree : +:/ramdisk +├── a.py +├── b.py +└── foo + ├── aa.py + └── ba.py +----- +path = '.' +tree :. +:/ramdisk +├── a.py +├── b.py +└── foo + ├── aa.py + └── ba.py +----- +path = ':.' +tree :. +:/ramdisk +├── a.py +├── b.py +└── foo + ├── aa.py + └── ba.py +----- +multiple trees +mkdir :bar +touch :bar/aaa.py +touch :bar/bbbb.py +mkdir :bar/baz +touch :bar/baz/aaa.py +touch :bar/baz/bbbb.py +mkdir :bar/baz/quux +touch :bar/baz/quux/aaa.py +touch :bar/baz/quux/bbbb.py +mkdir :bar/baz/quux/xen +touch :bar/baz/quux/xen/aaa.py +tree : +:/ramdisk +├── a.py +├── b.py +├── bar +│ ├── aaa.py +│ ├── baz +│ │ ├── aaa.py +│ │ ├── bbbb.py +│ │ └── quux +│ │ ├── aaa.py +│ │ ├── bbbb.py +│ │ └── xen +│ │ └── aaa.py +│ └── bbbb.py +└── foo + ├── aa.py + └── ba.py +----- +single path +tree :foo +:foo +├── aa.py +└── ba.py +----- +multiple paths +tree :foo +:foo +├── aa.py +└── ba.py +tree :bar +:bar +├── aaa.py +├── baz +│ ├── aaa.py +│ ├── bbbb.py +│ └── quux +│ ├── aaa.py +│ ├── bbbb.py +│ └── xen +│ └── aaa.py +└── bbbb.py +----- +subtree +tree :bar/baz +:bar/baz +├── aaa.py +├── bbbb.py +└── quux + ├── aaa.py + ├── bbbb.py + └── xen + └── aaa.py +----- +mountpoint +tree :/ramdisk +:/ramdisk +├── a.py +├── b.py +├── bar +│ ├── aaa.py +│ ├── baz +│ │ ├── aaa.py +│ │ ├── bbbb.py +│ │ └── quux +│ │ ├── aaa.py +│ │ ├── bbbb.py +│ │ └── xen +│ │ └── aaa.py +│ └── bbbb.py +└── foo + ├── aa.py + └── ba.py +----- +non-existent folder : error +tree :not_there +mpremote: tree: 'not_there' is not a directory +expect error: 1 +----- +file : error +tree :a.py +mpremote: tree: 'a.py' is not a directory +expect error: 1 +----- +tree -s : +cp ${TMP}/data : +tree : +:/ramdisk +├── [ 0] a.py +├── [ 0] b.py +├── bar +│ ├── [ 0] aaa.py +│ ├── baz +│ │ ├── [ 0] aaa.py +│ │ ├── [ 0] bbbb.py +│ │ └── quux +│ │ ├── [ 0] aaa.py +│ │ ├── [ 0] bbbb.py +│ │ └── xen +│ │ └── [ 0] aaa.py +│ └── [ 0] bbbb.py +├── data +│ ├── [ 20] file1.txt +│ ├── [ 204] file2.txt +│ ├── [ 1096] file3.txt +│ └── [ 2192] file4.txt +└── foo + ├── [ 0] aa.py + └── [ 0] ba.py +----- +tree -s +tree : +:/ramdisk +├── [ 0] a.py +├── [ 0] b.py +├── bar +│ ├── [ 0] aaa.py +│ ├── baz +│ │ ├── [ 0] aaa.py +│ │ ├── [ 0] bbbb.py +│ │ └── quux +│ │ ├── [ 0] aaa.py +│ │ ├── [ 0] bbbb.py +│ │ └── xen +│ │ └── [ 0] aaa.py +│ └── [ 0] bbbb.py +├── data +│ ├── [ 20] file1.txt +│ ├── [ 204] file2.txt +│ ├── [ 1096] file3.txt +│ └── [ 2192] file4.txt +└── foo + ├── [ 0] aa.py + └── [ 0] ba.py +----- +tree : +:/ramdisk +├── [ 0] a.py +├── [ 0] b.py +├── bar +│ ├── [ 0] aaa.py +│ ├── baz +│ │ ├── [ 0] aaa.py +│ │ ├── [ 0] bbbb.py +│ │ └── quux +│ │ ├── [ 0] aaa.py +│ │ ├── [ 0] bbbb.py +│ │ └── xen +│ │ └── [ 0] aaa.py +│ └── [ 0] bbbb.py +├── data +│ ├── [ 20] file1.txt +│ ├── [ 204] file2.txt +│ ├── [ 1.1K] file3.txt +│ └── [ 2.1K] file4.txt +└── foo + ├── [ 0] aa.py + └── [ 0] ba.py +----- +usage: fs [--help] [--recursive | --no-recursive] [--force | --no-force] + [--verbose | --no-verbose] [--size | --human] + command path [path ...] ... +fs: error: argument --human/-h: not allowed with argument --size/-s +expect error: 2 +----- |