summaryrefslogtreecommitdiffstatshomepage
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rwxr-xr-xtools/ci.sh42
-rw-r--r--tools/gen-cpydiff.py32
-rw-r--r--tools/mpremote/mpremote/commands.py49
-rw-r--r--tools/mpremote/mpremote/main.py34
-rwxr-xr-xtools/mpremote/tests/test_filesystem.sh3
-rw-r--r--tools/mpremote/tests/test_filesystem.sh.exp2
-rwxr-xr-xtools/mpremote/tests/test_fs_tree.sh114
-rw-r--r--tools/mpremote/tests/test_fs_tree.sh.exp225
-rwxr-xr-xtools/mpy_ld.py82
-rwxr-xr-xtools/pyboard.py66
-rwxr-xr-xtools/verifygitlog.py12
11 files changed, 597 insertions, 64 deletions
diff --git a/tools/ci.sh b/tools/ci.sh
index a4bd43567c..ea67e2c104 100755
--- a/tools/ci.sh
+++ b/tools/ci.sh
@@ -93,23 +93,30 @@ function ci_code_size_build {
function code_size_build_step {
COMMIT=$1
OUTFILE=$2
- IGNORE_ERRORS=$3
echo "Building ${COMMIT}..."
git checkout --detach $COMMIT
git submodule update --init $SUBMODULES
git show -s
tools/metrics.py clean $PORTS_TO_CHECK
- tools/metrics.py build $PORTS_TO_CHECK | tee $OUTFILE || $IGNORE_ERRORS
+ tools/metrics.py build $PORTS_TO_CHECK | tee $OUTFILE
+ return $?
}
+ # Allow errors from tools/metrics.py to propagate out of the pipe above.
+ set -o pipefail
+
# build reference, save to size0
# ignore any errors with this build, in case master is failing
- code_size_build_step $REFERENCE ~/size0 true
+ code_size_build_step $REFERENCE ~/size0
# build PR/branch, save to size1
- code_size_build_step $COMPARISON ~/size1 false
+ code_size_build_step $COMPARISON ~/size1
+ STATUS=$?
+ set +o pipefail
unset -f code_size_build_step
+
+ return $STATUS
}
########################################################################################
@@ -159,7 +166,7 @@ function ci_cc3200_build {
# ports/esp32
# GitHub tag of ESP-IDF to use for CI (note: must be a tag or a branch)
-IDF_VER=v5.2.2
+IDF_VER=v5.4.1
PYTHON=$(command -v python3 2> /dev/null)
PYTHON_VER=$(${PYTHON:-python} --version | cut -d' ' -f2)
@@ -524,38 +531,25 @@ function ci_native_mpy_modules_build {
else
arch=$1
fi
- for natmod in features1 features3 features4 heapq re
+ for natmod in deflate features1 features3 features4 framebuf heapq random re
do
- make -C examples/natmod/$natmod clean
+ make -C examples/natmod/$natmod ARCH=$arch clean
make -C examples/natmod/$natmod ARCH=$arch
done
- # deflate, framebuf, and random currently cannot build on xtensa due to
- # some symbols that have been removed from the compiler's runtime, in
- # favour of being provided from ROM.
- if [ $arch != "xtensa" ]; then
- for natmod in deflate framebuf random
- do
- make -C examples/natmod/$natmod clean
- make -C examples/natmod/$natmod ARCH=$arch
- done
- fi
-
# features2 requires soft-float on armv7m, rv32imc, and xtensa. On armv6m
# the compiler generates absolute relocations in the object file
# referencing soft-float functions, which is not supported at the moment.
- make -C examples/natmod/features2 clean
+ make -C examples/natmod/features2 ARCH=$arch clean
if [ $arch = "rv32imc" ] || [ $arch = "armv7m" ] || [ $arch = "xtensa" ]; then
make -C examples/natmod/features2 ARCH=$arch MICROPY_FLOAT_IMPL=float
elif [ $arch != "armv6m" ]; then
make -C examples/natmod/features2 ARCH=$arch
fi
- # btree requires thread local storage support on rv32imc, whilst on xtensa
- # it relies on symbols that are provided from ROM but not exposed to
- # natmods at the moment.
- if [ $arch != "rv32imc" ] && [ $arch != "xtensa" ]; then
- make -C examples/natmod/btree clean
+ # btree requires thread local storage support on rv32imc.
+ if [ $arch != "rv32imc" ]; then
+ make -C examples/natmod/btree ARCH=$arch clean
make -C examples/natmod/btree ARCH=$arch
fi
}
diff --git a/tools/gen-cpydiff.py b/tools/gen-cpydiff.py
index 2f9394deea..3bb928090b 100644
--- a/tools/gen-cpydiff.py
+++ b/tools/gen-cpydiff.py
@@ -45,6 +45,12 @@ else:
CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3")
MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", "../ports/unix/build-standard/micropython")
+# Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale
+os.environ["PYTHONIOENCODING"] = "utf-8"
+
+# Set PYTHONUNBUFFERED so that CPython will interleave stdout & stderr without buffering
+os.environ["PYTHONUNBUFFERED"] = "a non-empty string"
+
TESTPATH = "../tests/cpydiff"
DOCPATH = "../docs/genrst"
SRCDIR = "../docs/differences"
@@ -111,7 +117,7 @@ def run_tests(tests):
results = []
for test in tests:
test_fullpath = os.path.join(TESTPATH, test.name)
- with open(test_fullpath, "rb") as f:
+ with open(test_fullpath, "r") as f:
input_py = f.read()
process = subprocess.Popen(
@@ -119,20 +125,22 @@ def run_tests(tests):
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
- stderr=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ encoding="utf-8",
)
- output_cpy = [com.decode("utf8") for com in process.communicate(input_py)]
+ output_cpy = process.communicate(input_py)[0]
process = subprocess.Popen(
MICROPYTHON,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
- stderr=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ encoding="utf-8",
)
- output_upy = [com.decode("utf8") for com in process.communicate(input_py)]
+ output_upy = process.communicate(input_py)[0]
- if output_cpy[0] == output_upy[0] and output_cpy[1] == output_upy[1]:
+ if output_cpy == output_upy:
print("Error: Test has same output in CPython vs MicroPython: " + test_fullpath)
same_results = True
else:
@@ -211,6 +219,8 @@ def gen_rst(results):
class_ = []
for output in results:
section = output.class_.split(",")
+ if len(section) < 2:
+ raise SystemExit("Each item must have at least 2 categories")
for i in range(len(section)):
section[i] = section[i].rstrip()
if section[i] in CLASSMAP:
@@ -220,8 +230,8 @@ def gen_rst(results):
filename = section[i].replace(" ", "_").lower()
rst = open(os.path.join(DOCPATH, filename + ".rst"), "w")
rst.write(HEADER)
- rst.write(section[i] + "\n")
- rst.write(RSTCHARS[0] * len(section[i]))
+ rst.write(section[0] + "\n")
+ rst.write(RSTCHARS[0] * len(section[0]) + "\n\n")
rst.write(time.strftime("\nGenerated %a %d %b %Y %X UTC\n\n", time.gmtime()))
# If a file docs/differences/<filename>_preamble.txt exists
# then its output is inserted after the top-level heading,
@@ -239,16 +249,16 @@ def gen_rst(results):
class_ = section
rst.write(".. _cpydiff_%s:\n\n" % os.path.splitext(output.name)[0])
rst.write(output.desc + "\n")
- rst.write("~" * len(output.desc) + "\n\n")
+ rst.write(RSTCHARS[min(i + 1, len(RSTCHARS) - 1)] * len(output.desc) + "\n\n")
if output.cause != "Unknown":
rst.write("**Cause:** " + output.cause + "\n\n")
if output.workaround != "Unknown":
rst.write("**Workaround:** " + output.workaround + "\n\n")
rst.write("Sample code::\n\n" + indent(output.code, TAB) + "\n")
- output_cpy = indent("".join(output.output_cpy[0:2]), TAB).rstrip()
+ output_cpy = indent(output.output_cpy, TAB).rstrip()
output_cpy = ("::\n\n" if output_cpy != "" else "") + output_cpy
- output_upy = indent("".join(output.output_upy[0:2]), TAB).rstrip()
+ output_upy = indent(output.output_upy, TAB).rstrip()
output_upy = ("::\n\n" if output_upy != "" else "") + output_upy
table = gen_table([["CPy output:", output_cpy], ["uPy output:", output_upy]])
rst.write(table)
diff --git a/tools/mpremote/mpremote/commands.py b/tools/mpremote/mpremote/commands.py
index 452384728a..428600baf4 100644
--- a/tools/mpremote/mpremote/commands.py
+++ b/tools/mpremote/mpremote/commands.py
@@ -334,6 +334,49 @@ def do_filesystem_recursive_rm(state, path, args):
print(f"removed: '{path}'")
+def human_size(size, decimals=1):
+ for unit in ['B', 'K', 'M', 'G', 'T']:
+ if size < 1024.0 or unit == 'T':
+ break
+ size /= 1024.0
+ return f"{size:.{decimals}f}{unit}" if unit != 'B' else f"{int(size)}"
+
+
+def do_filesystem_tree(state, path, args):
+ """Print a tree of the device's filesystem starting at path."""
+ connectors = ("├── ", "└── ")
+
+ def _tree_recursive(path, prefix=""):
+ entries = state.transport.fs_listdir(path)
+ entries.sort(key=lambda e: e.name)
+ for i, entry in enumerate(entries):
+ connector = connectors[1] if i == len(entries) - 1 else connectors[0]
+ is_dir = entry.st_mode & 0x4000 # Directory
+ size_str = ""
+ # most MicroPython filesystems don't support st_size on directories, reduce clutter
+ if entry.st_size > 0 or not is_dir:
+ if args.size:
+ size_str = f"[{entry.st_size:>9}] "
+ elif args.human:
+ size_str = f"[{human_size(entry.st_size):>6}] "
+ print(f"{prefix}{connector}{size_str}{entry.name}")
+ if is_dir:
+ _tree_recursive(
+ _remote_path_join(path, entry.name),
+ prefix + (" " if i == len(entries) - 1 else "│ "),
+ )
+
+ if not path or path == ".":
+ path = state.transport.exec("import os;print(os.getcwd())").strip().decode("utf-8")
+ if not (path == "." or state.transport.fs_isdir(path)):
+ raise CommandError(f"tree: '{path}' is not a directory")
+ if args.verbose:
+ print(f":{path} on {state.transport.device_name}")
+ else:
+ print(f":{path}")
+ _tree_recursive(path)
+
+
def do_filesystem(state, args):
state.ensure_raw_repl()
state.did_action()
@@ -361,8 +404,8 @@ def do_filesystem(state, args):
# leading ':' if the user included them.
paths = [path[1:] if path.startswith(":") else path for path in paths]
- # ls implicitly lists the cwd.
- if command == "ls" and not paths:
+ # ls and tree implicitly lists the cwd.
+ if command in ("ls", "tree") and not paths:
paths = [""]
try:
@@ -404,6 +447,8 @@ def do_filesystem(state, args):
)
else:
do_filesystem_cp(state, path, cp_dest, len(paths) > 1, not args.force)
+ elif command == "tree":
+ do_filesystem_tree(state, path, args)
except OSError as er:
raise CommandError("{}: {}: {}.".format(command, er.strerror, os.strerror(er.errno)))
except TransportError as er:
diff --git a/tools/mpremote/mpremote/main.py b/tools/mpremote/mpremote/main.py
index b30a1a2135..f8c913d26d 100644
--- a/tools/mpremote/mpremote/main.py
+++ b/tools/mpremote/mpremote/main.py
@@ -181,7 +181,11 @@ def argparse_rtc():
def argparse_filesystem():
- cmd_parser = argparse.ArgumentParser(description="execute filesystem commands on the device")
+ cmd_parser = argparse.ArgumentParser(
+ description="execute filesystem commands on the device",
+ add_help=False,
+ )
+ cmd_parser.add_argument("--help", action="help", help="show this help message and exit")
_bool_flag(cmd_parser, "recursive", "r", False, "recursive (for cp and rm commands)")
_bool_flag(
cmd_parser,
@@ -197,10 +201,26 @@ def argparse_filesystem():
None,
"enable verbose output (defaults to True for all commands except cat)",
)
+ size_group = cmd_parser.add_mutually_exclusive_group()
+ size_group.add_argument(
+ "--size",
+ "-s",
+ default=False,
+ action="store_true",
+ help="show file size in bytes(tree command only)",
+ )
+ size_group.add_argument(
+ "--human",
+ "-h",
+ default=False,
+ action="store_true",
+ help="show file size in a more human readable way (tree command only)",
+ )
+
cmd_parser.add_argument(
"command",
nargs=1,
- help="filesystem command (e.g. cat, cp, sha256sum, ls, rm, rmdir, touch)",
+ help="filesystem command (e.g. cat, cp, sha256sum, ls, rm, rmdir, touch, tree)",
)
cmd_parser.add_argument("path", nargs="+", help="local and remote paths")
return cmd_parser
@@ -355,6 +375,7 @@ _BUILTIN_COMMAND_EXPANSIONS = {
"rmdir": "fs rmdir",
"sha256sum": "fs sha256sum",
"touch": "fs touch",
+ "tree": "fs tree",
# Disk used/free.
"df": [
"exec",
@@ -552,8 +573,13 @@ def main():
command_args = remaining_args
extra_args = []
- # Special case: "fs ls" allowed have no path specified.
- if cmd == "fs" and len(command_args) == 1 and command_args[0] == "ls":
+ # Special case: "fs ls" and "fs tree" can have only options and no path specified.
+ if (
+ cmd == "fs"
+ and len(command_args) >= 1
+ and command_args[0] in ("ls", "tree")
+ and sum(1 for a in command_args if not a.startswith('-')) == 1
+ ):
command_args.append("")
# Use the command-specific argument parser.
diff --git a/tools/mpremote/tests/test_filesystem.sh b/tools/mpremote/tests/test_filesystem.sh
index a29015e987..13c5394f71 100755
--- a/tools/mpremote/tests/test_filesystem.sh
+++ b/tools/mpremote/tests/test_filesystem.sh
@@ -237,3 +237,6 @@ echo -----
# try to delete existing folder in mounted filesystem
$MPREMOTE mount "${TMP}" + rm -rv :package || echo "expect error"
echo -----
+# fs without command should raise error
+$MPREMOTE fs 2>/dev/null || echo "expect error: $?"
+echo -----
diff --git a/tools/mpremote/tests/test_filesystem.sh.exp b/tools/mpremote/tests/test_filesystem.sh.exp
index 3d9d0fe9ae..63411580b7 100644
--- a/tools/mpremote/tests/test_filesystem.sh.exp
+++ b/tools/mpremote/tests/test_filesystem.sh.exp
@@ -272,3 +272,5 @@ rm :package
mpremote: rm -r not permitted on /remote directory
expect error
-----
+expect error: 2
+-----
diff --git a/tools/mpremote/tests/test_fs_tree.sh b/tools/mpremote/tests/test_fs_tree.sh
new file mode 100755
index 0000000000..d7fc433ae6
--- /dev/null
+++ b/tools/mpremote/tests/test_fs_tree.sh
@@ -0,0 +1,114 @@
+#!/bin/bash
+set -e
+
+# Creates a RAM disk big enough to hold two copies of the test directory
+# structure.
+cat << EOF > "${TMP}/ramdisk.py"
+class RAMBlockDev:
+ def __init__(self, block_size, num_blocks):
+ self.block_size = block_size
+ self.data = bytearray(block_size * num_blocks)
+
+ def readblocks(self, block_num, buf):
+ for i in range(len(buf)):
+ buf[i] = self.data[block_num * self.block_size + i]
+
+ def writeblocks(self, block_num, buf):
+ for i in range(len(buf)):
+ self.data[block_num * self.block_size + i] = buf[i]
+
+ def ioctl(self, op, arg):
+ if op == 4: # get number of blocks
+ return len(self.data) // self.block_size
+ if op == 5: # get block size
+ return self.block_size
+
+import os
+
+bdev = RAMBlockDev(512, 50)
+os.VfsFat.mkfs(bdev)
+os.mount(bdev, '/ramdisk')
+os.chdir('/ramdisk')
+EOF
+
+# setup
+echo -----
+$MPREMOTE run "${TMP}/ramdisk.py"
+$MPREMOTE resume ls
+
+echo -----
+echo "empty tree"
+$MPREMOTE resume tree :
+
+echo -----
+$MPREMOTE resume touch :a.py + touch :b.py
+$MPREMOTE resume mkdir :foo + touch :foo/aa.py + touch :foo/ba.py
+
+echo "small tree - :"
+$MPREMOTE resume tree :
+
+echo -----
+echo "no path"
+$MPREMOTE resume tree
+
+echo -----
+echo "path = '.'"
+$MPREMOTE resume tree .
+
+echo -----
+echo "path = ':.'"
+$MPREMOTE resume tree :.
+
+
+echo -----
+echo "multiple trees"
+$MPREMOTE resume mkdir :bar + touch :bar/aaa.py + touch :bar/bbbb.py
+$MPREMOTE resume mkdir :bar/baz + touch :bar/baz/aaa.py + touch :bar/baz/bbbb.py
+$MPREMOTE resume mkdir :bar/baz/quux + touch :bar/baz/quux/aaa.py + touch :bar/baz/quux/bbbb.py
+$MPREMOTE resume mkdir :bar/baz/quux/xen + touch :bar/baz/quux/xen/aaa.py
+
+$MPREMOTE resume tree
+
+echo -----
+echo single path
+$MPREMOTE resume tree :foo
+
+echo -----
+echo "multiple paths"
+$MPREMOTE resume tree :foo :bar
+
+echo -----
+echo "subtree"
+$MPREMOTE resume tree bar/baz
+
+echo -----
+echo mountpoint
+$MPREMOTE resume tree :/ramdisk
+
+echo -----
+echo non-existent folder : error
+$MPREMOTE resume tree :not_there || echo "expect error: $?"
+
+echo -----
+echo file : error
+$MPREMOTE resume tree :a.py || echo "expect error: $?"
+
+echo -----
+echo "tree -s :"
+mkdir -p "${TMP}/data"
+dd if=/dev/zero of="${TMP}/data/file1.txt" bs=1 count=20 > /dev/null 2>&1
+dd if=/dev/zero of="${TMP}/data/file2.txt" bs=1 count=204 > /dev/null 2>&1
+dd if=/dev/zero of="${TMP}/data/file3.txt" bs=1 count=1096 > /dev/null 2>&1
+dd if=/dev/zero of="${TMP}/data/file4.txt" bs=1 count=2192 > /dev/null 2>&1
+
+$MPREMOTE resume cp -r "${TMP}/data" :
+$MPREMOTE resume tree -s :
+echo -----
+echo "tree -s"
+$MPREMOTE resume tree -s
+echo -----
+$MPREMOTE resume tree --human :
+echo -----
+$MPREMOTE resume tree -s --human : || echo "expect error: $?"
+echo -----
+
diff --git a/tools/mpremote/tests/test_fs_tree.sh.exp b/tools/mpremote/tests/test_fs_tree.sh.exp
new file mode 100644
index 0000000000..9a67883b1c
--- /dev/null
+++ b/tools/mpremote/tests/test_fs_tree.sh.exp
@@ -0,0 +1,225 @@
+-----
+ls :
+-----
+empty tree
+tree :
+:/ramdisk
+-----
+touch :a.py
+touch :b.py
+mkdir :foo
+touch :foo/aa.py
+touch :foo/ba.py
+small tree - :
+tree :
+:/ramdisk
+├── a.py
+├── b.py
+└── foo
+ ├── aa.py
+ └── ba.py
+-----
+no path
+tree :
+:/ramdisk
+├── a.py
+├── b.py
+└── foo
+ ├── aa.py
+ └── ba.py
+-----
+path = '.'
+tree :.
+:/ramdisk
+├── a.py
+├── b.py
+└── foo
+ ├── aa.py
+ └── ba.py
+-----
+path = ':.'
+tree :.
+:/ramdisk
+├── a.py
+├── b.py
+└── foo
+ ├── aa.py
+ └── ba.py
+-----
+multiple trees
+mkdir :bar
+touch :bar/aaa.py
+touch :bar/bbbb.py
+mkdir :bar/baz
+touch :bar/baz/aaa.py
+touch :bar/baz/bbbb.py
+mkdir :bar/baz/quux
+touch :bar/baz/quux/aaa.py
+touch :bar/baz/quux/bbbb.py
+mkdir :bar/baz/quux/xen
+touch :bar/baz/quux/xen/aaa.py
+tree :
+:/ramdisk
+├── a.py
+├── b.py
+├── bar
+│ ├── aaa.py
+│ ├── baz
+│ │ ├── aaa.py
+│ │ ├── bbbb.py
+│ │ └── quux
+│ │ ├── aaa.py
+│ │ ├── bbbb.py
+│ │ └── xen
+│ │ └── aaa.py
+│ └── bbbb.py
+└── foo
+ ├── aa.py
+ └── ba.py
+-----
+single path
+tree :foo
+:foo
+├── aa.py
+└── ba.py
+-----
+multiple paths
+tree :foo
+:foo
+├── aa.py
+└── ba.py
+tree :bar
+:bar
+├── aaa.py
+├── baz
+│ ├── aaa.py
+│ ├── bbbb.py
+│ └── quux
+│ ├── aaa.py
+│ ├── bbbb.py
+│ └── xen
+│ └── aaa.py
+└── bbbb.py
+-----
+subtree
+tree :bar/baz
+:bar/baz
+├── aaa.py
+├── bbbb.py
+└── quux
+ ├── aaa.py
+ ├── bbbb.py
+ └── xen
+ └── aaa.py
+-----
+mountpoint
+tree :/ramdisk
+:/ramdisk
+├── a.py
+├── b.py
+├── bar
+│ ├── aaa.py
+│ ├── baz
+│ │ ├── aaa.py
+│ │ ├── bbbb.py
+│ │ └── quux
+│ │ ├── aaa.py
+│ │ ├── bbbb.py
+│ │ └── xen
+│ │ └── aaa.py
+│ └── bbbb.py
+└── foo
+ ├── aa.py
+ └── ba.py
+-----
+non-existent folder : error
+tree :not_there
+mpremote: tree: 'not_there' is not a directory
+expect error: 1
+-----
+file : error
+tree :a.py
+mpremote: tree: 'a.py' is not a directory
+expect error: 1
+-----
+tree -s :
+cp ${TMP}/data :
+tree :
+:/ramdisk
+├── [ 0] a.py
+├── [ 0] b.py
+├── bar
+│ ├── [ 0] aaa.py
+│ ├── baz
+│ │ ├── [ 0] aaa.py
+│ │ ├── [ 0] bbbb.py
+│ │ └── quux
+│ │ ├── [ 0] aaa.py
+│ │ ├── [ 0] bbbb.py
+│ │ └── xen
+│ │ └── [ 0] aaa.py
+│ └── [ 0] bbbb.py
+├── data
+│ ├── [ 20] file1.txt
+│ ├── [ 204] file2.txt
+│ ├── [ 1096] file3.txt
+│ └── [ 2192] file4.txt
+└── foo
+ ├── [ 0] aa.py
+ └── [ 0] ba.py
+-----
+tree -s
+tree :
+:/ramdisk
+├── [ 0] a.py
+├── [ 0] b.py
+├── bar
+│ ├── [ 0] aaa.py
+│ ├── baz
+│ │ ├── [ 0] aaa.py
+│ │ ├── [ 0] bbbb.py
+│ │ └── quux
+│ │ ├── [ 0] aaa.py
+│ │ ├── [ 0] bbbb.py
+│ │ └── xen
+│ │ └── [ 0] aaa.py
+│ └── [ 0] bbbb.py
+├── data
+│ ├── [ 20] file1.txt
+│ ├── [ 204] file2.txt
+│ ├── [ 1096] file3.txt
+│ └── [ 2192] file4.txt
+└── foo
+ ├── [ 0] aa.py
+ └── [ 0] ba.py
+-----
+tree :
+:/ramdisk
+├── [ 0] a.py
+├── [ 0] b.py
+├── bar
+│ ├── [ 0] aaa.py
+│ ├── baz
+│ │ ├── [ 0] aaa.py
+│ │ ├── [ 0] bbbb.py
+│ │ └── quux
+│ │ ├── [ 0] aaa.py
+│ │ ├── [ 0] bbbb.py
+│ │ └── xen
+│ │ └── [ 0] aaa.py
+│ └── [ 0] bbbb.py
+├── data
+│ ├── [ 20] file1.txt
+│ ├── [ 204] file2.txt
+│ ├── [ 1.1K] file3.txt
+│ └── [ 2.1K] file4.txt
+└── foo
+ ├── [ 0] aa.py
+ └── [ 0] ba.py
+-----
+usage: fs [--help] [--recursive | --no-recursive] [--force | --no-force]
+ [--verbose | --no-verbose] [--size | --human]
+ command path [path ...] ...
+fs: error: argument --human/-h: not allowed with argument --size/-s
+expect error: 2
+-----
diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py
index a47653f900..6518037f2e 100755
--- a/tools/mpy_ld.py
+++ b/tools/mpy_ld.py
@@ -402,6 +402,7 @@ class LinkEnv:
self.known_syms = {} # dict of symbols that are defined
self.unresolved_syms = [] # list of unresolved symbols
self.mpy_relocs = [] # list of relocations needed in the output .mpy file
+ self.externs = {} # dict of externally-defined symbols
def check_arch(self, arch_name):
if arch_name != self.arch.name:
@@ -491,10 +492,14 @@ def populate_got(env):
sym = got_entry.sym
if hasattr(sym, "resolved"):
sym = sym.resolved
- sec = sym.section
- addr = sym["st_value"]
- got_entry.sec_name = sec.name
- got_entry.link_addr += sec.addr + addr
+ if sym.name in env.externs:
+ got_entry.sec_name = ".external.fixed_addr"
+ got_entry.link_addr = env.externs[sym.name]
+ else:
+ sec = sym.section
+ addr = sym["st_value"]
+ got_entry.sec_name = sec.name
+ got_entry.link_addr += sec.addr + addr
# Get sorted GOT, sorted by external, text, rodata, bss so relocations can be combined
got_list = sorted(
@@ -520,6 +525,9 @@ def populate_got(env):
dest = int(got_entry.name.split("+")[1], 16) // env.arch.word_size
elif got_entry.sec_name == ".external.mp_fun_table":
dest = got_entry.sym.mp_fun_table_offset
+ elif got_entry.sec_name == ".external.fixed_addr":
+ # Fixed-address symbols should not be relocated.
+ continue
elif got_entry.sec_name.startswith(".text"):
dest = ".text"
elif got_entry.sec_name.startswith(".rodata"):
@@ -1207,6 +1215,9 @@ def link_objects(env, native_qstr_vals_len):
sym.section = env.obj_table_section
elif sym.name in env.known_syms:
sym.resolved = env.known_syms[sym.name]
+ elif sym.name in env.externs:
+ # Fixed-address symbols do not need pre-processing.
+ continue
else:
if sym.name in fun_table:
sym.section = mp_fun_table_sec
@@ -1214,6 +1225,15 @@ def link_objects(env, native_qstr_vals_len):
else:
undef_errors.append("{}: undefined symbol: {}".format(sym.filename, sym.name))
+ for sym in env.externs:
+ if sym in env.known_syms:
+ log(
+ LOG_LEVEL_1,
+ "Symbol {} is a fixed-address symbol at {:08x} and is also provided from an object file".format(
+ sym, env.externs[sym]
+ ),
+ )
+
if undef_errors:
raise LinkError("\n".join(undef_errors))
@@ -1456,6 +1476,9 @@ def do_link(args):
log(LOG_LEVEL_2, "qstr vals: " + ", ".join(native_qstr_vals))
env = LinkEnv(args.arch)
try:
+ if args.externs:
+ env.externs = parse_linkerscript(args.externs)
+
# Load object files
for fn in args.files:
with open(fn, "rb") as f:
@@ -1484,6 +1507,50 @@ def do_link(args):
sys.exit(1)
+def parse_linkerscript(source):
+ # This extracts fixed-address symbol lists from linkerscripts, only parsing
+ # a small subset of all possible directives. Right now the only
+ # linkerscript file this is really tested against is the ESP8266's builtin
+ # ROM functions list ($SDK/ld/eagle.rom.addr.v6.ld).
+ #
+ # The parser should be able to handle symbol entries inside ESP-IDF's ROM
+ # symbol lists for the ESP32 range of MCUs as well (see *.ld files in
+ # $SDK/components/esp_rom/<name>/).
+
+ symbols = {}
+
+ LINE_REGEX = re.compile(
+ r'^(?P<weak>PROVIDE\()?' # optional weak marker start
+ r'(?P<symbol>[a-zA-Z_]\w*)' # symbol name
+ r'=0x(?P<address>[\da-fA-F]{1,8})*' # symbol address
+ r'(?(weak)\));$', # optional weak marker end and line terminator
+ re.ASCII,
+ )
+
+ inside_comment = False
+ for line in (line.strip() for line in source.readlines()):
+ if line.startswith('/*') and not inside_comment:
+ if not line.endswith('*/'):
+ inside_comment = True
+ continue
+ if inside_comment:
+ if line.endswith('*/'):
+ inside_comment = False
+ continue
+ if line.startswith('//'):
+ continue
+ match = LINE_REGEX.match(''.join(line.split()))
+ if not match:
+ continue
+ tokens = match.groupdict()
+ symbol = tokens['symbol']
+ address = int(tokens['address'], 16)
+ if symbol in symbols:
+ raise ValueError(f"Symbol {symbol} already defined")
+ symbols[symbol] = address
+ return symbols
+
+
def main():
import argparse
@@ -1500,6 +1567,13 @@ def main():
cmd_parser.add_argument(
"--output", "-o", default=None, help="output .mpy file (default to input with .o->.mpy)"
)
+ cmd_parser.add_argument(
+ "--externs",
+ "-e",
+ type=argparse.FileType("rt"),
+ default=None,
+ help="linkerscript providing fixed-address symbols to augment symbol resolution",
+ )
cmd_parser.add_argument("files", nargs="+", help="input files")
args = cmd_parser.parse_args()
diff --git a/tools/pyboard.py b/tools/pyboard.py
index d49365f617..40928e8bbb 100755
--- a/tools/pyboard.py
+++ b/tools/pyboard.py
@@ -267,7 +267,15 @@ class ProcessPtyToTerminal:
class Pyboard:
def __init__(
- self, device, baudrate=115200, user="micro", password="python", wait=0, exclusive=True
+ self,
+ device,
+ baudrate=115200,
+ user="micro",
+ password="python",
+ wait=0,
+ exclusive=True,
+ timeout=None,
+ write_timeout=5,
):
self.in_raw_repl = False
self.use_raw_paste = True
@@ -283,7 +291,12 @@ class Pyboard:
import serial.tools.list_ports
# Set options, and exclusive if pyserial supports it
- serial_kwargs = {"baudrate": baudrate, "interCharTimeout": 1}
+ serial_kwargs = {
+ "baudrate": baudrate,
+ "timeout": timeout,
+ "write_timeout": write_timeout,
+ "interCharTimeout": 1,
+ }
if serial.__version__ >= "3.3":
serial_kwargs["exclusive"] = exclusive
@@ -323,14 +336,25 @@ class Pyboard:
def close(self):
self.serial.close()
- def read_until(self, min_num_bytes, ending, timeout=10, data_consumer=None):
- # if data_consumer is used then data is not accumulated and the ending must be 1 byte long
+ def read_until(
+ self, min_num_bytes, ending, timeout=10, data_consumer=None, timeout_overall=None
+ ):
+ """
+ min_num_bytes: Obsolete.
+ ending: Return if 'ending' matches.
+ timeout [s]: Return if timeout between characters. None: Infinite timeout.
+ timeout_overall [s]: Return not later than timeout_overall. None: Infinite timeout.
+ data_consumer: Use callback for incoming characters.
+ If data_consumer is used then data is not accumulated and the ending must be 1 byte long
+
+ It is not visible to the caller why the function returned. It could be ending or timeout.
+ """
assert data_consumer is None or len(ending) == 1
+ assert isinstance(timeout, (type(None), int, float))
+ assert isinstance(timeout_overall, (type(None), int, float))
- data = self.serial.read(min_num_bytes)
- if data_consumer:
- data_consumer(data)
- timeout_count = 0
+ data = b""
+ begin_overall_s = begin_char_s = time.monotonic()
while True:
if data.endswith(ending):
break
@@ -341,15 +365,25 @@ class Pyboard:
data = new_data
else:
data = data + new_data
- timeout_count = 0
+ begin_char_s = time.monotonic()
else:
- timeout_count += 1
- if timeout is not None and timeout_count >= 100 * timeout:
+ if timeout is not None and time.monotonic() >= begin_char_s + timeout:
+ break
+ if (
+ timeout_overall is not None
+ and time.monotonic() >= begin_overall_s + timeout_overall
+ ):
break
time.sleep(0.01)
return data
- def enter_raw_repl(self, soft_reset=True):
+ def enter_raw_repl(self, soft_reset=True, timeout_overall=10):
+ try:
+ self._enter_raw_repl_unprotected(soft_reset, timeout_overall)
+ except OSError as er:
+ raise PyboardError("could not enter raw repl: {}".format(er))
+
+ def _enter_raw_repl_unprotected(self, soft_reset, timeout_overall):
self.serial.write(b"\r\x03") # ctrl-C: interrupt any running program
# flush input (without relying on serial.flushInput())
@@ -361,7 +395,9 @@ class Pyboard:
self.serial.write(b"\r\x01") # ctrl-A: enter raw REPL
if soft_reset:
- data = self.read_until(1, b"raw REPL; CTRL-B to exit\r\n>")
+ data = self.read_until(
+ 1, b"raw REPL; CTRL-B to exit\r\n>", timeout_overall=timeout_overall
+ )
if not data.endswith(b"raw REPL; CTRL-B to exit\r\n>"):
print(data)
raise PyboardError("could not enter raw repl")
@@ -371,12 +407,12 @@ class Pyboard:
# Waiting for "soft reboot" independently to "raw REPL" (done below)
# allows boot.py to print, which will show up after "soft reboot"
# and before "raw REPL".
- data = self.read_until(1, b"soft reboot\r\n")
+ data = self.read_until(1, b"soft reboot\r\n", timeout_overall=timeout_overall)
if not data.endswith(b"soft reboot\r\n"):
print(data)
raise PyboardError("could not enter raw repl")
- data = self.read_until(1, b"raw REPL; CTRL-B to exit\r\n")
+ data = self.read_until(1, b"raw REPL; CTRL-B to exit\r\n", timeout_overall=timeout_overall)
if not data.endswith(b"raw REPL; CTRL-B to exit\r\n"):
print(data)
raise PyboardError("could not enter raw repl")
diff --git a/tools/verifygitlog.py b/tools/verifygitlog.py
index 5234611983..dba6ebd6de 100755
--- a/tools/verifygitlog.py
+++ b/tools/verifygitlog.py
@@ -105,8 +105,12 @@ def verify_message_body(raw_body, err):
# Message body lines.
for line in raw_body[2:]:
- # Long lines with URLs are exempt from the line length rule.
- if len(line) >= 76 and "://" not in line:
+ # Long lines with URLs or human names are exempt from the line length rule.
+ if len(line) >= 76 and not (
+ "://" in line
+ or line.startswith("Co-authored-by: ")
+ or line.startswith("Signed-off-by: ")
+ ):
err.error("Message lines should be 75 or less characters: " + line)
if not raw_body[-1].startswith("Signed-off-by: ") or "@" not in raw_body[-1]:
@@ -116,8 +120,8 @@ def verify_message_body(raw_body, err):
def verify_subject_line_prefix(prefix, err):
ext = (".c", ".h", ".cpp", ".js", ".rst", ".md")
- if prefix.startswith("."):
- err.error('Subject prefix cannot begin with ".".')
+ if prefix.startswith((".", "/")):
+ err.error('Subject prefix cannot begin with "." or "/".')
if prefix.endswith("/"):
err.error('Subject prefix cannot end with "/".')
generated by cgit v1.2.3 (git 2.39.1) at 2025-06-10 16:31:22 +0000