esp32/tools: Add metrics_esp32 size comparison script.

Signed-off-by: Angus Gratton <angus@redyak.com.au>

1 files changed, 192 insertions, 0 deletions

diff --git a/ports/esp32/tools/metrics_esp32.py b/ports/esp32/tools/metrics_esp32.py
new file mode 100755
index 0000000000..66a6a588ba
--- /dev/null
+++ b/ports/esp32/tools/metrics_esp32.py
@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+# MIT license; Copyright (c) 2024 Angus Gratton
+#
+# This is a utility script for MicroPython maintainers, similar to tools/metrics.py
+# but particular to this port. It's for measuring the impact of an ESP-IDF update or
+# config change at a high level.
+#
+# Specifically, it builds the esp32 MicroPython port for a collection of boards
+# and outputs a Markdown table of binary sizes, static IRAM size, and static
+# DRAM size (the latter generally inversely correlates to free heap at runtime.)
+#
+# To use:
+#
+# 1) Need to not be in an ESP-IDF venv already (i.e. don't source export.sh),
+#    but IDF_PATH has to be set.
+#
+# 2) Choose the versions you want to test and the board/variant pairs by
+#    editing the tuples below.
+#
+# 3) The IDF install script sometimes fails if it has to downgrade a package
+#    within a minor version. The "nuclear option" is to delete all the install
+#    environments and have this script recreate them as it runs:
+#    rm -rf ~/.espressif/python_env/*
+#
+# 4) Run this script from the ports/esp32 directory, i.e.:
+#    ./tools/metrics_esp32.py
+#
+# 5) If all goes well, it will run for a while and then print a Markdown
+#    formatted table of binary sizes, sorted by board+variant.
+#
+# Note that for ESP32-S3 and C3, IRAM and DRAM are exchangeable so the IRAM size
+# column of the table is really D/IRAM.
+import os
+import re
+import sys
+import subprocess
+from dataclasses import dataclass
+
+IDF_VERS = ("v5.2.2",)
+
+BUILDS = (
+    ("ESP32_GENERIC", ""),
+    ("ESP32_GENERIC", "D2WD"),
+    ("ESP32_GENERIC", "SPIRAM"),
+    ("ESP32_GENERIC_S3", ""),
+    ("ESP32_GENERIC_S3", "SPIRAM_OCT"),
+)
+
+
+@dataclass
+class BuildSizes:
+    idf_ver: str
+    board: str
+    variant: str
+    bin_size: str = ""
+    dram_size: str = ""
+    iram_size: str = ""
+
+    def print_summary(self, include_ver=False):
+        print(f"BOARD={self.board} BOARD_VARIANT={self.variant}")
+        if include_ver:
+            print(f"IDF_VER {self.idf_ver}")
+        print(f"Binary size: {self.bin_size}")
+        print(f"IRAM size: {self.iram_size}")
+        print(f"DRAM size: {self.dram_size}")
+
+    def print_table_heading():
+        print(
+            "| BOARD | BOARD_VARIANT | IDF Version | Binary Size | Static IRAM Size | Static DRAM Size |"
+        )
+        print(
+            "|-------|---------------|-------------|-------------|------------------|------------------|"
+        )
+
+    def print_table_row(self, print_board):
+        print(
+            "| "
+            + " | ".join(
+                (
+                    self.board if print_board else "",
+                    self.variant if print_board else "",
+                    self.idf_ver,
+                    self.bin_size,
+                    self.iram_size,
+                    self.dram_size,
+                )
+            )
+            + " |"
+        )
+
+    def __lt__(self, other):
+        """sort by board, then variant, then IDF version to get an easy
+        to compare table"""
+        return (self.board, self.variant, self.idf_ver) < (
+            other.board,
+            other.variant,
+            other.idf_ver,
+        )
+
+    def build_dir(self):
+        if self.variant:
+            return f"build-{self.board}_{self.variant}"
+        else:
+            return f"build-{self.board}"
+
+    def run_make(self, target):
+        env = dict(os.environ)
+        env["BOARD"] = self.board
+        env["BOARD_VARIANT"] = self.variant
+
+        try:
+            # IDF version changes as we go, so re-export the environment each time
+            cmd = f"source $IDF_PATH/export.sh; make {target}"
+            return subprocess.check_output(
+                cmd, shell=True, env=env, stderr=subprocess.STDOUT
+            ).decode()
+        except subprocess.CalledProcessError as e:
+            err_file = f"{self.build_dir()}/make-{target}-failed-{self.idf_ver}.log"
+            print(f"'make {target}' failed, writing to log to {err_file}", file=sys.stderr)
+            with open(err_file, "w") as f:
+                f.write(e.output.decode())
+            raise
+
+    def make_size(self):
+        try:
+            size_out = self.run_make("size")
+            # "Used static DRAM:" or "Used stat D/IRAM:"
+            RE_DRAM = r"Used stat(?:ic)? D.*: *(\d+) bytes"
+            RE_IRAM = r"Used static IRAM: *(\d+) bytes"
+            RE_BIN = r"Total image size: *(\d+) bytes"
+            self.dram_size = re.search(RE_DRAM, size_out).group(1)
+            self.iram_size = re.search(RE_IRAM, size_out).group(1)
+            self.bin_size = re.search(RE_BIN, size_out).group(1)
+        except subprocess.CalledProcessError:
+            self.bin_size = "build failed"
+
+
+def main(do_clean):
+    if "IDF_PATH" not in os.environ:
+        raise RuntimeError("IDF_PATH must be set")
+
+    sizes = []
+    for idf_ver in IDF_VERS:
+        switch_ver(idf_ver)
+        for board, variant in BUILDS:
+            print(f"Building '{board}'/'{variant}'...", file=sys.stderr)
+            result = BuildSizes(idf_ver, board, variant)
+            result.run_make("clean")
+            result.make_size()
+            result.print_summary()
+            sizes.append(result)
+
+    # print everything again as a table sorted by board+variant
+    last_bv = ""
+    BuildSizes.print_table_heading()
+    for build_sizes in sorted(sizes):
+        bv = (build_sizes.board, build_sizes.variant)
+        build_sizes.print_table_row(last_bv != bv)
+        last_bv = bv
+
+
+def idf_git(*commands):
+    try:
+        subprocess.check_output(
+            ["git"] + list(commands), cwd=os.environ["IDF_PATH"], stderr=subprocess.STDOUT
+        )
+    except subprocess.CalledProcessError as e:
+        print(f"git {' '.join(commands)} failed:")
+        print(e.output.decode())
+        raise
+
+
+def idf_install():
+    try:
+        subprocess.check_output(
+            ["bash", "install.sh"], cwd=os.environ["IDF_PATH"], stderr=subprocess.STDOUT
+        )
+    except subprocess.CalledProcessError as e:
+        print("IDF install.sh failed:")
+        print(e.output.decode())
+        raise
+
+
+def switch_ver(idf_ver):
+    print(f"Switching version to {idf_ver}...", file=sys.stderr)
+    idf_git("switch", "--detach", idf_ver)
+    idf_git("submodule", "update", "--init", "--recursive")
+    idf_install()
+
+
+if __name__ == "__main__":
+    main("--no-clean" not in sys.argv)