Commander1024/ESP32_Serial_Swiss_Army_Knife
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate to IDF 5.5.3 candidate

Browse Source 
Pin PlatformIO packages and toolchains, rebase protected SDK
overrides, and add WebSocket receive regression coverage. Document
isolated candidate validation, archive provenance, and remaining gates.
This commit is contained in:
Commander1024
2026-09-18 14:23:13 +02:00
parent cdc4d4a8df
commit 797d2681ac
32 changed files with 1394 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/idf_candidate/artifacts.json
+58
View File
@@ -0,0 +1,58 @@
{
"review_date": "2026-09-16",
"host": "linux_x86_64",
"artifacts": [
{
"package": "platformio/platform/espressif32",
"version": "6.13.0",
"registry_url": "https://api.registry.platformio.org/v3/packages/platformio/platform/espressif32",
"name": "espressif32-6.13.0.tar.gz",
"size": 1009115,
"checksum": {
"sha256": "5d1032b43828773ba87cf2e509432202c0bfe64f7304b58c9d669f13b116c6e0"
},
"system": "*",
"download_url": "https://dl.registry.platformio.org/download/platformio/platform/espressif32/6.13.0/espressif32-6.13.0.tar.gz"
},
{
"package": "platformio/tool/framework-espidf",
"version": "3.50503.0",
"registry_url": "https://api.registry.platformio.org/v3/packages/platformio/tool/framework-espidf",
"name": "framework-espidf-3.50503.0.tar.gz",
"size": 76402966,
"checksum": {
"sha256": "8353f6fd5030dd7e662500891428fad15d46efd7e4b718cab2fe6bfb9e7f13fc"
},
"system": "*",
"download_url": "https://dl.registry.platformio.org/download/platformio/tool/framework-espidf/3.50503.0/framework-espidf-3.50503.0.tar.gz"
},
{
"package": "platformio/tool/toolchain-xtensa-esp-elf",
"version": "14.2.0+20251107",
"registry_url": "https://api.registry.platformio.org/v3/packages/platformio/tool/toolchain-xtensa-esp-elf",
"name": "toolchain-xtensa-esp-elf-linux_x86_64-14.2.0+20251107.tar.gz",
"size": 322439270,
"checksum": {
"sha256": "a5de49ce3299b0d9253ab6a423648bc23113db96b34a7cc8e57702cae1bb190e"
},
"system": [
"linux_x86_64"
],
"download_url": "https://dl.registry.platformio.org/download/platformio/tool/toolchain-xtensa-esp-elf/14.2.0+20251107/toolchain-xtensa-esp-elf-linux_x86_64-14.2.0+20251107.tar.gz"
},
{
"package": "platformio/tool/toolchain-riscv32-esp",
"version": "14.2.0+20251107",
"registry_url": "https://api.registry.platformio.org/v3/packages/platformio/tool/toolchain-riscv32-esp",
"name": "toolchain-riscv32-esp-linux_x86_64-14.2.0+20251107.tar.gz",
"size": 592894688,
"checksum": {
"sha256": "1af8e233931500b8712079808e4974413d95d3601d03275dff79665c436e9d33"
},
"system": [
"linux_x86_64"
],
"download_url": "https://dl.registry.platformio.org/download/platformio/tool/toolchain-riscv32-esp/14.2.0+20251107/toolchain-riscv32-esp-linux_x86_64-14.2.0+20251107.tar.gz"
}
]
}
tools/idf_candidate/prepare.py
+139
View File
@@ -0,0 +1,139 @@
#!/usr/bin/env python3
"""Verify official candidate archives; optionally fetch or prepare an isolated smoke project."""
import argparse
import hashlib
import json
import pathlib
import platform
import tarfile
import urllib.parse
import urllib.request
ROOT = pathlib.Path(__file__).resolve().parents[2]
BASE = ROOT / ".pio/idf-candidate-5.5.3"
LOCK = pathlib.Path(__file__).with_name("artifacts.json")
class RegistryRedirects(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
check_url(newurl)
return super().redirect_request(req, fp, code, msg, headers, newurl)
def check_url(url):
parsed = urllib.parse.urlsplit(url)
if parsed.scheme != "https" or parsed.hostname not in {
"dl.registry.platformio.org", "dl.registry.nm1.platformio.org",
}:
raise ValueError("Unapproved artifact host: " + url)
def verify(path, item):
digest = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(block)
if path.stat().st_size != item["size"] or digest.hexdigest() != item["checksum"]["sha256"]:
raise ValueError("Artifact size/hash mismatch: " + str(path))
def member(archive, name):
matches = [m for m in archive.getmembers() if m.name.removeprefix("./") == name and m.isfile()]
if len(matches) != 1:
raise ValueError("Missing/ambiguous archive member: " + name)
return archive.extractfile(matches[0]).read()
def require(condition):
if not condition:
raise ValueError("Candidate package contract mismatch")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--fetch", action="store_true", help="Download absent pinned archives; never install shared packages")
parser.add_argument("--prepare", action="store_true", help="Create a fresh, separate smoke project after verifying all four archives")
parser.add_argument("--sdk-only", action="store_true", help="Verify/download platform and SDK only, not toolchains; cannot prepare")
args = parser.parse_args()
if args.prepare and args.sdk_only:
parser.error("--prepare requires both toolchain archives")
if platform.system() != "Linux" or platform.machine() != "x86_64":
parser.error("This lock is for Linux x86_64 only")
# Do not follow a user-created candidate/cache symlink outside the workspace.
for path in (ROOT / ".pio", BASE, BASE / "archives"):
if path.is_symlink():
raise ValueError("Refusing symlink: " + str(path))
path.mkdir(exist_ok=True)
items = json.loads(LOCK.read_text())["artifacts"]
selected = items[:2] if args.sdk_only else items
opener = urllib.request.build_opener(RegistryRedirects())
for item in selected:
path = BASE / "archives" / item["name"]
if path.is_symlink():
raise ValueError("Refusing symlink: " + str(path))
if not path.exists() and args.fetch:
check_url(item["download_url"])
temporary = path.with_suffix(".partial")
with temporary.open("xb") as target:
try:
with opener.open(item["download_url"], timeout=60) as source:
total = 0
while block := source.read(1024 * 1024):
total += len(block)
if total > item["size"]:
raise ValueError("Download exceeds pinned size")
target.write(block)
except BaseException:
temporary.unlink()
raise
try:
verify(temporary, item)
temporary.rename(path)
finally:
temporary.unlink(missing_ok=True)
verify(path, item)
print("SHA256 PASS", item["name"], flush=True)
with tarfile.open(BASE / "archives" / items[0]["name"]) as archive:
manifest = json.loads(member(archive, "platform.json"))
require(manifest["version"] == "6.13.0")
require(manifest["packages"]["framework-espidf"]["version"] == "~3.50503.0")
require(manifest["packages"]["toolchain-xtensa-esp-elf"]["version"] == "14.2.0+20251107")
with tarfile.open(BASE / "archives" / items[1]["name"]) as archive:
require(json.loads(member(archive, "package.json"))["version"] == "3.50503.0")
tools = json.loads(member(archive, "tools/tools.json"))
xtensa = next(t for t in tools["tools"] if t["name"] == "xtensa-esp-elf")
require(any(v["name"] == "esp-14.2.0_20251107" and v["status"] == "recommended" for v in xtensa["versions"]))
header = member(archive, "components/wpa_supplicant/esp_supplicant/src/esp_wifi_driver.h").decode()
require("(*wpa_ap_get_wpa_ie)(size_t *len)" in header)
expected = {
"core": "9f7b14a8bf6eec64973da8adc65d35b5ba9bee49",
"espnow": "132b4f67e339ca2081d2add91c14eefa39476ff9",
"mesh": "2e9dc1c8c7afbf033337b4175032e9b1161e3262",
"net80211": "2800d447ec385d33869373696ba8191292647694",
"pp": "8944bcad7371621045f376cf74c62fde6f368cbb",
"smartconfig": "4dc759e25617aa00b9e12887fd092a1d5780a170",
"wapi": "65655b6feab0572a6e8a1200946d53a21f3f4722",
}
for name, digest in expected.items():
data = member(archive, "components/esp_wifi/lib/esp32s3/lib" + name + ".a")
require(hashlib.sha1(b"blob " + str(len(data)).encode() + b"\0" + data).hexdigest() == digest)
print("Official manifest/toolchain, fixed callback ABI and all seven release Wi-Fi archives PASS")
if args.prepare:
project = BASE / "smoke"
project.mkdir(exist_ok=False)
(project / "src").mkdir()
(project / "src/main.c").write_text("void app_main(void) {}\n")
(project / "src/CMakeLists.txt").write_text('idf_component_register(SRCS "main.c" INCLUDE_DIRS ".")\n')
(project / "CMakeLists.txt").write_text('cmake_minimum_required(VERSION 3.16)\ninclude($ENV{IDF_PATH}/tools/cmake/project.cmake)\nproject(idf_candidate_smoke)\n')
config = "[platformio]\ncore_dir = " + str(BASE / "core") + "\n\n[env:candidate]\n"
config += "platform = " + (BASE / "archives" / items[0]["name"]).as_uri().replace("%2B", "+").replace("%", "%%") + "\n"
config += "board = esp32-s3-devkitc-1\nframework = espidf\nplatform_packages =\n"
for item in items[1:]:
config += " " + item["package"].split("/")[-1] + " @ " + (BASE / "archives" / item["name"]).as_uri().replace("%2B", "+").replace("%", "%%") + "\n"
(project / "platformio.ini").write_text(config)
print("Prepared", project)
print("Smoke only: not production sources, board customization, managed dependencies or security overrides.")
if __name__ == "__main__":
main()
tools/idf_candidate/stage_app.py
+74
View File
@@ -0,0 +1,74 @@
#!/usr/bin/env python3
"""Stage current tracked working-tree inputs plus explicitly reviewed test inputs."""
import argparse
import hashlib
import json
import pathlib
import subprocess
ROOT = pathlib.Path(__file__).resolve().parents[2]
BASE = ROOT / ".pio/idf-candidate-5.5.3"
EXACT = {"CMakeLists.txt", "platformio.ini", "extra_script.py", "partitions.csv",
"sdkconfig.defaults", "dependencies.lock", "tools/security_overrides.py"}
PREFIXES = ("src/", "cmake/", "boards/", "tests/", "tools/wolfssh_order/")
UNTRACKED = {"tests/sdk_security_overrides/ws.c"}
def stage(destination, root=ROOT, base=BASE, items=None):
# A single fresh child prevents traversal into archives/core or old app outputs.
if destination in {"", ".", ".."} or pathlib.Path(destination).name != destination:
raise ValueError("Destination must be a single fresh candidate directory name")
dest = base / destination
for path in (root / ".pio", base, dest):
if path.is_symlink():
raise ValueError("Refusing symlink destination: " + str(path))
if base.resolve() != root.resolve() / ".pio/idf-candidate-5.5.3":
raise ValueError("Candidate root is not confined to the project")
if dest.exists():
raise FileExistsError(dest)
if items is None:
items = json.loads(pathlib.Path(__file__).with_name("artifacts.json").read_text())["artifacts"]
tracked = subprocess.check_output(["git", "--no-pager", "ls-files", "-z"], cwd=root).decode().split("\0")
names = {name for name in tracked if name in EXACT or name.startswith(PREFIXES)}
names.update(name for name in UNTRACKED if (root / name).exists())
inputs = {}
for name in sorted(names):
source = root / name
if source.is_symlink() or source.resolve() != root.resolve() / name:
raise ValueError("Refusing symlink source: " + name)
inputs[name] = source.read_bytes()
config = inputs["platformio.ini"].decode()
replacements = {"[platformio]\n": "[platformio]\ncore_dir = " + str(base / "core") + "\n"}
for index, item in enumerate(items):
package = "platformio/" + item["package"].split("/")[-1]
uri = (base / "archives" / item["name"]).as_uri().replace("%2B", "+").replace("%", "%%")
replacements[package + "@" + item["version"]] = uri if index == 0 else package.split("/")[-1] + " @ " + uri
for old, new in replacements.items():
if config.count(old) != 1:
raise ValueError("Missing/ambiguous platform configuration anchor: " + old)
config = config.replace(old, new, 1)
inputs["platformio.ini"] = config.encode()
dest.mkdir(exist_ok=False)
manifest = {}
for name, data in inputs.items():
target = dest / name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(data)
manifest[name] = hashlib.sha256(data).hexdigest()
(dest / "app-source-manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
print("Staged", len(manifest), "working-tree inputs in", dest)
print("No existing sdkconfig, secrets, .git, cache or managed components copied.")
print("Copy the six hash-verified locked managed-component directories separately before building.")
return dest
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--destination", required=True, help="Fresh direct child name under .pio/idf-candidate-5.5.3 (e.g. app-validated)")
args = parser.parse_args()
stage(args.destination)
if __name__ == "__main__":
main()
tools/idf_candidate/test_prepare.py
+98
View File
@@ -0,0 +1,98 @@
"""Offline fail-closed helper checks; does not download/install/build."""
import hashlib
import io
import pathlib
import tarfile
import tempfile
import unittest
import json
from unittest import mock
import stage_app
import prepare
class Contracts(unittest.TestCase):
def test_hash_and_size(self):
with tempfile.TemporaryDirectory() as directory:
path = pathlib.Path(directory) / "artifact"
path.write_bytes(b"candidate")
item = {"size": 9, "checksum": {"sha256": hashlib.sha256(b"candidate").hexdigest()}}
prepare.verify(path, item)
path.write_bytes(b"Candidate")
with self.assertRaises(ValueError):
prepare.verify(path, item)
path.write_bytes(b"candidate-extra")
with self.assertRaises(ValueError):
prepare.verify(path, item)
def test_network_allowlist(self):
prepare.check_url("https://dl.registry.platformio.org/download/a")
prepare.check_url("https://dl.registry.nm1.platformio.org/download/a")
for url in ("http://dl.registry.platformio.org/a", "https://example.com/a", "https://dl.registry.platformio.org.evil/a"):
with self.assertRaises(ValueError):
prepare.check_url(url)
def test_contract_failure(self):
with self.assertRaises(ValueError):
prepare.require(False)
def test_member_ambiguity(self):
stream = io.BytesIO()
with tarfile.open(fileobj=stream, mode="w") as archive:
for name in ("package.json", "./package.json"):
entry = tarfile.TarInfo(name)
entry.size = 2
archive.addfile(entry, io.BytesIO(b"{}"))
stream.seek(0)
with tarfile.open(fileobj=stream) as archive:
with self.assertRaises(ValueError):
prepare.member(archive, "package.json")
with self.assertRaises(ValueError):
prepare.member(archive, "missing")
class Staging(unittest.TestCase):
def test_fresh_snapshot_and_manifest(self):
with tempfile.TemporaryDirectory() as directory:
root = pathlib.Path(directory)
base = root / ".pio/idf-candidate-5.5.3"
base.mkdir(parents=True)
(base / "app").mkdir()
(base / "app/sentinel").write_text("old")
(root / "platformio.ini").write_text("[platformio]\nplatform = platformio/espressif32@6.13.0\n")
(root / "src").mkdir()
(root / "src/main.c").write_text("modified working tree")
(root / "src/secret").write_text("never copy")
ws = root / "tests/sdk_security_overrides/ws.c"
ws.parent.mkdir(parents=True)
ws.write_text("explicit fixture")
items = [{"package": "platformio/platform/espressif32", "version": "6.13.0", "name": "platform.tar.gz"}]
with mock.patch.object(stage_app.subprocess, "check_output", return_value=b"platformio.ini\0src/main.c\0"):
dest = stage_app.stage("app-validated", root, base, items)
manifest = json.loads((dest / "app-source-manifest.json").read_text())
for name, digest in manifest.items():
self.assertEqual(hashlib.sha256((dest / name).read_bytes()).hexdigest(), digest)
self.assertIn(str(base / "core"), (dest / "platformio.ini").read_text())
self.assertEqual((dest / "src/main.c").read_text(), "modified working tree")
self.assertTrue((dest / "tests/sdk_security_overrides/ws.c").exists())
self.assertFalse((dest / "src/secret").exists())
self.assertEqual((base / "app/sentinel").read_text(), "old")
for name in ("app", "app-validated"):
with self.assertRaises(FileExistsError):
stage_app.stage(name, root, base, items)
for name in ("../escape", "/tmp/escape", "", ".", "..", "app/nested"):
with self.assertRaises(ValueError):
stage_app.stage(name, root, base, items)
(base / "linked").symlink_to(root, target_is_directory=True)
with self.assertRaises(ValueError):
stage_app.stage("linked", root, base, items)
(root / "src/main.c").unlink()
(root / "src/main.c").symlink_to(root / "platformio.ini")
with self.assertRaises(ValueError):
stage_app.stage("unsafe", root, base, items)
self.assertFalse((base / "unsafe").exists())
if __name__ == "__main__":
unittest.main()