tools/check_source_package.py

Source at 4d73c9795adca3e49eea74d1d2cbbbbdcbea0b6c

SHA-256: 28d36450f5003c5fc10904059c040fe9db4cca1c4c1dcd899259b1bf9421aa59

#!/usr/bin/env python3
"""Build and consume exact Hex source artifacts through a private local registry."""

from __future__ import annotations

import argparse
from functools import partial
import hashlib
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
import io
import json
import os
from pathlib import Path
import platform
import shutil
import subprocess
import tarfile
import tempfile
import threading
import time
import urllib.request
from urllib.parse import unquote

from check_package import exercise_consumer


ROOT = Path(__file__).resolve().parent.parent
TOOLS = ROOT / "_build/source-package-tools"
PINS = json.loads((ROOT / "tools/package-tool.json").read_text())["packages"]
APP = "libtmux_erlang"


def digest(path: Path) -> str:
    return hashlib.sha256(path.read_bytes()).hexdigest()


def pinned_tarball(pin: dict) -> Path:
    return TOOLS / f"{pin['name']}-{pin['version']}.tar"


def runtime() -> dict:
    expression = (
        'R = erlang:system_info(otp_release), '
        '{ok, V} = file:read_file(filename:join([code:root_dir(), "releases", R, "OTP_VERSION"])), '
        'io:format("~s", [json:encode(#{otp => string:trim(V), '
        'erts => list_to_binary(erlang:system_info(version)), '
        'architecture => list_to_binary(erlang:system_info(system_architecture))})]), halt().'
    )
    return json.loads(subprocess.check_output(
        ["erl", "-noshell", "-noinput", "-eval", expression], text=True, timeout=10))


def prepared_files() -> dict:
    return {p.relative_to(TOOLS).as_posix(): digest(p)
            for p in sorted((TOOLS / "hex_core").rglob("*")) if p.is_file()}


def source_identity() -> dict:
    def git(*args):
        result = subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True)
        return result.stdout.strip() if result.returncode == 0 else None
    return {"head": git("rev-parse", "--verify", "HEAD"),
            "index_tree": git("write-tree"), "status": git("status", "--porcelain"),
            "artifact_identity": "Package SHA256 values bind the actual compiled source inputs"}


def prepare() -> None:
    """Provision test-only tooling separately from the measured outer check."""
    TOOLS.mkdir(parents=True, exist_ok=True)
    for pin in PINS:
        path = pinned_tarball(pin)
        if path.is_file() and digest(path) == pin["sha256"]:
            continue
        with urllib.request.urlopen("https://repo.hex.pm/tarballs/" + path.name,
                                    timeout=30) as response:
            data = response.read(8 * 1024 * 1024 + 1)
        if hashlib.sha256(data).hexdigest() != pin["sha256"]:
            raise RuntimeError(f"tool checksum mismatch: {path.name}")
        path.write_bytes(data)
    core = TOOLS / "hex_core"
    if core.exists():
        shutil.rmtree(core)
    with tarfile.open(pinned_tarball(PINS[1])) as outer:
        packed = outer.extractfile("contents.tar.gz").read()
    with tarfile.open(fileobj=io.BytesIO(packed), mode="r:gz") as source:
        source.extractall(core, filter="data")
    ebin = core / "ebin"
    ebin.mkdir(exist_ok=True)
    subprocess.run(["erlc", "-o", str(core / "src"),
                    str(core / "src/safe_erl_term.xrl")], check=True)
    subprocess.run(["erlc", "-Werror", "-pa", str(ebin),
                    "-I", str(core / "src"), "-o", str(ebin),
                    *map(str, sorted((core / "src").glob("*.erl")))], check=True)
    manifest = {"pins_sha256": digest(ROOT / "tools/package-tool.json"),
                "runtime": runtime(), "files": prepared_files()}
    (TOOLS / "prepared.json").write_text(json.dumps(manifest, indent=2) + "\n")
    print("Prepared pinned Hex tooling; no project package was published.")


def check_contents(path: Path, app: str) -> dict:
    with tarfile.open(path) as outer:
        packed = outer.extractfile("contents.tar.gz").read()
    with tarfile.open(fileobj=io.BytesIO(packed), mode="r:gz") as source:
        names = source.getnames()
        forbidden = [n for n in names if n.startswith(("priv/", "ebin/", "_build/"))
                     or n.endswith((".beam", ".o", ".so"))]
        if forbidden:
            raise RuntimeError(f"prebuilt files in {app}: {forbidden}")
        required = {"LICENSE", "CHANGELOG.md", "README.md", "rebar.config",
                    "assets/logo.svg", "assets/logo-128.png", "assets/logo-512.png",
                    f"src/{app}.app.src"}
        required.update({"c_src/owned_io.c", "c_src/Makefile", "c_src/LICENSE",
                         "c_src/provenance.json"})
        if missing := required - set(names):
            raise RuntimeError(f"missing source-package files in {app}: {sorted(missing)}")
        return {"sha256": digest(path), "files": names}


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--prepare", action="store_true")
    args = parser.parse_args()
    if args.prepare:
        prepare()
        return
    hex_ebin = TOOLS / "hex_core/ebin"
    started = time.monotonic()
    output = ROOT / "_build/source-package-check"
    attempt = output / f"run-{time.time_ns()}"
    attempt.mkdir(parents=True)
    (output / "receipt.json").unlink(missing_ok=True)
    receipt = {"result": "FAIL", "platform": platform.platform(),
               "machine": platform.machine(), "tools": PINS, "commands": [],
               "source_sha256": {name: digest(ROOT / name) for name in (
                   "tools/check_source_package.py", "tools/source_package_registry.escript",
                   "tools/check_package.py", "tools/package-tool.json",
                   ".tool-versions")},
               "packages": {}, "publication": "UNVERIFIED; local artifacts only"}
    env = dict(os.environ, ERL_FLAGS=os.environ.get("ERL_FLAGS", "+S 4:4"))
    for key in ("ERL_LIBS", "TMUX", "TMUX_PANE", "REBAR_CONFIG", "HEX_CDN",
                "HEX_UNSAFE_REGISTRY", "HEX_NO_VERIFY_REPO_ORIGIN",
                "HEX_TRUSTED_MIRROR_URL", "HEX_MIRROR_URL"):
        env.pop(key, None)

    def run(command, cwd, environment=env, *, expected_failure=False):
        before = time.monotonic()
        timeout_error = None
        try:
            result = subprocess.run(command, cwd=cwd, env=environment, capture_output=True,
                                    text=True, timeout=30)
            code, text = result.returncode, result.stdout + result.stderr
        except subprocess.TimeoutExpired as error:
            timeout_error = error
            code = 124
            text = (error.stdout or b"").decode(errors="replace")
            text += (error.stderr or b"").decode(errors="replace")
        log = attempt / f"command-{len(receipt['commands']) + 1}.log"
        log.write_text(text)
        receipt["commands"].append({"command": command, "seconds": time.monotonic() - before,
                                    "exit_status": code, "timed_out": timeout_error is not None,
                                    "expected_failure": expected_failure,
                                    "log": log.relative_to(ROOT).as_posix()})
        if timeout_error is not None:
            raise timeout_error
        if bool(code) != expected_failure:
            raise RuntimeError(text[-8000:])
        return text if expected_failure else result.stdout

    try:
        for pin in PINS:
            if not pinned_tarball(pin).is_file() or digest(pinned_tarball(pin)) != pin["sha256"]:
                raise RuntimeError("Run tools/check_source_package.py --prepare first")
        manifest = json.loads((TOOLS / "prepared.json").read_text())
        current_runtime = runtime()
        expected = {"pins_sha256": digest(ROOT / "tools/package-tool.json"),
                    "runtime": current_runtime, "files": prepared_files()}
        if manifest != expected or not (hex_ebin / "hex_registry.beam").is_file():
            raise RuntimeError("Prepared tool source/BEAM/runtime mismatch; repeat --prepare")
        receipt["prepared_tool_sha256"] = digest(TOOLS / "prepared.json")
        receipt["runtime"] = current_runtime
        receipt["source"] = source_identity()
        with tempfile.TemporaryDirectory(prefix="erlang-source-consumer-") as directory:
            temp = Path(directory)
            public = temp / "registry/public"
            tarballs = public / "tarballs"
            tarballs.mkdir(parents=True)
            for pin in PINS:
                shutil.copyfile(pinned_tarball(pin), tarballs / pinned_tarball(pin).name)

            def registry():
                return json.loads(run(["escript", str(ROOT / "tools/source_package_registry.escript"),
                                       str(hex_ebin), str(public.parent)], temp))

            registry()
            requests = []

            class Handler(SimpleHTTPRequestHandler):
                def log_message(self, *_args):
                    pass

                def end_headers(self):
                    path = Path(self.translate_path(self.path))
                    if path.is_file():
                        self.send_header("ETag", '"' + digest(path) + '"')
                    super().end_headers()

                def do_GET(self):
                    if len(requests) >= 256:
                        self.send_error(429)
                        return
                    requests.append(self.path)
                    super().do_GET()

            server = ThreadingHTTPServer(("127.0.0.1", 0), partial(Handler, directory=str(public)))
            thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05})
            thread.start()
            try:
                url = f"http://127.0.0.1:{server.server_port}"
                # Rebar3 3.28 overrides the hexpm URL after reading repo config.
                env["HEX_CDN"] = url
                pem = (public / "public_key").read_text()
                repo_config = ('{rebar_packages_cdn, ' + json.dumps(url) + '}.\n'
                               '{hex, [{repos, replace, [#{name => <<"hexpm">>, '
                               f'repo_url => <<"{url}">>, repo_public_key => <<'
                               + json.dumps(pem) + '>>}]}]}.\n')
                build_config = temp / "build-config"
                config_file = build_config / ".config/rebar3/rebar.config"
                config_file.parent.mkdir(parents=True)
                config_file.write_text(
                    '{plugins, [{rebar3_hex, "7.5.0"}]}.\n' + repo_config)
                env.update(REBAR_GLOBAL_CONFIG_DIR=str(build_config),
                           REBAR_CACHE_DIR=str(temp / "build-cache"),
                           REBAR_GLOBAL_CACHE_DIR=str(temp / "build-cache"))
                stage = temp / "build" / APP
                shutil.copytree(ROOT / "apps" / APP, stage,
                                ignore=shutil.ignore_patterns("_build", "priv", "test", "ebin"))
                run(["rebar3", "hex", "build", "package"], stage)
                artifacts = list((stage / "_build/default/lib" / APP / "hex").glob("*.tar"))
                if len(artifacts) != 1:
                    raise RuntimeError(f"expected one actual Hex artifact for {APP}")
                artifact = attempt / artifacts[0].name
                shutil.copyfile(artifacts[0], artifact)
                receipt["packages"][APP] = check_contents(artifact, APP)
                shutil.copyfile(artifact, tarballs / artifact.name)
                metadata = registry()
                current = next(m for m in metadata if m["name"] == APP)
                receipt["packages"][APP]["metadata"] = current
                if current["requirements"]:
                    raise RuntimeError("core must have no Hex runtime dependencies")

                # This consumer has no source checkout, plugin or build cache.
                consumer = temp / "consumer"
                consumer.mkdir()
                config = temp / "consumer-config"
                config_file = config / ".config/rebar3/rebar.config"
                config_file.parent.mkdir(parents=True)
                config_file.write_text(repo_config)
                version = receipt["packages"][APP]["metadata"]["version"]
                (consumer / "rebar.config").write_text(
                    '{deps, [{libtmux_erlang, ' + json.dumps(version) + '}]}.\n')
                env.update(REBAR_GLOBAL_CONFIG_DIR=str(config),
                           REBAR_CACHE_DIR=str(temp / "consumer-cache"),
                           REBAR_GLOBAL_CACHE_DIR=str(temp / "consumer-cache"))
                first_request = len(requests)
                failed_env = dict(env, CC=str(temp / "missing-c-compiler"))
                failure = run(["rebar3", "compile"], consumer, failed_env, expected_failure=True)
                helper = consumer / "_build/default/lib" / APP / "priv/tmux_owned_io"
                if "missing-c-compiler" not in failure or helper.exists():
                    raise RuntimeError("missing compiler did not fail before helper installation")
                receipt["missing_compiler_rejected"] = True
                run(["rebar3", "compile"], consumer)
                receipt["installed"] = json.loads(run([
                    "escript", str(ROOT / "tools/source_package_registry.escript"),
                    "installed", str(consumer)], consumer))
                shutil.copyfile(consumer / "rebar.lock", attempt / "consumer-rebar.lock")
                installed = consumer / "_build/default/lib"
                if not (installed / APP / "ebin" / (APP + ".app")).is_file():
                    raise RuntimeError(f"dependency not compiled: {APP}")
                name = next(a.name for a in attempt.glob(APP + "-*.tar"))
                if "/tarballs/" + name not in map(unquote, requests[first_request:]):
                    raise RuntimeError(f"consumer did not fetch source artifact: {APP}")
                receipt["consumer"] = exercise_consumer(
                    ROOT, consumer, installed / APP / "ebin")
                receipt["requests"] = requests
                receipt["helper_built_from_source"] = True
                receipt["source_only_consumer"] = True
            finally:
                receipt["requests"] = requests
                server.shutdown()
                thread.join(timeout=1)
                server.server_close()
                receipt["registry_retired"] = not thread.is_alive()
                if thread.is_alive():
                    raise RuntimeError("registry did not retire")
        receipt["temporary_consumer_removed"] = not temp.exists()
        receipt["result"] = "PASS"
    except (OSError, RuntimeError, subprocess.SubprocessError) as error:
        receipt["error"] = str(error)
        if isinstance(error, subprocess.TimeoutExpired):
            receipt["owned_cleanup"] = "UNVERIFIED after forced check-runner timeout"
    finally:
        receipt["seconds"] = time.monotonic() - started
        text = json.dumps(receipt, indent=2) + "\n"
        (attempt / "receipt.json").write_text(text)
        (output / "receipt.json").write_text(text)
    print(text)
    raise SystemExit(0 if receipt["result"] == "PASS" else 1)


if __name__ == "__main__":
    main()