#!/bin/sh # macos-remote-build -- run the signed, notarized macOS build on the mac that # holds the signing identity, and bring the artifacts home. # # contrib/macos-remote-build --builder # this tree, as it is # contrib/macos-remote-build --ref v0.2.0 # a pushed tag, from a clean tree # contrib/macos-remote-build --target pkg # any Makefile.macos target # # Normally you do not run this by hand: `make macos-dist` forwards to it through # Makefile.macos, which is where the builder's name and the signing identity # live. Those are properties of a machine and an Apple account, so they are not # in the repo; this script is the mechanism, and holds neither. # # Two source modes, because the two things you do are not the same thing: # # working tree (default) rsync what is in front of you, dirty or not. This # is for "does it sign, does Apple accept it" while # you are still changing the code. The version string # says -dirty and that is the honest answer. # # --ref the builder resets a pristine checkout to a ref the # server already has, and builds from scratch. This is # what a release is: an artifact you can point at a # commit id, which is also what contrib/brew-release # needs to be true. It refuses a ref the remote has # never heard of rather than build the wrong tree. # # Everything it needs on the far side is put there by contrib/macos-builder-setup. set -eu BUILDER="${SLOSH_BUILDER:-}" DIR="${SLOSH_BUILDER_DIR:-src/slosh}" TARGET=dist REF="" root="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" fail() { printf 'macos-remote-build: %s\n' "$*" >&2; exit 1; } say() { printf ' %s\n' "$*"; } step() { printf '\n== %s\n' "$*"; } while [ $# -gt 0 ]; do case "$1" in --builder) [ $# -ge 2 ] || fail "--builder needs a host"; BUILDER="$2"; shift 2 ;; --builder=*) BUILDER="${1#--builder=}"; shift ;; --dir) [ $# -ge 2 ] || fail "--dir needs a path"; DIR="$2"; shift 2 ;; --dir=*) DIR="${1#--dir=}"; shift ;; --ref) [ $# -ge 2 ] || fail "--ref needs a ref"; REF="$2"; shift 2 ;; --ref=*) REF="${1#--ref=}"; shift ;; --target) [ $# -ge 2 ] || fail "--target needs a target"; TARGET="$2"; shift 2 ;; --target=*) TARGET="${1#--target=}"; shift ;; -h|--help) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; *) fail "unknown argument $1" ;; esac done [ -n "$BUILDER" ] \ || fail "which builder? pass --builder , or set SLOSH_BUILDER. Normally Makefile.macos passes it for you: make macos-dist" [ -f "$root/Makefile.macos" ] \ || fail "no Makefile.macos in $root -- it names the signing identity and the builder, and is deliberately not in the repo. See contrib/macos-builder-setup --help" step "builder $BUILDER:$DIR" ssh -o BatchMode=yes -o ConnectTimeout=15 "$BUILDER" true \ || fail "cannot ssh to '$BUILDER' without a prompt -- check ~/.ssh/config and your agent" ssh "$BUILDER" "test -d '$DIR/.git'" \ || fail "no checkout at $BUILDER:$DIR -- run contrib/macos-builder-setup $BUILDER" say "reachable, checkout present" # macos-release.mk is not in the repo -- it is the one file that is not -- so a # clean checkout on the builder does not have one and it is pushed from here. # Only that file: Makefile.macos and these scripts are tracked, so the reset # below brings the builder the versions belonging to the commit being released, # which is what should sign it. Copying ours over would override that with # whatever the laptop happens to have, and make a release checkout dirty -- # which is a failure below. push_signing_flow() { [ -f "$root/macos-release.mk" ] \ || fail "no macos-release.mk here -- it names the identity and the builder: cp macos-release.mk.example macos-release.mk" rsync -q "$root/macos-release.mk" "$BUILDER:$DIR/macos-release.mk" } if [ -n "$REF" ]; then step "resolving $REF against origin" ssh "$BUILDER" "cd '$DIR' && git fetch -q --tags --prune origin" \ || fail "git fetch failed on the builder" # ls-remote, not rev-parse. A release must be a tree the server publishes, # and rev-parse answers from local refs first -- the builder's checkout is # rsynced from a laptop in the other mode, so refs/heads/master there can # be an unpushed commit. Asking that would quietly notarize a build nobody # else can reproduce, under a name that claims otherwise. Same resolution # order as contrib/brew-release, so the zip and the formula agree. full="$(ssh "$BUILDER" "cd '$DIR' && git ls-remote origin 'refs/tags/$REF^{}' 'refs/tags/$REF' 'refs/heads/$REF' 2>/dev/null | head -1 | cut -f1")" if [ -n "$full" ]; then say "$REF -> $full (published by origin)" else # Not a name origin publishes. A raw commit id is still allowed, but # only once we can see it on a remote branch. full="$(ssh "$BUILDER" "cd '$DIR' && git rev-parse -q --verify '$REF^{commit}' 2>/dev/null || true")" [ -n "$full" ] || fail "'$REF' is not a tag, branch or commit the builder can resolve -- is it pushed?" ssh "$BUILDER" "cd '$DIR' && test -n \"\$(git branch -r --contains '$full' 2>/dev/null)\"" \ || fail "'$REF' ($full) is on no branch origin publishes -- push it before releasing it" say "$REF -> $full (commit, reachable from a remote branch)" fi step "checking out $full on the builder" # -e keeps our own untracked flow files; no -x, so build/ and the vendored # zig-out survive and a release build is a recompile, not a re-download. ssh "$BUILDER" "cd '$DIR' \ && git reset -q --hard '$full' \ && git clean -qfd -e macos-release.mk -e dist \ && make -s clean" || fail "could not put the builder on $full" push_signing_flow # What the artifact will be stamped with, from the builder's own git. If # this does not name the commit we just asked for, stop: every later step # would be labelling the wrong tree. desc="$(ssh "$BUILDER" "cd '$DIR' && git describe --always --dirty")" say "builder is at $desc" case "$desc" in *dirty*) fail "the builder's tree is dirty after a reset -- look at it by hand" ;; esac ssh "$BUILDER" "cd '$DIR' && test \"\$(git rev-parse HEAD)\" = '$full'" \ || fail "the builder is not on $full after the reset" else step "syncing the working tree" # --delete so a file removed here is removed there; build outputs and the # vendored zig caches are excluded, which also protects them from --delete. # .git goes too, so `git describe` on the builder tells the truth about the # tree it was handed. Its info/exclude does not: that names the untracked # files *the builder* keeps, and overwriting it with ours makes them show # up as debris in its `git status` and as fair game for `git clean`. # # The three vendor/libghostty-vt caches below must stay in step with the # .gitignore entries of the same name -- they are gigabytes of gitignored # zig cache, and when zig-pkg/ (350M) was in .gitignore but missing here, # every sync silently re-pushed it and looked like a hang. web/ (the # landing page, ~430M with the demo) and tests/fuzz/ (the ~120M corpus) # are tracked but are not inputs to `make all`, which is all the builder # runs, so they are excluded too: sync what the signed binary needs. rsync -a --delete \ --exclude '/build/' --exclude '/dist/' --exclude '/.venv/' \ --exclude '/.git/info/exclude' \ --exclude 'vendor/libghostty-vt/zig-out/' \ --exclude 'vendor/libghostty-vt/.zig-cache/' \ --exclude 'vendor/libghostty-vt/zig-pkg/' \ --exclude '/web/' \ --exclude '/tests/fuzz/' \ --exclude '.DS_Store' \ "$root/" "$BUILDER:$DIR/" || fail "rsync failed" # zig-pkg is excluded above (and protected from --delete) because it is # 350M of vendored zig packages -- but it is an *input* to `make all`, not # a cache: a vendor bump adds packages, and a builder missing one cannot # build (v0.1.7 died on a translate_c that a once-interrupted sync had # left as empty directories). So it is synced separately, additively: # rsync sends only what changed, which after the first push is nothing, # and no --delete means a partial local tree cannot maim the builder's. if [ -d "$root/vendor/libghostty-vt/zig-pkg" ]; then rsync -a "$root/vendor/libghostty-vt/zig-pkg/" \ "$BUILDER:$DIR/vendor/libghostty-vt/zig-pkg/" \ || fail "rsync of zig-pkg failed" fi # web/ and tests/fuzz/ are excluded above but *tracked*, and .git went # over whole -- so the builder's stale copies of them read as local edits, # and `git describe --dirty` stamps a clean release tree -dirty (which is # exactly what named slosh-0.1.1-dirty-macos-arm64.zip before this line). # Restoring them from the synced .git costs no network and keeps the # describe honest; a tree that is *really* dirty here still says so. ssh "$BUILDER" "cd '$DIR' && git checkout -q -- web tests/fuzz 2>/dev/null || true" say "synced $(cd "$root" && git rev-parse --short HEAD 2>/dev/null || echo '?')$(cd "$root" && git diff --quiet 2>/dev/null || echo ' (dirty)')" fi step "make -f Makefile.macos $TARGET" ssh "$BUILDER" "cd '$DIR' && make -f Makefile.macos $TARGET" \ || fail "the build failed on $BUILDER (nothing was fetched)" # ── verify, on the builder ────────────────────────────────────────────────── # codesign and spctl are macOS tools, and the builder is the mac -- so the # artifact is verified there, before it is fetched. That is not a concession: # it is what lets this script be run from anywhere. Driven from a Linux box the # verdict is identical, because it is the same command on the same machine, and # all this end needs is ssh, rsync and something that can hash a file. step "verifying on the builder" # The script goes to a file and is fed to ssh as stdin, rather than living in a # heredoc inside $( ). bash 3.2 -- which is what macOS ships, and what runs # this file there -- mis-parses a `case` inside a command substitution: the # unbalanced `)` of each pattern ends the substitution early. Writing it out # sidesteps a bug that produces a syntax error thirty lines from its cause. vscript="$(mktemp)" trap 'rm -f "$vscript"' EXIT INT TERM cat > "$vscript" <<'EOS' set -eu art="$(ls -t dist/*.zip dist/*.pkg 2>/dev/null | head -1 || true)" [ -n "$art" ] || { echo "!! no artifact in dist/"; exit 1; } echo "ARTIFACT $art" case "$art" in *.zip) tmp="$(mktemp -d)" trap 'rm -rf "$tmp"' EXIT INT TERM ditto -x -k "$art" "$tmp" bin="$(find "$tmp" -type f -perm -u+x | head -1)" [ -n "$bin" ] || { echo "!! the zip holds no executable"; exit 1; } codesign --verify --strict "$bin" 2>&1 || { echo "!! not validly signed"; exit 1; } echo "SIGNED ok" # The question a stranger's mac actually asks. Not `spctl --type exec`: # that only understands app bundles and answers "the code is valid but does # not seem to be an app" for every CLI tool ever signed, which reads as a # failure and is not one. A downloaded binary is assessed as a quarantined # file against its primary signature, and that path looks Apple's ticket up # online -- so "Notarized Developer ID" means the notarization is real and # live, not merely that we submitted something. # # But it is looked up *online*, and a bare CLI binary cannot be stapled # (stapler only takes .pkg/.dmg/.app), so there is nothing on disk to carry # the ticket -- Gatekeeper must fetch it from Apple's CloudKit, which lags # the notary's own "Accepted" by seconds to a couple of minutes. Checking # once, immediately, therefore races that propagation and reports # "Unnotarized Developer ID" for a build Apple has already accepted. So # poll: succeed the moment the ticket shows up, and if it still has not # after a couple of minutes, say so and let the release stand -- the notary # verdict above is the authoritative one, and codesign already vouched for # the signature. Any *other* rejection still fails hard. xattr -w com.apple.quarantine "0083;00000000;macos-remote-build;" "$bin" 2>/dev/null || true v=""; ok=""; n=0 while :; do v="$(spctl -a -t open --context context:primary-signature -v "$bin" 2>&1 || true)" case "$v" in *"Notarized Developer ID"*) ok=yes; break ;; esac n=$((n + 1)); [ "$n" -ge 8 ] && break sleep 15 done if [ -n "$ok" ]; then echo "GATEKEEPER accepted, Notarized Developer ID" else case "$v" in *"Unnotarized Developer ID"*) echo "GATEKEEPER pending: notary accepted it, but the online ticket"\ " was not visible after ${n}x15s -- it will verify once it propagates" ;; *) echo "!! gatekeeper rejected it: $v"; exit 1 ;; esac fi ;; *.pkg) pkgutil --check-signature "$art" >/dev/null 2>&1 || { echo "!! pkg not signed"; exit 1; } echo "SIGNED ok" spctl --assess --type install -v "$art" >/dev/null 2>&1 \ && echo "GATEKEEPER accepted for install" \ || { echo "!! gatekeeper rejected the pkg"; exit 1; } ;; esac echo "SHA256 $(shasum -a 256 "$art" | cut -d' ' -f1)" EOS verify="$(ssh "$BUILDER" "cd '$DIR' && bash -s" < "$vscript")" \ || fail "verification failed on the builder: $verify" artifact="$(printf '%s\n' "$verify" | sed -n 's/^ARTIFACT //p')" remote_sha="$(printf '%s\n' "$verify" | sed -n 's/^SHA256 //p')" printf '%s\n' "$verify" | sed -n 's/^SIGNED / signature /p; s/^GATEKEEPER / gatekeeper: /p' case "$verify" in *"GATEKEEPER pending:"*) printf ' !! note: notarization was ACCEPTED, but Gatekeeper could not see the\n' >&2 printf ' ticket online yet. This is propagation lag on Apple'\''s side, not a\n' >&2 printf ' signing failure; the artifact is good and will verify shortly.\n' >&2 ;; esac [ -n "$artifact" ] || fail "the builder named no artifact" step "fetching artifacts" mkdir -p "$root/dist" rsync -a --include '*/' --include '*.zip' --include '*.pkg' --exclude '*' \ "$BUILDER:$DIR/dist/" "$root/dist/" || fail "could not fetch dist/" local_artifact="$root/dist/$(basename "$artifact")" [ -f "$local_artifact" ] || fail "$artifact did not arrive in dist/" say "$(basename "$local_artifact") $(wc -c < "$local_artifact" | tr -d ' ') bytes" # The builder vouched for those exact bytes. Hashing them here is the whole of # what this end can honestly check -- and it is enough: same hash, same verdict. sha256_of() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | cut -d' ' -f1 elif command -v shasum >/dev/null 2>&1; then shasum -a 256 "$1" | cut -d' ' -f1 elif command -v openssl >/dev/null 2>&1; then openssl dgst -sha256 "$1" | sed 's/.*= *//' else return 1 fi } if local_sha="$(sha256_of "$local_artifact")" && [ -n "$local_sha" ]; then [ "$local_sha" = "$remote_sha" ] \ || fail "the artifact changed in transit: builder $remote_sha here $local_sha" say "sha256 $local_sha (matches what the builder verified)" else say "no sha256 tool here -- fetched, but transport is unverified" fi printf '\nartifact: %s\n' "$local_artifact"