From b8c99eb44f1563f09bfe9af8e789d963738bb6e7 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Wed, 26 Aug 2026 14:01:47 +0200 Subject: [PATCH 01/11] infra: reduce .sources/ to five submodules, watch the rest for releases Eleven repos were vendored as submodules purely so agents could check a fact against them. None of their bytes reach the built site, so each one was a recurring bump PR with nothing to show for it, and several drifted years of releases behind unnoticed. They are now watched instead: .sources/upstream.json records the ref the docs are verified against, scripts/check-upstream-releases.mjs compares that against the newest matching tag, and a weekly workflow opens one issue per repo that moved, carrying the ref delta and a summary of what changed. Agents verify against the pinned ref via raw.githubusercontent.com rather than main, so a review stays reproducible and a page cannot document a flag from a release whose doc links have not been adapted yet. --- .agents/submodule-bumping.md | 84 -------- .agents/upstream-tracking.md | 184 ++++++++++++++++++ .github/workflows/upstream-releases.yml | 91 +++++++++ .gitignore | 3 + .gitmodules | 36 ---- .sources/VERSIONS | 53 ++--- .sources/candid | 1 - .sources/cdk-rs | 1 - .sources/chain-fusion-signer | 1 - .sources/ic-pub-key | 1 - .sources/icp-cli | 1 - .sources/icp-cli-recipes | 1 - .sources/icp-cli-templates | 1 - .sources/icp-js-sdk-docs | 1 - .sources/motoko-core | 1 - .sources/papi | 1 - .sources/response-verification | 1 - .sources/upstream.json | 112 +++++++++++ AGENTS.md | 85 +++++--- CONTRIBUTING.md | 7 +- package.json | 3 +- scripts/check-upstream-releases.mjs | 248 ++++++++++++++++++++++++ 22 files changed, 715 insertions(+), 202 deletions(-) delete mode 100644 .agents/submodule-bumping.md create mode 100644 .agents/upstream-tracking.md create mode 100644 .github/workflows/upstream-releases.yml delete mode 160000 .sources/candid delete mode 160000 .sources/cdk-rs delete mode 160000 .sources/chain-fusion-signer delete mode 160000 .sources/ic-pub-key delete mode 160000 .sources/icp-cli delete mode 160000 .sources/icp-cli-recipes delete mode 160000 .sources/icp-cli-templates delete mode 160000 .sources/icp-js-sdk-docs delete mode 160000 .sources/motoko-core delete mode 160000 .sources/papi delete mode 160000 .sources/response-verification create mode 100644 .sources/upstream.json create mode 100644 scripts/check-upstream-releases.mjs diff --git a/.agents/submodule-bumping.md b/.agents/submodule-bumping.md deleted file mode 100644 index 5847a7cf..00000000 --- a/.agents/submodule-bumping.md +++ /dev/null @@ -1,84 +0,0 @@ -# Submodule bumping - -Only the project maintainer bumps submodule refs. Reference this file when asked to bump a submodule. - -## Step 0 — Determine the new ref - -- **Release-pinned repos:** `git ls-remote --tags origin` — pin to the highest version tag's commit. -- **main/master repos:** fetch and checkout `origin/main` or `origin/master`. - -## General checklist (all submodules) - -1. Identify changes: `git -C .sources/ log --oneline ..` -2. Grep docs pages for content derived from that submodule; update affected pages -3. Check open PRs — post a bump-notice comment if the bump may affect pages under review (format below) -4. Update `.sources/VERSIONS` for release-pinned repos -5. Note the bump in the PR description - -**Bump-notice PR comment:** -```bash -gh pr comment --body "$(cat <<'EOF' - -`` was bumped to ``. The following content on this PR may be outdated: -- [specific item and why] - -Please review before merging. -EOF -)" -``` - -## Per-submodule additional checks - -| Submodule | Extra checks on bump | -|---|---| -| `motoko` | **Automated** — `.github/workflows/sync-motoko.yml` opens a PR with the bump, synced docs, and VERSIONS update already committed. Review the content diff and merge. Also grep all Motoko code blocks for changed/removed API signatures. | -| `motoko-core` | Grep all Motoko code blocks for changed/removed API signatures | -| `cdk-rs` | Check `ic-cdk`, `ic-cdk-timers`, `ic-cdk-macros` API changes — grep all Rust code blocks | -| `icp-cli` | Grep all CLI examples for changed/removed commands or flags. If the release introduces a new minor version (e.g. `v0.3.x`), update all CLI doc link slugs — see "Link adaptation for icp-cli" below. | -| `icskills` | Check for changed canister IDs or code patterns | -| `examples` | Verify linked files still exist at the same path | -| `icp-cli-recipes` | Check for renamed or removed recipes referenced in docs | -| `icp-cli-templates` | Check for renamed or restructured templates | -| `icp-js-sdk-docs` | Unzip and compare API signatures; check `versions.json` for new releases | -| `candid` | Check for spec changes affecting the Candid reference or type-mapping examples | -| `response-verification` | Check for API changes affecting certified variables patterns | -| `dotskills` | Check if the `technical-documentation` skill changed in ways that affect review criteria | -| `internetidentity` | Run `npm run sync:ii-spec` — syncs `ii-spec.mdx` → `docs/references/internet-identity-spec.md` and `vc-spec.md` → `docs/references/verifiable-credentials-spec.md`. If the script exits with a warning about unhandled links, add the new pattern to `linkMap` (ii-spec) or `vcLinkMap` (vc-spec) in `scripts/sync-ii-spec.mjs`. Pin to the latest `release-YYYY-MM-DD` tag. The **Sync II spec** workflow (`.github/workflows/sync-ii-spec.yml`) runs automatically; trigger manually for early sync. | -| `chain-fusion-signer` | Check for changed canister IDs, API methods, or key derivation patterns | -| `papi` | Check for changed payment interface or cycle cost model | -| `ic-pub-key` | Check for changed CLI flags or commands | - -## Link adaptation for `icp-cli` - -All CLI docs links use a versioned slug (e.g. `https://cli.internetcomputer.org/0.2/...`). When bumped to a new minor version: - -1. Determine the new slug: it is the `major.minor` of the release tag you pinned (e.g. `v1.1.0` → `1.1`). Do **not** read it from `.sources/icp-cli/docs-site/versions.json` — at a release tag that file still lists the *previous* slug, since the docs-site version bump lands as a follow-up commit after the tag. Confirm the new slug is live by opening the docs-site root (`https://cli.internetcomputer.org/`), which redirects to the latest version. -2. Verify all linked paths still exist in the new submodule before replacing: - ```bash - grep -roh "cli\.internetcomputer\.org/[0-9][.0-9]*/[^\"' )#]*" docs/ --include="*.md" --include="*.mdx" \ - | sed 's|cli\.internetcomputer\.org/[0-9][.0-9]*/||' | sort -u | grep -v "^$" \ - | while read p; do - [ -f ".sources/icp-cli/docs/${p}.md" ] || echo "MISSING: $p" - done - ``` - Resolve any MISSING paths manually before proceeding. -3. Replace the slug across all files (per-file loop — portable across GNU and BSD/macOS `sed`, whose `-i` syntax differs): - ```bash - old=1.0; new=1.1 - grep -rl "cli.internetcomputer.org/${old}/" docs/ --include="*.md" --include="*.mdx" | while IFS= read -r f; do - sed -i.bak "s|cli.internetcomputer.org/${old}/|cli.internetcomputer.org/${new}/|g" "$f" && rm -f "$f.bak" - done - ``` -4. Run `npm run build` to confirm no broken links. - -## Link adaptation for `internet-identity-spec.md` and `verifiable-credentials-spec.md` - -Both are handled automatically by `npm run sync:ii-spec`. If a new unhandled link pattern appears, the script exits with a warning — add it to `linkMap` (ii-spec) or `vcLinkMap` (vc-spec) in `scripts/sync-ii-spec.mjs`, then re-run. Use `grep -r "{#}" docs/references/ic-interface-spec/` to find which file owns a given anchor. - -## Shallow clone resolution - -If a shallow clone can't resolve a pinned commit: -```bash -git -C .sources/ fetch --unshallow -git -C .sources/ checkout -``` diff --git a/.agents/upstream-tracking.md b/.agents/upstream-tracking.md new file mode 100644 index 00000000..03b24c00 --- /dev/null +++ b/.agents/upstream-tracking.md @@ -0,0 +1,184 @@ +# Upstream tracking + +How this repo stays current with the projects it documents. Replaces the old +`submodule-bumping.md`, which assumed every upstream was a submodule. + +Upstream repos fall into two groups, and the group decides the procedure. + +| | Vendored (submodule) | Watched (not vendored) | +|---|---|---| +| Which | `motoko`, `internetidentity`, `examples`, `icskills`, `dotskills` | everything in `.sources/upstream.json` | +| Why | their bytes reach the built site or the agent workflow | nothing they contain is published | +| Pin lives in | git (`.gitmodules`) plus a label in `.sources/VERSIONS` | `pinned` in `.sources/upstream.json` | +| Moved by | a bump PR (below) | a bump PR triggered by an **Upstream release check** issue | +| Verified against | the checked-out working tree | the pinned ref, fetched on demand | + +## Why only five are vendored + +A submodule is justified when the repo's content is part of the build or the +agent workflow, because then it has to be on disk anyway: + +- `motoko` — synced into `docs/languages/motoko/` **and** resolved at build time + by 52 `` file includes. +- `internetidentity` — `scripts/sync-ii-spec.mjs` generates two reference pages + from it. +- `examples` — `plugins/remark-snippet.mjs` extracts `snippet=` code at build + time. Only one page uses it today; more are planned. +- `icskills` — 16 of the 17 symlinks in `.agents/skills/` point into it. +- `dotskills` — the 17th (`technical-documentation`). + +Every other repo was only ever read to check a fact. Vendoring those meant a +recurring bump PR per repo with nothing to show for it, which is how several of +them ended up years of releases behind without anyone noticing. + +## Watched repos + +### Verifying a fact against a watched repo + +Read the file at the **pinned** ref, not at `main`: + +```bash +# The pinned ref for each repo is in .sources/upstream.json +curl -sL https://raw.githubusercontent.com/dfinity/icp-cli/v1.1.0/docs/reference/cli.md +``` + +Use `raw.githubusercontent.com`, not `gh api .../contents/...`: the API returns +base64 that gets truncated by the CLI wrapper, and a truncated flag reference is +worse than none because it looks complete. + +Verify against the pinned ref rather than `main` for two reasons. A review stays +reproducible, and a page cannot end up documenting a flag from a release whose +doc links have not been adapted yet. + +Many of these repos also publish authoritative docs (the `reference` field in +`upstream.json`: mops.one, docs.rs, js.icp.build, cli.internetcomputer.org). +Those are the right thing to *link* readers to. For verifying a claim, prefer the +pinned source file, because a published site always shows "latest". + +### When an upstream moves + +`.github/workflows/upstream-releases.yml` runs weekly and opens one issue per +repo that has moved past its pin, labelled `upstream--`. The issue +carries the ref delta, a compare link, a summary of what changed in the repo's +`verify` file when one is configured, and the `affects` note saying which pages +to re-check. + +To close one: + +1. Review the changes against the pages named in the issue. +2. Update those pages if anything they state has changed. +3. Set `pinned` to the new ref in `.sources/upstream.json`, in the same PR. + +If nothing in the docs is affected, bump the pin alone and say so in the PR. The +pin means "the docs were checked against this ref", so it should not move +without someone having looked. + +Run it locally at any time: + +```bash +node scripts/check-upstream-releases.mjs # all repos +node scripts/check-upstream-releases.mjs --repo dfinity/icp-cli +``` + +Exit code 0 means nothing moved, 1 means at least one did (bodies are written to +`.upstream-checks/`), 2 means a check errored. + +### Adding a repo to the watch list + +Add an entry to `.sources/upstream.json`. `track: "release"` needs a +`tagPattern` that matches only the tags that count as a release for the thing we +document — several of these repos tag per crate or per recipe, so an unanchored +pattern picks up the wrong series. `track: "commit"` is for repos with no tags at +all. Set `verify` when a single file carries the API surface we check against; +its diff becomes the issue's review payload. + +### `icp-cli`: link slug adaptation + +All CLI docs links use a versioned slug (`https://cli.internetcomputer.org/1.3/...`). +When `icp-cli` moves to a new minor: + +1. The slug is the `major.minor` of the release (`v1.3.0` → `1.3`). Confirm it is + live by opening the docs-site root, which redirects to the latest version. +2. Verify every linked path and anchor resolves at the new slug **before** + replacing. Check the live site, not a repo tree: that validates the published + URL, its trailing-slash behaviour, and the anchor. + ```bash + grep -roh "cli\.internetcomputer\.org/[0-9][.0-9]*/[^\"' )#]*" docs/ \ + | sed 's|cli\.internetcomputer\.org/[0-9][.0-9]*/||' | sort -u | grep -v "^$" \ + | while read -r p; do + code=$(curl -sSL -o /dev/null -w "%{http_code}" "https://cli.internetcomputer.org//$p") + [ "$code" = "200" ] || echo "MISSING ($code): $p" + done + ``` + For deep links, also confirm the anchor exists: + ```bash + curl -sL "https://cli.internetcomputer.org//reference/cli/" | grep -o 'id="icp-cycles"' + ``` +3. Replace the slug across all files (per-file loop, because GNU and BSD `sed` + disagree on `-i`): + ```bash + old=1.1; new=1.3 + grep -rl "cli.internetcomputer.org/${old}/" docs/ | while IFS= read -r f; do + sed -i.bak "s|cli.internetcomputer.org/${old}/|cli.internetcomputer.org/${new}/|g" "$f" && rm -f "$f.bak" + done + ``` +4. Update the slug named in the AGENTS.md linking rule. +5. Run `npm run build`. + +## Vendored submodules + +Only the project maintainer bumps submodule refs. + +### Determine the new ref + +- **Release-pinned** (`motoko`, `internetidentity`): `git ls-remote --tags origin`, + pin to the highest version tag's commit. +- **Branch-tracking** (`examples`, `icskills`, `dotskills`): fetch and check out + `origin/main` or `origin/master`. + +### Checklist + +1. Identify changes: `git -C .sources/ log --oneline ..` +2. Grep `docs/` for content derived from that submodule; update affected pages +3. Check open PRs — post a bump notice if the bump may affect pages under review +4. Update `.sources/VERSIONS` for the release-pinned ones +5. Note the bump in the PR description + +**Bump-notice PR comment:** +```bash +gh pr comment --body "$(cat <<'EOF' + +`` was bumped to ``. The following content on this PR may be outdated: +- [specific item and why] + +Please review before merging. +EOF +)" +``` + +### Per-submodule extra checks + +| Submodule | Extra checks on bump | +|---|---| +| `motoko` | **Automated** — `.github/workflows/sync-motoko.yml` opens a PR with the bump, the synced docs, and the VERSIONS update already committed. Review the content diff and merge. Also grep Motoko code blocks for changed API signatures. | +| `internetidentity` | Run `npm run sync:ii-spec`. If it warns about an unhandled link, add the pattern to `linkMap` (ii-spec) or `vcLinkMap` (vc-spec) in `scripts/sync-ii-spec.mjs`. Pin to the latest `release-YYYY-MM-DD` tag. `.github/workflows/sync-ii-spec.yml` runs this automatically; trigger it manually for an early sync. | +| `examples` | Verify every `snippet=` path and `#region` marker still resolves — a missing region is a build error. | +| `icskills` | Check for changed canister IDs or code patterns; the 16 skill symlinks in `.agents/skills/` follow the pin. | +| `dotskills` | Check whether the `technical-documentation` skill changed in ways that affect review criteria. | + +### Link adaptation for the synced specs + +`internet-identity-spec.md` and `verifiable-credentials-spec.md` are handled by +`npm run sync:ii-spec`. If a new unhandled link pattern appears the script exits +with a warning; add it to `linkMap` or `vcLinkMap` in `scripts/sync-ii-spec.mjs` +and re-run. Use `grep -r "{#}" docs/references/ic-interface-spec/` to +find which file owns an anchor. + +### Shallow clone resolution + +If a shallow clone cannot resolve a pinned commit: + +```bash +git -C .sources/ fetch --unshallow +git -C .sources/ checkout +``` diff --git a/.github/workflows/upstream-releases.yml b/.github/workflows/upstream-releases.yml new file mode 100644 index 00000000..c1df63fb --- /dev/null +++ b/.github/workflows/upstream-releases.yml @@ -0,0 +1,91 @@ +name: Upstream release check + +# Watches the repos in .sources/upstream.json (the ones that are deliberately +# not submodules) and opens an issue when one moves past the ref the docs are +# verified against. See .agents/upstream-tracking.md. + +on: + schedule: + - cron: '0 8 * * 1' # Weekly on Monday + workflow_dispatch: + inputs: + repo: + description: 'Check a single repo (owner/name), or leave empty for all' + required: false + +permissions: + contents: read + issues: write + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - uses: actions/setup-node@v4 + with: + node-version: '22' + + - name: Create GitHub App Token + uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 + id: app-token + with: + client-id: ${{ vars.PR_AUTOMATION_BOT_PUBLIC_CLIENT_ID }} + private-key: ${{ secrets.PR_AUTOMATION_BOT_PUBLIC_PRIVATE_KEY }} + + - name: Check upstream refs + id: check + run: | + ARGS="" + if [ -n "${{ github.event.inputs.repo }}" ]; then + ARGS="--repo ${{ github.event.inputs.repo }}" + fi + set +e + node scripts/check-upstream-releases.mjs $ARGS --out-dir .upstream-checks > moved.tsv + rc=$? + set -e + case $rc in + 0) echo "moved=false" >> $GITHUB_OUTPUT ;; + 1) echo "moved=true" >> $GITHUB_OUTPUT ;; + # 2 means at least one check errored. Any repo that did move still + # wrote its body, so surface those rather than failing the run + # outright, and mark the job failed at the end. + 2) echo "moved=true" >> $GITHUB_OUTPUT; echo "partial=true" >> $GITHUB_OUTPUT ;; + *) exit $rc ;; + esac + + - name: Open or refresh an issue per moved repo + if: steps.check.outputs.moved == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + # One issue per upstream, labelled `upstream-` so a later run can + # find and supersede it. An issue whose title already matches is left + # alone, so a repo that stays ahead does not generate weekly noise. + while IFS=$'\t' read -r slug title; do + [ -n "$slug" ] || continue + label="upstream-${slug}" + + gh label create "$label" --description "Upstream ref moved: ${slug}" --color ededed 2>/dev/null || true + + existing=$(gh issue list --label "$label" --state open --limit 1 --json number,title --jq 'first // empty') + if [ -n "$existing" ]; then + num=$(echo "$existing" | jq -r .number) + old=$(echo "$existing" | jq -r .title) + if [ "$old" = "$title" ]; then + echo "#$num already tracks '$title' — skipping" + continue + fi + gh issue close "$num" --comment "Superseded: a newer ref is available. Closing in favour of a fresh issue; target the new ref directly, there is no need to step through the one this issue tracked." + echo "Closed superseded #$num" + fi + + gh issue create --title "$title" --body-file ".upstream-checks/${slug}.md" --label "$label" + done < moved.tsv + + - name: Fail if a check errored + if: steps.check.outputs.partial == 'true' + run: | + echo "At least one upstream check failed; see the 'Check upstream refs' step." + exit 1 diff --git a/.gitignore b/.gitignore index 799f0e3f..4d78dbaf 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ public/docs-manifest.json # Skills install artifacts (npx skills add writes here; committed skills live in .agents/skills/) /skills/ skills-lock.json + +# Upstream release check output (scripts/check-upstream-releases.mjs) +.upstream-checks/ diff --git a/.gitmodules b/.gitmodules index 337c3e30..5aaf182a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,14 +1,3 @@ -[submodule ".sources/icp-cli"] - path = .sources/icp-cli - url = git@github.com:dfinity/icp-cli.git -[submodule ".sources/icp-cli-recipes"] - path = .sources/icp-cli-recipes - url = git@github.com:dfinity/icp-cli-recipes.git - branch = main -[submodule ".sources/icp-cli-templates"] - path = .sources/icp-cli-templates - url = git@github.com:dfinity/icp-cli-templates.git - branch = main [submodule ".sources/icskills"] path = .sources/icskills url = git@github.com:dfinity/icskills.git @@ -17,37 +6,12 @@ path = .sources/examples url = git@github.com:dfinity/examples.git branch = master -[submodule ".sources/icp-js-sdk-docs"] - path = .sources/icp-js-sdk-docs - url = git@github.com:dfinity/icp-js-sdk-docs.git - branch = main [submodule ".sources/motoko"] path = .sources/motoko url = git@github.com:caffeinelabs/motoko.git -[submodule ".sources/motoko-core"] - path = .sources/motoko-core - url = git@github.com:caffeinelabs/motoko-core.git -[submodule ".sources/cdk-rs"] - path = .sources/cdk-rs - url = git@github.com:dfinity/cdk-rs.git -[submodule ".sources/candid"] - path = .sources/candid - url = git@github.com:dfinity/candid.git -[submodule ".sources/response-verification"] - path = .sources/response-verification - url = git@github.com:dfinity/response-verification.git [submodule ".sources/dotskills"] path = .sources/dotskills url = git@github.com:vincentkoc/dotskills.git -[submodule ".sources/chain-fusion-signer"] - path = .sources/chain-fusion-signer - url = git@github.com:dfinity/chain-fusion-signer.git -[submodule ".sources/papi"] - path = .sources/papi - url = git@github.com:dfinity/papi.git -[submodule ".sources/ic-pub-key"] - path = .sources/ic-pub-key - url = git@github.com:dfinity/ic-pub-key.git [submodule ".sources/internetidentity"] path = .sources/internetidentity url = https://github.com/dfinity/internet-identity.git diff --git a/.sources/VERSIONS b/.sources/VERSIONS index a24a9b49..5958b5ff 100644 --- a/.sources/VERSIONS +++ b/.sources/VERSIONS @@ -1,8 +1,11 @@ -# Pinned submodule versions (release-pinned repos only) +# Pinned submodule versions # ------------------------------------------------------- -# Repos that track main/master are NOT listed here — they have no fixed release -# version. Those repos are: examples, icskills, dotskills, -# icp-cli-recipes, icp-cli-templates, icp-js-sdk-docs. +# This file covers the FIVE submodules under .sources/ — the repos whose bytes +# reach the built site or the agent workflow. Every other upstream repo is +# watched rather than vendored; its pin lives in .sources/upstream.json. +# +# Submodules that track a branch are NOT listed here (they have no fixed +# release version): examples, icskills, dotskills. # # internetidentity — uses date-based release tags (release-YYYY-MM-DD). # Only ii-spec.mdx, vc-spec.md, and internet_identity.did are synced. @@ -11,6 +14,10 @@ # The automated workflow (.github/workflows/sync-ii-spec.yml) finds the # latest release-YYYY-MM-DD tag and only opens a PR if those files changed. # +# motoko — the release tag commit may not resolve in a shallow clone. +# The merge commit of the release PR represents the same state and is used +# instead. This is expected behaviour with --depth 1 submodule init. +# # FORMAT: <7-char hash> # # The hash is the authoritative ref — it is what .gitmodules pins and what @@ -18,48 +25,18 @@ # Verify with: git -C .sources/ rev-parse HEAD # # ------------------------------------------------------- -# AGENTS — when bumping a release-pinned submodule: +# AGENTS — when bumping a submodule: # 1. Find the latest release tag: # git -C .sources/ ls-remote --tags origin # 2. Fetch and pin to that tag's commit (NOT the tip of main): # git -C .sources/ fetch origin refs/tags/ # git -C .sources/ checkout FETCH_HEAD # 3. Update the version label and hash in this file -# 4. Follow the full checklist in AGENTS.md "Bumping submodules" +# 4. Follow the full checklist in .agents/upstream-tracking.md # # Do NOT pin past the latest release — unreleased changes must not # appear in docs before they ship to users. # ------------------------------------------------------- -# REPO-SPECIFIC NOTES -# -# cdk-rs — monorepo with per-crate tags using different prefixes: -# ic-cdk: plain semver tags (0.19.0, 0.18.x, ...) -# ic-cdk-timers: timers-1.0.0, timers-0.12.x, ... -# ic-cdk-executor: executor-2.0.0, executor-1.0.x, ... -# Releases are usually batched: a single merge commit releases all crates -# together. Individual crate tags may point to slightly earlier commits within -# the release PR branch. Pin to the combined release merge commit for -# coherence, and list all crate versions explicitly below. -# When bumping: check if timers/executor also released alongside ic-cdk. -# -# candid — uses date-based release tags (no semver git tags exist). -# The candid crate, candid_parser, and didc tool each have their own semver -# in Cargo.toml. Check these after bumping: -# find .sources/candid -name "Cargo.toml" | xargs grep -l "^name = \"candid\"$" | xargs grep "^version" -# find .sources/candid -name "Cargo.toml" | xargs grep -l "name = \"didc\"" | xargs grep "^version" -# -# motoko — the release tag commit may not resolve in a shallow clone. -# The merge commit of the release PR represents the same state and is used -# instead. This is expected behaviour with --depth 1 submodule init. -# ------------------------------------------------------- -chain-fusion-signer v0.4.0 a18c52a -papi v0.1.1 168bc9d -ic-pub-key v1.0.1 f89fa55 -icp-cli v1.1.0 fe4af7c -motoko v1.14.1 58c3239 -motoko-core v2.4.0 cd37dbf -cdk-rs ic-cdk v0.20.1 / ic-cdk-timers v1.0.0 / ic-cdk-executor v2.0.0 317f55c -candid 2025-12-18 # candid v0.10.20, didc v0.5.4 2e4a2cf -response-verification v3.1.0 18c5a37 -internetidentity release-2026-08-21 4c934d1f +motoko v1.14.1 58c3239 +internetidentity release-2026-08-21 4c934d1f diff --git a/.sources/candid b/.sources/candid deleted file mode 160000 index 2e4a2cf6..00000000 --- a/.sources/candid +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2e4a2cf6a66593e2cb3289535899eb232635bef2 diff --git a/.sources/cdk-rs b/.sources/cdk-rs deleted file mode 160000 index 317f55c0..00000000 --- a/.sources/cdk-rs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 317f55c08e8f4384552d04c97f91e122b84abf1a diff --git a/.sources/chain-fusion-signer b/.sources/chain-fusion-signer deleted file mode 160000 index a18c52ae..00000000 --- a/.sources/chain-fusion-signer +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a18c52ae389eb6625a29433e5f41f1b146290d68 diff --git a/.sources/ic-pub-key b/.sources/ic-pub-key deleted file mode 160000 index f89fa550..00000000 --- a/.sources/ic-pub-key +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f89fa550c1f5619d8af546aea235841eed702e51 diff --git a/.sources/icp-cli b/.sources/icp-cli deleted file mode 160000 index fe4af7c5..00000000 --- a/.sources/icp-cli +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fe4af7c5600f14aab3c3f0ded7cf9da4f288571f diff --git a/.sources/icp-cli-recipes b/.sources/icp-cli-recipes deleted file mode 160000 index 8dda3d08..00000000 --- a/.sources/icp-cli-recipes +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8dda3d080702c6b8df69aa5082bacd1b1fb1439b diff --git a/.sources/icp-cli-templates b/.sources/icp-cli-templates deleted file mode 160000 index 9f1e8194..00000000 --- a/.sources/icp-cli-templates +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9f1e81941db9742d07ddae76dd2ac50d3841711e diff --git a/.sources/icp-js-sdk-docs b/.sources/icp-js-sdk-docs deleted file mode 160000 index 17e4346a..00000000 --- a/.sources/icp-js-sdk-docs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 17e4346a456bf4f8a2f526c192fe0d6461158bff diff --git a/.sources/motoko-core b/.sources/motoko-core deleted file mode 160000 index cd37dbfb..00000000 --- a/.sources/motoko-core +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cd37dbfb6ae63552b1151324cf7e3ae461278b4c diff --git a/.sources/papi b/.sources/papi deleted file mode 160000 index 168bc9de..00000000 --- a/.sources/papi +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 168bc9de51d7439af13ec0d455663d9691accd4a diff --git a/.sources/response-verification b/.sources/response-verification deleted file mode 160000 index 18c5a37f..00000000 --- a/.sources/response-verification +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 18c5a37f58a9453d6332da38c6e6f5adcc72aac2 diff --git a/.sources/upstream.json b/.sources/upstream.json new file mode 100644 index 00000000..110d56e1 --- /dev/null +++ b/.sources/upstream.json @@ -0,0 +1,112 @@ +{ + "$comment": [ + "Upstream repos that are NOT vendored as submodules. They are watched instead:", + "scripts/check-upstream-releases.mjs compares `pinned` against the newest matching", + "tag, and .github/workflows/upstream-releases.yml opens an issue when they differ.", + "See AGENTS.md 'Source material' and .agents/upstream-tracking.md.", + "", + "`pinned` is the ref the docs were last verified against. Agents verify claims", + "against THIS ref rather than against main, so that a review is reproducible and a", + "page cannot document a flag from a release whose links we have not adapted yet.", + "Moving `pinned` is deliberate and comes with re-checking the affected pages.", + "", + "Fields:", + " repo owner/name on GitHub", + " pinned the ref docs are currently verified against", + " track 'release' (tags matching tagPattern) or 'commit' (repo has no tags)", + " tagPattern anchored JS regex selecting the tags that count as a release", + " verify file whose diff between the two refs is the review payload (optional)", + " affects what to re-check in docs/ when this moves", + " reference authoritative published docs for the topic, for readers and agents" + ], + "watched": [ + { + "repo": "dfinity/icp-cli", + "pinned": "v1.1.0", + "track": "release", + "tagPattern": "^v\\d+\\.\\d+\\.\\d+$", + "verify": "docs/reference/cli.md", + "affects": "Every `icp` command and flag in docs/. A new minor also moves the cli.internetcomputer.org link slug (AGENTS.md 'Linking rules'). NOTE: the doc links were moved to the 1.3 slug in #354, but the commands themselves were last verified against v1.1.0, so the first bump of this pin carries that verification pass.", + "reference": "https://cli.internetcomputer.org/" + }, + { + "repo": "dfinity/icp-cli-recipes", + "pinned": "static-site-v0.3.3", + "track": "release", + "tagPattern": "^(static-site|asset-canister|motoko|rust|prebuilt)-v\\d+\\.\\d+\\.\\d+$", + "affects": "Recipe versions in `icp.yaml` examples. Each recipe releases on its own tag, so the tag prefix says which one moved. NOTE: docs currently cite `@dfinity/asset-canister@v2.2.1` while v2.3.0 is out, and `@dfinity/static-site` is not documented at all yet (dfinity/certified-assets#124).", + "reference": "https://github.com/dfinity/icp-cli-recipes/releases" + }, + { + "repo": "dfinity/icp-cli-templates", + "pinned": "9f1e819", + "track": "commit", + "affects": "Template layout and the recipes templates ship with, referenced from getting-started/ and guides/frontends/. All frontend templates moved to `@dfinity/static-site` after this pin.", + "reference": "https://github.com/dfinity/icp-cli-templates" + }, + { + "repo": "dfinity/motoko-core", + "pinned": "v2.4.0", + "track": "release", + "tagPattern": "^v\\d+\\.\\d+\\.\\d+$", + "affects": "Every `mo:core` import and API signature in Motoko code blocks.", + "reference": "https://mops.one/core/docs" + }, + { + "repo": "dfinity/cdk-rs", + "pinned": "0.20.1", + "track": "release", + "tagPattern": "^\\d+\\.\\d+\\.\\d+$", + "affects": "`ic-cdk` APIs in Rust code blocks. `ic-cdk-timers` and `ic-cdk-executor` tag separately (`timers-*`, `executor-*`) and usually release in the same batch, so check whether they moved too. Bare semver tags are `ic-cdk` itself.", + "reference": "https://docs.rs/ic-cdk/latest/ic_cdk/" + }, + { + "repo": "dfinity/candid", + "pinned": "2025-12-18", + "track": "release", + "tagPattern": "^\\d{4}-\\d{2}-\\d{2}$", + "verify": "spec/Candid.md", + "affects": "references/candid-spec.md and the Candid type-mapping tables. The repo uses date tags; the candid, candid_parser, and didc crates carry their own semver in Cargo.toml.", + "reference": "https://github.com/dfinity/candid/blob/master/spec/Candid.md" + }, + { + "repo": "dfinity/response-verification", + "pinned": "3.1.0", + "track": "release", + "tagPattern": "^\\d+\\.\\d+\\.\\d+$", + "affects": "guides/frontends/certification.md and guides/backends/certified-variables.md: the `ic-http-certification`, `ic-asset-certification`, and `@dfinity/certificate-verification` APIs.", + "reference": "https://docs.rs/ic-http-certification/latest/ic_http_certification/" + }, + { + "repo": "dfinity/chain-fusion-signer", + "pinned": "v0.4.0", + "track": "release", + "tagPattern": "^v\\d+\\.\\d+\\.\\d+$", + "affects": "guides/chain-fusion/chain-fusion-signer.md: canister IDs, API methods, key derivation.", + "reference": "https://github.com/dfinity/chain-fusion-signer" + }, + { + "repo": "dfinity/papi", + "pinned": "v0.1.1", + "track": "release", + "tagPattern": "^v\\d+\\.\\d+\\.\\d+$", + "affects": "Payment interface and cycle cost model wherever PAPI is referenced.", + "reference": "https://github.com/dfinity/papi" + }, + { + "repo": "dfinity/ic-pub-key", + "pinned": "v1.0.1", + "track": "release", + "tagPattern": "^v\\d+\\.\\d+\\.\\d+$", + "affects": "guides/chain-fusion/offline-key-derivation.md: CLI flags and the library API.", + "reference": "https://github.com/dfinity/ic-pub-key" + }, + { + "repo": "dfinity/icp-js-sdk-docs", + "pinned": "17e4346", + "track": "commit", + "affects": "`@icp-sdk/core` and `@icp-sdk/canisters` APIs in JavaScript and TypeScript code blocks.", + "reference": "https://js.icp.build" + } + ] +} diff --git a/AGENTS.md b/AGENTS.md index ee60073e..7a5944f2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,12 +15,15 @@ Tasks come on-demand — no GitHub issue required unless proposing a structural Load `technical-documentation`, `icp-brand-voice`, and the relevant icskill before writing (see "Skills" for the topic-to-skill mapping). Then: -- Read relevant source material from `.sources/` to verify CLI commands, API signatures, and technical claims (see "Source material" below for what to consult per topic). -- **CLI commands:** verify every flag against `.sources/icp-cli/docs/reference/cli.md` — never guess syntax. +- Verify CLI commands, API signatures, and technical claims against the source material for that topic (see "Source material" below). Never verify from memory. +- **CLI commands:** verify every flag against `docs/reference/cli.md` in `dfinity/icp-cli` at the ref pinned in `.sources/upstream.json` — never guess syntax: + ```bash + curl -sL https://raw.githubusercontent.com/dfinity/icp-cli//docs/reference/cli.md + ``` - **Internal links:** run `ls ` before adding any link. Always use `.md` extension, even for `.mdx` targets. - **External URLs:** use the "Linking rules" table below. Verify any URL not in the table — do not guess. -- **Flag uncertainty:** add `` next to any claim you can't verify from `.sources/`. Never silently guess. -- **Do not invent command output** — copy from `.sources/` READMEs or test fixtures, or write ``. +- **Flag uncertainty:** add `` next to any claim you can't verify against a pinned source. Never silently guess. +- **Do not invent command output** — copy from an upstream README or test fixture, or write ``. - **`.md` → `.mdx` conversion:** if a page needs multi-language tabs, rename `.md` → `.mdx`, add `import { Tabs, TabItem } from '@astrojs/starlight/components';` after the frontmatter, and convert `` comments to `{/* */}`. Astro resolves `.md` links to `.mdx` files — no link updates needed. - For pages that closely track a specific upstream file, add at the bottom: ``. Skip for pages that draw from multiple sources or are fully original. - Follow the "Content rules" section below. @@ -47,7 +50,7 @@ Only when explicitly asked. Load `technical-documentation` and the relevant icsk *Mechanical checks:* 1. **Internal links** — `ls` every `[text](path.md)` target. Flag as broken only if neither `.md` nor `.mdx` exists. 2. **External URLs** — verify against the linking rules table below. -3. **CLI commands** — verify against `.sources/icp-cli/docs/reference/cli.md`. +3. **CLI commands** — verify against `docs/reference/cli.md` in `dfinity/icp-cli` at the pinned ref (see "Source material"). 4. **Frontmatter** — title and description present and consistent with body. 5. **Rules compliance** — no `dfx`, no `.mdx` without interactive components, relative `.md` links, `mo:core` not `mo:base`. @@ -55,7 +58,7 @@ Only when explicitly asked. Load `technical-documentation` and the relevant icsk 6. **Reader test** — does the opening deliver on the title's promise and stand alone without assumed prior context? 7. **Funnel** — orient → explain/instruct → next steps. Flag buried leads and pages that end without direction. 8. **Scanability** — can a developer get the gist from headings and bold text alone? -9. **Accuracy** — cross-check technical claims against `.sources/`. Flag anything wrong or outdated. +9. **Accuracy** — cross-check technical claims against the pinned source material. Flag anything wrong or outdated. 10. **Developer empathy** — does it address what a developer will actually struggle with? Post using this format: @@ -171,7 +174,7 @@ EOF - Link to `internetcomputer.org/docs/` (retired) or `learn.internetcomputer.org` (content is now in this repo under `docs/concepts/`) - Link to internal pages that don't exist — run `ls ` before linking. Links to `.mdx` files use `.md` extension. - Link to an internal page without checking for a relevant section anchor — read the target page to find the most specific section that fits, then derive the anchor slug from its heading (lowercase, spaces → `-`, special chars stripped). -- Link to `https://cli.internetcomputer.org/` bare root — use the versioned path. Current slug: `1.1` (the `major.minor` of the pinned icp-cli release). Do not trust `.sources/icp-cli/docs-site/versions.json` here: at a release tag it still lists the *previous* slug, because the docs-site version bump lands as a follow-up commit after the tag. Confirm the live slug at the docs-site root (it redirects to the latest version). +- Link to `https://cli.internetcomputer.org/` bare root — use the versioned path. Current slug: `1.3`. It is the `major.minor` of the latest icp-cli release, and the docs always track the latest. Do not read it from the repo's `docs-site/versions.json`: at a release tag that file still lists the *previous* slug, because the docs-site version bump lands as a follow-up commit after the tag. Confirm the live slug at the docs-site root (it redirects to the latest version), and see `.agents/upstream-tracking.md` for the full slug-bump procedure. - Link externally when an internal page exists — check `docs/` first - Write em-dashes (`—`) or use `--` as prose punctuation — use colon, semicolon, or parentheses instead. (`--` is fine inside code blocks as a CLI flag or comment.) - Rename Candid field names, management canister API identifiers, or example repo names — these are protocol-level identifiers @@ -182,10 +185,10 @@ EOF - `docs/` — All documentation (`.md` by default). `src/content/docs/` symlinks here. - `docs/languages/motoko/` — Auto-synced from `caffeinelabs/motoko` (do not edit directly) -- `docs/guides/tools/migrating-from-dfx.md` — Synced from `dfinity/icp-cli` (do not edit directly) -- `.sources/` — Pinned source submodules (read-only) +- `docs/references/internet-identity-spec.md`, `docs/references/verifiable-credentials-spec.md` — Synced from `dfinity/internet-identity` (do not edit directly) +- `.sources/` — Vendored submodules, read-only, plus `upstream.json` (watched repos) and `VERSIONS` (submodule pins) - `.agents/skills/` — Agent skill files. Run `git submodule update --init --depth 1` if broken. -- `.agents/submodule-bumping.md` — Procedures for bumping source submodules (maintainer use) +- `.agents/upstream-tracking.md` — How upstreams are tracked and bumped (maintainer use) ## Project structure @@ -210,36 +213,60 @@ docs/ ## Source material -All source repos are pinned as git submodules under `.sources/`. Always read from `.sources/` — never from local clones, `gh api`, or training data. Do not modify `.sources/` — it is read-only. +Upstream repos are tracked two ways. Which one decides where you read from. + +**Vendored as submodules** — five repos, because their bytes reach the built site +or the agent workflow. Read them from disk; do not edit them. ```bash git submodule update --init --depth 1 # do NOT use --recursive ``` -Consult the relevant submodule when writing or reviewing: - | Topic | Submodule | |-------|-----------| -| CLI commands and flags | `.sources/icp-cli/` — verify against `.sources/icp-cli/docs/reference/cli.md` | -| Motoko APIs (`mo:core`) | `.sources/motoko-core/` | -| Motoko compiler / syntax | `.sources/motoko/` | -| Rust CDK (`ic-cdk`, `ic-cdk-timers`) | `.sources/cdk-rs/` | -| JavaScript / TypeScript SDK | `.sources/icp-js-sdk-docs/` — unzip: `unzip -p .sources/icp-js-sdk-docs/public//latest.zip ` | -| Code examples | `.sources/examples/` | -| CLI recipes | `.sources/icp-cli-recipes/` | -| Project templates | `.sources/icp-cli-templates/` | +| Motoko compiler / syntax, synced Motoko pages | `.sources/motoko/` | +| Internet Identity and VC specs | `.sources/internetidentity/` | +| Code examples (`snippet=`, ``) | `.sources/examples/` | | Canister IDs and skill files | `.sources/icskills/` | -| Candid spec | `.sources/candid/` | -| Certified variables | `.sources/response-verification/` | -| Chain Fusion Signer | `.sources/chain-fusion-signer/` | -| PAPI (payment API) | `.sources/papi/` | -| `@dfinity/ic-pub-key` CLI | `.sources/ic-pub-key/` | -| Internet Identity spec | `.sources/internetidentity/` | | Technical documentation skill | `.sources/dotskills/` | -For current pinned versions, see `.sources/VERSIONS`. +Pinned versions: [`.sources/VERSIONS`](.sources/VERSIONS). + +**Watched, not vendored** — everything else. Nothing they contain is published, +so the repo records a pinned ref instead of a copy, and a weekly workflow opens +an issue when one moves. Read the file you need at the **pinned ref** in +[`.sources/upstream.json`](.sources/upstream.json): + +```bash +curl -sL https://raw.githubusercontent.com/// +``` -For submodule bump procedures, see [`.agents/submodule-bumping.md`](.agents/submodule-bumping.md). +Use `raw.githubusercontent.com`, not `gh api .../contents/...`: the API returns +base64 that gets truncated, and a truncated reference is worse than none because +it looks complete. Read the pinned ref rather than `main`, so a review is +reproducible and a page cannot document something from a release whose links we +have not adapted yet. + +| Topic | Repo | Verify against | +|-------|------|----------------| +| CLI commands and flags | `dfinity/icp-cli` | `docs/reference/cli.md` | +| CLI recipes | `dfinity/icp-cli-recipes` | `recipes//` | +| Project templates | `dfinity/icp-cli-templates` | the template's `icp.yaml` | +| Motoko APIs (`mo:core`) | `dfinity/motoko-core` | `src/` | +| Rust CDK (`ic-cdk`, `ic-cdk-timers`) | `dfinity/cdk-rs` | the crate's `src/` and `CHANGELOG.md` | +| JavaScript / TypeScript SDK | `dfinity/icp-js-sdk-docs` | `public//latest.zip` | +| Candid spec | `dfinity/candid` | `spec/Candid.md` | +| Certified variables | `dfinity/response-verification` | `packages//README.md` | +| Chain Fusion Signer | `dfinity/chain-fusion-signer` | the canister's `.did` | +| PAPI (payment API) | `dfinity/papi` | `README.md` | +| `@dfinity/ic-pub-key` CLI | `dfinity/ic-pub-key` | `README.md` | + +Several of these also publish authoritative docs (the `reference` field in +`upstream.json`). Those are what to **link readers to**; for verifying a claim, +prefer the pinned source file, since a published site always shows "latest". + +For the tracking and bump procedures, see +[`.agents/upstream-tracking.md`](.agents/upstream-tracking.md). ## Skills diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fa9e619a..b2f0192a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,11 +102,12 @@ The build generates `/llms.txt` and per-page `.md` endpoints from your content. ## Source material -`.sources/` contains pinned git submodules that agents use as ground truth when writing and reviewing content — CLI references, API signatures, skill files, and code examples. +Upstream repos are tracked two ways, because most of them are only ever read to check a fact. -**Do not edit files in `.sources/` directly.** They are read-only references; changes go to the upstream repos. +- **Vendored as submodules** (`.sources/motoko`, `internetidentity`, `examples`, `icskills`, `dotskills`) — five repos whose content reaches the built site or the agent workflow. **Do not edit files in `.sources/` directly**; they are read-only, and changes go to the upstream repo. Pins are in [`.sources/VERSIONS`](.sources/VERSIONS). +- **Watched, not vendored** — everything else, listed in [`.sources/upstream.json`](.sources/upstream.json) with the ref the docs are verified against. A weekly workflow opens an issue when one of them ships something newer. -Current pinned release versions are in [`.sources/VERSIONS`](.sources/VERSIONS). Bumping a submodule is a maintainer task — follow [`.agents/submodule-bumping.md`](.agents/submodule-bumping.md) for the full procedure. +Bumping either is a maintainer task — follow [`.agents/upstream-tracking.md`](.agents/upstream-tracking.md) for the procedure. ## Synced content diff --git a/package.json b/package.json index f310b9a6..9b625ea7 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "astro": "astro", "validate": "node scripts/validate.js --all", "sync:motoko": "./scripts/sync-motoko.sh", - "sync:ii-spec": "./scripts/sync-ii-spec.mjs" + "sync:ii-spec": "./scripts/sync-ii-spec.mjs", + "check:upstream": "node scripts/check-upstream-releases.mjs" }, "dependencies": { "@astrojs/markdown-remark": "^7.2.2", diff --git a/scripts/check-upstream-releases.mjs b/scripts/check-upstream-releases.mjs new file mode 100644 index 00000000..a22b7c5a --- /dev/null +++ b/scripts/check-upstream-releases.mjs @@ -0,0 +1,248 @@ +#!/usr/bin/env node +/** + * Check the upstream repos listed in `.sources/upstream.json` for a newer ref + * than the one the docs are pinned to, and write an issue body for each. + * + * These repos are deliberately NOT submodules: nothing they contain is + * published, so the repo watches their releases instead of vendoring them. + * See AGENTS.md "Source material" and .agents/upstream-tracking.md. + * + * Usage: + * node scripts/check-upstream-releases.mjs [--repo owner/name] [--out-dir DIR] + * + * Writes DIR/.md per repo that moved and prints one `slugtitle` + * line per issue to stdout, for the workflow to consume. Exit codes: + * 0 nothing moved + * 1 at least one repo moved (bodies written) + * 2 a check failed (network, bad config) + * + * Needs `git` on PATH and network access. No GitHub token required: tags come + * from `git ls-remote` and file diffs from raw.githubusercontent.com. + */ + +import { execFileSync } from 'node:child_process'; +import { readFileSync, writeFileSync, mkdirSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = resolve(fileURLToPath(import.meta.url), '..', '..'); +const CONFIG = join(ROOT, '.sources', 'upstream.json'); + +const args = process.argv.slice(2); +const only = argValue('--repo'); +const outDir = argValue('--out-dir') ?? join(ROOT, '.upstream-checks'); + +function argValue(flag) { + const i = args.indexOf(flag); + return i === -1 ? undefined : args[i + 1]; +} + +/** + * Compare two refs the way a human reads a version: numeric runs compare as + * numbers, everything else as text. Needed because these repos tag in at least + * four shapes (`v1.3.0`, `0.20.1`, `static-site-v0.3.3`, `2025-12-18`) and none + * of them sort correctly as plain strings — lexicographically `0.9.4` beats + * `0.20.1`, which would silently report a downgrade as the latest release. + */ +function compareRefs(a, b) { + const split = (s) => s.split(/(\d+)/).filter((p) => p !== ''); + const pa = split(a); + const pb = split(b); + for (let i = 0; i < Math.max(pa.length, pb.length); i++) { + const x = pa[i]; + const y = pb[i]; + if (x === undefined) return -1; + if (y === undefined) return 1; + const nx = /^\d+$/.test(x); + const ny = /^\d+$/.test(y); + if (nx && ny) { + const d = Number(x) - Number(y); + if (d !== 0) return d; + } else if (x !== y) { + return x < y ? -1 : 1; + } + } + return 0; +} + +function git(...argv) { + return execFileSync('git', argv, { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }); +} + +function remoteTags(repo) { + const out = git('ls-remote', '--tags', `https://github.com/${repo}.git`); + return out + .split('\n') + .map((l) => l.split('refs/tags/')[1]) + .filter((t) => t && !t.endsWith('^{}')); +} + +function defaultBranchHead(repo) { + // `ls-remote --symref HEAD` reports both the branch name and its sha, so the + // default branch does not have to be hardcoded per repo. + const out = git('ls-remote', '--symref', `https://github.com/${repo}.git`, 'HEAD'); + const branch = out.match(/^ref:\s+refs\/heads\/(\S+)\s+HEAD$/m)?.[1] ?? 'HEAD'; + const sha = out.match(/^([0-9a-f]{40})\s+HEAD$/m)?.[1]; + return { branch, sha }; +} + +async function fetchFile(repo, ref, path) { + const url = `https://raw.githubusercontent.com/${repo}/${ref}/${path}`; + const res = await fetch(url); + if (!res.ok) return null; + return res.text(); +} + +/** Unified-diff-free summary: which headings and how many lines changed. */ +function summarizeChange(oldText, newText) { + const oldLines = oldText.split('\n'); + const newLines = newText.split('\n'); + const oldSet = new Set(oldLines); + const newSet = new Set(newLines); + const added = newLines.filter((l) => l.trim() && !oldSet.has(l)); + const removed = oldLines.filter((l) => l.trim() && !newSet.has(l)); + const heading = (l) => /^#{1,6}\s/.test(l); + return { + addedHeadings: added.filter(heading), + removedHeadings: removed.filter(heading), + addedCount: added.length, + removedCount: removed.length, + }; +} + +function slugFor(repo) { + return repo.replace('/', '-'); +} + +async function checkOne(entry) { + const { repo, pinned, track, tagPattern, verify, affects, reference } = entry; + + let latest; + let kind; + if (track === 'release') { + if (!tagPattern) throw new Error(`${repo}: track "release" needs a tagPattern`); + const re = new RegExp(tagPattern); + const matching = remoteTags(repo).filter((t) => re.test(t)); + if (matching.length === 0) { + throw new Error(`${repo}: no tags match ${tagPattern}`); + } + matching.sort(compareRefs); + latest = matching[matching.length - 1]; + kind = 'release'; + } else if (track === 'commit') { + const { branch, sha } = defaultBranchHead(repo); + if (!sha) throw new Error(`${repo}: could not resolve HEAD`); + latest = sha.slice(0, 7); + kind = `commit on ${branch}`; + } else { + throw new Error(`${repo}: unknown track "${track}"`); + } + + if (compareRefs(latest, pinned) <= 0) return null; + + const title = `chore: upstream ${repo} moved to ${latest}`; + const lines = [ + `\`${repo}\` has moved past the ref the docs are verified against.`, + '', + `| | |`, + `|---|---|`, + `| Pinned in \`.sources/upstream.json\` | \`${pinned}\` |`, + `| Latest ${kind} | \`${latest}\` |`, + `| Compare | https://github.com/${repo}/compare/${pinned}...${latest} |`, + ]; + if (reference) lines.push(`| Published reference | ${reference} |`); + lines.push(''); + lines.push('## What to re-check'); + lines.push(''); + lines.push(affects ?? 'No notes recorded for this repo.'); + + if (verify) { + const [oldText, newText] = await Promise.all([ + fetchFile(repo, pinned, verify), + fetchFile(repo, latest, verify), + ]); + lines.push(''); + lines.push(`## \`${verify}\``); + lines.push(''); + if (!oldText || !newText) { + lines.push(`Could not fetch \`${verify}\` at both refs, so no diff summary. Compare manually:`); + lines.push(''); + lines.push('```bash'); + lines.push(`diff <(curl -sL https://raw.githubusercontent.com/${repo}/${pinned}/${verify}) \\`); + lines.push(` <(curl -sL https://raw.githubusercontent.com/${repo}/${latest}/${verify})`); + lines.push('```'); + } else if (oldText === newText) { + lines.push('Unchanged between the two refs.'); + } else { + const s = summarizeChange(oldText, newText); + lines.push(`${s.addedCount} line(s) added, ${s.removedCount} removed.`); + if (s.addedHeadings.length) { + lines.push(''); + lines.push('New sections:'); + lines.push(''); + for (const h of s.addedHeadings) lines.push(`- \`${h.trim()}\``); + } + if (s.removedHeadings.length) { + lines.push(''); + lines.push('Removed sections:'); + lines.push(''); + for (const h of s.removedHeadings) lines.push(`- \`${h.trim()}\``); + } + lines.push(''); + lines.push('Full diff:'); + lines.push(''); + lines.push('```bash'); + lines.push(`diff <(curl -sL https://raw.githubusercontent.com/${repo}/${pinned}/${verify}) \\`); + lines.push(` <(curl -sL https://raw.githubusercontent.com/${repo}/${latest}/${verify})`); + lines.push('```'); + } + } + + lines.push(''); + lines.push('## How to close this'); + lines.push(''); + lines.push(`1. Review the changes above against the pages named under "What to re-check".`); + lines.push(`2. Update those pages if anything they state has changed.`); + lines.push(`3. Set \`pinned\` to \`${latest}\` in \`.sources/upstream.json\` in the same PR.`); + lines.push(''); + lines.push('If nothing in the docs is affected, bump the pin on its own and say so in the PR.'); + lines.push(''); + lines.push('Procedure: `.agents/upstream-tracking.md`'); + + return { slug: slugFor(repo), title, body: lines.join('\n') }; +} + +const config = JSON.parse(readFileSync(CONFIG, 'utf8')); +const entries = config.watched.filter((e) => !only || e.repo === only); +if (only && entries.length === 0) { + console.error(`No entry for --repo ${only} in ${CONFIG}`); + process.exit(2); +} + +const results = []; +let failed = false; +for (const entry of entries) { + try { + const r = await checkOne(entry); + if (r) { + results.push(r); + console.error(`moved: ${entry.repo} ${entry.pinned} -> ${r.title.split(' ').pop()}`); + } else { + console.error(`current: ${entry.repo} ${entry.pinned}`); + } + } catch (e) { + failed = true; + console.error(`FAILED: ${entry.repo}: ${e.message}`); + } +} + +if (results.length > 0) { + mkdirSync(outDir, { recursive: true }); + for (const r of results) { + writeFileSync(join(outDir, `${r.slug}.md`), r.body + '\n'); + process.stdout.write(`${r.slug}\t${r.title}\n`); + } +} + +if (failed) process.exit(2); +process.exit(results.length > 0 ? 1 : 0); From 43057d87041735cb7f99e13d5cae6f19a4fe70ab Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Wed, 26 Aug 2026 14:40:18 +0200 Subject: [PATCH 02/11] infra: extend the upstream check to branch-tracking submodules examples, icskills, and dotskills had no notification when their pin fell behind: motoko and internetidentity are covered by their own sync workflows, and the release check only looked at the non-vendored repos. They are now in upstream.json as a vendored group whose pin is read from the gitlink, so git stays the single source of truth for it. icskills and dotskills are both currently behind. --- .agents/upstream-tracking.md | 32 +++++++---- .sources/upstream.json | 39 ++++++++++++-- AGENTS.md | 5 +- scripts/check-upstream-releases.mjs | 83 ++++++++++++++++++++++++++--- 4 files changed, 138 insertions(+), 21 deletions(-) diff --git a/.agents/upstream-tracking.md b/.agents/upstream-tracking.md index 03b24c00..7cd03db3 100644 --- a/.agents/upstream-tracking.md +++ b/.agents/upstream-tracking.md @@ -1,22 +1,25 @@ # Upstream tracking -How this repo stays current with the projects it documents. Replaces the old -`submodule-bumping.md`, which assumed every upstream was a submodule. +How this repo stays current with the projects it documents. Upstream repos fall into two groups, and the group decides the procedure. | | Vendored (submodule) | Watched (not vendored) | |---|---|---| -| Which | `motoko`, `internetidentity`, `examples`, `icskills`, `dotskills` | everything in `.sources/upstream.json` | +| Which | `motoko`, `internetidentity`, `examples`, `icskills`, `dotskills` | the `watched` array in `.sources/upstream.json` | | Why | their bytes reach the built site or the agent workflow | nothing they contain is published | -| Pin lives in | git (`.gitmodules`) plus a label in `.sources/VERSIONS` | `pinned` in `.sources/upstream.json` | -| Moved by | a bump PR (below) | a bump PR triggered by an **Upstream release check** issue | +| Pin lives in | git (the gitlink), plus a label in `.sources/VERSIONS` for the release-pinned ones | `pinned` in `.sources/upstream.json` | +| Moved by | a bump PR (below) | a bump PR, triggered by an **Upstream release check** issue | | Verified against | the checked-out working tree | the pinned ref, fetched on demand | +Every one of them is checked weekly. `motoko` and `internetidentity` have their +own sync workflows that open the bump PR directly; the rest are covered by +**Upstream release check**, which opens an issue. + ## Why only five are vendored -A submodule is justified when the repo's content is part of the build or the -agent workflow, because then it has to be on disk anyway: +A submodule is justified only when the repo's content is part of the build or +the agent workflow, because then it has to be on disk anyway: - `motoko` — synced into `docs/languages/motoko/` **and** resolved at build time by 52 `` file includes. @@ -27,9 +30,9 @@ agent workflow, because then it has to be on disk anyway: - `icskills` — 16 of the 17 symlinks in `.agents/skills/` point into it. - `dotskills` — the 17th (`technical-documentation`). -Every other repo was only ever read to check a fact. Vendoring those meant a -recurring bump PR per repo with nothing to show for it, which is how several of -them ended up years of releases behind without anyone noticing. +Everything else is only ever read to check a fact, which needs a pinned ref +rather than a copy. Add a submodule only if something in the build or the agent +workflow has to open its files. ## Watched repos @@ -129,6 +132,15 @@ When `icp-cli` moves to a new minor: Only the project maintainer bumps submodule refs. +`examples`, `icskills`, and `dotskills` track a branch and are checked by the +same **Upstream release check** workflow, which opens an issue when the gitlink +falls behind that branch. Their pin lives in git, so `upstream.json` records only +the branch to compare against and what a bump affects. + +`motoko` and `internetidentity` are not in `upstream.json`: `sync-motoko.yml` and +`sync-ii-spec.yml` already check for a new release, run the sync, and open the +bump PR with the result. + ### Determine the new ref - **Release-pinned** (`motoko`, `internetidentity`): `git ls-remote --tags origin`, diff --git a/.sources/upstream.json b/.sources/upstream.json index 110d56e1..8d7aab32 100644 --- a/.sources/upstream.json +++ b/.sources/upstream.json @@ -1,8 +1,18 @@ { "$comment": [ - "Upstream repos that are NOT vendored as submodules. They are watched instead:", - "scripts/check-upstream-releases.mjs compares `pinned` against the newest matching", - "tag, and .github/workflows/upstream-releases.yml opens an issue when they differ.", + "Every upstream this repo tracks, in two groups.", + "", + "`vendored` are submodules whose pin has no sync workflow of its own. Their pin", + "is the gitlink in git, not a field here, so it has one source of truth; the entry", + "records which branch to compare against and what a bump affects. `motoko` and", + "`internetidentity` are absent on purpose: sync-motoko.yml and sync-ii-spec.yml", + "already check for a new release and open the bump PR themselves.", + "", + "`watched` are repos that are not vendored at all. Nothing they contain is", + "published, so this file records the ref the docs are verified against.", + "", + "scripts/check-upstream-releases.mjs checks both groups and", + ".github/workflows/upstream-releases.yml opens an issue per repo that moved.", "See AGENTS.md 'Source material' and .agents/upstream-tracking.md.", "", "`pinned` is the ref the docs were last verified against. Agents verify claims", @@ -19,6 +29,29 @@ " affects what to re-check in docs/ when this moves", " reference authoritative published docs for the topic, for readers and agents" ], + "vendored": [ + { + "path": ".sources/examples", + "repo": "dfinity/examples", + "track": "branch", + "branch": "master", + "affects": "Every `snippet=` path and `#region` marker resolved at build time by plugins/remark-snippet.mjs. A renamed region is a build error, so verify each one after bumping." + }, + { + "path": ".sources/icskills", + "repo": "dfinity/icskills", + "track": "branch", + "branch": "main", + "affects": "The 16 skill symlinks in .agents/skills/ follow this pin, so a bump changes the guidance agents load. Check for changed canister IDs and code patterns." + }, + { + "path": ".sources/dotskills", + "repo": "vincentkoc/dotskills", + "track": "branch", + "branch": "main", + "affects": "The `technical-documentation` skill in .agents/skills/. Check whether the review criteria changed." + } + ], "watched": [ { "repo": "dfinity/icp-cli", diff --git a/AGENTS.md b/AGENTS.md index 7a5944f2..d9c69d7c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -230,7 +230,10 @@ git submodule update --init --depth 1 # do NOT use --recursive | Canister IDs and skill files | `.sources/icskills/` | | Technical documentation skill | `.sources/dotskills/` | -Pinned versions: [`.sources/VERSIONS`](.sources/VERSIONS). +Pinned versions: [`.sources/VERSIONS`](.sources/VERSIONS). `motoko` and +`internetidentity` are release-checked and synced by their own workflows; +`examples`, `icskills`, and `dotskills` track a branch and are checked by the +weekly **Upstream release check**. **Watched, not vendored** — everything else. Nothing they contain is published, so the repo records a pinned ref instead of a copy, and a weekly workflow opens diff --git a/scripts/check-upstream-releases.mjs b/scripts/check-upstream-releases.mjs index a22b7c5a..5bae348e 100644 --- a/scripts/check-upstream-releases.mjs +++ b/scripts/check-upstream-releases.mjs @@ -1,10 +1,11 @@ #!/usr/bin/env node /** - * Check the upstream repos listed in `.sources/upstream.json` for a newer ref - * than the one the docs are pinned to, and write an issue body for each. + * Check every upstream in `.sources/upstream.json` for a newer ref than the one + * the docs are pinned to, and write an issue body for each that moved. * - * These repos are deliberately NOT submodules: nothing they contain is - * published, so the repo watches their releases instead of vendoring them. + * Covers both groups in that file: `vendored` submodules whose pin has no sync + * workflow of its own (their pin is read from the gitlink, so git stays the + * single source of truth), and `watched` repos that are not vendored at all. * See AGENTS.md "Source material" and .agents/upstream-tracking.md. * * Usage: @@ -86,6 +87,18 @@ function defaultBranchHead(repo) { return { branch, sha }; } +function gitlinkSha(path) { + // The committed submodule pointer, readable without initializing the + // submodule, so this works on a bare checkout in CI. + const out = git('ls-tree', 'HEAD', path); + return out.match(/^\d+ commit ([0-9a-f]{40})\t/)?.[1]; +} + +function branchHead(repo, branch) { + const out = git('ls-remote', `https://github.com/${repo}.git`, `refs/heads/${branch}`); + return out.match(/^([0-9a-f]{40})/)?.[1]; +} + async function fetchFile(repo, ref, path) { const url = `https://raw.githubusercontent.com/${repo}/${ref}/${path}`; const res = await fetch(url); @@ -114,6 +127,45 @@ function slugFor(repo) { return repo.replace('/', '-'); } +function checkVendored(entry) { + const { path, repo, branch, affects } = entry; + const pinnedSha = gitlinkSha(path); + if (!pinnedSha) throw new Error(`${path}: not a submodule in this commit`); + const headSha = branchHead(repo, branch); + if (!headSha) throw new Error(`${repo}: no branch ${branch}`); + if (pinnedSha === headSha) return null; + + const pinned = pinnedSha.slice(0, 7); + const latest = headSha.slice(0, 7); + const name = path.replace(/^\.sources\//, ''); + const body = [ + `The \`${path}\` submodule is behind \`${repo}@${branch}\`.`, + '', + '| | |', + '|---|---|', + `| Pinned (gitlink) | \`${pinned}\` |`, + `| Branch head | \`${latest}\` |`, + `| Compare | https://github.com/${repo}/compare/${pinned}...${latest} |`, + '', + '## What to re-check', + '', + affects ?? 'No notes recorded for this submodule.', + '', + '## How to close this', + '', + '```bash', + `git -C ${path} fetch origin ${branch}`, + `git -C ${path} checkout ${latest}`, + '```', + '', + 'Then work through the submodule checklist and commit the new pointer.', + '', + 'Procedure: `.agents/upstream-tracking.md`', + ].join('\n'); + + return { slug: `submodule-${name}`, title: `chore: submodule ${name} is behind ${repo}@${branch}`, body }; +} + async function checkOne(entry) { const { repo, pinned, track, tagPattern, verify, affects, reference } = entry; @@ -213,15 +265,32 @@ async function checkOne(entry) { } const config = JSON.parse(readFileSync(CONFIG, 'utf8')); -const entries = config.watched.filter((e) => !only || e.repo === only); -if (only && entries.length === 0) { +const vendored = (config.vendored ?? []).filter((e) => !only || e.repo === only); +const watched = (config.watched ?? []).filter((e) => !only || e.repo === only); +if (only && vendored.length + watched.length === 0) { console.error(`No entry for --repo ${only} in ${CONFIG}`); process.exit(2); } const results = []; let failed = false; -for (const entry of entries) { + +for (const entry of vendored) { + try { + const r = checkVendored(entry); + if (r) { + results.push(r); + console.error(`behind: ${entry.path} -> ${entry.repo}@${entry.branch}`); + } else { + console.error(`current: ${entry.path}`); + } + } catch (e) { + failed = true; + console.error(`FAILED: ${entry.path}: ${e.message}`); + } +} + +for (const entry of watched) { try { const r = await checkOne(entry); if (r) { From dbcdfee284c7a7e2892ba668c21fc91c3cafe53f Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Wed, 26 Aug 2026 15:42:25 +0200 Subject: [PATCH 03/11] infra: mirror IC skills from the registry, vendor technical-documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skills came from two submodules through a double symlink chain (.claude/skills -> .agents/skills -> .sources//skills). They now come from skills.internetcomputer.org via a SessionStart hook, matching dfinity/examples: a differential sync keyed on each skill's published hash, offline-safe, and it only prunes skills it installed itself. That drops the icskills submodule, the symlink farm, and a watcher entry, and raises coverage from 16 skills to 29 — including static-site, custom-domains, encrypted-maps, service-discoverability, and mops-cli. It also corrects two stale names in the AGENTS.md table: asset-canister no longer exists (static-site covers it, including the legacy recipe and migration) and vetkd is now vetkeys. technical-documentation is not on the IC registry. It changed once since the pin and only to switch its license field to MIT, so its 11 text files are vendored next to the two brand skills that were already committed, and the dotskills submodule goes too. Three submodules remain: motoko, internetidentity, examples. --- .agents/skills/asset-canister | 1 - .agents/skills/canister-security | 1 - .agents/skills/certified-variables | 1 - .agents/skills/ckbtc | 1 - .agents/skills/cycles-management | 1 - .agents/skills/evm-rpc | 1 - .agents/skills/https-outcalls | 1 - .agents/skills/ic-dashboard | 1 - .agents/skills/icp-cli | 1 - .agents/skills/icrc-ledger | 1 - .agents/skills/internet-identity | 1 - .agents/skills/multi-canister | 1 - .agents/skills/sns-launch | 1 - .agents/skills/stable-memory | 1 - .agents/skills/technical-documentation | 1 - .agents/skills/vetkd | 1 - .agents/skills/wallet-integration | 1 - .agents/upstream-tracking.md | 32 ++- .claude/settings.json | 12 + .claude/skills/asset-canister | 1 - .claude/skills/canister-security | 1 - .claude/skills/certified-variables | 1 - .claude/skills/ckbtc | 1 - .claude/skills/cycles-management | 1 - .claude/skills/evm-rpc | 1 - .claude/skills/https-outcalls | 1 - .claude/skills/ic-dashboard | 1 - .claude/skills/icp-cli | 1 - .claude/skills/icrc-ledger | 1 - .claude/skills/internet-identity | 1 - .claude/skills/multi-canister | 1 - .claude/skills/sns-launch | 1 - .claude/skills/stable-memory | 1 - .claude/skills/technical-documentation | 1 - .../skills/technical-documentation/SKILL.md | 75 +++++++ .../agents/docs-framework-agent.md | 29 +++ .../agents/governance-agent.md | 27 +++ .../agents/inventory-agent.md | 28 +++ .../agents/openai.yaml | 10 + .../agents/synthesis-agent.md | 25 +++ .../references/agent-and-contributing.md | 144 ++++++++++++ .../references/build.md | 113 ++++++++++ .../references/principles.md | 52 +++++ .../references/review.md | 117 ++++++++++ .../references/tooling.md | 32 +++ .claude/skills/vetkd | 1 - .claude/skills/wallet-integration | 1 - .claude/sync-ic-skills.sh | 206 ++++++++++++++++++ .gitignore | 8 +- .gitmodules | 7 - .sources/VERSIONS | 8 +- .sources/dotskills | 1 - .sources/icskills | 1 - .sources/upstream.json | 14 -- AGENTS.md | 40 ++-- CONTRIBUTING.md | 4 +- 56 files changed, 924 insertions(+), 95 deletions(-) delete mode 120000 .agents/skills/asset-canister delete mode 120000 .agents/skills/canister-security delete mode 120000 .agents/skills/certified-variables delete mode 120000 .agents/skills/ckbtc delete mode 120000 .agents/skills/cycles-management delete mode 120000 .agents/skills/evm-rpc delete mode 120000 .agents/skills/https-outcalls delete mode 120000 .agents/skills/ic-dashboard delete mode 120000 .agents/skills/icp-cli delete mode 120000 .agents/skills/icrc-ledger delete mode 120000 .agents/skills/internet-identity delete mode 120000 .agents/skills/multi-canister delete mode 120000 .agents/skills/sns-launch delete mode 120000 .agents/skills/stable-memory delete mode 120000 .agents/skills/technical-documentation delete mode 120000 .agents/skills/vetkd delete mode 120000 .agents/skills/wallet-integration delete mode 120000 .claude/skills/asset-canister delete mode 120000 .claude/skills/canister-security delete mode 120000 .claude/skills/certified-variables delete mode 120000 .claude/skills/ckbtc delete mode 120000 .claude/skills/cycles-management delete mode 120000 .claude/skills/evm-rpc delete mode 120000 .claude/skills/https-outcalls delete mode 120000 .claude/skills/ic-dashboard delete mode 120000 .claude/skills/icp-cli delete mode 120000 .claude/skills/icrc-ledger delete mode 120000 .claude/skills/internet-identity delete mode 120000 .claude/skills/multi-canister delete mode 120000 .claude/skills/sns-launch delete mode 120000 .claude/skills/stable-memory delete mode 120000 .claude/skills/technical-documentation create mode 100644 .claude/skills/technical-documentation/SKILL.md create mode 100644 .claude/skills/technical-documentation/agents/docs-framework-agent.md create mode 100644 .claude/skills/technical-documentation/agents/governance-agent.md create mode 100644 .claude/skills/technical-documentation/agents/inventory-agent.md create mode 100644 .claude/skills/technical-documentation/agents/openai.yaml create mode 100644 .claude/skills/technical-documentation/agents/synthesis-agent.md create mode 100644 .claude/skills/technical-documentation/references/agent-and-contributing.md create mode 100644 .claude/skills/technical-documentation/references/build.md create mode 100644 .claude/skills/technical-documentation/references/principles.md create mode 100644 .claude/skills/technical-documentation/references/review.md create mode 100644 .claude/skills/technical-documentation/references/tooling.md delete mode 120000 .claude/skills/vetkd delete mode 120000 .claude/skills/wallet-integration create mode 100755 .claude/sync-ic-skills.sh delete mode 160000 .sources/dotskills delete mode 160000 .sources/icskills diff --git a/.agents/skills/asset-canister b/.agents/skills/asset-canister deleted file mode 120000 index d41dc471..00000000 --- a/.agents/skills/asset-canister +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/asset-canister \ No newline at end of file diff --git a/.agents/skills/canister-security b/.agents/skills/canister-security deleted file mode 120000 index d557d634..00000000 --- a/.agents/skills/canister-security +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/canister-security \ No newline at end of file diff --git a/.agents/skills/certified-variables b/.agents/skills/certified-variables deleted file mode 120000 index d96071ae..00000000 --- a/.agents/skills/certified-variables +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/certified-variables \ No newline at end of file diff --git a/.agents/skills/ckbtc b/.agents/skills/ckbtc deleted file mode 120000 index 8c94b4d5..00000000 --- a/.agents/skills/ckbtc +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/ckbtc \ No newline at end of file diff --git a/.agents/skills/cycles-management b/.agents/skills/cycles-management deleted file mode 120000 index d7cefc76..00000000 --- a/.agents/skills/cycles-management +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/cycles-management \ No newline at end of file diff --git a/.agents/skills/evm-rpc b/.agents/skills/evm-rpc deleted file mode 120000 index 577f324f..00000000 --- a/.agents/skills/evm-rpc +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/evm-rpc \ No newline at end of file diff --git a/.agents/skills/https-outcalls b/.agents/skills/https-outcalls deleted file mode 120000 index 40419bbd..00000000 --- a/.agents/skills/https-outcalls +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/https-outcalls \ No newline at end of file diff --git a/.agents/skills/ic-dashboard b/.agents/skills/ic-dashboard deleted file mode 120000 index eb367be0..00000000 --- a/.agents/skills/ic-dashboard +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/ic-dashboard \ No newline at end of file diff --git a/.agents/skills/icp-cli b/.agents/skills/icp-cli deleted file mode 120000 index 456cc8c4..00000000 --- a/.agents/skills/icp-cli +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/icp-cli \ No newline at end of file diff --git a/.agents/skills/icrc-ledger b/.agents/skills/icrc-ledger deleted file mode 120000 index f61c3fe6..00000000 --- a/.agents/skills/icrc-ledger +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/icrc-ledger \ No newline at end of file diff --git a/.agents/skills/internet-identity b/.agents/skills/internet-identity deleted file mode 120000 index 256a9080..00000000 --- a/.agents/skills/internet-identity +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/internet-identity \ No newline at end of file diff --git a/.agents/skills/multi-canister b/.agents/skills/multi-canister deleted file mode 120000 index 78c01fd8..00000000 --- a/.agents/skills/multi-canister +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/multi-canister \ No newline at end of file diff --git a/.agents/skills/sns-launch b/.agents/skills/sns-launch deleted file mode 120000 index 7d479745..00000000 --- a/.agents/skills/sns-launch +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/sns-launch \ No newline at end of file diff --git a/.agents/skills/stable-memory b/.agents/skills/stable-memory deleted file mode 120000 index 7a4d806a..00000000 --- a/.agents/skills/stable-memory +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/stable-memory \ No newline at end of file diff --git a/.agents/skills/technical-documentation b/.agents/skills/technical-documentation deleted file mode 120000 index cd3d9d72..00000000 --- a/.agents/skills/technical-documentation +++ /dev/null @@ -1 +0,0 @@ -../../.sources/dotskills/skills/technical-documentation \ No newline at end of file diff --git a/.agents/skills/vetkd b/.agents/skills/vetkd deleted file mode 120000 index 9608b171..00000000 --- a/.agents/skills/vetkd +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/vetkd \ No newline at end of file diff --git a/.agents/skills/wallet-integration b/.agents/skills/wallet-integration deleted file mode 120000 index 0a3615a2..00000000 --- a/.agents/skills/wallet-integration +++ /dev/null @@ -1 +0,0 @@ -../../.sources/icskills/skills/wallet-integration \ No newline at end of file diff --git a/.agents/upstream-tracking.md b/.agents/upstream-tracking.md index 7cd03db3..17306edc 100644 --- a/.agents/upstream-tracking.md +++ b/.agents/upstream-tracking.md @@ -6,7 +6,7 @@ Upstream repos fall into two groups, and the group decides the procedure. | | Vendored (submodule) | Watched (not vendored) | |---|---|---| -| Which | `motoko`, `internetidentity`, `examples`, `icskills`, `dotskills` | the `watched` array in `.sources/upstream.json` | +| Which | `motoko`, `internetidentity`, `examples` | the `watched` array in `.sources/upstream.json` | | Why | their bytes reach the built site or the agent workflow | nothing they contain is published | | Pin lives in | git (the gitlink), plus a label in `.sources/VERSIONS` for the release-pinned ones | `pinned` in `.sources/upstream.json` | | Moved by | a bump PR (below) | a bump PR, triggered by an **Upstream release check** issue | @@ -16,23 +16,22 @@ Every one of them is checked weekly. `motoko` and `internetidentity` have their own sync workflows that open the bump PR directly; the rest are covered by **Upstream release check**, which opens an issue. -## Why only five are vendored +## Why only three are vendored -A submodule is justified only when the repo's content is part of the build or -the agent workflow, because then it has to be on disk anyway: +A submodule is justified only when the repo's content is resolved during the +build, because then it has to be on disk at a known ref: - `motoko` — synced into `docs/languages/motoko/` **and** resolved at build time by 52 `` file includes. - `internetidentity` — `scripts/sync-ii-spec.mjs` generates two reference pages from it. - `examples` — `plugins/remark-snippet.mjs` extracts `snippet=` code at build - time. Only one page uses it today; more are planned. -- `icskills` — 16 of the 17 symlinks in `.agents/skills/` point into it. -- `dotskills` — the 17th (`technical-documentation`). + time. -Everything else is only ever read to check a fact, which needs a pinned ref -rather than a copy. Add a submodule only if something in the build or the agent -workflow has to open its files. +Everything else is read to check a fact, which needs a pinned ref rather than a +copy, or is a skill, which the session-start sync mirrors from +[skills.internetcomputer.org](https://skills.internetcomputer.org) (see AGENTS.md +"Skills"). Add a submodule only if the build has to open its files. ## Watched repos @@ -132,10 +131,10 @@ When `icp-cli` moves to a new minor: Only the project maintainer bumps submodule refs. -`examples`, `icskills`, and `dotskills` track a branch and are checked by the -same **Upstream release check** workflow, which opens an issue when the gitlink -falls behind that branch. Their pin lives in git, so `upstream.json` records only -the branch to compare against and what a bump affects. +`examples` tracks a branch and is checked by the same **Upstream release check** +workflow, which opens an issue when the gitlink falls behind that branch. Its pin +lives in git, so `upstream.json` records only the branch to compare against and +what a bump affects. `motoko` and `internetidentity` are not in `upstream.json`: `sync-motoko.yml` and `sync-ii-spec.yml` already check for a new release, run the sync, and open the @@ -145,8 +144,7 @@ bump PR with the result. - **Release-pinned** (`motoko`, `internetidentity`): `git ls-remote --tags origin`, pin to the highest version tag's commit. -- **Branch-tracking** (`examples`, `icskills`, `dotskills`): fetch and check out - `origin/main` or `origin/master`. +- **Branch-tracking** (`examples`): fetch and check out `origin/master`. ### Checklist @@ -175,8 +173,6 @@ EOF | `motoko` | **Automated** — `.github/workflows/sync-motoko.yml` opens a PR with the bump, the synced docs, and the VERSIONS update already committed. Review the content diff and merge. Also grep Motoko code blocks for changed API signatures. | | `internetidentity` | Run `npm run sync:ii-spec`. If it warns about an unhandled link, add the pattern to `linkMap` (ii-spec) or `vcLinkMap` (vc-spec) in `scripts/sync-ii-spec.mjs`. Pin to the latest `release-YYYY-MM-DD` tag. `.github/workflows/sync-ii-spec.yml` runs this automatically; trigger it manually for an early sync. | | `examples` | Verify every `snippet=` path and `#region` marker still resolves — a missing region is a build error. | -| `icskills` | Check for changed canister IDs or code patterns; the 16 skill symlinks in `.agents/skills/` follow the pin. | -| `dotskills` | Check whether the `technical-documentation` skill changed in ways that affect review criteria. | ### Link adaptation for the synced specs diff --git a/.claude/settings.json b/.claude/settings.json index 7773ff5a..85480250 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -21,5 +21,17 @@ "Bash(cargo audit*)", "Bash(cargo check*)" ] + }, + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "bash .claude/sync-ic-skills.sh" + } + ] + } + ] } } diff --git a/.claude/skills/asset-canister b/.claude/skills/asset-canister deleted file mode 120000 index 78d7fcbf..00000000 --- a/.claude/skills/asset-canister +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/asset-canister \ No newline at end of file diff --git a/.claude/skills/canister-security b/.claude/skills/canister-security deleted file mode 120000 index 9d10dec3..00000000 --- a/.claude/skills/canister-security +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/canister-security \ No newline at end of file diff --git a/.claude/skills/certified-variables b/.claude/skills/certified-variables deleted file mode 120000 index e4455894..00000000 --- a/.claude/skills/certified-variables +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/certified-variables \ No newline at end of file diff --git a/.claude/skills/ckbtc b/.claude/skills/ckbtc deleted file mode 120000 index 53e7b9f5..00000000 --- a/.claude/skills/ckbtc +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/ckbtc \ No newline at end of file diff --git a/.claude/skills/cycles-management b/.claude/skills/cycles-management deleted file mode 120000 index 540644fc..00000000 --- a/.claude/skills/cycles-management +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/cycles-management \ No newline at end of file diff --git a/.claude/skills/evm-rpc b/.claude/skills/evm-rpc deleted file mode 120000 index c11b1091..00000000 --- a/.claude/skills/evm-rpc +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/evm-rpc \ No newline at end of file diff --git a/.claude/skills/https-outcalls b/.claude/skills/https-outcalls deleted file mode 120000 index 1d79f76a..00000000 --- a/.claude/skills/https-outcalls +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/https-outcalls \ No newline at end of file diff --git a/.claude/skills/ic-dashboard b/.claude/skills/ic-dashboard deleted file mode 120000 index ce0db851..00000000 --- a/.claude/skills/ic-dashboard +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/ic-dashboard \ No newline at end of file diff --git a/.claude/skills/icp-cli b/.claude/skills/icp-cli deleted file mode 120000 index 70d89294..00000000 --- a/.claude/skills/icp-cli +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/icp-cli \ No newline at end of file diff --git a/.claude/skills/icrc-ledger b/.claude/skills/icrc-ledger deleted file mode 120000 index 842ea880..00000000 --- a/.claude/skills/icrc-ledger +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/icrc-ledger \ No newline at end of file diff --git a/.claude/skills/internet-identity b/.claude/skills/internet-identity deleted file mode 120000 index 1988f369..00000000 --- a/.claude/skills/internet-identity +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/internet-identity \ No newline at end of file diff --git a/.claude/skills/multi-canister b/.claude/skills/multi-canister deleted file mode 120000 index f40efd41..00000000 --- a/.claude/skills/multi-canister +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/multi-canister \ No newline at end of file diff --git a/.claude/skills/sns-launch b/.claude/skills/sns-launch deleted file mode 120000 index 43cd98bb..00000000 --- a/.claude/skills/sns-launch +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/sns-launch \ No newline at end of file diff --git a/.claude/skills/stable-memory b/.claude/skills/stable-memory deleted file mode 120000 index 933f8c13..00000000 --- a/.claude/skills/stable-memory +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/stable-memory \ No newline at end of file diff --git a/.claude/skills/technical-documentation b/.claude/skills/technical-documentation deleted file mode 120000 index a467b967..00000000 --- a/.claude/skills/technical-documentation +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/technical-documentation \ No newline at end of file diff --git a/.claude/skills/technical-documentation/SKILL.md b/.claude/skills/technical-documentation/SKILL.md new file mode 100644 index 00000000..686cee39 --- /dev/null +++ b/.claude/skills/technical-documentation/SKILL.md @@ -0,0 +1,75 @@ +--- +name: technical-documentation +description: Build and review high-quality technical docs as well as agent instruction files in your repository. +license: MIT +metadata: + source: "https://github.com/vincentkoc/dotskills" +--- + +# Technical Documentation + +## Purpose + +Produce and review technical documentation that is clear, actionable, and maintainable for both humans and agents, including contributor-governance files and agent instruction files. + +## When to use + +- Creating or overhauling docs in an existing product/codebase (brownfield). +- Building evergreen docs meant to stay accurate and reusable over time. +- Reviewing doc diffs for structure, clarity, and operational correctness. +- Running full-repo documentation audits that must include both governance files and product docs surfaces (`docs/`, `README*`, `.md/.mdx/.mdc`, Fern/Sphinx/Mintlify-style sources). +- Updating or reviewing AGENTS.md and/or CONTRIBUTING.md to keep agent and contributor workflows aligned with current repo practices. +- Improving repository onboarding/docs that include contribution instructions, issue templates, PR flow, and review gates. +- Designing governance documentation strategy for repos with alias instruction files (for example `CLAUDE.md`, `AGENT.md`, `.cursorrules`, `.cursor/rules/*`, `.agent/`, `.agents/`, `.pi/`) where `CLAUDE.md` is treated as a canonical policy source and `AGENTS.md` should be kept as compatibility alias if present. +- Diagnosing agent-file drift where teams had to prompt iteratively to surface missing files, broken commands, or policy conflicts. + +## Workflow + +1. Classify task: `build` or `review`; context: `brownfield` or `evergreen`. +2. Inventory full documentation scope early (governance + product docs): AGENTS/CONTRIBUTING/aliases plus docs directories, framework sources, and root/module READMEs. +3. Detect multilingual scope (README/docs in multiple languages) and define required parity level. +4. Read `references/agent-and-contributing.md` for agent instruction and `CONTRIBUTING.md` workflow rules (inventory, canonical/alias mapping, dual-mode balance, deliverable standards, and precedence/conflict handling). +5. Read `references/principles.md` for the governing ruleset (Matt Palmer & OpenAI). +6. For build tasks, follow `references/build.md`. +7. For review tasks, follow `references/review.md` and proactively detect issues without waiting for repeated prompts. +8. For complex or high-risk tasks (build or review), it is acceptable to run longer, deeper, and more exhaustive investigations when needed for confidence. +9. When available, use sub-agents for bounded parallel discovery/review work, then merge outputs into one coherent final deliverable. +10. Use `references/tooling.md` when platform/tooling choices affect recommendations. +11. Run a proactive issue sweep for both governance and docs-content surfaces, and fix high-confidence defects in the same pass unless explicitly asked for report-only mode. +12. In brownfield mode, prioritize compatibility with current docs IA, tooling, and release state. +13. In evergreen mode, prioritize timeless wording, update strategy, and durable structure. +14. Return deliverables plus validation notes, parity status, and remaining gaps. + +## Sub-agent orchestration guidance + +Prefer sub-agents when the repo is large or the requested change set is broad; use them by default for repo-wide, multi-framework, or high-conflict work. + +- `inventory-agent` -> `agents/inventory-agent.md` (`fast` / Claude `haiku`): file/config discovery, coverage map, and missing-path checks. +- `governance-agent` -> `agents/governance-agent.md` (`thinking` / Claude `sonnet`): AGENTS/CONTRIBUTING/alias precedence, conflicts, and policy drift. +- `docs-framework-agent` -> `agents/docs-framework-agent.md` (`thinking` / Claude `sonnet`): framework config, relative path base, and file-path vs URL-path mapping checks. +- `synthesis-agent` -> `agents/synthesis-agent.md` (`long` / Claude `opus`): merge sub-agent outputs into one prioritized fix plan and unified precedence model. + +## Inputs + +- Doc type (tutorial, how-to, reference, explanation) and audience. +- File scope or diff scope. +- Docs framework/tooling constraints (Fern, Mintlify, Sphinx, etc.). +- Build/review mode and brownfield/evergreen intent. +- Target agent and human compatibility intent. +- Docs framework surfaces in scope (for example Fern, Sphinx, Mintlify, Markdown/MDX/MDC/RST/RSC files). +- Desired investigation depth/time budget (quick pass vs exhaustive review). +- Execution mode (`single-agent` or `sub-agent-assisted` when available). +- Remediation mode (`apply-fixes` by default, or `report-only` when requested). +- Multilingual scope: source-of-truth language, target locales, and parity expectations. + +## Outputs + +- Updated draft or review findings with clear next actions. +- Validation notes (what was checked, what remains). +- Navigation/maintenance recommendations for long-term quality. +- Governance-doc alignment summary when AGENTS/CONTRIBUTING were touched. +- Agent instruction-surface map (primary file, alias files, Codex/Claude/Cursor handling plan). +- Documentation-surface coverage map (what was reviewed under `/docs`, README hierarchy, and framework-specific source trees). +- Autodetected issue list with applied fixes (or explicit report-only findings). +- Delegation notes when sub-agents were used (scope delegated and how findings were merged). +- Multilingual parity note (in-sync, partial with rationale, or intentionally divergent). diff --git a/.claude/skills/technical-documentation/agents/docs-framework-agent.md b/.claude/skills/technical-documentation/agents/docs-framework-agent.md new file mode 100644 index 00000000..2051d711 --- /dev/null +++ b/.claude/skills/technical-documentation/agents/docs-framework-agent.md @@ -0,0 +1,29 @@ +--- +name: docs-framework-agent +description: Thinking-focused docs framework checker for config-relative paths and route/file mapping consistency. +model: sonnet +tools: + - Read + - Glob + - Grep +permissionMode: default +maxTurns: 10 +--- + +You are the docs-framework sub-agent for technical documentation. + +Goals: +- validate framework config-driven docs behavior +- prevent path-mapping drift between source files and published routes + +Tasks: +- detect and read framework config first (Fern/Sphinx/Mintlify/custom) +- resolve paths relative to the declaring file/config +- validate both maps: + - config -> file exists + - config/nav/routing -> URL path is valid and consistent + +Return: +- config files reviewed +- path assumptions made +- mismatches (`missing file`, `stale route`, `wrong base path`) diff --git a/.claude/skills/technical-documentation/agents/governance-agent.md b/.claude/skills/technical-documentation/agents/governance-agent.md new file mode 100644 index 00000000..1950e13d --- /dev/null +++ b/.claude/skills/technical-documentation/agents/governance-agent.md @@ -0,0 +1,27 @@ +--- +name: governance-agent +description: Thinking-focused governance reviewer for AGENTS/CONTRIBUTING/alias precedence, conflict detection, and policy drift analysis. +model: sonnet +tools: + - Read + - Glob + - Grep +permissionMode: default +maxTurns: 10 +--- + +You are the governance sub-agent for technical documentation. + +Goals: +- validate AGENTS/CONTRIBUTING/alias alignment and precedence +- identify policy drift and conflicting instructions + +Tasks: +- determine canonical instruction source and alias compatibility mapping +- detect conflicts across nested scope files and tool-specific rule consumers +- validate command examples against stated governance expectations + +Return: +- precedence model +- conflict list with severity +- recommended low-risk remediations diff --git a/.claude/skills/technical-documentation/agents/inventory-agent.md b/.claude/skills/technical-documentation/agents/inventory-agent.md new file mode 100644 index 00000000..babf1e8b --- /dev/null +++ b/.claude/skills/technical-documentation/agents/inventory-agent.md @@ -0,0 +1,28 @@ +--- +name: inventory-agent +description: Fast repo-surface discovery for technical documentation audits. Use for coverage mapping and missing-path detection before deeper review. +model: haiku +tools: + - Read + - Glob + - Grep + - LS +permissionMode: default +maxTurns: 6 +--- + +You are the inventory sub-agent for technical documentation. + +Goals: +- enumerate governance and docs-content surfaces in scope +- detect missing files, broken references, and obvious command/path failures + +Tasks: +- map `AGENTS.md`/`CONTRIBUTING.md`/aliases and docs surfaces (`docs/**`, README hierarchy, `.md/.mdx/.mdc/.rst/.rsc`) +- list framework config files discovered (Fern/Sphinx/Mintlify or equivalent) +- report hard failures only, with exact file paths + +Return: +- coverage map +- missing/broken path list +- unresolved blockers diff --git a/.claude/skills/technical-documentation/agents/openai.yaml b/.claude/skills/technical-documentation/agents/openai.yaml new file mode 100644 index 00000000..903f204a --- /dev/null +++ b/.claude/skills/technical-documentation/agents/openai.yaml @@ -0,0 +1,10 @@ +interface: + display_name: "Technical Documentation" + short_description: "Build and review technical documentation for brownfield and evergreen systems." + icon_small: "./assets/icon.jpg" + icon_large: "./assets/icon.jpg" + brand_color: "#111827" + default_prompt: "Build or review technical documentation with a clear, maintainable, and production-ready workflow." + +policy: + allow_implicit_invocation: true diff --git a/.claude/skills/technical-documentation/agents/synthesis-agent.md b/.claude/skills/technical-documentation/agents/synthesis-agent.md new file mode 100644 index 00000000..c1ff5f39 --- /dev/null +++ b/.claude/skills/technical-documentation/agents/synthesis-agent.md @@ -0,0 +1,25 @@ +--- +name: synthesis-agent +description: Long-context synthesis agent that merges sub-agent outputs into one prioritized and deduplicated documentation action plan. +model: opus +tools: + - Read +permissionMode: default +maxTurns: 12 +--- + +You are the synthesis sub-agent for technical documentation. + +Goal: +- merge sub-agent outputs into one coherent, non-duplicated action plan + +Tasks: +- prioritize blockers first, then non-blocking improvements +- normalize to one precedence model for governance decisions +- remove duplicated recommendations and contradictory fixes +- keep final output concise and execution-ready + +Return: +- prioritized fix plan +- validation summary (done vs pending) +- explicit remaining gaps/blockers diff --git a/.claude/skills/technical-documentation/references/agent-and-contributing.md b/.claude/skills/technical-documentation/references/agent-and-contributing.md new file mode 100644 index 00000000..56050e40 --- /dev/null +++ b/.claude/skills/technical-documentation/references/agent-and-contributing.md @@ -0,0 +1,144 @@ +# AGENT and CONTRIBUTING Principles + +This reference consolidates the core rules for agent-policy and contributor-governance docs. + +You must: +1. Discover repo-level and nested instruction files with: + `rg --files -g 'AGENTS.md' -g 'CONTRIBUTING.md' -g 'CLAUDE.md' -g 'AGENT.md' -g '.cursor/rules/*' -g '.cursorrules' -g '.agent/**' -g '.agents/**' -g '.pi/**' -g 'AGENTS.*.md'` +2. Read the root and nearest-scope `AGENTS.md`/`CONTRIBUTING.md` pair before editing. +3. If alias files exist, normalize to one canonical source (`AGENTS.md` preferred when present; otherwise nearest alias), plus compatibility pointers or explicit symlink notes. +4. Document conflicting instructions and precedence decisions. + +## GitHub + AGENTS baseline + +Source: https://docs.github.com/en/communities/setting-up-your-project-for-healthy-contributions/setting-guidelines-for-repository-contributors +Source: https://agents.md/ +Source: https://github.blog/ai-and-ml/github-copilot/how-to-write-a-great-agents-md-lessons-from-over-2500-repositories/ +Source: https://cobusgreyling.substack.com/p/what-is-agentsmd +Source: https://www.infoq.com/news/2025/08/agents-md/ + +Use these as default operating principles: + +1. Keep `CONTRIBUTING.md` discoverable and actionable (`.github`, root, or `docs`). +2. Keep agent instructions concrete: real commands, real paths, clear boundaries. +3. Use explicit behavior boundaries for agents: `Always`, `Ask first`, `Never`. +4. Keep contributor and agent rules aligned with actual repository workflows. +5. Ensure clear guidance is provided to agents on if, when and how to raise issues and pull requests. + +## Canonical and alias policy + +Source: https://agents.md/ +Source: https://github.blog/ai-and-ml/github-copilot/how-to-write-a-great-agents-md-lessons-from-over-2500-repositories/ + +1. Treat `AGENTS.md` as canonical when present. +2. If `AGENTS.md` is absent, treat the nearest alias file as canonical. +3. Keep compatibility surfaces explicit: `AGENTS.md`, `AGENT.md`, `.cursorrules`, `.cursor/rules/*`, `.agent/`, `.agents/`, `.pi/`. +4. If aliases are used, document how they map back to canonical policy (or symlink when supported). +5. When repos use `.agents/` as canonical rule storage, keep `.cursor` as a compatibility symlink to `.agents` for Cursor rule auto-loading. +6. Keep policy DRY: store one shared policy core and expose it via aliases/symlinks instead of duplicating rule text. + +## Context-awareness by agent platform + +Source: https://github.com/vercel-labs/agent-skills/blob/main/AGENTS.md +Source: https://github.com/openai/codex/blob/main/AGENTS.md + +1. For Cursor and Claude-style glob consumers, keep rule files narrow and bounded. +2. Avoid over-referencing large path sets that inflate context for glob-based agents. +3. For Codex-style workflows, prefer explicit file references and deterministic commands. +4. Keep long runbooks outside top-level policy files; link to scoped docs. +5. Ensure all agents have a happy path regardless so ensuring everything works across Codex, Claude and other coding agents. + +## Symlink and compatibility operations + +1. Preferred layout for multi-agent compatibility: + - canonical rule directory: `.agents/` + - Cursor compatibility path: `.cursor -> .agents` symlink + - canonical policy doc: `AGENTS.md` pointing to `.agents` paths where relevant +2. Validate symlink state before finalizing changes: + - if `.agents/` exists and `.cursor` is missing, create `.cursor` symlink to `.agents` + - if `.cursor` is a symlink to another target, fix target or document why it must differ + - if `.cursor` is a real directory/file, treat as migration conflict and ask before replacement +3. Validate rule payload through the canonical directory: + - rules: `.agents/rules/*.mdc` with valid frontmatter (`description`, `globs`, `alwaysApply` as needed) + - commands: `.agents/commands/*.md` when command routing is used + - MCP config: `.agents/mcp.json` when MCP is in scope +4. Keep Codex behavior explicit: + - `AGENTS.md` is primary for Codex repository instructions + - `.cursor` compatibility is for Cursor auto-loading and does not replace canonical AGENTS policy +5. Record applied symlink fixes and unresolved compatibility gaps in validation notes. + +## Dual-mode and deliverable standards + +Source: https://github.blog/ai-and-ml/github-copilot/how-to-write-a-great-agents-md-lessons-from-over-2500-repositories/ +Source: https://agents.md/ +Source: https://github.com/openai/codex/blob/main/AGENTS.md +Source: https://github.com/vercel-labs/agent-skills/blob/main/AGENTS.md + +1. Author one shared policy core (same commands, boundaries, and precedence) for all agents. +2. For Cursor/Claude-style agents, expose that core through glob-driven and bounded files (small `AGENTS.md`/rule surface). +3. For Codex, expose that same core through explicit file references with precise scope. +4. Where styles diverge, prefer the smallest common structure that satisfies both and avoid duplicating policy text. +5. Treat AGENTS/CONTRIBUTING as first-class deliverables when in scope. +6. Preserve required structure, constraints, and examples from existing files. +7. Align wording and commands with active repository instructions. + +## Proactive issue discovery and remediation + +Source: https://github.blog/ai-and-ml/github-copilot/how-to-write-a-great-agents-md-lessons-from-over-2500-repositories/ +Source: https://github.com/openai/codex/blob/main/AGENTS.md +Source: https://github.com/vercel-labs/agent-skills/blob/main/AGENTS.md + +1. Run a conflict matrix review across AGENTS/aliases/CONTRIBUTING and related command/rule docs before finalizing. +2. Treat the following as high-priority defects: missing referenced files, non-existent setup commands, command scope mismatches, and branch/commit policy conflicts. +3. Do not stop at caveat-only notes when a low-risk fix is clear; apply the fix in the same pass. +4. If a canonical entry file is missing (for example a directory `README.md` that docs depend on), create a minimal actionable file and update references. +5. Long-running investigations are acceptable when needed to uncover cross-file drift, especially in agent-instruction ecosystems. + +## Discovery + +1. Agents prefer simple terminal commands so having a well defined `make *` or `npm run *` is ideal +2. Agents can discover terminal commands through shell completion so providing shell completion helps + +## CONTRIBUTING size and scope control + +Source: https://contributing.md/how-to-build-contributing-md/ +Source: https://blog.codacy.com/best-practices-to-manage-an-open-source-project +Source: https://mozillascience.github.io/working-open-workshop/contributing/ +Source: https://github.com/openclaw/openclaw/blob/main/CONTRIBUTING.md + +1. Keep root `CONTRIBUTING.md` focused on setup, issue flow, PR flow, testing, and review gates. +2. Use issue/PR template links instead of embedding every process detail inline. +3. When the file grows too large, split by domain and link from root. +4. Move any large content into docs if avalible (for example Mintlify/Fern/Sphinx workflows) to avoid large contributor guide. +5. Optimize for agent/machine readability as well as humans. + +## Example repos to emulate + +Source: https://github.com/openclaw/openclaw/blob/main/AGENTS.md +Source: https://github.com/openclaw/openclaw/blob/main/CONTRIBUTING.md +Source: https://github.com/openclaw/openclaw/blob/main/VISION.md +Source: https://github.com/openai/codex/blob/main/AGENTS.md +Source: https://github.com/processing/p5.js/blob/main/AGENTS.md +Source: https://github.com/vercel-labs/agent-skills/blob/main/AGENTS.md +Source: https://github.com/agentsmd/agents.md/blob/main/AGENTS.md +Source: https://github.com/rails/rails/blob/main/CONTRIBUTING.md +Source: https://github.com/kubernetes/kubernetes/blob/master/CONTRIBUTING.md +Source: https://github.com/atom/atom/blob/master/CONTRIBUTING.md +Source: https://github.com/github/docs/blob/main/CONTRIBUTING.md +Source: https://github.com/facebook/react/blob/main/CONTRIBUTING.md + +1. OpenClaw: strong real-world alias policy and AGENTS/CONTRIBUTING/VISION cohesion. +2. OpenAI Codex: strict command discipline and explicit scope control. +3. p5.js: explicit AI-policy guardrails in agent instructions. +4. Vercel + agentsmd spec: compact, context-efficient AGENTS patterns. +5. Rails/Kubernetes/Atom/GitHub Docs/React: contributor guidance patterns at different project scales. + +## Practical merge policy + +When these rules conflict: + +1. Preserve contributor and reader task success first. +2. Preserve instruction clarity and unambiguous boundaries second. +3. Preserve long-term maintainability and context-efficiency third. +4. Add extra agent optimization only if it does not reduce human clarity or there is explict need. +5. Use your judgement as the expert. diff --git a/.claude/skills/technical-documentation/references/build.md b/.claude/skills/technical-documentation/references/build.md new file mode 100644 index 00000000..8d0150f3 --- /dev/null +++ b/.claude/skills/technical-documentation/references/build.md @@ -0,0 +1,113 @@ +# Build Docs Playbook + +Read `principles.md` first, then follow this execution flow. + +## 1. Detect and align agent instruction and governance instructions + +- Use `references/agent-and-contributing.md` as the source of truth for inventory, canonical/alias mapping, and precedence/conflict handling. +- Apply the symlink compatibility policy when in scope (`.agents` canonical directory with `.cursor` compatibility symlink when required by tooling). +- Long-running and extensive build investigations are acceptable when needed to resolve ambiguous or conflicting documentation sources. +- When available, use sub-agents for bounded parallel inventory/cross-check tasks and merge results into one canonical decision set. +- Capture required constraints before writing: + - nested-agent rules, command/test requirements, PR workflow, and style checks. +- Use the same command and validation expectations in proposed snippets and examples. + +## 2. Inventory product documentation surfaces (not governance only) + +- For repo-wide builds, include docs content surfaces in addition to AGENTS/CONTRIBUTING. +- Inventory docs files and frameworks in scope (examples): `README*.md`, `docs/**`, `**/*.md`, `**/*.mdx`, `**/*.mdc`, `**/*.rst`, `**/*.rsc`, Fern/Mintlify config, Sphinx `conf.py`. +- Build a coverage map before drafting so governance and product docs are both represented. +- If scope is ambiguous, default to broader docs discovery first, then narrow intentionally. + +## 3. Framework config and path mapping rules + +- Detect framework/config first (for example Fern config, Sphinx `conf.py`, Mintlify config, or equivalent). +- Resolve every referenced path relative to the file/config that declares it, not assumed repo root. +- Treat filesystem paths and published URL routes as separate mappings; do not infer one from the other without config evidence. +- Validate both layers: + - config -> file exists on disk + - config/nav/routing -> URL path is consistent and reachable +- Record path-mapping assumptions and mismatches in handoff (`missing file`, `stale route`, `wrong base path`). + +## 4. Define intent and success + +- Audience, prerequisites, and job-to-be-done. +- Expected reader outcome immediately after completion. +- Doc type: tutorial, how-to, reference, explanation. +- Success criteria: what must be true after publish. + +## 5. Build structure before prose + +- Follow the funnel: what/why, quickstart, next steps. +- Keep headings informative and scannable. +- Open each section with the takeaway sentence. +- Add decision points with concrete branch guidance. + +## 6. Build AGENTS.md and CONTRIBUTING.md intentionally + +- Keep AGENTS.md structure consistent with `agents.md` ecosystem patterns: + - include YAML frontmatter when present in repo style (`name`, `description`). + - state persona scope and explicit instruction boundaries: `Always`, `Ask first`, `Never`. + - include concrete commands and representative code examples. +- For CONTRIBUTING.md, prioritize issue triage flow, PR expectations, setup/test commands, and review gates. +- Add `Code of Conduct`, `Testing`, `Local checks`, and `PR expectations` sections when missing but required by the repo. +- If CONTRIBUTING.md is becoming too large, split by scope into linked docs (for example, framework/tool-specific setup and release workflows) and keep the root file as a concise entry point. +- Keep cross-file consistency: links from CONTRIBUTING.md to AGENTS.md (and vice versa) should be accurate and non-circular. +- If multiple AGENTS.md files exist, document the directory-level scope and avoid conflicting advice. +- If a required canonical entry file is missing (for example referenced `README.md` under a major directory), create the file in the same pass instead of adding a caveat-only note. +- For new entry files, keep them minimal and actionable: purpose, prerequisites, concrete run commands, and pointers to deeper docs. + +## 7. Keep agent context tight + +- Author once, expose twice: + - keep one shared policy core and avoid duplicating guidance in separate agent-specific files. + - publish that core through bounded glob-friendly files for Cursor/Claude plus explicit path references for Codex. +- For Cursor and Claude-style agents, avoid broad references. Use minimal globbing and narrow rule files that each serve one concern (for example, repo-wide setup, test rules, security checks). +- Keep AGENTS and alias files short-to-medium; move detailed runbooks to linked docs. +- For Codex, prefer explicit file references and concrete paths for exact reuse. +- Avoid adding unrelated historical or process details to avoid token/context drift during future tool reads. + +## 8. Brownfield build mode + +- Match existing terminology, navigation, and component patterns. +- Preserve existing IA unless there is a documented migration plan. +- For rewrites, include a migration note from old to new paths. +- Prefer smallest safe change set that improves utility. + +## 9. Evergreen build mode + +- Prefer stable concepts over release-tied narrative. +- Isolate volatile details under clearly marked version sections. +- Include maintenance signals: owners, refresh triggers, stale criteria. +- Include lifecycle notes: deprecation and replacement paths. + +## 10. Writing constraints + +- Use precise language and short, imperative instructions. +- Keep code examples copy-ready and self-contained. +- Include common failure modes and safe defaults. +- Avoid placeholder guidance that cannot be executed. + +## 11. Agent and automation readiness + +- Keep key facts in text (not image-only). +- Prefer structured lists/tables when choices matter. +- Add links and anchors that allow deterministic navigation. +- Document what can be checked automatically in CI. + +## 12. Build validation + +- Validate commands and snippets where possible. +- Verify links and references in changed sections. +- Run a reference existence sweep for every path/command you introduced. +- Verify docs-framework consistency when in scope (for example Sphinx/Fern config and referenced doc paths). + +## 13. Multilingual parity mode (when applicable) + +- Pick one source-of-truth language for technical accuracy and release timing. +- Define parity target: full parity, staged parity, or intentional divergence per section. +- Keep structure aligned across locales (headings, anchors, section order) when possible. +- Preserve command/code correctness first; localize explanatory text second. +- If parity is not feasible, add a visible note with missing scope and expected sync window. +- Run a locale parity check for changed sections (added/removed steps, warnings, prerequisites). +- Record unresolved checks explicitly in handoff. diff --git a/.claude/skills/technical-documentation/references/principles.md b/.claude/skills/technical-documentation/references/principles.md new file mode 100644 index 00000000..522842d7 --- /dev/null +++ b/.claude/skills/technical-documentation/references/principles.md @@ -0,0 +1,52 @@ +# Documentation Principles + +This reference consolidates the core rules used by this skill. + +## Matt Palmer: 8 rules for better docs + +Source: https://mattpalmer.io/posts/2025/10/8-rules-for-better-docs/ + +Use these as default operating principles: + +1. Write for humans, optimize for agents. +2. Start with a funnel: what/why, quickstart, next steps. +3. Use Diataxis to scaffold content. +4. Write with AI, but structure for agents. +5. Offload routine docs operations to background agents. +6. Automate quality with CI. +7. Automate scaffolding and repetitive workflow tasks. +8. Make contribution easy and visible. + +## OpenAI cookbook: what makes documentation good + +Source: https://cookbook.openai.com/articles/what_makes_documentation_good + +Key quality constraints: + +- Prefer specific and accurate terminology over niche jargon. +- Keep examples self-contained and minimize dependencies. +- Prioritize high-value topics over edge-case depth. +- Do not teach unsafe patterns (for example, exposed secrets). +- Open with context that helps readers orient quickly. +- Apply empathy and override rigid rules when it clearly improves outcomes. + +## Practical merge policy + +When these rules conflict: + +1. Preserve reader task success first. +2. Preserve structural clarity second. +3. Preserve long-term maintainability third. +4. Add agent optimization only if it does not reduce human clarity. + +For agent-instructions and contributor-governance specifics (AGENTS/aliases/CONTRIBUTING), use `references/agent-and-contributing.md` as the detailed additional source of truth. + +## Execution policy for this skill + +- Long-running and extensive investigations are allowed for both build and review work when needed to resolve ambiguity or cross-file drift. +- Use sub-agents when available for bounded parallel discovery, verification, or cross-source comparison. +- Keep one merged outcome: sub-agent outputs must be normalized into a single consistent recommendation/fix set. + +## Multilingual parity rule + +When docs exist in multiple languages, target cross-locale parity for task-critical content (steps, warnings, prerequisites, and limits). If full parity is not possible, publish explicit parity status and sync intent. diff --git a/.claude/skills/technical-documentation/references/review.md b/.claude/skills/technical-documentation/references/review.md new file mode 100644 index 00000000..3cfc3e7e --- /dev/null +++ b/.claude/skills/technical-documentation/references/review.md @@ -0,0 +1,117 @@ +# Review Docs Playbook + +Read `principles.md` first, then apply this checklist. + +## 1. Scope and classification + +- Identify doc type and target audience. +- Confirm brownfield vs evergreen intent. +- Confirm expected outcome for the reader. +- For full-repo reviews, explicitly include both governance surfaces and product-doc surfaces (`docs/`, README trees, `.md/.mdx/.mdc`, `.rst/.rsc`, framework docs configs). + +## 2. Investigation behavior + +- Proactively find issues and risks without waiting for repeated prompts. +- If there are signals of deeper problems, continue investigation beyond the first pass. +- Long-running and extensive investigations are acceptable when needed for confidence and correctness. +- When available, use sub-agents for bounded parallel discovery (for example file-inventory, command validation, or cross-doc consistency checks), then merge to one final issue set. +- When no issues are found, state that explicitly and call out residual risks or validation gaps. +- Default to `apply-fixes` for high-confidence documentation defects unless the user explicitly requests `report-only`. +- Do not stop at AGENTS/CONTRIBUTING checks when the task is documentation-wide; continue into docs-content and docs-framework surfaces. + +## 3. Governance surface review + +- Use `references/agent-and-contributing.md` as the source of truth for inventory, canonical/alias mapping, and precedence/conflict handling. +For AGENTS.md: + +- confirm persona intent, scope, and command/tool boundaries are explicit. +- check frontmatter style matches repo conventions when present. +- ensure `Always`, `Ask first`, and `Never` boundaries are present when expected. +- require concrete command examples and repo-specific paths to avoid ambiguity. + +For CONTRIBUTING.md: + +- verify issue/PR workflow is complete and actionable. +- ensure local setup, lint/test commands, and review criteria are accurate. +- ensure governance does not conflict with nested AGENTS instructions. +- flag oversized files that should be split into linked section docs (for example tool-specific setup and release docs). + +For agent-platform awareness: + +- confirm references are minimal and scoped for Cursor/Claude glob behavior. +- confirm Codex-facing guidance uses explicit file references. +- confirm both surfaces represent the same shared policy core (commands, boundaries, and precedence), not divergent guidance. +- audit `.agents`/`.cursor` compatibility behavior: + - verify canonical rule directory and symlink state match repo policy + - verify symlink target integrity and platform/tooling expectations + - verify AGENTS policy references remain canonical for Codex even when `.cursor` compatibility exists +- check for context bloat from duplicated policy statements across agent and contributor files. +- check for conflicting rules, skills and agent instructions +- check for conflicting information in agent instructions vs codebase +- check for broken or missing referenced files (for example README/index files named as canonical entry points). +- check for setup/command drift (for example non-existent install commands, root-level commands that should be module-scoped). + +## 4. Product documentation surface review + +- Verify docs IA coverage across root/module `README*` files and `docs/**` trees. +- Review framework-native docs sources in scope (for example Fern, Mintlify, Sphinx, MkDocs) and ensure guidance matches actual source-of-truth files. +- Check `.md/.mdx/.mdc/.rst/.rsc` for stale commands, missing prerequisites, and broken cross-links. +- Confirm referenced doc paths and anchors exist. +- Flag docs that should be split/merged to improve discoverability and maintenance. + +## 5. Framework config and path mapping checks + +- Detect and read framework config first (for example Fern config, Sphinx `conf.py`, Mintlify config, or equivalent). +- Resolve path references relative to the declaring file/config. +- Treat filesystem paths and published URL routes as separate maps; verify both. +- Flag path-map drift explicitly (`missing file`, `stale route`, `wrong base path`). + +## 6. Structural review + +- Funnel check: what/why, quickstart, next steps. +- Validate heading flow and navigation discoverability. +- Flag critical content trapped in images or buried sections. +- Check Diataxis alignment and split mixed-purpose sections. + +## 7. Writing quality review + +- Check for concise, scannable paragraphs. +- Remove ambiguous pronouns and undefined terms. +- Verify examples are executable and scoped correctly. +- Verify tone is directive, technical, and non-hand-wavy. + +## 8. Brownfield review mode + +- Verify compatibility with existing docs IA and conventions. +- Verify anchors, redirects, and cross-doc links remain valid. +- Flag regressions in onboarding and task completion paths. +- Ensure changed terminology is intentionally propagated. + +## 9. Evergreen review mode + +- Flag date-stamped or brittle wording without version scope. +- Check ownership and refresh signals are present. +- Ensure recommendations remain valid after routine product evolution. +- Flag missing deprecation/migration guidance. + +## 10. Tooling and platform review + +Read `tooling.md` if platform fit is uncertain. + +- Check whether content uses platform primitives effectively. +- Flag structure that fights the chosen docs platform. +- Recommend targeted platform-aware improvements. + +## 11. Multilingual parity review (when applicable) + +- Confirm declared source-of-truth language and expected parity policy. +- Compare changed sections across locales for step/order/warning drift. +- Flag missing updates to prerequisites, version notes, limits, and safety guidance. +- Allow intentional divergence only when rationale is explicit and user-impact is low. +- Require a reader-visible status note when locale parity is partial. + +## 12. Output format + +1. Blocking issues (file + required fix) +2. Non-blocking improvements +3. Validation notes (done vs pending) diff --git a/.claude/skills/technical-documentation/references/tooling.md b/.claude/skills/technical-documentation/references/tooling.md new file mode 100644 index 00000000..a3a8705c --- /dev/null +++ b/.claude/skills/technical-documentation/references/tooling.md @@ -0,0 +1,32 @@ +# Documentation Tooling Guide + +Source: https://www.mintlify.com/blog/top-7-api-documentation-tools-of-2025 + +Use this file when deciding build/review expectations for doc platforms. + +## Tool-selection checkpoints + +- Existing stack lock-in: do not force migration for minor gains. +- API workflow depth: generated references, OpenAPI support, testability. +- Collaboration model: docs-as-code, review workflow, versioning. +- Runtime quality: search, navigation, and copy-ready code snippets. +- AI readiness: structured content, stable URLs, machine-friendly layout yet human readable. +- Human readiness: reading complexity, reading UX, navigation depth, minimize jargon. + +## Apply in brownfield mode + +- Prioritize compatibility with the current platform. +- Use available components and style conventions before introducing new patterns. +- Propose migration only when current constraints block critical outcomes. + +## Apply in evergreen mode + +- Favor platforms and templates that make routine updates low-friction. +- Standardize section templates to reduce drift. +- Capture ownership, update cadence, and stale-content detection rules. + +## Review implications + +- Check whether content uses platform primitives correctly (tabs, callouts, endpoint blocks). +- Flag docs that are technically correct but hard to scan in the chosen platform. +- Recommend platform-specific improvements only when they reduce cognitive load. diff --git a/.claude/skills/vetkd b/.claude/skills/vetkd deleted file mode 120000 index 5bce56b3..00000000 --- a/.claude/skills/vetkd +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/vetkd \ No newline at end of file diff --git a/.claude/skills/wallet-integration b/.claude/skills/wallet-integration deleted file mode 120000 index b19dec04..00000000 --- a/.claude/skills/wallet-integration +++ /dev/null @@ -1 +0,0 @@ -../../.agents/skills/wallet-integration \ No newline at end of file diff --git a/.claude/sync-ic-skills.sh b/.claude/sync-ic-skills.sh new file mode 100755 index 00000000..fcc380d3 --- /dev/null +++ b/.claude/sync-ic-skills.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# sync-ic-skills.sh — mirror the latest Internet Computer skills into .claude/skills/ +# +# Differential sync: fetches the discovery index once and re-downloads only the +# skills whose published `hash` changed (or are new). Skills already at the current +# hash are skipped entirely — no per-file downloads. Prints a one-line summary only +# when something actually changed. +# +# Idempotent and offline-safe. Only skills this script installed are ever pruned, +# so your own local skills are never touched. +set -euo pipefail + +BASE="https://skills.internetcomputer.org/.well-known/skills" +INDEX_URL="$BASE/index.json" +DEST=".claude/skills" +MANIFEST="$DEST/.ic-managed.json" # { "": "" } of skills this script manages + +mkdir -p "$DEST" + +# --- Temp files. NEW_MANIFEST is built up as we go, then swapped in atomically. +# STAGING holds the skill dir currently being downloaded, so the trap can +# remove a half-written skill if the run is interrupted. --- +TMP_INDEX="$(mktemp)" +NEW_MANIFEST="$(mktemp)" +STAGING="" +trap 'rm -f "$TMP_INDEX" "$NEW_MANIFEST"; [ -n "$STAGING" ] && rm -rf "$STAGING"' EXIT + +# Remove any staging dirs left by a previously interrupted run — an in-progress +# download is always safe to discard. (.old-* backups are handled by the recovery +# step below, which never deletes one that is still the only copy of a skill.) +rm -rf "${DEST:?}"/.staging-* 2>/dev/null || true + +# --- Path-safety guards. `name` and `f` come from the remote index and flow into +# rm -rf / mv / file writes, so reject anything that could escape $DEST. --- +is_safe_name() { # a flat skill slug: non-empty, no slash, no ".." + case "$1" in + ""|.|..|*/*|*..*) return 1 ;; + *) return 0 ;; + esac +} +is_safe_relpath() { # a file path within a skill: subdirs ok, but not absolute or ".." + case "$1" in + ""|/*|*..*) return 1 ;; + *) return 0 ;; + esac +} + +# --- Recover from a run interrupted mid-swap. A `.old-.` dir is the +# previous good copy of , moved aside just before its swap. If that swap +# never finished (the skill dir is now missing), restore it; otherwise it is +# stale and safe to drop. This runs BEFORE the index fetch, so an interrupted +# skill is restored even on an offline run — keeping the cached copy available. --- +for backup in "$DEST"/.old-*; do + [ -e "$backup" ] || continue # unmatched glob stays literal — skip + bname="$(basename "$backup")"; bname="${bname#.old-}"; bname="${bname%.*}" + if is_safe_name "$bname" && [ ! -e "$DEST/$bname" ]; then + mv "$backup" "$DEST/$bname" + echo "[autosync-ic-skills] recovered '$bname' from an interrupted sync" >&2 + else + rm -rf "$backup" + fi +done + +# --- Fetch the index. On any network failure, keep cached skills and exit cleanly. --- +if ! curl -fsSL --max-time 20 "$INDEX_URL" -o "$TMP_INDEX"; then + echo "[autosync-ic-skills] could not reach $INDEX_URL — keeping cached skills" >&2 + exit 0 +fi + +# --- jq is required to parse the index. If absent, warn and exit without failing. --- +if ! command -v jq >/dev/null 2>&1; then + echo "[autosync-ic-skills] 'jq' not found — install jq to enable IC skill sync" >&2 + exit 0 +fi + +# --- Previously-managed skill names. Supports the legacy manifest format +# (a bare array of names, no hashes) as well as the current object form. --- +managed_names() { + [ -f "$MANIFEST" ] || return 0 + jq -r 'if type == "object" then keys[] elif type == "array" then .[] else empty end' \ + "$MANIFEST" 2>/dev/null || true +} + +# --- Stored hash for a skill, or empty if unknown (new skill, or legacy manifest). --- +stored_hash() { + [ -f "$MANIFEST" ] || return 0 + jq -r --arg n "$1" 'if type == "object" then (.[$n] // "") else "" end' \ + "$MANIFEST" 2>/dev/null || true +} + +# --- Append a name->hash pair to the new manifest being built. --- +record() { + local tmp; tmp="$(mktemp)" + jq --arg n "$1" --arg h "$2" '.[$n] = $h' "$NEW_MANIFEST" > "$tmp" && mv "$tmp" "$NEW_MANIFEST" +} + +NEW_NAMES="$(jq -r '.skills[].name' "$TMP_INDEX")" +MANAGED="$(managed_names)" +echo '{}' > "$NEW_MANIFEST" + +# --- Prune: drop previously-managed skills that are no longer in the index. --- +removed=0 +while IFS= read -r old; do + [ -n "$old" ] || continue + is_safe_name "$old" || { echo "[autosync-ic-skills] skipping unsafe managed name: $old" >&2; continue; } + if ! grep -qxF "$old" <<<"$NEW_NAMES"; then + rm -rf "${DEST:?}/$old" + removed=$((removed + 1)) + echo "[autosync-ic-skills] removed: $old" >&2 + fi +done <<<"$MANAGED" + +# --- Sync: download only skills whose hash changed (new / hashless always download). --- +added=0; updated=0; unchanged=0 +while IFS= read -r entry; do + name="$(jq -r '.name' <<<"$entry")" + [ -n "$name" ] && [ "$name" != "null" ] || continue + is_safe_name "$name" || { echo "[autosync-ic-skills] skipping skill with unsafe name: $name" >&2; continue; } + new_hash="$(jq -r '.hash // ""' <<<"$entry")" + old_hash="$(stored_hash "$name")" + + # Skip when the hash is known, unchanged, and the files are already on disk. + if [ -n "$new_hash" ] && [ "$new_hash" = "$old_hash" ] && [ -d "$DEST/$name" ]; then + unchanged=$((unchanged + 1)) + record "$name" "$new_hash" + continue + fi + + # Otherwise download this skill into a fresh staging dir, then swap it in + # atomically. A clean staging dir means an intra-skill file rename or removal + # leaves no orphaned files behind, and a mid-download failure keeps the existing + # copy intact — the swap happens only after every file downloaded successfully. + ok=1 + STAGING="$(mktemp -d "${DEST}/.staging-${name}.XXXXXX")" + while IFS= read -r f; do + [ -n "$f" ] || continue + if ! is_safe_relpath "$f"; then + echo "[autosync-ic-skills] warning: unsafe file path in $name: $f — skipping skill" >&2 + ok=0 + break + fi + mkdir -p "$(dirname "$STAGING/$f")" # files may live in subdirs (e.g. scripts/) + if ! curl -fsSL --max-time 20 "$BASE/$name/$f" -o "$STAGING/$f"; then + echo "[autosync-ic-skills] warning: failed to fetch $name/$f" >&2 + ok=0 + break + fi + done < <(jq -r '.files[]?' <<<"$entry") + + if [ "$ok" -eq 1 ]; then + # Swap in the fresh copy. Move any existing dir aside first, move the new one + # into place, and only then drop the old copy — so a failed swap restores the + # existing copy intact, while files removed or renamed upstream don't survive. + backup="" + if [ -e "$DEST/$name" ]; then + backup="${DEST}/.old-${name}.$$" + rm -rf "$backup" + mv "$DEST/$name" "$backup" + fi + if mv "$STAGING" "$DEST/$name"; then + STAGING="" + [ -n "$backup" ] && rm -rf "$backup" + # Record the new hash so the next run can skip this skill. A hashless server + # records an empty hash, which never equals new_hash -> always re-downloads. + record "$name" "$new_hash" + if grep -qxF "$name" <<<"$MANAGED"; then + updated=$((updated + 1)) + else + added=$((added + 1)) + fi + else + # Swap failed: restore any existing copy and retry on the next run. + echo "[autosync-ic-skills] warning: failed to install $name — kept any existing copy; will retry next run" >&2 + [ -n "$backup" ] && mv "$backup" "$DEST/$name" + rm -rf "$STAGING" + STAGING="" + record "$name" "$old_hash" + fi + else + # Download incomplete: discard the staging dir, keep the existing skill dir + # untouched, and keep the old hash so the next run retries this skill. + rm -rf "$STAGING" + STAGING="" + record "$name" "$old_hash" + fi +done < <(jq -c '.skills[]' "$TMP_INDEX") + +# --- Swap in the updated manifest. --- +mv "$NEW_MANIFEST" "$MANIFEST" + +# --- Report only when something changed; stay silent on a no-op sync. --- +# SessionStart hook stdout/stderr is NOT shown in the Claude Code UI — only JSON +# fields are surfaced. We emit a single JSON object on stdout: +# - systemMessage -> rendered to the USER as a visible system notice +# - additionalContext -> injected into Claude's context so it can mention it too +if [ $((added + updated + removed)) -gt 0 ]; then + summary="[autosync-ic-skills] ${added} added, ${updated} updated, ${removed} removed (${unchanged} unchanged) in $DEST" + jq -n --arg msg "$summary" '{ + systemMessage: $msg, + hookSpecificOutput: { + reloadSkills: true, + hookEventName: "SessionStart", + additionalContext: $msg + } + }' +fi diff --git a/.gitignore b/.gitignore index 4d78dbaf..9c4d5b33 100644 --- a/.gitignore +++ b/.gitignore @@ -28,9 +28,15 @@ public/docs-manifest.json # macOS-specific files .DS_Store -# Claude Code (local-only files; .claude/settings.json and .claude/skills/ are committed) +# Claude Code. settings.json, the skill autosync script, and the three skills +# maintained here are committed; skills fetched by the autosync are not. .claude/settings.local.json .claude/worktrees/ +.claude/skills/* +!.claude/skills/icp-brand-design/ +!.claude/skills/icp-brand-voice/ +!.claude/skills/technical-documentation/ +.claude/skills/.ic-managed.json # Skills install artifacts (npx skills add writes here; committed skills live in .agents/skills/) /skills/ diff --git a/.gitmodules b/.gitmodules index 5aaf182a..3a4fd0c0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,3 @@ -[submodule ".sources/icskills"] - path = .sources/icskills - url = git@github.com:dfinity/icskills.git - branch = main [submodule ".sources/examples"] path = .sources/examples url = git@github.com:dfinity/examples.git @@ -9,9 +5,6 @@ [submodule ".sources/motoko"] path = .sources/motoko url = git@github.com:caffeinelabs/motoko.git -[submodule ".sources/dotskills"] - path = .sources/dotskills - url = git@github.com:vincentkoc/dotskills.git [submodule ".sources/internetidentity"] path = .sources/internetidentity url = https://github.com/dfinity/internet-identity.git diff --git a/.sources/VERSIONS b/.sources/VERSIONS index 5958b5ff..5bf95c15 100644 --- a/.sources/VERSIONS +++ b/.sources/VERSIONS @@ -1,11 +1,11 @@ # Pinned submodule versions # ------------------------------------------------------- -# This file covers the FIVE submodules under .sources/ — the repos whose bytes -# reach the built site or the agent workflow. Every other upstream repo is -# watched rather than vendored; its pin lives in .sources/upstream.json. +# This file covers the three submodules under .sources/ — the repos whose +# content is resolved during the build. Every other upstream repo is watched +# rather than vendored; its pin lives in .sources/upstream.json. # # Submodules that track a branch are NOT listed here (they have no fixed -# release version): examples, icskills, dotskills. +# release version): examples. # # internetidentity — uses date-based release tags (release-YYYY-MM-DD). # Only ii-spec.mdx, vc-spec.md, and internet_identity.did are synced. diff --git a/.sources/dotskills b/.sources/dotskills deleted file mode 160000 index a44218fb..00000000 --- a/.sources/dotskills +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a44218fb73f1ab5ef597d66ab3d9eb651f075f9c diff --git a/.sources/icskills b/.sources/icskills deleted file mode 160000 index d8dad4c7..00000000 --- a/.sources/icskills +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d8dad4c75046e85906fc08e8f02994231a09c1ee diff --git a/.sources/upstream.json b/.sources/upstream.json index 8d7aab32..acd9df36 100644 --- a/.sources/upstream.json +++ b/.sources/upstream.json @@ -36,20 +36,6 @@ "track": "branch", "branch": "master", "affects": "Every `snippet=` path and `#region` marker resolved at build time by plugins/remark-snippet.mjs. A renamed region is a build error, so verify each one after bumping." - }, - { - "path": ".sources/icskills", - "repo": "dfinity/icskills", - "track": "branch", - "branch": "main", - "affects": "The 16 skill symlinks in .agents/skills/ follow this pin, so a bump changes the guidance agents load. Check for changed canister IDs and code patterns." - }, - { - "path": ".sources/dotskills", - "repo": "vincentkoc/dotskills", - "track": "branch", - "branch": "main", - "affects": "The `technical-documentation` skill in .agents/skills/. Check whether the review criteria changed." } ], "watched": [ diff --git a/AGENTS.md b/AGENTS.md index d9c69d7c..e19fa6a4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -167,7 +167,7 @@ EOF - Reference `dfx` — it is deprecated and banned - Use `mo:base` — use `mo:core` instead. Critical replacements: `Buffer` → `List`, `HashMap`/`TrieMap`/`Trie`/`RBTree` → `Map`, `Deque` → `Queue`, `OrderedMap` → `pure/Map`, `OrderedSet` → `pure/Set` - Create `.mdx` without a clear need for interactive components -- Duplicate content that lives in external docs (icp-cli site, JS SDK docs, icskills) +- Duplicate content that lives in external docs (icp-cli site, JS SDK docs, the IC skills) - Edit synced files directly (`docs/languages/motoko/`, `docs/guides/tools/migrating-from-dfx.md`) - Nest sidebar items more than 3 levels deep - Add `Co-Authored-By` or any AI attribution to commits or PR descriptions @@ -187,7 +187,7 @@ EOF - `docs/languages/motoko/` — Auto-synced from `caffeinelabs/motoko` (do not edit directly) - `docs/references/internet-identity-spec.md`, `docs/references/verifiable-credentials-spec.md` — Synced from `dfinity/internet-identity` (do not edit directly) - `.sources/` — Vendored submodules, read-only, plus `upstream.json` (watched repos) and `VERSIONS` (submodule pins) -- `.agents/skills/` — Agent skill files. Run `git submodule update --init --depth 1` if broken. +- `.claude/skills/` — Skills. IC skills are mirrored by `.claude/sync-ic-skills.sh` and not committed; `icp-brand-design`, `icp-brand-voice`, and `technical-documentation` are maintained here - `.agents/upstream-tracking.md` — How upstreams are tracked and bumped (maintainer use) ## Project structure @@ -227,13 +227,11 @@ git submodule update --init --depth 1 # do NOT use --recursive | Motoko compiler / syntax, synced Motoko pages | `.sources/motoko/` | | Internet Identity and VC specs | `.sources/internetidentity/` | | Code examples (`snippet=`, ``) | `.sources/examples/` | -| Canister IDs and skill files | `.sources/icskills/` | -| Technical documentation skill | `.sources/dotskills/` | Pinned versions: [`.sources/VERSIONS`](.sources/VERSIONS). `motoko` and `internetidentity` are release-checked and synced by their own workflows; -`examples`, `icskills`, and `dotskills` track a branch and are checked by the -weekly **Upstream release check**. +`examples` tracks a branch and is checked by the weekly **Upstream release +check**. Canister IDs and code patterns are in the skills (see "Skills"). **Watched, not vendored** — everything else. Nothing they contain is published, so the repo records a pinned ref instead of a copy, and a weekly workflow opens @@ -273,33 +271,49 @@ For the tracking and bump procedures, see ## Skills -Load skills matching the task before starting any content work. Run `git submodule update --init --depth 1` if skills appear as broken symlinks. +Load skills matching the task before starting any content work. + +Skills live in `.claude/skills/`. The IC skills are mirrored from +[skills.internetcomputer.org](https://skills.internetcomputer.org) by +`.claude/sync-ic-skills.sh`, which runs on session start and re-downloads only +what changed. They are not committed. Three skills are maintained in this repo +and are committed: `icp-brand-design`, `icp-brand-voice`, and +`technical-documentation`. The sync only ever prunes skills it installed itself, +so those three are never touched. If the sync cannot reach the registry it keeps +whatever is already on disk, so an offline session still has skills. Always load for content writing: - **`technical-documentation`** — quality and structure - **`icp-brand-voice`** — vocabulary, banned terms, voice -Load the icskill matching the page topic: +Load the skill matching the page topic: -| Topic | icskill | -|-------|---------| +| Topic | Skill | +|-------|-------| | Bitcoin / ckBTC | `ckbtc` | | Ethereum / EVM | `evm-rpc` | | Certified variables | `certified-variables` | | HTTPS outcalls | `https-outcalls` | | SNS / governance | `sns-launch` | | Identity / auth | `internet-identity` | +| Agent sign-in / web identity | `agent-web-identity` | | Multi-canister | `multi-canister` | | ICRC tokens / ledger | `icrc-ledger` | | CLI / tooling | `icp-cli` | -| Frontend / asset canister | `asset-canister` | +| Motoko package management | `mops-cli` | +| Motoko language | `writing-motoko`, `migrating-motoko-actors` | +| Frontend / static site hosting | `static-site` | +| Custom domains | `custom-domains` | +| Agent discoverability | `service-discoverability` | | Cycles / billing | `cycles-management` | | Stable memory | `stable-memory` | | Security | `canister-security` | | Wallet / DeFi | `wallet-integration` | -| vetKD / encryption | `vetkd` | +| vetKeys / encryption | `vetkeys`, `encrypted-maps` | +| Dashboard APIs | `ic-dashboard` | -Topics without a dedicated icskill: on-chain AI, randomness/VRF, timers, Candid, chain-key tokens. +Topics with no dedicated skill: on-chain AI, randomness/VRF, timers, Candid, +chain-key tokens. For design work (CSS, UI, marketing copy), also load `icp-brand-design`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b2f0192a..a727e2a8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,7 +61,7 @@ Each top-level section has a specific purpose. Match your content accordingly: ### Don't - Reference `dfx` — it is deprecated. CI will reject it. - Use `.mdx` without a clear need for interactive components (default to `.md`) -- Duplicate content that lives in external docs (icp-cli, JS SDK, icskills) +- Duplicate content that lives in external docs (icp-cli, JS SDK, the IC skills) - Nest sidebar items more than 3 levels deep - Add images without alt text - Write for a specific framework version — always describe "latest" @@ -104,7 +104,7 @@ The build generates `/llms.txt` and per-page `.md` endpoints from your content. Upstream repos are tracked two ways, because most of them are only ever read to check a fact. -- **Vendored as submodules** (`.sources/motoko`, `internetidentity`, `examples`, `icskills`, `dotskills`) — five repos whose content reaches the built site or the agent workflow. **Do not edit files in `.sources/` directly**; they are read-only, and changes go to the upstream repo. Pins are in [`.sources/VERSIONS`](.sources/VERSIONS). +- **Vendored as submodules** (`.sources/motoko`, `internetidentity`, `examples`) — three repos whose content is resolved during the build. **Do not edit files in `.sources/` directly**; they are read-only, and changes go to the upstream repo. Pins are in [`.sources/VERSIONS`](.sources/VERSIONS). - **Watched, not vendored** — everything else, listed in [`.sources/upstream.json`](.sources/upstream.json) with the ref the docs are verified against. A weekly workflow opens an issue when one of them ships something newer. Bumping either is a maintainer task — follow [`.agents/upstream-tracking.md`](.agents/upstream-tracking.md) for the procedure. From 2428f4c2affb72178359d4f4b22bac1fc7782983 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Wed, 26 Aug 2026 15:46:42 +0200 Subject: [PATCH 04/11] chore: carry the MIT notice with the vendored skill MIT requires the copyright and permission notice to travel with a copy. The skill's frontmatter recorded its license and source but not the notice itself, so the upstream license text now sits alongside the files and NOTICE records the attribution. --- .agents/upstream-tracking.md | 5 +++++ .../skills/technical-documentation/LICENSE | 21 +++++++++++++++++++ NOTICE | 21 +++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 .claude/skills/technical-documentation/LICENSE create mode 100644 NOTICE diff --git a/.agents/upstream-tracking.md b/.agents/upstream-tracking.md index 17306edc..3c86536f 100644 --- a/.agents/upstream-tracking.md +++ b/.agents/upstream-tracking.md @@ -33,6 +33,11 @@ copy, or is a skill, which the session-start sync mirrors from [skills.internetcomputer.org](https://skills.internetcomputer.org) (see AGENTS.md "Skills"). Add a submodule only if the build has to open its files. +Third-party content that is copied in rather than referenced must carry its +license with it: the license text alongside the files, and an entry in +[`NOTICE`](../NOTICE). A license that forbids that is a reason to reference the +upstream instead of copying it. + ## Watched repos ### Verifying a fact against a watched repo diff --git a/.claude/skills/technical-documentation/LICENSE b/.claude/skills/technical-documentation/LICENSE new file mode 100644 index 00000000..a7cb85a0 --- /dev/null +++ b/.claude/skills/technical-documentation/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Vincent Koc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000..6d160aba --- /dev/null +++ b/NOTICE @@ -0,0 +1,21 @@ +Copyright 2026 DFINITY Stiftung + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +-------------------------------------------------------------------------------- + +Third-party content included in this repository: + +.claude/skills/technical-documentation/ + From https://github.com/vincentkoc/dotskills, MIT License, + Copyright (c) 2026 Vincent Koc. The full license text is included in that + directory. The skill's own frontmatter records its license and source. From fa5b844c954183a5bd4e61e5f65c27ea967687eb Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Wed, 26 Aug 2026 16:12:48 +0200 Subject: [PATCH 05/11] docs: state how skills are consumed outside Claude Code The session-start hook is Claude Code only and automatic skill loading is harness-specific everywhere, so nothing loads skills automatically elsewhere. What is portable is where to get them: the IC skills from the registry index by URL, since they are gitignored and absent from a fresh clone, and the three repo-maintained skills from their committed path. Also records that the sync replaces a local skill whose name is later published to the registry, so publishing one means deleting the local copy in the same change. --- AGENTS.md | 44 +++++++++++++++++++++++++++++++++++++------- CONTRIBUTING.md | 3 ++- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e19fa6a4..50a868f1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -273,14 +273,44 @@ For the tracking and bump procedures, see Load skills matching the task before starting any content work. -Skills live in `.claude/skills/`. The IC skills are mirrored from +Skills live in `.claude/skills/` and come from two places. + +**The IC skills** are mirrored from [skills.internetcomputer.org](https://skills.internetcomputer.org) by -`.claude/sync-ic-skills.sh`, which runs on session start and re-downloads only -what changed. They are not committed. Three skills are maintained in this repo -and are committed: `icp-brand-design`, `icp-brand-voice`, and -`technical-documentation`. The sync only ever prunes skills it installed itself, -so those three are never touched. If the sync cannot reach the registry it keeps -whatever is already on disk, so an offline session still has skills. +`.claude/sync-ic-skills.sh`, which runs on session start, re-downloads only what +changed, and keeps whatever is on disk when the registry is unreachable. They are +**not committed**, so a skill changing upstream leaves nothing to review. + +**Three skills are maintained in this repo** and are committed: +`icp-brand-design`, `icp-brand-voice`, and `technical-documentation`. None of +them is on the registry. The sync only prunes skills it installed itself, so it +never touches them. + +Read a skill's `SKILL.md` from `.claude/skills//` at any point; the files +are plain markdown with frontmatter and nothing about them is Claude-specific. + +### Skills outside Claude Code + +The session-start hook is Claude Code only, and automatic skill loading is +specific to every harness, so nothing loads skills automatically elsewhere. What +is portable is where to get them: + +- **The IC skills:** fetch `https://skills.internetcomputer.org/.well-known/skills/index.json` + once, then fetch the matching skill's `SKILL.md` before writing. No repo state + is involved, and it is always current. This is the path to use in Cursor, + Copilot, Codex, and anything else that is not Claude Code. Do not read the IC + skills from `.claude/skills/` outside Claude Code: they are gitignored, so a + fresh clone has none of them until a sync has run. +- **The three repo-maintained skills:** read them from `.claude/skills//`. + They are in git, so they are present in every clone. + +To force a refresh in a running Claude Code session, or to populate the +directory by hand, run `bash .claude/sync-ic-skills.sh`. + +If one of the three repo-maintained names is ever published to the registry, the +sync replaces the local copy with the published one. That is the right outcome +when we publish a skill deliberately, so delete the local copy in the same change +rather than working around it. Always load for content writing: - **`technical-documentation`** — quality and structure diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a727e2a8..cd069fe1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -116,7 +116,8 @@ Some files are auto-synced from other repositories. Currently synced: - `docs/languages/motoko/` — from `caffeinelabs/motoko` -- `docs/guides/tools/migrating-from-dfx.md` — from `dfinity/icp-cli` +- `docs/references/internet-identity-spec.md`, `docs/references/verifiable-credentials-spec.md` — from `dfinity/internet-identity` +- `.claude/skills/` (except `icp-brand-design`, `icp-brand-voice`, and `technical-documentation`) — from [skills.internetcomputer.org](https://skills.internetcomputer.org), refreshed on session start and not committed. See AGENTS.md "Skills" for how to consume them outside Claude Code. ## Review ownership From af0897ea177485469649f71e11cdabf8c16615e5 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Wed, 26 Aug 2026 16:19:08 +0200 Subject: [PATCH 06/11] fix: guard against a pin that is not a real tag, track ic-cdk by crate version The cdk-rs pin was 0.20.1, which is the published crate version, not a git tag: the repo's newest bare-semver tag is 0.19.0, two minors behind. Compared against tags, the pin could never be overtaken, so the check reported 'current' forever for the repo that governs every Rust code block. A release pin that is not among the matched tags is now a loud failure rather than silence, and cdk-rs tracks crates.io, which is where its releases actually appear. Auditing the other ten pins found no further cases. motoko-core and response-verification now diff their changelogs, which names the APIs that changed instead of leaving 'check every signature' as the whole task. --- .sources/upstream.json | 12 +++++++----- scripts/check-upstream-releases.mjs | 26 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/.sources/upstream.json b/.sources/upstream.json index acd9df36..f4087ee5 100644 --- a/.sources/upstream.json +++ b/.sources/upstream.json @@ -69,14 +69,15 @@ "track": "release", "tagPattern": "^v\\d+\\.\\d+\\.\\d+$", "affects": "Every `mo:core` import and API signature in Motoko code blocks.", - "reference": "https://mops.one/core/docs" + "reference": "https://mops.one/core/docs", + "verify": "Changelog.md" }, { "repo": "dfinity/cdk-rs", "pinned": "0.20.1", - "track": "release", - "tagPattern": "^\\d+\\.\\d+\\.\\d+$", - "affects": "`ic-cdk` APIs in Rust code blocks. `ic-cdk-timers` and `ic-cdk-executor` tag separately (`timers-*`, `executor-*`) and usually release in the same batch, so check whether they moved too. Bare semver tags are `ic-cdk` itself.", + "track": "crate", + "crate": "ic-cdk", + "affects": "`ic-cdk` APIs in Rust code blocks. The repo stopped tagging releases (its newest bare-semver tag is two minors behind the published crate), so the crate version on crates.io is the release identity. Read the sections newer than the pin in https://github.com/dfinity/cdk-rs/blob/master/ic-cdk/CHANGELOG.md, then grep docs/ for the symbols they name. `ic-cdk-timers` and `ic-cdk-executor` version separately; check whether they moved too.", "reference": "https://docs.rs/ic-cdk/latest/ic_cdk/" }, { @@ -94,7 +95,8 @@ "track": "release", "tagPattern": "^\\d+\\.\\d+\\.\\d+$", "affects": "guides/frontends/certification.md and guides/backends/certified-variables.md: the `ic-http-certification`, `ic-asset-certification`, and `@dfinity/certificate-verification` APIs.", - "reference": "https://docs.rs/ic-http-certification/latest/ic_http_certification/" + "reference": "https://docs.rs/ic-http-certification/latest/ic_http_certification/", + "verify": "CHANGELOG.md" }, { "repo": "dfinity/chain-fusion-signer", diff --git a/scripts/check-upstream-releases.mjs b/scripts/check-upstream-releases.mjs index 5bae348e..b773033b 100644 --- a/scripts/check-upstream-releases.mjs +++ b/scripts/check-upstream-releases.mjs @@ -99,6 +99,17 @@ function branchHead(repo, branch) { return out.match(/^([0-9a-f]{40})/)?.[1]; } +async function cratesIoNewest(crate) { + const res = await fetch(`https://crates.io/api/v1/crates/${crate}`, { + headers: { 'User-Agent': 'dfinity-developer-docs upstream check' }, + }); + if (!res.ok) throw new Error(`crates.io returned ${res.status} for ${crate}`); + const body = await res.json(); + const v = body?.crate?.newest_version; + if (!v) throw new Error(`crates.io gave no newest_version for ${crate}`); + return v; +} + async function fetchFile(repo, ref, path) { const url = `https://raw.githubusercontent.com/${repo}/${ref}/${path}`; const res = await fetch(url); @@ -178,9 +189,24 @@ async function checkOne(entry) { if (matching.length === 0) { throw new Error(`${repo}: no tags match ${tagPattern}`); } + // A pin that is not one of the matched tags can never be overtaken by them, + // so the repo would report "current" forever. Fail instead of going quiet. + if (!matching.includes(pinned)) { + throw new Error( + `${repo}: pinned "${pinned}" is not a tag matching ${tagPattern} ` + + `(newest matching tag is "${[...matching].sort(compareRefs).pop()}"). ` + + `Fix the pin, the pattern, or switch this entry to another track.` + ); + } matching.sort(compareRefs); latest = matching[matching.length - 1]; kind = 'release'; + } else if (track === 'crate') { + // Some repos publish releases to a package registry without tagging them, + // so the crate version is the release identity and tags lag behind it. + if (!entry.crate) throw new Error(`${repo}: track "crate" needs a crate name`); + latest = await cratesIoNewest(entry.crate); + kind = `crates.io release of ${entry.crate}`; } else if (track === 'commit') { const { branch, sha } = defaultBranchHead(repo); if (!sha) throw new Error(`${repo}: could not resolve HEAD`); From 78c31e6bf19989061edae0b192bbc3a609e872e2 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Wed, 26 Aug 2026 16:33:36 +0200 Subject: [PATCH 07/11] fix: track registry-published upstreams, add the shared upstream-sync label Auditing every watched repo against its real publishing channel found a second case beyond cdk-rs: icp-js-sdk-docs carries no tags, so its commit was standing in for an @icp-sdk/core release. Tracking npm instead shows the docs are verified against 5.4.0 while 6.1.0 is out, a major bump across every JS code block. The pin was derived from public/core/versions.json at the previously pinned commit rather than guessed. Registry-tracked entries get a registry link instead of a GitHub compare view and no file diff, because a package version is not a ref git can resolve; both were producing broken links and a failed fetch. Every issue now also carries upstream-sync, so they are findable as a class rather than only per repo. --- .agents/upstream-tracking.md | 31 ++++++++++++++++++++----- .github/workflows/upstream-releases.yml | 6 ++++- .sources/upstream.json | 7 +++--- scripts/check-upstream-releases.mjs | 29 ++++++++++++++++++++--- 4 files changed, 60 insertions(+), 13 deletions(-) diff --git a/.agents/upstream-tracking.md b/.agents/upstream-tracking.md index 3c86536f..47ee114f 100644 --- a/.agents/upstream-tracking.md +++ b/.agents/upstream-tracking.md @@ -92,12 +92,31 @@ Exit code 0 means nothing moved, 1 means at least one did (bodies are written to ### Adding a repo to the watch list -Add an entry to `.sources/upstream.json`. `track: "release"` needs a -`tagPattern` that matches only the tags that count as a release for the thing we -document — several of these repos tag per crate or per recipe, so an unanchored -pattern picks up the wrong series. `track: "commit"` is for repos with no tags at -all. Set `verify` when a single file carries the API surface we check against; -its diff becomes the issue's review payload. +Add an entry to `.sources/upstream.json`, declaring where its releases actually +appear: + +| `track` | Latest ref comes from | Use it when | +|---|---|---| +| `release` | git tags matching `tagPattern` | tags are the release identity | +| `crate` | crates.io `newest_version` for `crate` | the repo publishes without tagging | +| `npm` | the npm registry `latest` for `package` | same, for a JS package | +| `commit` | the default branch head | the repo has no releases at all | + +Two traps this has already hit: + +- **`tagPattern` must be anchored and specific.** Several of these repos tag per + crate or per recipe, so a loose pattern picks the wrong series and reports a + version from a different artifact. +- **A `release` pin must be a real tag.** A pin that no tag can overtake makes + the repo report "current" for ever. The script fails loudly on this rather than + going quiet, which is how `cdk-rs` was caught pinned to a crate version that + its tags were two minors behind. + +Set `verify` when a single file carries the surface we check against; its diff +becomes the issue's review payload, and a changelog is usually the best choice +because it names what changed instead of leaving "check every signature" as the +task. It only applies to `release` and `commit` tracks, since a registry version +is not a ref git can resolve. ### `icp-cli`: link slug adaptation diff --git a/.github/workflows/upstream-releases.yml b/.github/workflows/upstream-releases.yml index c1df63fb..b2a6d7d6 100644 --- a/.github/workflows/upstream-releases.yml +++ b/.github/workflows/upstream-releases.yml @@ -67,6 +67,10 @@ jobs: [ -n "$slug" ] || continue label="upstream-${slug}" + # `upstream-sync` groups every issue this workflow opens so they are + # findable as a class; `upstream-` identifies the one repo, so a + # later run can find and supersede its own issue. + gh label create upstream-sync --description "An upstream release needs the docs re-checked" --color 0e8a16 2>/dev/null || true gh label create "$label" --description "Upstream ref moved: ${slug}" --color ededed 2>/dev/null || true existing=$(gh issue list --label "$label" --state open --limit 1 --json number,title --jq 'first // empty') @@ -81,7 +85,7 @@ jobs: echo "Closed superseded #$num" fi - gh issue create --title "$title" --body-file ".upstream-checks/${slug}.md" --label "$label" + gh issue create --title "$title" --body-file ".upstream-checks/${slug}.md" --label "$label" --label upstream-sync done < moved.tsv - name: Fail if a check errored diff --git a/.sources/upstream.json b/.sources/upstream.json index f4087ee5..e00f57eb 100644 --- a/.sources/upstream.json +++ b/.sources/upstream.json @@ -124,9 +124,10 @@ }, { "repo": "dfinity/icp-js-sdk-docs", - "pinned": "17e4346", - "track": "commit", - "affects": "`@icp-sdk/core` and `@icp-sdk/canisters` APIs in JavaScript and TypeScript code blocks.", + "pinned": "5.4.0", + "track": "npm", + "package": "@icp-sdk/core", + "affects": "`@icp-sdk/core` and `@icp-sdk/canisters` APIs in JavaScript and TypeScript code blocks. This repo is the docs mirror and carries no tags, so the released package version is the signal rather than a commit on it. `@icp-sdk/canisters` versions separately; check whether it moved too. The mirror's `public/core/versions.json` lists which versions it carries, and `public//latest.zip` holds the API docs to unzip.", "reference": "https://js.icp.build" } ] diff --git a/scripts/check-upstream-releases.mjs b/scripts/check-upstream-releases.mjs index b773033b..f5fa30cd 100644 --- a/scripts/check-upstream-releases.mjs +++ b/scripts/check-upstream-releases.mjs @@ -6,6 +6,11 @@ * Covers both groups in that file: `vendored` submodules whose pin has no sync * workflow of its own (their pin is read from the gitlink, so git stays the * single source of truth), and `watched` repos that are not vendored at all. + * + * A watched repo declares where its releases actually appear: `release` (git + * tags matching a pattern), `crate` (crates.io) or `npm` (the npm registry) for + * repos that publish without tagging, or `commit` for repos with no releases at + * all. * See AGENTS.md "Source material" and .agents/upstream-tracking.md. * * Usage: @@ -110,6 +115,14 @@ async function cratesIoNewest(crate) { return v; } +async function npmLatest(pkg) { + const res = await fetch(`https://registry.npmjs.org/${pkg}/latest`); + if (!res.ok) throw new Error(`npm returned ${res.status} for ${pkg}`); + const v = (await res.json())?.version; + if (!v) throw new Error(`npm gave no version for ${pkg}`); + return v; +} + async function fetchFile(repo, ref, path) { const url = `https://raw.githubusercontent.com/${repo}/${ref}/${path}`; const res = await fetch(url); @@ -182,6 +195,10 @@ async function checkOne(entry) { let latest; let kind; + // A registry-tracked ref is a package version, not something git can resolve, + // so those entries get a registry link instead of a compare view and no file + // diff. + let compare; if (track === 'release') { if (!tagPattern) throw new Error(`${repo}: track "release" needs a tagPattern`); const re = new RegExp(tagPattern); @@ -203,10 +220,16 @@ async function checkOne(entry) { kind = 'release'; } else if (track === 'crate') { // Some repos publish releases to a package registry without tagging them, - // so the crate version is the release identity and tags lag behind it. + // so the package version is the release identity and tags lag behind it. if (!entry.crate) throw new Error(`${repo}: track "crate" needs a crate name`); latest = await cratesIoNewest(entry.crate); kind = `crates.io release of ${entry.crate}`; + compare = `https://crates.io/crates/${entry.crate}`; + } else if (track === 'npm') { + if (!entry.package) throw new Error(`${repo}: track "npm" needs a package name`); + latest = await npmLatest(entry.package); + kind = `npm release of ${entry.package}`; + compare = `https://www.npmjs.com/package/${entry.package}`; } else if (track === 'commit') { const { branch, sha } = defaultBranchHead(repo); if (!sha) throw new Error(`${repo}: could not resolve HEAD`); @@ -226,7 +249,7 @@ async function checkOne(entry) { `|---|---|`, `| Pinned in \`.sources/upstream.json\` | \`${pinned}\` |`, `| Latest ${kind} | \`${latest}\` |`, - `| Compare | https://github.com/${repo}/compare/${pinned}...${latest} |`, + `| ${compare ? 'Registry' : 'Compare'} | ${compare ?? `https://github.com/${repo}/compare/${pinned}...${latest}`} |`, ]; if (reference) lines.push(`| Published reference | ${reference} |`); lines.push(''); @@ -234,7 +257,7 @@ async function checkOne(entry) { lines.push(''); lines.push(affects ?? 'No notes recorded for this repo.'); - if (verify) { + if (verify && !compare) { const [oldText, newText] = await Promise.all([ fetchFile(repo, pinned, verify), fetchFile(repo, latest, verify), From 910b2cfe52f1f6d8ef55b28da171fe63039e07fe Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Wed, 26 Aug 2026 16:40:32 +0200 Subject: [PATCH 08/11] fix: watch the JS SDK repos instead of the docs mirror icp-js-sdk-docs is the delivery site for js.icp.build and carries the API docs for six libraries as zips. A commit on it means the docs site was rebuilt, not that an SDK released, so it was a proxy for the thing we actually document. The two packages the docs use are watched directly, and they need different signals: icp-js-core tags match its npm versions, so tags are authoritative and its CHANGELOG is the review payload, while icp-js-canisters publishes 3.6.0 with its newest semver tag at v1.0.0, so npm is the signal there. Both pins come from what the docs assert: v5.4.0 from the mirror's public/core/versions.json at the previously pinned commit, and 3.5.0 from the stated minimum in references/application-canisters.md. --- .sources/upstream.json | 19 ++++++++++++++----- AGENTS.md | 3 ++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.sources/upstream.json b/.sources/upstream.json index e00f57eb..3dd23e1f 100644 --- a/.sources/upstream.json +++ b/.sources/upstream.json @@ -123,12 +123,21 @@ "reference": "https://github.com/dfinity/ic-pub-key" }, { - "repo": "dfinity/icp-js-sdk-docs", - "pinned": "5.4.0", + "repo": "dfinity/icp-js-core", + "pinned": "v5.4.0", + "track": "release", + "tagPattern": "^v\\d+\\.\\d+\\.\\d+$", + "verify": "CHANGELOG.md", + "affects": "`@icp-sdk/core` APIs in JavaScript and TypeScript code blocks: agents, identity, Candid, and the canister-env helpers. Tags match the published npm versions.", + "reference": "https://js.icp.build/core/" + }, + { + "repo": "dfinity/icp-js-canisters", + "pinned": "3.5.0", "track": "npm", - "package": "@icp-sdk/core", - "affects": "`@icp-sdk/core` and `@icp-sdk/canisters` APIs in JavaScript and TypeScript code blocks. This repo is the docs mirror and carries no tags, so the released package version is the signal rather than a commit on it. `@icp-sdk/canisters` versions separately; check whether it moved too. The mirror's `public/core/versions.json` lists which versions it carries, and `public//latest.zip` holds the API docs to unzip.", - "reference": "https://js.icp.build" + "package": "@icp-sdk/canisters", + "affects": "`@icp-sdk/canisters` APIs, notably `AssetManager` in references/application-canisters.md and guides/frontends/asset-canister.md, which states a minimum of 3.5.0. This repo's semver tags lag its published versions, so npm is the release signal. Changelog: https://github.com/dfinity/icp-js-canisters/blob/main/CHANGELOG.md", + "reference": "https://js.icp.build/canisters/" } ] } diff --git a/AGENTS.md b/AGENTS.md index 50a868f1..ff3d245e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -255,7 +255,8 @@ have not adapted yet. | Project templates | `dfinity/icp-cli-templates` | the template's `icp.yaml` | | Motoko APIs (`mo:core`) | `dfinity/motoko-core` | `src/` | | Rust CDK (`ic-cdk`, `ic-cdk-timers`) | `dfinity/cdk-rs` | the crate's `src/` and `CHANGELOG.md` | -| JavaScript / TypeScript SDK | `dfinity/icp-js-sdk-docs` | `public//latest.zip` | +| JS SDK core (`@icp-sdk/core`) | `dfinity/icp-js-core` | `src/`, `CHANGELOG.md` | +| JS SDK canisters (`@icp-sdk/canisters`) | `dfinity/icp-js-canisters` | `src/`, `CHANGELOG.md` | | Candid spec | `dfinity/candid` | `spec/Candid.md` | | Certified variables | `dfinity/response-verification` | `packages//README.md` | | Chain Fusion Signer | `dfinity/chain-fusion-signer` | the canister's `.did` | From dbfbfd1e4364d4f5615d2d1949862c2d5397425a Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Wed, 26 Aug 2026 18:24:03 +0200 Subject: [PATCH 09/11] fix: compare commit-tracked refs by identity, correct four doc mismatches Addresses the Copilot review on #358. The commit track ordered short SHAs with the version comparator, so a real update whose SHA happened to sort lower was reported as current. Demonstrated: pinned ef03698 with head 9f1e819 returned 'current'. SHAs have no order, so the pin either prefixes the head or it does not, and links now use the full SHA while only the display is shortened. The link row was inferred from whether a registry URL was set, which mislabelled the commit track's GitHub compare view as 'Registry'; it is now named explicitly, and the verify-diff guard keys off that name rather than the URL. Also: upstream.json documented only two of the four tracks, VERSIONS claimed 7-char hashes while internetidentity uses 8, the label wording omitted both the shared upstream-sync label and the vendored form, and the workflow's jq filter uses an explicit index. --- .agents/upstream-tracking.md | 5 ++++- .github/workflows/upstream-releases.yml | 2 +- .sources/VERSIONS | 2 +- .sources/upstream.json | 5 ++++- scripts/check-upstream-releases.mjs | 18 ++++++++++++++---- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/.agents/upstream-tracking.md b/.agents/upstream-tracking.md index 47ee114f..9054f24a 100644 --- a/.agents/upstream-tracking.md +++ b/.agents/upstream-tracking.md @@ -65,7 +65,10 @@ pinned source file, because a published site always shows "latest". ### When an upstream moves `.github/workflows/upstream-releases.yml` runs weekly and opens one issue per -repo that has moved past its pin, labelled `upstream--`. The issue +repo that has moved past its pin. Every issue carries `upstream-sync`, so they +are findable as a class, plus a per-upstream label so a later run can supersede +its own issue: `upstream-dfinity-` for a watched repo and +`upstream-submodule-` for a vendored one. The issue carries the ref delta, a compare link, a summary of what changed in the repo's `verify` file when one is configured, and the `affects` note saying which pages to re-check. diff --git a/.github/workflows/upstream-releases.yml b/.github/workflows/upstream-releases.yml index b2a6d7d6..0c984738 100644 --- a/.github/workflows/upstream-releases.yml +++ b/.github/workflows/upstream-releases.yml @@ -73,7 +73,7 @@ jobs: gh label create upstream-sync --description "An upstream release needs the docs re-checked" --color 0e8a16 2>/dev/null || true gh label create "$label" --description "Upstream ref moved: ${slug}" --color ededed 2>/dev/null || true - existing=$(gh issue list --label "$label" --state open --limit 1 --json number,title --jq 'first // empty') + existing=$(gh issue list --label "$label" --state open --limit 1 --json number,title --jq '.[0] // empty') if [ -n "$existing" ]; then num=$(echo "$existing" | jq -r .number) old=$(echo "$existing" | jq -r .title) diff --git a/.sources/VERSIONS b/.sources/VERSIONS index 5bf95c15..635c66d5 100644 --- a/.sources/VERSIONS +++ b/.sources/VERSIONS @@ -18,7 +18,7 @@ # The merge commit of the release PR represents the same state and is used # instead. This is expected behaviour with --depth 1 submodule init. # -# FORMAT: <7-char hash> +# FORMAT: # # The hash is the authoritative ref — it is what .gitmodules pins and what # git checks out. The version label is the human-readable release name. diff --git a/.sources/upstream.json b/.sources/upstream.json index 3dd23e1f..ba46fc44 100644 --- a/.sources/upstream.json +++ b/.sources/upstream.json @@ -23,7 +23,10 @@ "Fields:", " repo owner/name on GitHub", " pinned the ref docs are currently verified against", - " track 'release' (tags matching tagPattern) or 'commit' (repo has no tags)", + " track where releases appear: 'release' (git tags matching tagPattern),", + " 'crate' (crates.io, needs `crate`), 'npm' (npm registry, needs", + " `package`), or 'commit' (no releases; the default branch head).", + " Entries in `vendored` use 'branch' and name the branch instead.", " tagPattern anchored JS regex selecting the tags that count as a release", " verify file whose diff between the two refs is the review payload (optional)", " affects what to re-check in docs/ when this moves", diff --git a/scripts/check-upstream-releases.mjs b/scripts/check-upstream-releases.mjs index f5fa30cd..5271a582 100644 --- a/scripts/check-upstream-releases.mjs +++ b/scripts/check-upstream-releases.mjs @@ -197,8 +197,10 @@ async function checkOne(entry) { let kind; // A registry-tracked ref is a package version, not something git can resolve, // so those entries get a registry link instead of a compare view and no file - // diff. + // diff. `moved` is set by tracks whose refs are not version-ordered. let compare; + let compareLabel = 'Compare'; + let moved; if (track === 'release') { if (!tagPattern) throw new Error(`${repo}: track "release" needs a tagPattern`); const re = new RegExp(tagPattern); @@ -225,21 +227,29 @@ async function checkOne(entry) { latest = await cratesIoNewest(entry.crate); kind = `crates.io release of ${entry.crate}`; compare = `https://crates.io/crates/${entry.crate}`; + compareLabel = 'Registry'; } else if (track === 'npm') { if (!entry.package) throw new Error(`${repo}: track "npm" needs a package name`); latest = await npmLatest(entry.package); kind = `npm release of ${entry.package}`; compare = `https://www.npmjs.com/package/${entry.package}`; + compareLabel = 'Registry'; } else if (track === 'commit') { const { branch, sha } = defaultBranchHead(repo); if (!sha) throw new Error(`${repo}: could not resolve HEAD`); latest = sha.slice(0, 7); kind = `commit on ${branch}`; + // Commit SHAs have no order, so "newer" cannot be a comparison: the pin is + // a prefix of the head or it is not. Ordering them would silently report a + // real update as current whenever the new SHA happened to sort lower. + moved = !sha.startsWith(pinned); + // Links resolve against the full SHA; only the display is shortened. + compare = `https://github.com/${repo}/compare/${pinned}...${sha}`; } else { throw new Error(`${repo}: unknown track "${track}"`); } - if (compareRefs(latest, pinned) <= 0) return null; + if (!(moved ?? compareRefs(latest, pinned) > 0)) return null; const title = `chore: upstream ${repo} moved to ${latest}`; const lines = [ @@ -249,7 +259,7 @@ async function checkOne(entry) { `|---|---|`, `| Pinned in \`.sources/upstream.json\` | \`${pinned}\` |`, `| Latest ${kind} | \`${latest}\` |`, - `| ${compare ? 'Registry' : 'Compare'} | ${compare ?? `https://github.com/${repo}/compare/${pinned}...${latest}`} |`, + `| ${compareLabel} | ${compare ?? `https://github.com/${repo}/compare/${pinned}...${latest}`} |`, ]; if (reference) lines.push(`| Published reference | ${reference} |`); lines.push(''); @@ -257,7 +267,7 @@ async function checkOne(entry) { lines.push(''); lines.push(affects ?? 'No notes recorded for this repo.'); - if (verify && !compare) { + if (verify && compareLabel !== 'Registry') { const [oldText, newText] = await Promise.all([ fetchFile(repo, pinned, verify), fetchFile(repo, latest, verify), From f9d4281730cda3d5b2d8840549c00fe944994c64 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Wed, 26 Aug 2026 18:27:07 +0200 Subject: [PATCH 10/11] infra: watch only the six upstreams whose releases can invalidate much Twelve watched entries meant a weekly check on repos the docs barely draw on. Measured footprint in docs/ decided each one: icp-cli 51 pages, 461 hits, plus the link slug motoko-core 63 pages, 328 hits cdk-rs 47 pages, 336 hits, plus pinned versions in prose icp-js-core 15 pages, 47 hits icp-cli-recipes 11 pages, 26 hits, all pinned recipe versions in prose icp-js-canisters 4 pages, 8 hits, with a stated minimum version The other six move to a group: templates (linked, not restated), candid (a rarely-moving spec), response-verification (two pages that link to docs.rs), chain-fusion-signer (seven mentions), papi (one page), ic-pub-key (three pages). They keep their entry and their verify path so an agent knows where to look, but carry no pin and produce no issue: verify against the latest release on demand, since the surface is small enough that the next edit to those pages catches drift. First run drops from ten issues to five. --- .agents/upstream-tracking.md | 30 +++++----- .sources/upstream.json | 107 ++++++++++++++++------------------- AGENTS.md | 25 +++++--- 3 files changed, 82 insertions(+), 80 deletions(-) diff --git a/.agents/upstream-tracking.md b/.agents/upstream-tracking.md index 9054f24a..f088c095 100644 --- a/.agents/upstream-tracking.md +++ b/.agents/upstream-tracking.md @@ -2,19 +2,23 @@ How this repo stays current with the projects it documents. -Upstream repos fall into two groups, and the group decides the procedure. - -| | Vendored (submodule) | Watched (not vendored) | -|---|---|---| -| Which | `motoko`, `internetidentity`, `examples` | the `watched` array in `.sources/upstream.json` | -| Why | their bytes reach the built site or the agent workflow | nothing they contain is published | -| Pin lives in | git (the gitlink), plus a label in `.sources/VERSIONS` for the release-pinned ones | `pinned` in `.sources/upstream.json` | -| Moved by | a bump PR (below) | a bump PR, triggered by an **Upstream release check** issue | -| Verified against | the checked-out working tree | the pinned ref, fetched on demand | - -Every one of them is checked weekly. `motoko` and `internetidentity` have their -own sync workflows that open the bump PR directly; the rest are covered by -**Upstream release check**, which opens an issue. +Upstream repos fall into three groups, and the group decides the procedure. + +| | Vendored (submodule) | Watched | Reference | +|---|---|---|---| +| Which | `motoko`, `internetidentity`, `examples` | the `watched` array in `.sources/upstream.json` | the `reference` array | +| Why | the build opens their files | a release can silently invalidate a lot of published content | drawn on too lightly for a weekly issue | +| Pin | the gitlink | `pinned` in `upstream.json` | none; verify against the latest release | +| Notified | yes | yes | no | + +Deciding between the last two is a judgment about blast radius, and the `why` +field on each `reference` entry records the footprint that decided it. Promote an +entry to `watched` if its footprint grows. + +`motoko` and `internetidentity` have their own weekly sync workflows that open +the bump PR directly. `examples` and the `watched` repos are covered by the +weekly **Upstream release check**, which opens an issue. `reference` repos are +not checked at all. ## Why only three are vendored diff --git a/.sources/upstream.json b/.sources/upstream.json index ba46fc44..83f28925 100644 --- a/.sources/upstream.json +++ b/.sources/upstream.json @@ -1,6 +1,6 @@ { "$comment": [ - "Every upstream this repo tracks, in two groups.", + "Every upstream this repo tracks, in three groups.", "", "`vendored` are submodules whose pin has no sync workflow of its own. Their pin", "is the gitlink in git, not a field here, so it has one source of truth; the entry", @@ -8,19 +8,21 @@ "`internetidentity` are absent on purpose: sync-motoko.yml and sync-ii-spec.yml", "already check for a new release and open the bump PR themselves.", "", - "`watched` are repos that are not vendored at all. Nothing they contain is", - "published, so this file records the ref the docs are verified against.", + "`watched` are repos where a release can silently invalidate a lot of published", + "content, so they get a pinned ref and a weekly issue when they move. Verify", + "claims against the pinned ref, not against main, so a review is reproducible.", "", - "scripts/check-upstream-releases.mjs checks both groups and", + "`reference` are repos the docs draw on too lightly to be worth a weekly issue:", + "each `why` records the footprint that decided it. They carry no pin. Verify", + "against the latest release on demand; the surface is small enough that the next", + "edit to those pages catches any drift. Promote one to `watched` if its footprint", + "grows.", + "", + "scripts/check-upstream-releases.mjs checks `vendored` and `watched` only, and", ".github/workflows/upstream-releases.yml opens an issue per repo that moved.", "See AGENTS.md 'Source material' and .agents/upstream-tracking.md.", "", - "`pinned` is the ref the docs were last verified against. Agents verify claims", - "against THIS ref rather than against main, so that a review is reproducible and a", - "page cannot document a flag from a release whose links we have not adapted yet.", - "Moving `pinned` is deliberate and comes with re-checking the affected pages.", - "", - "Fields:", + "Fields (watched):", " repo owner/name on GitHub", " pinned the ref docs are currently verified against", " track where releases appear: 'release' (git tags matching tagPattern),", @@ -28,9 +30,9 @@ " `package`), or 'commit' (no releases; the default branch head).", " Entries in `vendored` use 'branch' and name the branch instead.", " tagPattern anchored JS regex selecting the tags that count as a release", - " verify file whose diff between the two refs is the review payload (optional)", + " verify file whose diff between the two refs is the review payload", " affects what to re-check in docs/ when this moves", - " reference authoritative published docs for the topic, for readers and agents" + " reference authoritative published docs for the topic" ], "vendored": [ { @@ -59,13 +61,6 @@ "affects": "Recipe versions in `icp.yaml` examples. Each recipe releases on its own tag, so the tag prefix says which one moved. NOTE: docs currently cite `@dfinity/asset-canister@v2.2.1` while v2.3.0 is out, and `@dfinity/static-site` is not documented at all yet (dfinity/certified-assets#124).", "reference": "https://github.com/dfinity/icp-cli-recipes/releases" }, - { - "repo": "dfinity/icp-cli-templates", - "pinned": "9f1e819", - "track": "commit", - "affects": "Template layout and the recipes templates ship with, referenced from getting-started/ and guides/frontends/. All frontend templates moved to `@dfinity/static-site` after this pin.", - "reference": "https://github.com/dfinity/icp-cli-templates" - }, { "repo": "dfinity/motoko-core", "pinned": "v2.4.0", @@ -84,63 +79,59 @@ "reference": "https://docs.rs/ic-cdk/latest/ic_cdk/" }, { - "repo": "dfinity/candid", - "pinned": "2025-12-18", + "repo": "dfinity/icp-js-core", + "pinned": "v5.4.0", "track": "release", - "tagPattern": "^\\d{4}-\\d{2}-\\d{2}$", + "tagPattern": "^v\\d+\\.\\d+\\.\\d+$", + "verify": "CHANGELOG.md", + "affects": "`@icp-sdk/core` APIs in JavaScript and TypeScript code blocks: agents, identity, Candid, and the canister-env helpers. Tags match the published npm versions.", + "reference": "https://js.icp.build/core/" + }, + { + "repo": "dfinity/icp-js-canisters", + "pinned": "3.5.0", + "track": "npm", + "package": "@icp-sdk/canisters", + "affects": "`@icp-sdk/canisters` APIs, notably `AssetManager` in references/application-canisters.md and guides/frontends/asset-canister.md, which states a minimum of 3.5.0. This repo's semver tags lag its published versions, so npm is the release signal. Changelog: https://github.com/dfinity/icp-js-canisters/blob/main/CHANGELOG.md", + "reference": "https://js.icp.build/canisters/" + } + ], + "reference": [ + { + "repo": "dfinity/icp-cli-templates", + "verify": "the template's `icp.yaml`", + "why": "Docs link to templates rather than restating them: 12 mentions over 8 pages, no version asserted.", + "reference": "https://github.com/dfinity/icp-cli-templates" + }, + { + "repo": "dfinity/candid", "verify": "spec/Candid.md", - "affects": "references/candid-spec.md and the Candid type-mapping tables. The repo uses date tags; the candid, candid_parser, and didc crates carry their own semver in Cargo.toml.", + "why": "references/candid-spec.md points at the spec and restates the type tables. The spec moves rarely (date tags, latest 2025-12-18).", "reference": "https://github.com/dfinity/candid/blob/master/spec/Candid.md" }, { "repo": "dfinity/response-verification", - "pinned": "3.1.0", - "track": "release", - "tagPattern": "^\\d+\\.\\d+\\.\\d+$", - "affects": "guides/frontends/certification.md and guides/backends/certified-variables.md: the `ic-http-certification`, `ic-asset-certification`, and `@dfinity/certificate-verification` APIs.", - "reference": "https://docs.rs/ic-http-certification/latest/ic_http_certification/", - "verify": "CHANGELOG.md" + "verify": "packages//README.md", + "why": "Two pages, guides/frontends/certification.md and guides/backends/certified-variables.md, both of which link to docs.rs for the API.", + "reference": "https://docs.rs/ic-http-certification/latest/ic_http_certification/" }, { "repo": "dfinity/chain-fusion-signer", - "pinned": "v0.4.0", - "track": "release", - "tagPattern": "^v\\d+\\.\\d+\\.\\d+$", - "affects": "guides/chain-fusion/chain-fusion-signer.md: canister IDs, API methods, key derivation.", + "verify": "the canister's `.did`", + "why": "Three pages, seven mentions.", "reference": "https://github.com/dfinity/chain-fusion-signer" }, { "repo": "dfinity/papi", - "pinned": "v0.1.1", - "track": "release", - "tagPattern": "^v\\d+\\.\\d+\\.\\d+$", - "affects": "Payment interface and cycle cost model wherever PAPI is referenced.", + "verify": "README.md", + "why": "One page mentions it.", "reference": "https://github.com/dfinity/papi" }, { "repo": "dfinity/ic-pub-key", - "pinned": "v1.0.1", - "track": "release", - "tagPattern": "^v\\d+\\.\\d+\\.\\d+$", - "affects": "guides/chain-fusion/offline-key-derivation.md: CLI flags and the library API.", + "verify": "README.md", + "why": "Three pages; guides/chain-fusion/offline-key-derivation.md is the only one with CLI detail.", "reference": "https://github.com/dfinity/ic-pub-key" - }, - { - "repo": "dfinity/icp-js-core", - "pinned": "v5.4.0", - "track": "release", - "tagPattern": "^v\\d+\\.\\d+\\.\\d+$", - "verify": "CHANGELOG.md", - "affects": "`@icp-sdk/core` APIs in JavaScript and TypeScript code blocks: agents, identity, Candid, and the canister-env helpers. Tags match the published npm versions.", - "reference": "https://js.icp.build/core/" - }, - { - "repo": "dfinity/icp-js-canisters", - "pinned": "3.5.0", - "track": "npm", - "package": "@icp-sdk/canisters", - "affects": "`@icp-sdk/canisters` APIs, notably `AssetManager` in references/application-canisters.md and guides/frontends/asset-canister.md, which states a minimum of 3.5.0. This repo's semver tags lag its published versions, so npm is the release signal. Changelog: https://github.com/dfinity/icp-js-canisters/blob/main/CHANGELOG.md", - "reference": "https://js.icp.build/canisters/" } ] } diff --git a/AGENTS.md b/AGENTS.md index ff3d245e..7e2c5ead 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -233,10 +233,10 @@ Pinned versions: [`.sources/VERSIONS`](.sources/VERSIONS). `motoko` and `examples` tracks a branch and is checked by the weekly **Upstream release check**. Canister IDs and code patterns are in the skills (see "Skills"). -**Watched, not vendored** — everything else. Nothing they contain is published, -so the repo records a pinned ref instead of a copy, and a weekly workflow opens -an issue when one moves. Read the file you need at the **pinned ref** in -[`.sources/upstream.json`](.sources/upstream.json): +**Watched** are the repos where a release can silently invalidate a lot of what +is published, so [`.sources/upstream.json`](.sources/upstream.json) records the +ref the docs are verified against and a weekly workflow opens an issue when one +moves. Read the file you need at that **pinned ref**: ```bash curl -sL https://raw.githubusercontent.com/// @@ -251,12 +251,19 @@ have not adapted yet. | Topic | Repo | Verify against | |-------|------|----------------| | CLI commands and flags | `dfinity/icp-cli` | `docs/reference/cli.md` | -| CLI recipes | `dfinity/icp-cli-recipes` | `recipes//` | -| Project templates | `dfinity/icp-cli-templates` | the template's `icp.yaml` | -| Motoko APIs (`mo:core`) | `dfinity/motoko-core` | `src/` | -| Rust CDK (`ic-cdk`, `ic-cdk-timers`) | `dfinity/cdk-rs` | the crate's `src/` and `CHANGELOG.md` | +| Recipe versions in `icp.yaml` | `dfinity/icp-cli-recipes` | `recipes//` | +| Motoko APIs (`mo:core`) | `dfinity/motoko-core` | `src/`, `Changelog.md` | +| Rust CDK (`ic-cdk`) | `dfinity/cdk-rs` | `ic-cdk/src/`, `ic-cdk/CHANGELOG.md` | | JS SDK core (`@icp-sdk/core`) | `dfinity/icp-js-core` | `src/`, `CHANGELOG.md` | | JS SDK canisters (`@icp-sdk/canisters`) | `dfinity/icp-js-canisters` | `src/`, `CHANGELOG.md` | + +**Reference** repos are drawn on too lightly to be worth a weekly issue, so they +carry no pin. Verify against the **latest** release on demand; the surface is +small enough that the next edit to those pages catches any drift. + +| Topic | Repo | Verify against | +|-------|------|----------------| +| Project templates | `dfinity/icp-cli-templates` | the template's `icp.yaml` | | Candid spec | `dfinity/candid` | `spec/Candid.md` | | Certified variables | `dfinity/response-verification` | `packages//README.md` | | Chain Fusion Signer | `dfinity/chain-fusion-signer` | the canister's `.did` | @@ -265,7 +272,7 @@ have not adapted yet. Several of these also publish authoritative docs (the `reference` field in `upstream.json`). Those are what to **link readers to**; for verifying a claim, -prefer the pinned source file, since a published site always shows "latest". +prefer the source file, since a published site always shows "latest". For the tracking and bump procedures, see [`.agents/upstream-tracking.md`](.agents/upstream-tracking.md). From ebce87f91fd5667c2bd53bda24f91f9119a0af85 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Wed, 26 Aug 2026 18:46:01 +0200 Subject: [PATCH 11/11] fix: use full SHAs where a machine consumes them Addresses the second Copilot review on #358. The vendored check had both full SHAs in hand but put 7-char prefixes in the compare link and, more importantly, in the git checkout command it asks a maintainer to run. Short SHAs are now display-only. The commit track's comment also claimed links resolve against the full SHA, while the pin side of the compare URL is whatever upstream.json records. The comment now describes that: the head side is full, the pin side stays short for readability, and GitHub resolves a prefix on either side. --- scripts/check-upstream-releases.mjs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/check-upstream-releases.mjs b/scripts/check-upstream-releases.mjs index 5271a582..a37c0d98 100644 --- a/scripts/check-upstream-releases.mjs +++ b/scripts/check-upstream-releases.mjs @@ -159,6 +159,8 @@ function checkVendored(entry) { if (!headSha) throw new Error(`${repo}: no branch ${branch}`); if (pinnedSha === headSha) return null; + // Short SHAs are for the table and the issue title; anything a machine + // consumes (the compare link, the checkout command) gets the full SHA. const pinned = pinnedSha.slice(0, 7); const latest = headSha.slice(0, 7); const name = path.replace(/^\.sources\//, ''); @@ -169,7 +171,7 @@ function checkVendored(entry) { '|---|---|', `| Pinned (gitlink) | \`${pinned}\` |`, `| Branch head | \`${latest}\` |`, - `| Compare | https://github.com/${repo}/compare/${pinned}...${latest} |`, + `| Compare | https://github.com/${repo}/compare/${pinnedSha}...${headSha} |`, '', '## What to re-check', '', @@ -179,7 +181,7 @@ function checkVendored(entry) { '', '```bash', `git -C ${path} fetch origin ${branch}`, - `git -C ${path} checkout ${latest}`, + `git -C ${path} checkout ${headSha}`, '```', '', 'Then work through the submodule checklist and commit the new pointer.', @@ -243,7 +245,9 @@ async function checkOne(entry) { // a prefix of the head or it is not. Ordering them would silently report a // real update as current whenever the new SHA happened to sort lower. moved = !sha.startsWith(pinned); - // Links resolve against the full SHA; only the display is shortened. + // The head side uses the full SHA. The pin side is whatever `upstream.json` + // records, kept short there for readability; GitHub's compare view resolves + // a short prefix on either side. compare = `https://github.com/${repo}/compare/${pinned}...${sha}`; } else { throw new Error(`${repo}: unknown track "${track}"`);