feat(seo): add a temporary sitemap of retired URLs

layers/registry/app/components/SkillDetail.vue

@@ -5,6 +5,7 @@ import { resolveSkillAuditOverview } from '~~/app/utils/skill-audit-overview'
 import { avatarProxyUrl, githubAvatarProxyUrl } from '#shared/image-proxy'
 import { skillPageUrl as exactSkillPageUrl, skillInstallCmd, skillRunCmd, skillRunPrompt } from '#shared/skill-commands'
 import { partitionMetadataEntries } from '../utils/skill-metadata'
+import { resolveSkillPageState } from '../utils/skill-page-state'
 import { resolveSkillRawUrl } from '../utils/skill-raw-url'
 import { resolveSkillTitle } from '../utils/skill-title'
 import SkillCommandPanel from './_SkillCommandPanel.vue'
@@ -790,10 +791,33 @@ const canonicalSkillPagePath = computed(() => {
   const canonical = duplicateGroup.value?.canonical
   return canonical?.registryPath ?? skillPagePath.value
 })
-const canonicalSkillPageUrl = computed(() => {
-  return `${siteOrigin}${canonicalSkillPagePath.value}`
+
+const pageState = computed(() => {
+  const skill = data.value
+  if (isMissingSkill.value)
+    return resolveSkillPageState({ _tag: 'missing' })
+  if (!skill)
+    return resolveSkillPageState({ _tag: error.value ? 'failed' : 'loading' })
+  return resolveSkillPageState({
+    _tag: 'loaded',
+    indexable: skill.seo.indexable,
+    sourceGone: skill.sourceGone,
+    registryPath: skill.registryPath,
+    duplicateCanonicalPath: isWeakerDuplicate.value ? canonicalSkillPagePath.value : null,
+  })
 })
 
+// Set the status while rendering on the server. A missing Skill keeps its
+// helpful page, but the response says 404 so Google drops the URL instead of
+// filing it as a soft 404.
+if (import.meta.server && pageState.value.status) {
+  const event = useRequestEvent()!
+  setResponseStatus(event, pageState.value.status)
+  // A 503 tells Google the failure is transient, so it retries the URL.
+  if (pageState.value.retryAfterSeconds)
+    useResponseHeader('Retry-After').value = String(pageState.value.retryAfterSeconds)
+}
+
 // 2026-08-22: the shared "A Claude Code skill for Cursor, Codex, and other
 // agents." suffix is gone. Identical boilerplate across 1,300+ meta
 // descriptions was the last scaled-content signature in Google snippets
@@ -884,22 +908,17 @@ const skillDescription = computed(() => {
 useSeoMeta({
   title: () => skillTitle.value,
   description: () => skillDescription.value,
-  robots: () => {
-    return data.value?.seo.indexable && !isWeakerDuplicate.value ? 'index,follow' : 'noindex,follow'
-  },
+  robots: () => pageState.value.robots ?? undefined,
   ogTitle: () => skillTitle.value,
   ogDescription: () => skillDescription.value,
   twitterTitle: () => skillTitle.value,
   twitterDescription: () => skillDescription.value,
 })
 
 useHead(computed(() => ({
-  link: [
-    {
-      rel: 'canonical',
-      href: canonicalSkillPageUrl.value,
-    },
-  ],
+  link: pageState.value.canonicalPath
+    ? [{ rel: 'canonical', href: `${siteOrigin}${pageState.value.canonicalPath}` }]
+    : [],
 })))
 </script>
 

layers/registry/app/utils/skill-page-state.ts

@@ -0,0 +1,68 @@
+/**
+ * What a `/gh/<owner>/<repo>/<name>` page knows about its Skill, as data.
+ *
+ * `missing` is a 404 from the Skill API: the registry holds no row. A Skill
+ * that is real on GitHub but outside the registry lands here too, because the
+ * skill route never proxies GitHub. The repository hub is the proxied surface.
+ */
+export type SkillPageInput
+  = | { _tag: 'loading' }
+    | { _tag: 'failed' }
+    | { _tag: 'missing' }
+    | {
+      _tag: 'loaded'
+      indexable: boolean
+      /** The SKILL.md was deleted upstream; the site keeps a tombstone. */
+      sourceGone: boolean
+      registryPath: string
+      /** Set when a stronger duplicate owns this content. */
+      duplicateCanonicalPath: string | null
+    }
+
+export interface SkillPageState {
+  _tag: 'loading' | 'failed' | 'missing' | 'gone' | 'duplicate' | 'indexable' | 'noindex'
+  /** HTTP status to force, or null to leave the response alone. */
+  status: 404 | 410 | 503 | null
+  /** Seconds for the `Retry-After` header. Set only with status 503. */
+  retryAfterSeconds: number | null
+  /** The robots directive, or null to emit none and let the site default apply. */
+  robots: 'index,follow' | 'noindex,follow' | null
+  /** Site path for the canonical link, or null to omit the link. */
+  canonicalPath: string | null
+}
+
+/** How long Google should wait before it retries a Skill page that errored. */
+export const SKILL_PAGE_RETRY_AFTER_SECONDS = 300
+
+/**
+ * Decide the status, robots directive, and canonical for a Skill page.
+ *
+ * A canonical must name a page that exists. The old code fell back to the
+ * site root when no Skill loaded, which told Google that every made-up URL
+ * duplicated the homepage. Google read those as soft 404s (2,621 pages on
+ * 2026-09-30). A page with no Skill now omits the canonical.
+ *
+ * `failed` is an API error that is not a 404: the registry could not answer,
+ * so the page may be real. It answers 503 with `Retry-After` and emits no
+ * robots directive, so Google retries and keeps the URL. `noindex` on a
+ * transient error could drop a real page (D1 overload, September 2026).
+ */
+export function resolveSkillPageState(input: SkillPageInput): SkillPageState {
+  switch (input._tag) {
+    case 'missing':
+      return { _tag: 'missing', status: 404, retryAfterSeconds: null, robots: 'noindex,follow', canonicalPath: null }
+    case 'loading':
+      return { _tag: 'loading', status: null, retryAfterSeconds: null, robots: 'noindex,follow', canonicalPath: null }
+    case 'failed':
+      return { _tag: 'failed', status: 503, retryAfterSeconds: SKILL_PAGE_RETRY_AFTER_SECONDS, robots: null, canonicalPath: null }
+    case 'loaded': {
+      if (input.sourceGone)
+        return { _tag: 'gone', status: 410, retryAfterSeconds: null, robots: 'noindex,follow', canonicalPath: input.registryPath }
+      if (input.duplicateCanonicalPath)
+        return { _tag: 'duplicate', status: null, retryAfterSeconds: null, robots: 'noindex,follow', canonicalPath: input.duplicateCanonicalPath }
+      return input.indexable
+        ? { _tag: 'indexable', status: null, retryAfterSeconds: null, robots: 'index,follow', canonicalPath: input.registryPath }
+        : { _tag: 'noindex', status: null, retryAfterSeconds: null, robots: 'noindex,follow', canonicalPath: input.registryPath }
+    }
+  }
+}

layers/registry/server/api/__sitemap__/retired.ts

@@ -0,0 +1,38 @@
+import { GONE_SKILLS_SQL } from '~~/server/utils/source-gone-skills'
+import { getDB } from '#server/utils/db'
+import retiredUrls from '../../data/retired-urls.json'
+import { buildRetiredSitemapEntries, isRetiredSitemapActive } from '../../utils/retired-sitemap'
+import { MARKETING_REDIRECTS } from '../../utils/tag-quality'
+
+// Temporary sitemap of retired URLs. Experiment E, remove 2026-11-11. The
+// reasons, the status guarantee and the removal steps are in
+// `../../utils/retired-sitemap.ts`.
+export default defineSitemapEventHandler(async (event) => {
+  // Past the removal date, skip the D1 queries too.
+  const now = new Date()
+  if (!isRetiredSitemapActive(now))
+    return []
+
+  const db = getDB(event)
+  const [gone, collections] = await Promise.all([
+    db.prepare(GONE_SKILLS_SQL).all<{ owner: string, repo: string, name: string }>(),
+    db.prepare(`
+      SELECT u.login, c.slug
+      FROM collections_v2 c
+      JOIN users u ON u.id = c.author_user_id
+      WHERE c.deleted_at IS NOT NULL
+        AND NOT EXISTS (
+          SELECT 1 FROM collections_v2 live
+          WHERE live.author_user_id = c.author_user_id AND live.slug = c.slug AND live.deleted_at IS NULL
+        )
+    `).all<{ login: string, slug: string }>(),
+  ])
+
+  return buildRetiredSitemapEntries({
+    now,
+    goneSkillPaths: (gone.results ?? []).map(row => `/gh/${row.owner}/${row.repo}/${row.name}`),
+    deletedCollectionPaths: (collections.results ?? []).map(row => `/@${row.login}/${row.slug}`),
+    // Tag pages that hand off to a category: a static 301 each.
+    probedPaths: [...retiredUrls.paths, ...Object.keys(MARKETING_REDIRECTS).map(slug => `/skills/tag/${slug}`)],
+  })
+})