perf: reduce sitemap generation overhead (#633)

Author: harlan-zw

src/prerender.ts

@@ -49,6 +49,7 @@ export function setupPrerenderHandler(_options: { runtimeConfig: ModuleRuntimeCo
   nuxt.options.nitro.prerender.routes = nuxt.options.nitro.prerender.routes.filter(r => r && !includesSitemapRoot(options.sitemapName, [r]))
 
   const runtimeAssetsPath = join(nuxt.options.rootDir, 'node_modules/.cache/nuxt/sitemap')
+  const localeCodes = options.autoI18n ? new Set(options.autoI18n.locales.map(l => l.code)) : undefined
 
   // Setup virtual module for reading sources - must be in nitro:config to be bundled
   nuxt.hooks.hook('nitro:config', (nitroConfig) => {
@@ -112,7 +113,7 @@ export async function readSourcesFromFilesystem(filename) {
       // we need to figure out which sitemap this belongs to
       if (options.autoI18n && Object.keys(options.sitemaps).length > 1) {
         const path = route.route
-        const match = splitForLocales(path, options.autoI18n.locales.map(l => l.code))
+        const match = splitForLocales(path, localeCodes!)
         // if it's missing a locale then we put it in the default locale sitemap
         const locale = match[0] || options.autoI18n.defaultLocale
         if (options.isI18nMapped) {

src/runtime/server/kit.ts

@@ -5,10 +5,16 @@ import { createRouter as createRadixRouter, toRouteMatcher } from 'radix3'
 import { parseURL, withoutBase, withoutTrailingSlash } from 'ufo'
 
 function withoutQuery(path: string): string {
-  return path.split('?')[0]!
+  const queryIndex = path.indexOf('?')
+  return queryIndex === -1 ? path : path.slice(0, queryIndex)
 }
 
+let cachedRouteRuleMatcher: ((pathOrUrl: string) => NitroRouteRules) | undefined
+
 export function createNitroRouteRuleMatcher(): (pathOrUrl: string) => NitroRouteRules {
+  if (!import.meta.dev && cachedRouteRuleMatcher)
+    return cachedRouteRuleMatcher
+
   const { nitro, app } = useRuntimeConfig()
   const _routeRulesMatcher = toRouteMatcher(
     createRadixRouter({
@@ -18,10 +24,13 @@ export function createNitroRouteRuleMatcher(): (pathOrUrl: string) => NitroRoute
       ),
     }),
   )
-  return (pathOrUrl: string) => {
+  const matcher = (pathOrUrl: string) => {
     const path = pathOrUrl[0] === '/' ? pathOrUrl : parseURL(pathOrUrl, app.baseURL).pathname
     return defu({}, ..._routeRulesMatcher.matchAll(
       withoutBase(withoutTrailingSlash(withoutQuery(path)), app.baseURL),
     ).reverse()) as NitroRouteRules
   }
+  if (!import.meta.dev)
+    cachedRouteRuleMatcher = matcher
+  return matcher
 }

src/runtime/server/plugins/compression.ts

@@ -12,7 +12,7 @@ function getPreferredEncoding(event: H3Event): 'gzip' | 'deflate' | null {
 }
 
 export default defineNitroPlugin((nitro) => {
-  nitro.hooks.hook('beforeResponse', async (event, response) => {
+  nitro.hooks.hook('beforeResponse', (event, response) => {
     if (!event.context._isSitemap || !response.body)
       return
 
@@ -21,8 +21,7 @@ export default defineNitroPlugin((nitro) => {
       return
 
     const body = typeof response.body === 'string' ? response.body : JSON.stringify(response.body)
-    const stream = new Blob([body]).stream().pipeThrough(new CompressionStream(encoding))
-    response.body = Buffer.from(await new Response(stream).arrayBuffer())
+    response.body = new Blob([body]).stream().pipeThrough(new CompressionStream(encoding))
     setResponseHeader(event, 'Content-Encoding', encoding)
   })
 })

src/runtime/server/sitemap/builder/sitemap-index.ts

@@ -146,13 +146,18 @@ async function buildSitemapIndexInternal(resolvers: NitroUrlResolvers, runtimeCo
 }
 
 export function urlsToIndexXml(sitemaps: SitemapIndexEntry[], resolvers: NitroUrlResolvers, { version, xsl, credits, minify }: Pick<ModuleRuntimeConfig, 'version' | 'xsl' | 'credits' | 'minify'>, errorInfo?: { messages: string[], urls: string[] }) {
-  const sitemapXml = sitemaps.map(e => [
-    '    <sitemap>',
-    `        <loc>${escapeValueForXml(e.sitemap)}</loc>`,
-    // lastmod is optional
-    e.lastmod ? `        <lastmod>${escapeValueForXml(e.lastmod)}</lastmod>` : false,
-    '    </sitemap>',
-  ].filter(Boolean).join('\n')).join('\n')
+  const NL = minify ? '' : '\n'
+  const I1 = minify ? '' : '    '
+  const I2 = minify ? '' : '        '
+  let sitemapXml = ''
+  for (const entry of sitemaps) {
+    if (sitemapXml)
+      sitemapXml += NL
+    sitemapXml += `${I1}<sitemap>${NL}${I2}<loc>${escapeValueForXml(entry.sitemap)}</loc>${NL}`
+    if (entry.lastmod)
+      sitemapXml += `${I2}<lastmod>${escapeValueForXml(entry.lastmod)}</lastmod>${NL}`
+    sitemapXml += `${I1}</sitemap>`
+  }
 
   const xmlParts = [
     '<?xml version="1.0" encoding="UTF-8"?>',
@@ -186,10 +191,7 @@ export function urlsToIndexXml(sitemaps: SitemapIndexEntry[], resolvers: NitroUr
     xmlParts.push(`<!-- XML Sitemap Index generated by @nuxtjs/sitemap v${version} at ${new Date().toISOString()} -->`)
   }
 
-  // Join with appropriate separator
-  return minify
-    ? xmlParts.join('').replace(/(?<!<[^>]*)\s(?![^<]*>)/g, '')
-    : xmlParts.join('\n')
+  return xmlParts.join(NL)
 }
 
 export async function buildSitemapIndex(resolvers: NitroUrlResolvers, runtimeConfig: ModuleRuntimeConfig, nitro?: NitroApp) {

src/runtime/server/sitemap/builder/sitemap.ts

@@ -37,22 +37,24 @@ export function resolveSitemapEntries(sitemap: SitemapDefinition, urls: SitemapU
     autoI18n,
     isI18nMapped,
   } = runtimeConfig
-  const filterPath = createPathFilter({
-    include: sitemap.include,
-    exclude: sitemap.exclude,
-  }, baseURL || '/')
+  const hasFilters = !!sitemap.include?.length || !!sitemap.exclude?.length
+  const filterPath = hasFilters
+    ? createPathFilter({
+        include: sitemap.include,
+        exclude: sitemap.exclude,
+      }, baseURL || '/')
+    : undefined
   // 1. normalise
-  const _urls = urls.map((_e) => {
+  const _urls: ResolvedSitemapUrl[] = []
+  for (const _e of urls) {
     const e = preNormalizeEntry(_e, resolvers)
-    if (!e.loc || !filterPath(e.loc))
-      return false
-    return e
-  }).filter(Boolean) as ResolvedSitemapUrl[]
+    if (e.loc && (!filterPath || filterPath(e.loc, e._path?.pathname)))
+      _urls.push(e)
+  }
 
-  let validI18nUrlsForTransform: NormalizedI18n[] = []
   const withoutPrefixPaths: Record<string, NormalizedI18n[]> = {}
   if (autoI18n && autoI18n.strategy !== 'no_prefix') {
-    const localeCodes = autoI18n.locales.map(l => l.code)
+    const localeCodes = new Set(autoI18n.locales.map(l => l.code))
     // Create locale lookup Map for O(1) access
     const localeByCode = new Map(autoI18n.locales.map(l => [l.code, l]))
     // Pre-check strategy once
@@ -63,11 +65,14 @@ export function resolveSitemapEntries(sitemap: SitemapDefinition, urls: SitemapU
     // Cache frequently accessed values
     const defaultLocale = autoI18n.defaultLocale
     const hasPages = !!autoI18n.pages
+    const sortedPageKeys = hasPages ? Object.keys(autoI18n.pages!).sort((a, b) => b.length - a.length) : undefined
     const hasDifferentDomains = !!autoI18n.differentDomains
 
-    validI18nUrlsForTransform = _urls.map((_e, i) => {
+    const validI18nUrlsForTransform: NormalizedI18n[] = []
+    for (let i = 0; i < _urls.length; i++) {
+      const _e = _urls[i]!
       if (_e._abs)
-        return false
+        continue
       const split = splitForLocales(_e._relativeLoc, localeCodes)
       let localeCode = split[0]
       const pathWithoutPrefix = split[1]
@@ -78,37 +83,33 @@ export function resolveSitemapEntries(sitemap: SitemapDefinition, urls: SitemapU
       // Use Map instead of find for O(1) lookup
       const locale = localeByCode.get(localeCode)
       if (!locale)
-        return false
+        continue
       e._locale = locale
       e._index = i
       e._key = `${e._sitemap || ''}${e._path?.pathname || '/'}${e._path?.search || ''}`
       withoutPrefixPaths[pathWithoutPrefix] = withoutPrefixPaths[pathWithoutPrefix] || []
       // need to make sure the locale doesn't already exist
       if (!withoutPrefixPaths[pathWithoutPrefix].some(e => e._locale.code === locale.code))
         withoutPrefixPaths[pathWithoutPrefix].push(e)
-      return e
-    }).filter(Boolean) as NormalizedI18n[]
+      validI18nUrlsForTransform.push(e)
+    }
 
     for (const e of validI18nUrlsForTransform) {
       // let's try and find other urls that we can use for alternatives
       if (!e._i18nTransform && !e.alternatives?.length) {
-        const alternatives = (withoutPrefixPaths[e._pathWithoutPrefix] || [])
-          .map((u) => {
-            const entries: AlternativeEntry[] = []
-            if (u._locale.code === defaultLocale) {
-              entries.push({
-                href: u.loc,
-                hreflang: 'x-default',
-              })
-            }
-            entries.push({
+        const alternatives: AlternativeEntry[] = []
+        for (const u of withoutPrefixPaths[e._pathWithoutPrefix] || []) {
+          if (u._locale.code === defaultLocale) {
+            alternatives.push({
               href: u.loc,
-              hreflang: u._locale._hreflang || defaultLocale,
+              hreflang: 'x-default',
             })
-            return entries
+          }
+          alternatives.push({
+            href: u.loc,
+            hreflang: u._locale._hreflang || defaultLocale,
           })
-          .flat()
-          .filter(Boolean) as AlternativeEntry[]
+        }
         if (alternatives.length)
           e.alternatives = alternatives
       }
@@ -135,7 +136,7 @@ export function resolveSitemapEntries(sitemap: SitemapDefinition, urls: SitemapU
         }
         else {
           // Find page mapping with support for dynamic routes
-          const pageMatch = hasPages ? findPageMapping(e._pathWithoutPrefix, autoI18n.pages!) : null
+          const pageMatch = hasPages ? findPageMapping(e._pathWithoutPrefix, autoI18n.pages!, sortedPageKeys) : null
           const pathSearch = e._path?.search || ''
           const pathWithoutPrefix = e._pathWithoutPrefix
 
@@ -191,7 +192,7 @@ export function resolveSitemapEntries(sitemap: SitemapDefinition, urls: SitemapU
                 href = joinURL('/', code, pathWithoutPrefix)
               }
 
-              if (!filterPath(href))
+              if (filterPath && !filterPath(href))
                 continue
               alternatives.push({
                 hreflang: locale._hreflang,
@@ -283,11 +284,18 @@ export async function buildResolvedSitemapUrls(
   const localeSitemapKeys = isI18nMapped && autoI18n ? autoI18n.locales.map(l => l._sitemap) : []
   if (isMultiSitemap) {
     const sitemapNames = Object.keys(sitemaps).filter(k => k !== 'index')
+    const validSitemapNames = new Set(sitemapNames)
+    if (isI18nMapped) {
+      for (const name of sitemapNames) {
+        const localeKey = resolveI18nSitemapLocaleKey(name, localeSitemapKeys)
+        if (localeKey)
+          validSitemapNames.add(localeKey)
+      }
+    }
     // @ts-expect-error loose typing
     const warnedSitemaps = nitro?._sitemapWarnedSitemaps || new Set<string>()
     for (const e of enhancedUrls) {
-      const hasMatchingSitemap = typeof e._sitemap === 'string'
-        && (sitemapNames.includes(e._sitemap) || (isI18nMapped && sitemapNames.some(name => resolveI18nSitemapLocaleKey(name, localeSitemapKeys) === e._sitemap)))
+      const hasMatchingSitemap = typeof e._sitemap === 'string' && validSitemapNames.has(e._sitemap)
       if (typeof e._sitemap === 'string' && !hasMatchingSitemap) {
         if (!warnedSitemaps.has(e._sitemap)) {
           warnedSitemaps.add(e._sitemap)
@@ -301,6 +309,7 @@ export async function buildResolvedSitemapUrls(
     }
   }
 
+  const matchedLocaleSitemap = isI18nMapped ? resolveI18nSitemapLocaleKey(matchName, localeSitemapKeys) : null
   const filteredUrls = enhancedUrls.filter((e) => {
     if (e._sitemap === false)
       return false
@@ -313,7 +322,7 @@ export async function buildResolvedSitemapUrls(
       // back to its locale key (longest match) so prefix-sharing locales don't collide,
       // e.g. a `zh` URL must not land in the `zh-Hant` sitemap.
       if (isI18nMapped)
-        return e._sitemap === resolveI18nSitemapLocaleKey(matchName, localeSitemapKeys)
+        return e._sitemap === matchedLocaleSitemap
       return false
     }
     return true

src/runtime/server/sitemap/builder/xml.ts

@@ -31,7 +31,10 @@ function buildUrlXml(url: ResolvedSitemapUrl, NL: string, I1: string, I2: string
   if (url.alternatives) {
     for (const alt of url.alternatives) {
       let attrs = ''
-      for (const [k, v] of Object.entries(alt)) attrs += ` ${k}="${xmlEscape(String(v))}"`
+      for (const k in alt) {
+        if (Object.hasOwn(alt, k))
+          attrs += ` ${k}="${xmlEscape(String(alt[k as keyof typeof alt]))}"`
+      }
       xml += `${I2}<xhtml:link rel="alternate"${attrs} />${NL}`
     }
   }

src/runtime/server/sitemap/nitro.ts

@@ -75,6 +75,9 @@ async function buildSitemapXml(event: H3Event, definition: SitemapDefinition, re
 
   const routeRuleMatcher = createNitroRouteRuleMatcher()
   const { autoI18n } = runtimeConfig
+  const localeCodes = autoI18n?.locales && autoI18n.strategy !== 'no_prefix'
+    ? new Set(autoI18n.locales.map(l => l.code))
+    : undefined
 
   // Process in place to avoid creating intermediate arrays
   let validCount = 0
@@ -89,8 +92,8 @@ async function buildSitemapXml(event: H3Event, definition: SitemapDefinition, re
     let routeRules = routeRuleMatcher(path)
 
     // Apply top-level path without prefix
-    if (autoI18n?.locales && autoI18n?.strategy !== 'no_prefix') {
-      const match = splitForLocales(path, autoI18n.locales.map(l => l.code))
+    if (localeCodes) {
+      const match = splitForLocales(path, localeCodes)
       const pathWithoutPrefix = match[1]
       if (pathWithoutPrefix && pathWithoutPrefix !== path)
         routeRules = defu(routeRules, routeRuleMatcher(pathWithoutPrefix))
@@ -102,8 +105,16 @@ async function buildSitemapXml(event: H3Event, definition: SitemapDefinition, re
     if (typeof routeRules.robots !== 'undefined' && !routeRules.robots)
       continue
 
-    const hasRobotsDisabled = Object.entries(routeRules.headers || {})
-      .some(([name, value]) => name.toLowerCase() === 'x-robots-tag' && value.toLowerCase().includes('noindex'))
+    let hasRobotsDisabled = false
+    const headers = routeRules.headers
+    if (headers) {
+      for (const name in headers) {
+        if (name.toLowerCase() === 'x-robots-tag' && headers[name]!.toLowerCase().includes('noindex')) {
+          hasRobotsDisabled = true
+          break
+        }
+      }
+    }
 
     if (routeRules.redirect || hasRobotsDisabled)
       continue
@@ -129,14 +140,34 @@ async function buildSitemapXml(event: H3Event, definition: SitemapDefinition, re
   // we need to normalize any new urls otherwise they won't appear in the final sitemap
   // Note this is risky and users should be using the sitemap:input hook for additions
   if (resolvedCtx.urls.length !== locSize) {
-    resolvedCtx.urls = resolvedCtx.urls.map(e => preNormalizeEntry(e, resolvers))
+    for (let i = 0; i < resolvedCtx.urls.length; i++)
+      resolvedCtx.urls[i] = preNormalizeEntry(resolvedCtx.urls[i]!, resolvers)
   }
 
   const maybeSort = (urls: ResolvedSitemapUrl[]) => runtimeConfig.sortEntries ? sortInPlace(urls) : urls
   // final urls
-  const defaults = definition.defaults || {}
-  const normalizedPreDedupe = resolvedCtx.urls.map(e => normaliseEntry(e, defaults, resolvers))
-  const urls = maybeSort(mergeOnKey(normalizedPreDedupe, '_key').map(e => normaliseEntry(e, defaults, resolvers)))
+  const defaults = definition.defaults
+  const normalizedPreDedupe = resolvedCtx.urls
+  // Repeated CMS/default timestamps are common and expensive to parse. Sample the leading entries
+  // so unique timestamp feeds avoid paying cache bookkeeping on every URL.
+  const firstLastmod = normalizedPreDedupe[0]?.lastmod ?? defaults?.lastmod
+  let cacheLastmod = normalizedPreDedupe.length > 1 && !!firstLastmod
+  for (let i = 1; cacheLastmod && i < Math.min(normalizedPreDedupe.length, 8); i++)
+    cacheLastmod = (normalizedPreDedupe[i]!.lastmod ?? defaults?.lastmod) === firstLastmod
+  const normaliseCache = cacheLastmod ? {} : undefined
+  for (let i = 0; i < normalizedPreDedupe.length; i++)
+    normalizedPreDedupe[i] = normaliseEntry(normalizedPreDedupe[i]!, defaults, resolvers, normaliseCache)
+  const duplicateKeys = new Set<string>()
+  const urls = mergeOnKey(normalizedPreDedupe, '_key', key => duplicateKeys.add(key))
+  // A merge can introduce duplicate alternatives/images/videos. Unique URLs are already fully
+  // normalized, so avoid cloning and resolving every nested entry a second time.
+  if (duplicateKeys.size) {
+    for (let i = 0; i < urls.length; i++) {
+      if (duplicateKeys.has(urls[i]!._key))
+        urls[i] = normaliseEntry(urls[i]!, defaults, resolvers, normaliseCache)
+    }
+  }
+  maybeSort(urls)
 
   // Check if this is a chunk request that would be empty
   if (definition._isChunking && definition.sitemapName.includes('-')) {