XML Sitemap Engineering: Dynamic Generation and Best Practices

<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <url>
    <loc>https://example.com/</loc>
    <lastmod>2026-09-15T08:30:00+00:00</lastmod>
    <changefreq>daily</changefreq>
    <priority>1.0</priority>
  </url>
  <url>
    <loc>https://example.com/blog/technical-seo-guide/</loc>
    <lastmod>2026-09-10T14:00:00+00:00</lastmod>
    <changefreq>weekly</changefreq>
    <priority>0.8</priority>
  </url>
</urlset>

// app/sitemap.ts — Next.js 14/15 built-in sitemap generation
import { MetadataRoute } from 'next';

// Database or CMS query for all published content
async function getAllPosts() {
  const posts = await db.query(`
    SELECT slug, updated_at FROM posts
    WHERE status = 'published'
    ORDER BY updated_at DESC
  `);
  return posts;
}

async function getAllProducts() {
  const products = await db.query(`
    SELECT slug, updated_at FROM products
    WHERE active = true
    ORDER BY updated_at DESC
  `);
  return products;
}

export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
  const posts = await getAllPosts();
  const products = await getAllProducts();

  const staticPages: MetadataRoute.Sitemap = [
    {
      url: 'https://example.com',
      lastModified: new Date(),
      changeFrequency: 'daily',
      priority: 1,
    },
    {
      url: 'https://example.com/about',
      lastModified: new Date('2026-01-15'),
      changeFrequency: 'monthly',
      priority: 0.5,
    },
  ];

  const blogPages: MetadataRoute.Sitemap = posts.map((post) => ({
    url: `https://example.com/blog/${post.slug}`,
    lastModified: new Date(post.updated_at),
    changeFrequency: 'weekly' as const,
    priority: 0.7,
  }));

  const productPages: MetadataRoute.Sitemap = products.map((product) => ({
    url: `https://example.com/products/${product.slug}`,
    lastModified: new Date(product.updated_at),
    changeFrequency: 'daily' as const,
    priority: 0.8,
  }));

  return [...staticPages, ...blogPages, ...productPages];
}

# sitemaps.py — Django sitemap framework
from django.contrib.sitemaps import Sitemap
from blog.models import Post
from products.models import Product

class PostSitemap(Sitemap):
    changefreq = 'weekly'
    priority = 0.7

    def items(self):
        return Post.objects.filter(
            status='published'
        ).order_by('-updated_at')

    def lastmod(self, obj):
        return obj.updated_at

    def location(self, obj):
        return f'/blog/{obj.slug}/'

class ProductSitemap(Sitemap):
    changefreq = 'daily'
    priority = 0.8

    def items(self):
        return Product.objects.filter(
            active=True
        ).order_by('-updated_at')

    def lastmod(self, obj):
        return obj.updated_at

    def location(self, obj):
        return f'/products/{obj.slug}/'

# urls.py
from django.contrib.sitemaps.views import sitemap

sitemaps = {
    'posts': PostSitemap,
    'products': ProductSitemap,
}

urlpatterns = [
    path('sitemap.xml', sitemap, {'sitemaps': sitemaps},
         name='django.contrib.sitemaps.views.sitemap'),
]

// routes/sitemap.ts — Express.js dynamic sitemap
import { Router, Request, Response } from 'express';
import { SitemapStream, streamToPromise } from 'sitemap';
import { createGzip } from 'zlib';

const router = Router();

router.get('/sitemap.xml', async (req: Request, res: Response) => {
  res.header('Content-Type', 'application/xml');
  res.header('Content-Encoding', 'gzip');

  const smStream = new SitemapStream({
    hostname: 'https://example.com',
  });
  const pipeline = smStream.pipe(createGzip());

  // Static pages
  smStream.write({ url: '/', lastmod: new Date().toISOString(), priority: 1.0 });
  smStream.write({ url: '/about/', lastmod: '2026-01-15', priority: 0.5 });

  // Dynamic pages from database
  const posts = await db.getAllPublishedPosts();
  for (const post of posts) {
    smStream.write({
      url: `/blog/${post.slug}/`,
      lastmod: post.updatedAt.toISOString(),
      priority: 0.7,
    });
  }

  const products = await db.getAllActiveProducts();
  for (const product of products) {
    smStream.write({
      url: `/products/${product.slug}/`,
      lastmod: product.updatedAt.toISOString(),
      priority: 0.8,
    });
  }

  smStream.end();
  const data = await streamToPromise(pipeline);
  res.send(data);
});

export default router;

<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <sitemap>
    <loc>https://example.com/sitemaps/products-1.xml</loc>
    <lastmod>2026-09-15T08:30:00+00:00</lastmod>
  </sitemap>
  <sitemap>
    <loc>https://example.com/sitemaps/products-2.xml</loc>
    <lastmod>2026-09-14T12:00:00+00:00</lastmod>
  </sitemap>
  <sitemap>
    <loc>https://example.com/sitemaps/blog.xml</loc>
    <lastmod>2026-09-15T10:15:00+00:00</lastmod>
  </sitemap>
  <sitemap>
    <loc>https://example.com/sitemaps/categories.xml</loc>
    <lastmod>2026-09-01T00:00:00+00:00</lastmod>
  </sitemap>
</sitemapindex>

# Python — Generate sub-sitemaps with batching
import math
from lxml import etree

SITEMAP_NS = 'http://www.sitemaps.org/schemas/sitemap/0.9'
BATCH_SIZE = 10_000

def generate_sub_sitemaps(products: list, output_dir: str) -> list:
    """Generate batched product sub-sitemaps, return sub-sitemap URLs."""
    sub_sitemaps = []
    total_batches = math.ceil(len(products) / BATCH_SIZE)

    for batch_num in range(total_batches):
        batch = products[batch_num * BATCH_SIZE:(batch_num + 1) * BATCH_SIZE]
        root = etree.Element('urlset', xmlns=SITEMAP_NS)

        for product in batch:
            url_elem = etree.SubElement(root, 'url')
            loc = etree.SubElement(url_elem, 'loc')
            loc.text = f'https://example.com/products/{product["slug"]}/'
            lastmod = etree.SubElement(url_elem, 'lastmod')
            lastmod.text = product['updated_at'].isoformat()

        filename = f'products-{batch_num + 1}.xml'
        filepath = f'{output_dir}/{filename}'
        tree = etree.ElementTree(root)
        tree.write(filepath, xml_declaration=True, encoding='UTF-8',
                   pretty_print=True)
        sub_sitemaps.append(f'https://example.com/sitemaps/{filename}')

    return sub_sitemaps

<!-- ❌ Mistake: lastmod set to build/deploy time for all pages -->
<!-- Every page shows today's date, regardless of actual content changes -->
<url>
  <loc>https://example.com/about/</loc>
  <lastmod>2026-09-15T08:30:00+00:00</lastmod> <!-- not changed since 2025 -->
</url>

<!-- ❌ Mistake: lastmod set to current timestamp on every request -->
<!-- Dynamic generation that always returns "now" -->

<!-- ✅ Correct: lastmod reflects the actual last content edit -->
<url>
  <loc>https://example.com/about/</loc>
  <lastmod>2025-03-20T14:00:00+00:00</lastmod> <!-- real last edit date -->
</url>

// Track actual content changes, not deployment times
interface ContentRecord {
  slug: string;
  content_hash: string;   // SHA-256 of the rendered content
  last_content_change: Date; // Updated ONLY when content_hash changes
  last_deployed: Date;     // Updated on every deployment
}

// In your sitemap generator, use last_content_change, NOT last_deployed
function getLastmod(record: ContentRecord): string {
  return record.last_content_change.toISOString();
}

# Django — Track content changes with a hash field
import hashlib
from django.db import models

class Post(models.Model):
    slug = models.SlugField(unique=True)
    body = models.TextField()
    content_hash = models.CharField(max_length=64, blank=True)
    content_changed_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def save(self, *args, **kwargs):
        new_hash = hashlib.sha256(self.body.encode()).hexdigest()
        if new_hash != self.content_hash:
            self.content_hash = new_hash
            self.content_changed_at = timezone.now()
        super().save(*args, **kwargs)

# Validate sitemap XML syntax
xmllint --noout --schema \
  https://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd \
  sitemap.xml

# Check that all sitemap URLs return 200
grep -oP '<loc>\K[^<]+' sitemap.xml | while read url; do
  status=$(curl -s -o /dev/null -w "%{http_code}" "$url")
  if [ "$status" != "200" ]; then
    echo "ERROR: $url returned $status"
  fi
done

# Check for URLs with unescaped ampersands
grep -n '&[^a]' sitemap.xml | grep -v '&amp;' | head -10

# GitHub Actions — Validate sitemap on every deployment
name: Validate Sitemap
on:
  push:
    branches: [main]

jobs:
  validate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Build site
        run: npm run build
      - name: Validate sitemap XML
        run: |
          xmllint --noout dist/sitemap.xml
          echo "✅ Sitemap XML is valid"
      - name: Check URL count
        run: |
          count=$(grep -c '<loc>' dist/sitemap.xml)
          echo "Sitemap contains $count URLs"
          if [ "$count" -gt 50000 ]; then
            echo "❌ ERROR: Sitemap exceeds 50,000 URL limit"
            exit 1
          fi

# robots.txt — Declare sitemap location
User-agent: *
Allow: /

Sitemap: https://example.com/sitemap.xml

# Notify Google of sitemap updates after publishing new content
curl "https://www.google.com/ping?sitemap=https://example.com/sitemap.xml"

# Note: Google has deprecated the ping endpoint for most use cases
# Submit via GSC API instead for reliable notification