Audit Canonical Tags and Noindex Directives Across Your Sitemap with Python (Standard Library Only)

#!/usr/bin/env python3
"""Audit canonical tags and noindex directives for every URL in a sitemap.

Standard library only. Usage:
    python3 canon_audit.py https://example.com/sitemap.xml --out canon_report.csv
"""
import argparse
import csv
import re
import sys
import urllib.error
import urllib.request
import urllib.robotparser
import xml.etree.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor
from html.parser import HTMLParser
from urllib.parse import urljoin, urlsplit, urlunsplit

UA = "canon-audit/1.0 (+stdlib python)"
SM_NS = "{http://www.sitemaps.org/schemas/sitemap/0.9}"
GOOGLE_NAMES = {"robots", "googlebot"}           # meta names Google web search reads
RULES_WITH_COLON = {"unavailable_after", "max-snippet",
                    "max-image-preview", "max-video-preview"}
IGNORED_CANON_ATTRS = ("hreflang", "lang", "media", "type")
# rules from Google's robots meta tag page, plus the common index/follow defaults
KNOWN_RULES = {"all", "noindex", "nofollow", "none", "nosnippet", "indexifembedded",
               "notranslate", "noimageindex", "index", "follow"}
INFO_ONLY = {"no_canonical", "unrecognized_rule"}  # reported, but not a failure


def fetch(url, timeout=15):
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    with urllib.request.urlopen(req, timeout=timeout) as resp:
        charset = resp.headers.get_content_charset() or "utf-8"
        return {"status": resp.status, "final_url": resp.geturl(),
                "headers": resp.headers,
                "body": resp.read().decode(charset, "replace")}


def sitemap_urls(sitemap_url, seen=None):
    """Return page URLs from a urlset, following sitemap indexes."""
    seen = seen if seen is not None else set()
    if sitemap_url in seen:
        return []
    seen.add(sitemap_url)
    root = ET.fromstring(fetch(sitemap_url)["body"].encode("utf-8"))
    if root.tag == SM_NS + "sitemapindex":
        urls = []
        for sm in root.findall(SM_NS + "sitemap"):
            loc = sm.find(SM_NS + "loc")
            if loc is not None and loc.text:
                urls.extend(sitemap_urls(loc.text.strip(), seen))
        return urls
    return [u.find(SM_NS + "loc").text.strip()
            for u in root.findall(SM_NS + "url")
            if u.find(SM_NS + "loc") is not None and u.find(SM_NS + "loc").text]


def norm(url):
    """Lowercase scheme and host, drop default ports. Keep path and query as-is."""
    p = urlsplit(url)
    host = (p.hostname or "").lower()
    if p.port and not ((p.scheme == "http" and p.port == 80)
                       or (p.scheme == "https" and p.port == 443)):
        host += f":{p.port}"
    return urlunsplit((p.scheme.lower(), host, p.path or "/", p.query, ""))


class HeadParser(HTMLParser):
    """Collect <link rel=canonical> and robots <meta> tags."""

    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.canonicals = []   # dicts: href, in_head, ignored_attr
        self.robots = []       # (meta name, content)
        self._skip = 0
        self._in_head = True   # until </head> or <body>

    def handle_starttag(self, tag, attrs):
        a = {k: (v or "") for k, v in attrs}
        if tag in ("script", "style", "template", "noscript"):
            self._skip += 1
            return
        if self._skip:
            return
        if tag == "body":
            self._in_head = False
        elif tag == "link" and "canonical" in a.get("rel", "").lower().split():
            self.canonicals.append({
                "href": a.get("href", "").strip(),
                "in_head": self._in_head,
                "ignored_attr": [k for k in IGNORED_CANON_ATTRS if k in a]})
        elif tag == "meta" and a.get("name", "").lower() in GOOGLE_NAMES | {"googlebot-news", "bingbot"}:
            self.robots.append((a["name"].lower(), a.get("content", "")))

    def handle_startendtag(self, tag, attrs):
        self.handle_starttag(tag, attrs)

    def handle_endtag(self, tag):
        if tag in ("script", "style", "template", "noscript"):
            self._skip = max(0, self._skip - 1)
        elif tag == "head":
            self._in_head = False


def split_rules(text):
    return {r.strip().lower() for r in text.split(",") if r.strip()}


def xrobots(values):
    """Split X-Robots-Tag values into (google_rules, {other_agent: rules})."""
    google, other = set(), {}
    for v in values:
        m = re.match(r"\s*([A-Za-z][\w-]*)\s*:\s*(.*)$", v)
        if m and m.group(1).lower() not in RULES_WITH_COLON:
            agent, rules = m.group(1).lower(), split_rules(m.group(2))
            if agent == "googlebot":
                google |= rules
            else:
                other.setdefault(agent, set()).update(rules)
        else:
            google |= split_rules(v)
    return google, other


def link_canonicals(values):
    """Return URLs from Link headers with rel=canonical."""
    out = []
    for v in values:
        for m in re.finditer(r"<([^>]*)>\s*((?:;[^,]*)*)", v):
            if re.search(r'rel\s*=\s*"?canonical"?', m.group(2), re.I):
                out.append(m.group(1).strip())
    return out


def is_noindex(rules):
    return bool(rules & {"noindex", "none"})


def audit(url, robots, sitemap_set):
    row = {"page": url, "status": "", "final_url": "", "google_robots": "",
           "canonical": "", "issues": []}
    issues = row["issues"]
    if robots is not None and not robots.can_fetch("Googlebot", url):
        issues.append("blocked_by_robots")
    try:
        r = fetch(url)
    except urllib.error.HTTPError as e:
        row["status"] = e.code
        issues.append("http_error")
        return row
    except (urllib.error.URLError, TimeoutError) as e:
        row["status"] = "error"
        issues.append(f"fetch_failed:{e}")
        return row
    row["status"], row["final_url"] = r["status"], r["final_url"]
    if norm(r["final_url"]) != norm(url):
        issues.append("redirected")

    # robots rules from headers and (for HTML) meta tags
    google, other = xrobots(r["headers"].get_all("X-Robots-Tag") or [])
    header_canon = [urljoin(r["final_url"], h)
                    for h in link_canonicals(r["headers"].get_all("Link") or [])]
    p = HeadParser()
    if "html" in (r["headers"].get_content_type() or ""):
        p.feed(r["body"])
        p.close()
    for name, content in p.robots:
        if name in GOOGLE_NAMES:
            google |= split_rules(content)
        else:
            other.setdefault(name, set()).update(split_rules(content))
    row["google_robots"] = ",".join(sorted(google))
    if any(r not in KNOWN_RULES and r.split(":")[0] not in RULES_WITH_COLON
           for r in google):
        issues.append("unrecognized_rule")
    if is_noindex(google):
        issues.append("noindex")
    if any(is_noindex(rules) for rules in other.values()):
        issues.append("noindex_other_agent")

    # canonical checks
    used = [c for c in p.canonicals if not c["ignored_attr"]]
    if any(c["ignored_attr"] for c in p.canonicals):
        issues.append("canonical_ignored_attr")
    if any(not c["in_head"] for c in used):
        issues.append("canonical_in_body")
    html_canon = [urljoin(r["final_url"], c["href"]) for c in used
                  if c["in_head"] and c["href"]]
    if any(not urlsplit(c["href"]).scheme for c in used if c["in_head"]):
        issues.append("relative_canonical")
    if len({norm(c) for c in html_canon}) > 1 or len({norm(c) for c in header_canon}) > 1:
        issues.append("multiple_canonicals")
    if html_canon and header_canon and norm(html_canon[0]) != norm(header_canon[0]):
        issues.append("canonical_header_conflict")
    canon = (html_canon or header_canon or [""])[0]
    row["canonical"] = canon
    if not canon:
        issues.append("no_canonical")
        return row
    if urlsplit(canon).fragment:
        issues.append("canonical_fragment")
    if urlsplit(url).scheme == "https" and urlsplit(canon).scheme == "http":
        issues.append("canonical_to_http")
    if norm(canon) != norm(url):
        issues.append("canonical_elsewhere")
        if norm(canon) not in sitemap_set:
            issues.append("canonical_not_in_sitemap")
        if is_noindex(google):
            issues.append("noindex_with_canonical")
    return row


def check_target(url):
    """Fetch a canonical target once; return issue codes for it."""
    try:
        r = fetch(url)
    except urllib.error.HTTPError as e:
        return [f"canonical_target_{e.code}"]
    except (urllib.error.URLError, TimeoutError):
        return ["canonical_target_unreachable"]
    out = []
    if norm(r["final_url"]) != norm(url):
        out.append("canonical_target_redirects")
    google, _ = xrobots(r["headers"].get_all("X-Robots-Tag") or [])
    p = HeadParser()
    p.feed(r["body"])
    for name, content in p.robots:
        if name in GOOGLE_NAMES:
            google |= split_rules(content)
    if is_noindex(google):
        out.append("canonical_target_noindex")
    return out


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("sitemap")
    ap.add_argument("--out", default="canon_report.csv")
    ap.add_argument("--workers", type=int, default=8)
    ap.add_argument("--limit", type=int, default=0, help="check only the first N URLs")
    ap.add_argument("--check-targets", action="store_true",
                    help="also fetch canonical targets that differ from the page URL")
    ap.add_argument("--no-robots", action="store_true", help="skip the robots.txt check")
    args = ap.parse_args()
    try:
        urls = sitemap_urls(args.sitemap)
    except (urllib.error.URLError, ET.ParseError) as e:
        print(f"Could not read sitemap: {e}", file=sys.stderr)
        return 2
    if args.limit:
        urls = urls[:args.limit]
    robots = None
    if not args.no_robots:
        robots = urllib.robotparser.RobotFileParser(urljoin(args.sitemap, "/robots.txt"))
        try:
            robots.read()
        except (urllib.error.URLError, TimeoutError):
            robots = None
    sitemap_set = {norm(u) for u in urls}
    print(f"Checking {len(urls)} URLs from {args.sitemap}")
    with ThreadPoolExecutor(max_workers=args.workers) as pool:
        rows = list(pool.map(lambda u: audit(u, robots, sitemap_set), urls))
        if args.check_targets:
            targets = sorted({r["canonical"] for r in rows
                              if "canonical_elsewhere" in r["issues"]})
            found = dict(zip(targets, pool.map(check_target, targets)))
            for r in rows:
                r["issues"] += found.get(r["canonical"], [])
    counts = {}
    for r in rows:
        for code in r["issues"]:
            counts[code] = counts.get(code, 0) + 1
        r["issues"] = ";".join(r["issues"])
    with open(args.out, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=["page", "status", "final_url",
                                          "google_robots", "canonical", "issues"])
        w.writeheader()
        w.writerows(rows)
    flagged = [r for r in rows if set(r["issues"].split(";")) - INFO_ONLY - {""}]
    print(f"{len(rows)} pages checked; {len(flagged)} with problems")
    for code, n in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])):
        print(f"  {code:<28} {n}")
    print(f"Report: {args.out}")
    return 1 if flagged else 0


if __name__ == "__main__":
    sys.exit(main())

"""Static server that can add response headers and redirects, for testing."""
import http.server
import sys

HEADERS = {
    "/docs/guide.pdf": [("X-Robots-Tag", "noindex")],
    "/search.html": [("X-Robots-Tag", "bingbot: noindex"),
                     ("X-Robots-Tag", "googlebot: nosnippet")],
    "/landing.html": [("Link", '<http://127.0.0.1:8767/landing-a.html>; rel="canonical"')],
}
REDIRECTS = {"/old-page.html": "/new-page.html"}


class Handler(http.server.SimpleHTTPRequestHandler):
    def do_GET(self):
        path = self.path.split("?")[0]
        if path in REDIRECTS:
            self.send_response(301)
            self.send_header("Location", REDIRECTS[path])
            self.end_headers()
            return
        super().do_GET()

    def end_headers(self):
        for k, v in HEADERS.get(self.path.split("?")[0], []):
            self.send_header(k, v)
        super().end_headers()

    def log_message(self, *args):
        pass


port = int(sys.argv[1]) if len(sys.argv) > 1 else 8767
http.server.ThreadingHTTPServer(("127.0.0.1", port), Handler).serve_forever()

cd site
python3 ../serve_fixture.py 8767

python3 canon_audit.py http://127.0.0.1:8767/sitemap.xml --out canon_report.csv
echo "exit code: $?"

Checking 17 URLs from http://127.0.0.1:8767/sitemap.xml
17 pages checked; 15 with problems
  canonical_elsewhere          5
  no_canonical                 5
  noindex                      5
  canonical_not_in_sitemap     4
  blocked_by_robots            1
  canonical_fragment           1
  canonical_header_conflict    1
  canonical_ignored_attr       1
  canonical_in_body            1
  http_error                   1
  multiple_canonicals          1
  noindex_other_agent          1
  noindex_with_canonical       1
  redirected                   1
  relative_canonical           1
  unrecognized_rule            1
Report: canon_report.csv
exit code: 1

page,status,final_url,google_robots,canonical,issues
http://127.0.0.1:8767/,200,http://127.0.0.1:8767/,,http://127.0.0.1:8767/,
http://127.0.0.1:8767/pricing.html,200,http://127.0.0.1:8767/pricing.html,noarchive,,unrecognized_rule;no_canonical
http://127.0.0.1:8767/thanks.html,200,http://127.0.0.1:8767/thanks.html,"nofollow,noindex",http://127.0.0.1:8767/thanks.html,noindex
http://127.0.0.1:8767/old-page.html,200,http://127.0.0.1:8767/new-page.html,,http://127.0.0.1:8767/new-page.html,redirected;canonical_elsewhere;canonical_not_in_sitemap
http://127.0.0.1:8767/features.html,200,http://127.0.0.1:8767/features.html,"max-snippet:120,noindex",http://127.0.0.1:8767/features.html,noindex;relative_canonical
http://127.0.0.1:8767/compare.html,200,http://127.0.0.1:8767/compare.html,,http://127.0.0.1:8767/compare.html,multiple_canonicals
http://127.0.0.1:8767/de.html,200,http://127.0.0.1:8767/de.html,,,canonical_ignored_attr;no_canonical
http://127.0.0.1:8767/landing.html,200,http://127.0.0.1:8767/landing.html,,http://127.0.0.1:8767/landing-b.html,canonical_header_conflict;canonical_elsewhere;canonical_not_in_sitemap
http://127.0.0.1:8767/staging-copy.html,200,http://127.0.0.1:8767/staging-copy.html,noindex,http://127.0.0.1:8767/,noindex;canonical_elsewhere;noindex_with_canonical
http://127.0.0.1:8767/search.html,200,http://127.0.0.1:8767/search.html,nosnippet,http://127.0.0.1:8767/search.html,noindex_other_agent
http://127.0.0.1:8767/faq.html,200,http://127.0.0.1:8767/faq.html,,http://127.0.0.1:8767/faq.html#top,canonical_fragment
http://127.0.0.1:8767/body-canon.html,200,http://127.0.0.1:8767/body-canon.html,,,canonical_in_body;no_canonical
http://127.0.0.1:8767/promo.html,200,http://127.0.0.1:8767/promo.html,,http://127.0.0.1:8767/promo-old.html,canonical_elsewhere;canonical_not_in_sitemap
http://127.0.0.1:8767/private/report.html,200,http://127.0.0.1:8767/private/report.html,noindex,,blocked_by_robots;noindex;no_canonical
http://127.0.0.1:8767/docs/guide.pdf,200,http://127.0.0.1:8767/docs/guide.pdf,noindex,,noindex;no_canonical
http://127.0.0.1:8767/missing.html,404,,,,http_error
http://127.0.0.1:8767/blog/post-a.html,200,http://127.0.0.1:8767/blog/post-a.html,,http://127.0.0.1:8767/blog/post-a/,canonical_elsewhere;canonical_not_in_sitemap

Checking 17 URLs from http://127.0.0.1:8767/sitemap.xml
17 pages checked; 15 with problems
  canonical_elsewhere          5
  no_canonical                 5
  noindex                      5
  canonical_not_in_sitemap     4
  blocked_by_robots            1
  canonical_fragment           1
  canonical_header_conflict    1
  canonical_ignored_attr       1
  canonical_in_body            1
  canonical_target_404         1
  canonical_target_noindex     1
  http_error                   1
  multiple_canonicals          1
  noindex_other_agent          1
  noindex_with_canonical       1
  redirected                   1
  relative_canonical           1
  unrecognized_rule            1
Report: canon_targets.csv
exit code: 1

$ grep canonical_target canon_targets.csv
http://127.0.0.1:8767/promo.html,200,http://127.0.0.1:8767/promo.html,,http://127.0.0.1:8767/promo-old.html,canonical_elsewhere;canonical_not_in_sitemap;canonical_target_noindex
http://127.0.0.1:8767/blog/post-a.html,200,http://127.0.0.1:8767/blog/post-a.html,,http://127.0.0.1:8767/blog/post-a/,canonical_elsewhere;canonical_not_in_sitemap;canonical_target_404

Could not read sitemap: HTTP Error 404: File not found
exit code: 2

python3 canon_audit.py https://appie.run/sitemap.xml --out appie_canon.csv --workers 2 --check-targets

Checking 74 URLs from https://appie.run/sitemap.xml
74 pages checked; 0 with problems
  no_canonical                 1
  unrecognized_rule            1
Report: appie_canon.csv
exit code: 0

page,status,final_url,google_robots,canonical,issues
https://appie.run/,200,https://appie.run/,"follow,index",https://appie.run/,
https://appie.run/pricing,200,https://appie.run/pricing,"follow,index",https://appie.run/pricing,
https://appie.run/llms.txt,200,https://appie.run/llms.txt,llms-txt,,unrecognized_rule;no_canonical

python3 canon_audit.py https://example.com/sitemap.xml --out canon_report.csv --workers 2 --limit 50