#!/usr/bin/env python3 """Measure how readable the most visited .hu sites are for AI assistants. python3 measure.py top-1m.csv 500 > results.csv top-1m.csv is a Tranco list (https://tranco-list.eu/). The published results used list 8P8NV and were collected on 2026-11-09 from 14:04 UTC. Per domain, four requests: the homepage, /robots.txt, /llms.txt, /sitemap.xml. Standard library only. Identifies itself; no bot user agents are faked. """ import csv, gzip, json, re, sys, time, zlib import urllib.request, urllib.error, ssl from concurrent.futures import ThreadPoolExecutor from html import unescape UA = 'Mozilla/5.0 (compatible; pappfer-research/1.0; +https://pappfer.hu/)' TIMEOUT = 20 MAX = 3_000_000 # Training crawlers vs the agents that fetch pages to answer a user's question. TRAINING = ['GPTBot', 'ClaudeBot', 'CCBot', 'Google-Extended', 'Applebot-Extended', 'meta-externalagent', 'Bytespider'] ANSWER = ['OAI-SearchBot', 'ChatGPT-User', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User'] SEARCH = ['Googlebot', 'Bingbot'] AGENTS = TRAINING + ANSWER + SEARCH ctx = ssl.create_default_context() def fetch(url): req = urllib.request.Request(url, headers={ 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.8', 'Accept-Language': 'hu,en;q=0.8', 'Accept-Encoding': 'gzip, deflate'}) t = time.time() try: with urllib.request.urlopen(req, timeout=TIMEOUT, context=ctx) as r: body = r.read(MAX) enc = r.headers.get('Content-Encoding', '') if enc == 'gzip': body = gzip.decompress(body) elif enc == 'deflate': body = zlib.decompress(body) return {'status': r.status, 'url': r.geturl(), 'type': r.headers.get('Content-Type', ''), 'body': body, 'ms': int((time.time() - t) * 1000), 'xrobots': r.headers.get('X-Robots-Tag', '')} except urllib.error.HTTPError as e: return {'status': e.code, 'url': url, 'type': e.headers.get('Content-Type', '') if e.headers else '', 'body': b'', 'ms': 0, 'xrobots': ''} except Exception as e: return {'status': 0, 'url': url, 'type': '', 'body': b'', 'ms': 0, 'error': type(e).__name__, 'xrobots': ''} def text(b): for enc in ('utf-8', 'cp1250', 'latin-1'): try: return b.decode(enc) except UnicodeDecodeError: pass return b.decode('utf-8', 'replace') # --- robots.txt, RFC 9309: groups by exact product token, longest match wins, ties allow def parse_robots(src): groups, cur, last_was_agent = [], None, False for line in src.splitlines(): line = line.split('#', 1)[0].strip() if ':' not in line: continue k, v = (x.strip() for x in line.split(':', 1)) k = k.lower() if k == 'user-agent': if not last_was_agent: cur = {'agents': [], 'rules': []} groups.append(cur) cur['agents'].append(v.lower()) last_was_agent = True elif k in ('allow', 'disallow') and cur is not None: cur['rules'].append((k, v)) last_was_agent = False else: last_was_agent = False return groups def pattern_matches(pat, path): if pat == '': return False rx = re.escape(pat).replace(r'\*', '.*') if rx.endswith(r'\$'): rx = rx[:-2] + '$' return re.match(rx, path) is not None def allowed(groups, agent, path='/'): a = agent.lower() rules = [r for g in groups if a in g['agents'] for r in g['rules']] if not any(a in g['agents'] for g in groups): rules = [r for g in groups if '*' in g['agents'] for r in g['rules']] best = None for k, v in rules: if pattern_matches(v, path): if best is None or len(v) > len(best[1]) or (len(v) == len(best[1]) and k == 'allow'): best = (k, v) return best is None or best[0] == 'allow' def named(groups, agent): return any(agent.lower() in g['agents'] for g in groups) def is_html(r): head = r['body'][:600].lstrip().lower() return 'html' in r['type'].lower() or head.startswith(b']*\blang=["\']?([a-zA-Z-]+)', html, re.I) row['lang'] = m.group(1).lower() if m else '' m = re.search(r']*>(.*?)', html, re.I | re.S) row['title'] = unescape(re.sub(r'\s+', ' ', m.group(1))).strip()[:120] if m else '' row['meta_description'] = int(bool(re.search(r']+name=["\']description["\'][^>]+content=["\'][^"\']{20,}', html, re.I) or re.search(r']+content=["\'][^"\']{20,}["\'][^>]+name=["\']description', html, re.I))) row['canonical'] = int(bool(re.search(r']+rel=["\']canonical', html, re.I))) row['hreflang'] = int('hreflang=' in low) row['meta_noindex'] = int(bool(re.search(r']+name=["\']robots["\'][^>]+noindex', html, re.I))) # JSON-LD types types = set() blocks = re.findall(r']+application/ld\+json[^>]*>(.*?)', html, re.I | re.S) def walk(o): if isinstance(o, dict): t = o.get('@type') for x in (t if isinstance(t, list) else [t]): if isinstance(x, str): types.add(x) for v in o.values(): walk(v) elif isinstance(o, list): for v in o: walk(v) bad = 0 for b in blocks: try: walk(json.loads(b.strip())) except Exception: bad += 1 row['jsonld_blocks'] = len(blocks) row['jsonld_invalid'] = bad row['jsonld_types'] = ' '.join(sorted(types)) row['microdata'] = int('itemtype=' in low) # Text a crawler that does not run JavaScript gets from the first response body = re.sub(r'<(script|style|noscript|template|svg)[^>]*>.*?', ' ', html, flags=re.I | re.S) body = re.sub(r']*>.*?', ' ', body, flags=re.I | re.S) words = re.findall(r'\w{2,}', unescape(re.sub(r'<[^>]+>', ' ', body))) row['words_no_js'] = len(words) row['h1'] = len(re.findall(r']', html, re.I)) # robots.txt rb = fetch(base + '/robots.txt') rtxt = text(rb['body']) if rb['status'] == 200 and not is_html(rb) else '' row['robots'] = int(bool(rtxt.strip())) g = parse_robots(rtxt) for a in AGENTS: row['ok_' + a] = int(allowed(g, a)) row['named_' + a] = int(named(g, a)) row['robots_sitemap'] = int(bool(re.search(r'^\s*sitemap\s*:', rtxt, re.I | re.M))) row['content_signal'] = int('content-signal' in rtxt.lower()) # llms.txt: must be text, not an HTML error page or SPA fallback lt = fetch(base + '/llms.txt') ltxt = text(lt['body'])[:2000] if lt['status'] == 200 and not is_html(lt) else '' row['llms_txt'] = int(bool(ltxt.strip()) and ltxt.lstrip().startswith('#')) row['llms_txt_any'] = int(bool(ltxt.strip())) sm = fetch(base + '/sitemap.xml') row['sitemap_xml'] = int(sm['status'] == 200 and b'<' in sm['body'][:200] and (b'urlset' in sm['body'][:3000] or b'sitemapindex' in sm['body'][:3000])) return row def main(): src, n = sys.argv[1], int(sys.argv[2]) hu = [] with open(src) as f: for line in f: rank, dom = line.strip().split(',', 1) if dom.endswith('.hu'): hu.append((int(rank), dom)) if len(hu) >= n: break with ThreadPoolExecutor(12) as ex: rows = list(ex.map(lambda x: measure(*x), hu)) keys = [] for r in rows: for k in r: if k not in keys: keys.append(k) w = csv.DictWriter(sys.stdout, fieldnames=keys) w.writeheader() for r in rows: w.writerow(r) if __name__ == '__main__': main()