#!/usr/bin/env python3 """Measure how readable the most visited .hu sites are for AI assistants. python3 measure.py top-1m.csv 500 > results.csv top-1m.csv is a Tranco list (https://tranco-list.eu/). The published results used list 8P8NV and were collected on 2026-11-09 from 14:04 UTC. Per domain, four requests: the homepage, /robots.txt, /llms.txt, /sitemap.xml. Standard library only. Identifies itself; no bot user agents are faked. """ import csv, gzip, json, re, sys, time, zlib import urllib.request, urllib.error, ssl from concurrent.futures import ThreadPoolExecutor from html import unescape UA = 'Mozilla/5.0 (compatible; pappfer-research/1.0; +https://pappfer.hu/)' TIMEOUT = 20 MAX = 3_000_000 # Training crawlers vs the agents that fetch pages to answer a user's question. TRAINING = ['GPTBot', 'ClaudeBot', 'CCBot', 'Google-Extended', 'Applebot-Extended', 'meta-externalagent', 'Bytespider'] ANSWER = ['OAI-SearchBot', 'ChatGPT-User', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User'] SEARCH = ['Googlebot', 'Bingbot'] AGENTS = TRAINING + ANSWER + SEARCH ctx = ssl.create_default_context() def fetch(url): req = urllib.request.Request(url, headers={ 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.8', 'Accept-Language': 'hu,en;q=0.8', 'Accept-Encoding': 'gzip, deflate'}) t = time.time() try: with urllib.request.urlopen(req, timeout=TIMEOUT, context=ctx) as r: body = r.read(MAX) enc = r.headers.get('Content-Encoding', '') if enc == 'gzip': body = gzip.decompress(body) elif enc == 'deflate': body = zlib.decompress(body) return {'status': r.status, 'url': r.geturl(), 'type': r.headers.get('Content-Type', ''), 'body': body, 'ms': int((time.time() - t) * 1000), 'xrobots': r.headers.get('X-Robots-Tag', '')} except urllib.error.HTTPError as e: return {'status': e.code, 'url': url, 'type': e.headers.get('Content-Type', '') if e.headers else '', 'body': b'', 'ms': 0, 'xrobots': ''} except Exception as e: return {'status': 0, 'url': url, 'type': '', 'body': b'', 'ms': 0, 'error': type(e).__name__, 'xrobots': ''} def text(b): for enc in ('utf-8', 'cp1250', 'latin-1'): try: return b.decode(enc) except UnicodeDecodeError: pass return b.decode('utf-8', 'replace') # --- robots.txt, RFC 9309: groups by exact product token, longest match wins, ties allow def parse_robots(src): groups, cur, last_was_agent = [], None, False for line in src.splitlines(): line = line.split('#', 1)[0].strip() if ':' not in line: continue k, v = (x.strip() for x in line.split(':', 1)) k = k.lower() if k == 'user-agent': if not last_was_agent: cur = {'agents': [], 'rules': []} groups.append(cur) cur['agents'].append(v.lower()) last_was_agent = True elif k in ('allow', 'disallow') and cur is not None: cur['rules'].append((k, v)) last_was_agent = False else: last_was_agent = False return groups def pattern_matches(pat, path): if pat == '': return False rx = re.escape(pat).replace(r'\*', '.*') if rx.endswith(r'\$'): rx = rx[:-2] + '$' return re.match(rx, path) is not None def allowed(groups, agent, path='/'): a = agent.lower() rules = [r for g in groups if a in g['agents'] for r in g['rules']] if not any(a in g['agents'] for g in groups): rules = [r for g in groups if '*' in g['agents'] for r in g['rules']] best = None for k, v in rules: if pattern_matches(v, path): if best is None or len(v) > len(best[1]) or (len(v) == len(best[1]) and k == 'allow'): best = (k, v) return best is None or best[0] == 'allow' def named(groups, agent): return any(agent.lower() in g['agents'] for g in groups) def is_html(r): head = r['body'][:600].lstrip().lower() return 'html' in r['type'].lower() or head.startswith(b']*\blang=["\']?([a-zA-Z-]+)', html, re.I) row['lang'] = m.group(1).lower() if m else '' m = re.search(r'