# Mark Siazon GitHub profile README - crawler hints
# Repo: https://github.com/Iron-Mark/Iron-Mark
# Entity @id: https://www.marksiazon.dev/#person
# Last updated: 2026-06-25

User-agent: *
Allow: /

# --- OpenAI ---
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# --- Anthropic ---
User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

# --- Perplexity ---
User-agent: PerplexityBot
Allow: /

# --- Google AI training / extended ---
User-agent: Google-Extended
Allow: /

# --- Apple ---
User-agent: Applebot-Extended
Allow: /

# --- Meta ---
User-agent: Meta-ExternalAgent
Allow: /

# --- Cohere ---
User-agent: cohere-ai
Allow: /

# --- Common crawlers (explicit allow for index files) ---
User-agent: CCBot
Allow: /llms.txt
Allow: /llms-index.json
Allow: /
Allow: /

User-agent: Bytespider
Allow: /llms.txt
Allow: /llms-index.json
Allow: /

# --- Sitemaps ---
Sitemap: https://iron-mark.github.io/Iron-Mark/sitemap.xml

# Portfolio (canonical site - separate robots.txt)
# https://www.marksiazon.dev/robots.txt
# https://www.marksiazon.dev/sitemap.xml