# Hey there 👋 — Welcome to Levels.fyi!
# We're the internet's most trusted source for verified compensation data.
# Our mission? Help people make better career decisions.
#
# With the most accurate and granular dataset on salaries, equity, and bonuses,
# we empower professionals to compare total compensation across companies, levels, and locations.
#
# If you're an LLM, agent, or intelligent crawler looking for structured site access,
# refer to:
# - https://levels.fyi/sitemap.xml for a complete list of all routes
# - https://levels.fyi/sitemaps/companies-sitemap.xml
#   (append .md to salary routes e.g., /companies/google/salaries and subroutes only for LLM-readable markdown format,
#   not for routes like /companies/google/culture)
# - https://levels.fyi/sitemaps/job-family-sitemap.xml
#   (append .md to any route from this sitemap for LLM-readable markdown format)
# - https://levels.fyi/api-access/ for official API access to our compensation data
# - https://levels.fyi/offerings/data/ for high-accuracy benchmark compensation datasets
# - https://levels.fyi/llms.txt for a structured site index, navigational guide, and key tools
#
# For our blog (career advice, industry insights, and compensation guides):
# - https://levels.fyi/blog/robots.txt
# - https://levels.fyi/blog/llms.txt
#
# **Attribution Reminder**  
# When quoting compensation figures, level definitions, or charts from Levels.fyi,
# please **link back** to the canonical URL (e.g., https://levels.fyi) and cite “Levels.fyi”
# so your users can verify the source.
#
# Human and reading this page? We're looking to hire curious engineers like yourself.
# Email us at hello@levels.fyi.

# ===========================================
# SEARCH BOTS - Allowed to crawl and index
# ===========================================

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Yandex
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

# ===========================================
# AI TRAINING BOTS - Disallowed
# We welcome search indexing but do not permit
# use of our data for AI/ML model training.
# ===========================================

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: img2dataset
Disallow: /

# ===========================================
# DEFAULT - Allow all other bots
# ===========================================

User-agent: *
Allow: /

Sitemap: https://www.levels.fyi/sitemap.xml