# ============================================================
# ylishi.tools — Robots & Content Signals Policy
# Updated: 2026-08-26
# ============================================================
# CONTENT SIGNALS (per robots.txt content-signal proposal):
#   search   : YES — building search index & providing search results
#   ai-input : YES — AI retrieval / RAG / real-time citation / generative
#              AI answer references
#   ai-train : YES — public knowledge content AND the public CC BY 4.0 fee
#              datasets (/rates/*.json, /data/rate-snapshot.json, /data/
#              rate-changelog.json) are explicitly open for training with
#              attribution. Core proprietary systems behind /api/,
#              /tools/*/api/ and the login area remain Disallow (see rules
#              below) — that is an access rule, not a content-signal change.
#   user data, login area, databases, APIs: NOT for collection
#   bulk/commercial scraping by competitors: PROHIBITED (see site terms)
# Any restriction expressed below is an express reservation of rights under
# Article 4 of EU Directive 2019/790 (CDSM).
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# ============================================================
# General crawler rules
# Public knowledge content (pages, blog, guides) is fully open.
# Core data sources are disallowed for everyone, AI included.
# ============================================================
User-agent: *
Allow: /
Allow: /blog/
Disallow: /api/
Disallow: /ops-y7k9-admin/
# 2026-09-16: internal daily SEO report dir (nginx 404s it; listed here so no
# crawler ever sees it published again)
Disallow: /check-daily/

Disallow: /tools/*/api/
Disallow: /premium?canceled=

Sitemap: https://ylishi.tools/sitemap_index.xml

# ============================================================
# AI crawlers — explicitly welcome on public content.
# (They remain subject to the Disallow rules above for core data.)
# ============================================================
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: CCBot
Allow: /