User-agent: *
Allow: /

# Disallow admin and sensitive areas
Disallow: /admin/
Disallow: /api/

# Allow authentication pages but block user account areas
Allow: /account/login/
Allow: /account/signup/
Allow: /account/reset-password/
Disallow: /account/

Disallow: /chat/
Disallow: /documents/
Disallow: /refinement/
Disallow: /download/
Disallow: /upload-file/
Disallow: /upload/
Disallow: /send-chat/
Disallow: /feedback/
Disallow: /stripe_webhook/
Disallow: /webhooks/
Disallow: /create-portal-session/
Disallow: /download-file/
Disallow: /save-referral/
Disallow: /check-payment/

# Allow all language versions (pattern matches /en/, /es/, /de/, etc.)
Allow: /*/

# Public pages (no language prefix needed - covered by Allow: / and Allow: /*/)
# Terms and policies is English-only
Allow: /terms-and-policies/

# Affiliate program page
Allow: /affiliate/

# Coupon code page
Allow: /coupon-code/

# Blog posts
Allow: /blog/
Allow: /blog/top-10-student-ai-tools-essay-thesis-writing/
Allow: /blog/how-to-write-a-literature-review/
Allow: /blog/how-to-write-a-research-question/
Allow: /blog/research-methodology-types-methods-examples/
Allow: /blog/how-to-write-a-thesis-statement/
Allow: /blog/strengths-and-weaknesses-of-research-methods/
Allow: /blog/how-to-cite-sources-apa-ieee-harvard-oxford-chicago/
Allow: /blog/how-to-write-an-annotated-bibliography/
Allow: /blog/how-to-structure-a-research-paper/
Allow: /blog/openai-prism-ai-latex-editor-review/
Allow: /blog/how-to-write-a-research-paper-abstract/
Allow: /blog/how-to-write-a-methodology-section/
Allow: /blog/how-to-write-a-results-section/
Allow: /blog/how-to-write-a-research-paper-conclusion/
Allow: /blog/how-to-write-a-research-hypothesis/
Allow: /blog/how-to-write-limitations-of-a-study/
Allow: /blog/how-to-write-a-discussion-section/
Allow: /blog/how-to-write-a-research-paper-introduction/
Allow: /blog/qualitative-vs-quantitative-research/
Allow: /blog/how-to-choose-a-thesis-topic/
Allow: /blog/how-to-write-a-thesis-proposal/
Allow: /blog/systematic-review-vs-literature-review/
Allow: /blog/thesisai-vs-claude-for-thesis-writing/
Allow: /blog/claude-research-vs-thesisai-literature-review/
Allow: /blog/how-to-paraphrase-in-academic-writing/
Allow: /blog/what-is-a-theoretical-framework/
Allow: /blog/how-to-do-a-thematic-analysis/
Allow: /blog/sampling-methods-in-research/
Allow: /blog/reliability-and-validity-in-research/
Allow: /blog/primary-vs-secondary-sources/
Allow: /blog/how-to-cite-a-website/
Allow: /blog/how-to-proofread-and-edit-your-thesis/
Allow: /blog/how-to-prepare-for-your-thesis-defense/
Allow: /blog/how-to-use-ai-ethically-in-academic-writing/
Allow: /blog/thesis-vs-dissertation/
Allow: /blog/independent-vs-dependent-variables/
Allow: /blog/how-to-choose-a-statistical-test/
Allow: /blog/how-to-find-a-research-gap/
Allow: /blog/how-to-cite-chatgpt-and-ai-tools/
Allow: /blog/claude-science-review/
Allow: /blog/gemini-for-science-review/
Allow: /blog/how-to-avoid-plagiarism/
Allow: /blog/research-aims-and-objectives/
Allow: /blog/research-philosophy/
Allow: /blog/mixed-methods-research/
Allow: /blog/data-collection-methods/

# Competitor comparison pages
Allow: /academia-edu/
Allow: /acm/
Allow: /anara-ai/
Allow: /answerthis-ai/
Allow: /arxiv/
Allow: /asta/
Allow: /brainscape/
Allow: /cambridge-core/
Allow: /citeseerx/
Allow: /claude-science/
Allow: /connected-papers/
Allow: /consensus-ai/
Allow: /coursehero/
Allow: /docsity/
Allow: /elicit/
Allow: /elsevier/
Allow: /endnote/
Allow: /explain-paper/
Allow: /figshare/
Allow: /gatsbi-ai/
Allow: /gemini-for-science/
Allow: /grammarly/
Allow: /ieee-xplore/
Allow: /inciteful/
Allow: /jenni-ai/
Allow: /julius-ai/
Allow: /justdone/
Allow: /litmaps/
Allow: /manus-ai/
Allow: /mdpi/
Allow: /mendeley/
Allow: /mindthegraph/
Allow: /openai-prism/
Allow: /orcid/
Allow: /overleaf/
Allow: /paperdigest/
Allow: /paperpal/
Allow: /petal-ai/
Allow: /proquest/
Allow: /pubmed/
Allow: /quillbot/
Allow: /quizlet/
Allow: /rdiscovery/
Allow: /research-rabbit/
Allow: /researchgate/
Allow: /scholarcy/
Allow: /scibbr/
Allow: /sciencedirect/
Allow: /scinito/
Allow: /scispace/
Allow: /scite-ai/
Allow: /scopus/
Allow: /scribd/
Allow: /semantic-scholar/
Allow: /sourcely/
Allow: /springerlink/
Allow: /ssrn/
Allow: /studocu/
Allow: /studydrive/
Allow: /studytexter/
Allow: /stuvia/
Allow: /thesify-ai/
Allow: /trinka/
Allow: /web-of-science/
Allow: /wiley-online-library/
Allow: /wolfram-alpha/
Allow: /writefull-ai/
Allow: /yomu-ai/
Allow: /zotero/

# ---------------------------------------------------------------------------
# AI crawlers, answer engines and user-triggered fetchers.
#
# Named explicitly because RFC 9309 has a crawler obey only the single most
# specific group that matches its token - so a bot that finds its own name here
# stops reading the "*" group entirely, and the Disallow rules above have to be
# repeated rather than inherited. Each vendor also runs several tokens with
# different jobs (Anthropic: ClaudeBot trains, Claude-SearchBot indexes,
# Claude-User fetches a page a person asked about), and each one needs its own
# line. All of them get the same answer: the public marketing surface is open,
# the authenticated app is not.
# ---------------------------------------------------------------------------
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: DuckAssistBot
User-agent: meta-externalagent
User-agent: CCBot
User-agent: Bytespider
User-agent: cohere-ai
User-agent: MistralAI-User
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /chat/
Disallow: /documents/
Disallow: /refinement/
Disallow: /download/
Disallow: /upload-file/
Disallow: /upload/
Disallow: /send-chat/
Disallow: /feedback/
Disallow: /stripe_webhook/
Disallow: /webhooks/
Disallow: /create-portal-session/
Disallow: /download-file/
Disallow: /save-referral/
Disallow: /check-payment/
Disallow: /account/
Allow: /account/login/
Allow: /account/signup/
Allow: /account/reset-password/

# LLM/agent crawl guide. Every page below also has a Markdown twin at the
# same path with .md in place of the trailing slash (/blog/foo.md), which
# is also what `Accept: text/markdown` returns.
Allow: /llms.txt
Allow: /.well-known/mcp.json
Allow: /*.md$
# LaTeX/BibTeX sources of the example documents. Crawlable so agents can read
# them; the responses carry X-Robots-Tag: noindex so they do not compete with
# the example page itself in search results.
Allow: /example/

# The OAuth consent and token endpoints. Crawling them accomplishes nothing and
# /oauth/authorize needs a login, so keep them out; the two metadata documents
# stay reachable because that is how a client discovers the server.
Disallow: /oauth/
Allow: /.well-known/oauth-protected-resource
Allow: /.well-known/oauth-authorization-server

# Sitemap
Sitemap: https://www.thesisai.io/sitemap.xml