User-agent: *
Allow: /

# Allow search engines to crawl all public pages
Allow: /blog/
Allow: /blog/*
Allow: /careers/
Allow: /careers/*
Allow: /portfolio/
Allow: /portfolio/*
Allow: /team/
Allow: /team/*
Allow: /images/
Allow: /images/*

# Allow specific API routes that are public
Allow: /api/blog/
Allow: /api/careers/
Allow: /api/portfolio/
Allow: /api/team/
Allow: /sitemap.xml/

# Allow specific blog posts
Allow: /blog/2025-06-24-why-businesses-need-ai-the-competitive-edge-of-the-future
Allow: /blog/2025-07-04-building-an-ai-customer-support-assistant-a-complete-guide-to-fine-tuning-llama-3-with-lora-and-qlora
Allow: /blog/2025-08-01-software-3-0-is-here-software-in-the-age-of-ai
Allow: /blog/2025-08-26-agentic-rag-building-the-next-generation-of-retrieval-augmented-generation-workflows-with-agentic-intelligence
Allow: /blog/2025-08-26-trip-planning-with-multi-agent-ai-architecture

# Allow specific portfolio items
Allow: /portfolio/2025-01-28-automated-death-certificate-verification
Allow: /portfolio/2025-01-28-handwritten-cheque-processor
Allow: /portfolio/2025-01-28-llm-powered-adverse-media-search
Allow: /portfolio/2025-01-28-merchant-category-code-identification
Allow: /portfolio/2025-01-28-unsupervised-product-matching
Allow: /portfolio/2025-01-28-vehicle-routing-and-packing-optimization
Allow: /portfolio/2025-08-26-agentic-rag
Allow: /portfolio/2025-08-31-intelligent-document-extraction-ide-

# Allow specific team members
Allow: /team/2025-01-27-abhikesh-nag
Allow: /team/2025-01-27-anuj-rana
Allow: /team/2025-01-27-dr-arun-b-ayyar
Allow: /team/2025-01-27-dr-bharath-bhikkaji
Allow: /team/2025-01-27-ganesh-ranganath
Allow: /team/2025-01-27-gaurav-shrivastav
Allow: /team/2025-01-27-nithesh-rajmohan
Allow: /team/2025-01-27-p-v-srihari
Allow: /team/2025-01-27-sanjay-sivakumar
Allow: /team/2025-01-27-sridhar-sriraman

# Allow specific career opportunities


# Disallow admin and internal API routes
Disallow: /admin/
Disallow: /admin/*
Disallow: /api/upload/
Disallow: /api/upload/*
Disallow: /api/populate-data/
Disallow: /api/populate-data/*
Disallow: /api/demo-request/
Disallow: /api/demo-request/*

# Disallow temporary and cache files
Disallow: /*.tmp
Disallow: /*.cache
Disallow: /tmp/
Disallow: /cache/
Disallow: /_next/
Disallow: /.next/

# Disallow common WordPress/CMS paths (security)
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: /administrator/
Disallow: /xmlrpc.php
Disallow: /wp-login.php

# Specific user agents
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 2

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: YandexBot
Allow: /
Crawl-delay: 2

User-agent: facebookexternalhit
Allow: /
Crawl-delay: 1

User-agent: LinkedInBot
Allow: /
Crawl-delay: 1

User-agent: TwitterBot
Allow: /
Crawl-delay: 1

# AI/ML crawlers
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 1

User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ClaudeBot
Allow: /
Crawl-delay: 1

User-agent: Claude-Web
Allow: /
Crawl-delay: 1

User-agent: PerplexityBot
Allow: /
Crawl-delay: 1

User-agent: YouBot
Allow: /
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /
Crawl-delay: 1

User-agent: OpenAI-SearchBot
Allow: /
Crawl-delay: 1

# Aggressive crawlers - slower crawl delay
User-agent: AhrefsBot
Allow: /
Crawl-delay: 10

User-agent: SemrushBot
Allow: /
Crawl-delay: 10

User-agent: MJ12bot
Allow: /
Crawl-delay: 10

User-agent: DotBot
Allow: /
Crawl-delay: 10

# Sitemap location
Sitemap: https://sfrpl.in/sitemap.xml

# Host directive
Host: sfrpl.in

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Additional information
# Website: https://sfrpl.in
# Contact: info@sfrpl.in
# Last updated: 2025-09-01
# This robots.txt file is for Santa Fe Research Private Limited (SFRPL)
# We welcome search engines and AI crawlers to index our public content
# Please respect our crawl-delay directives for optimal server performance
# This file is dynamically generated and includes all current content