# Robots.txt for localseoservicesnyc.com
# Last updated: 2026-01-09

User-agent: *
Allow: /

# Primary Sitemap
Sitemap: https://localseoservicesnyc.com/sitemap.xml

# Allow all important content pages
Allow: /services/
Allow: /locations/
Allow: /industry/
Allow: /blog/
Allow: /local-seo-guide
Allow: /local-seo-checklist
Allow: /local-seo-faq
Allow: /local-seo-tools
Allow: /local-seo-mistakes
Allow: /local-seo-audit
Allow: /local-rank-tracker
Allow: /seo-reporting-tools
Allow: /local-seo-packages
Allow: /local-directory-tracker
Allow: /local-business-events
Allow: /local-seo-vs-competitors
Allow: /seasonal-local-seo
Allow: /local-seo-case-studies
Allow: /case-studies
Allow: /about
Allow: /pricing
Allow: /best-seo-company-nyc
Allow: /seo-company-nyc
Allow: /best-seo-company-new-york
Allow: /best-nyc-seo-company
Allow: /search-engine-optimization-nyc

# Disallow admin, private, and technical paths
Disallow: /admin/
Disallow: /api/
Disallow: /*.json$
Disallow: /404
Disallow: /*?*
Disallow: /_
Disallow: /node_modules/
Disallow: /.git/
Disallow: /*.map$

# Allow common static assets for rendering
Allow: /assets/
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.ico$
Allow: /*.webp$
Allow: /*.woff2$
Allow: /manifest.json

# ==================== 2026 AI/LLM Bot Directives ====================
# Allow AI crawlers to index content for AI search features

User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: Google-Extended
Allow: /

User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: CCBot
Allow: /
Disallow: /admin/

User-agent: anthropic-ai
Allow: /
Disallow: /admin/

User-agent: ClaudeBot
Allow: /
Disallow: /admin/

User-agent: PerplexityBot
Allow: /
Disallow: /admin/

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Block aggressive/bad bots
User-agent: Bytespider
Disallow: /

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

# General crawl delay for respectful crawling
Crawl-delay: 1

# Canonical host
Host: https://localseoservicesnyc.com