deepsite / public /robots.txt
enzostvs's picture
enzostvs HF Staff
hf.co to huggingface.co
6cca3b1
User-agent: *
Allow: /
# Block access to API routes and private areas
Disallow: /api/
Disallow: /auth/
Disallow: /_next/
Disallow: /assets/
# Allow specific public API endpoints if needed
Allow: /api/public/
# Sitemap location
Sitemap: https://huggingface.co/deepsite/sitemap.xml
# Crawl-delay for respectful crawling
Crawl-delay: 1