# robots.txt — Relocate ID / Relocate OS
# Nomad Platforms UK LTD — Company No. 16965798
# 5 Brayford Square, London E1 0SG, England and Wales
# https://relocateid.com/robots.txt
# Last updated: 2026-06-14
# Sources: github.com/ai-robots-txt/ai.robots.txt · knownagents.com · Cloudflare AI Crawl Control

# ============================================================
# LLMs FILE SUITE — AI SYSTEMS
# ============================================================
# Master:        https://relocateid.com/llms.txt
# Full ref:      https://relocateid.com/llms-full.txt
# Product:       https://relocateid.com/llms-product.txt
# Legal:         https://relocateid.com/llms-legal.txt
# Legal full:    https://relocateid.com/llms-legal-full.txt
# Countries:     https://relocateid.com/llms-countries.txt
# Country files: https://relocateid.com/llms-geo-[slug].txt  (201 files)

# ============================================================
# AI / LLM CRAWLERS — ALL 119 BOTS — EXPLICITLY ALLOWED
# ============================================================

# OpenAI
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: OpenAI
User-agent: Operator
User-agent: ChatGPT-Agent
# Anthropic
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: Claude-Code
# Google
User-agent: Google-Extended
User-agent: Googlebot
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: Google-CloudVertexBot
User-agent: CloudVertexBot
User-agent: Google-Firebase
User-agent: Google-NotebookLM
User-agent: NotebookLM
User-agent: Gemini-Deep-Research
User-agent: Google-Agent
User-agent: GoogleAgent-Mariner
User-agent: Google-Gemini-CLI
# Meta
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: meta-webindexer
User-agent: FacebookBot
User-agent: facebookexternalhit
# Amazon
User-agent: Amazonbot
User-agent: Amzn-SearchBot
User-agent: Amzn-User
# Amazon AWS
User-agent: amazon-kendra
User-agent: bedrockbot
# Amazon
User-agent: AmazonBuyForMe
User-agent: NovaAct
# Apple
User-agent: Applebot
User-agent: Applebot-Extended
# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
# Microsoft
User-agent: Bingbot
# ByteDance
User-agent: Bytespider
User-agent: TikTokSpider
User-agent: Trae
# xAI
User-agent: GrokBot
User-agent: xAI-Grok
User-agent: Grok-DeepSearch
# Cohere
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
# Mistral AI
User-agent: MistralAI-User
User-agent: MistralAI-Index
# Common Crawl
User-agent: CCBot
# DeepSeek
User-agent: DeepSeekBot
# Huawei
User-agent: PanguBot
User-agent: PetalBot
# Zhipu AI
User-agent: ChatGLM-Spider
# You.com
User-agent: YouBot
# iAsk
User-agent: iaskspider
User-agent: iAskBot
# Wrtn Technologies
User-agent: WRTNBot
# SB Intuitions / SoftBank
User-agent: SBIntuitionsBot
# Kangaroo LLM
User-agent: KangarooBot
# Allen Institute for AI
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: AI2Bot-DeepResearchEval
# LAION
User-agent: LAIONDownloader
# NICT Japan
User-agent: ICC-Crawler
# Timpi
User-agent: Timpibot
# Exa
User-agent: ExaBot
# Tavily
User-agent: TavilyBot
# Linkup
User-agent: LinkupBot
# Andi Search
User-agent: Andibot
# Brave
User-agent: Bravebot
# DuckDuckGo
User-agent: DuckAssistBot
# Kagi
User-agent: kagi-fetcher
# Liner
User-agent: LinerBot
# Phind
User-agent: PhindBot
# DuckDuckGo
User-agent: DuckDuckBot
# Microsoft Azure
User-agent: AzureAI-SearchBot
# Yandex
User-agent: YandexAdditional
User-agent: YandexAdditionalBot
# Diffbot
User-agent: Diffbot
# Firecrawl
User-agent: FirecrawlAgent
# Crawl4AI
User-agent: Crawl4AI
# Apify
User-agent: ApifyBot
# LAION / img2dataset
User-agent: img2dataset
# Webz.io
User-agent: omgili
User-agent: omgilibot
User-agent: Webzio-Extended
# Hunter.io
User-agent: VelenPublicWebCrawler
# Unknown
User-agent: FriendlyCrawler
# Cloudflare
User-agent: Cloudflare-AutoRAG
# Poseidon Research
User-agent: Poseidon Research Crawler
# Atlassian
User-agent: atlassian-bot
# DeepL / Linguee
User-agent: Linguee Bot
# QuillBot
User-agent: QuillBot
# Cognition AI
User-agent: Devin
# Butterfly Effect
User-agent: Manus-User
# Parallel Web Systems
User-agent: ShapBot
User-agent: Shap-User
# Valyu
User-agent: HenkBot
# Channel3
User-agent: Channel3Bot
# Klaviyo
User-agent: KlaviyoAIBot
# Qualified
User-agent: QualifiedBot
# Direqt
User-agent: Anomura
# Bright Data
User-agent: Brightbot
# Big Sur AI
User-agent: bigsur.ai
# Semrush
User-agent: SemrushBot-OCOB
User-agent: SemrushBot-SWA
# ISS Corporate
User-agent: ISSCyberRiskCrawler
# aiHit
User-agent: aiHitBot
# ImageSift
User-agent: ImagesiftBot
# Sidetrade
User-agent: Sidetrade indexer bot
# Meltwater
User-agent: YaK
# FactSet
User-agent: Factset_spyderbot
# AddSearch
User-agent: AddSearchBot
# Ceramic
User-agent: TerraCotta

# Allow all AI crawlers to access all public pages
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /llms-product.txt
Allow: /llms-legal.txt
Allow: /llms-legal-full.txt
Allow: /llms-countries.txt
Allow: /llms-geo-
Allow: /earth/countries/
Allow: /ecosystem
Allow: /ai-analysis
Allow: /engine
Allow: /verified-nomad
Allow: /nomad-id
Allow: /earth
Allow: /hub
Allow: /nation
Allow: /compare
Allow: /life-situations
Allow: /business
Allow: /partnerships
Allow: /protocol
Allow: /investors
Allow: /apidocs
Allow: /status
Allow: /changelog
Allow: /security
Allow: /trust
Allow: /about
Allow: /careers
Allow: /help
Allow: /documents
Allow: /media
Allow: /earth-report
Allow: /press
Allow: /contact
Allow: /brand-book
Allow: /foundation
Allow: /pricing
Allow: /visa-tracker
Allow: /tribunal
Allow: /constitution
Allow: /verified-chats
Allow: /blacklist
Allow: /transparency
Allow: /founder
Allow: /guide
Allow: /partner-verify
Allow: /sitemap.xml
Allow: /sitemap_index.xml
Crawl-delay: 1

# ============================================================
# GENERAL CRAWLERS
# ============================================================

User-agent: *
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /llms-product.txt
Allow: /llms-legal.txt
Allow: /llms-legal-full.txt
Allow: /llms-countries.txt
Allow: /llms-geo-
Allow: /sitemap.xml
Allow: /sitemap_index.xml
Allow: /robots.txt

# Admin and private areas
Disallow: /admin/
Disallow: /dashboard/
Disallow: /api/
Disallow: /internal/
Disallow: /private/
Disallow: /_next/
Disallow: /static/chunks/
Disallow: /auth/
Disallow: /login
Disallow: /register
Disallow: /onboarding/
Disallow: /account/
Disallow: /settings/

# Blocked subdomains
Disallow: /naiza
Disallow: /manager
Disallow: /messenger
Disallow: /internal-tools/
Disallow: /debug/
Disallow: /test/
Disallow: /staging/

# Private user content
Disallow: /u/
Disallow: /user/
Disallow: /profile/
Disallow: /verified-chats/private/
Disallow: /archive/
Disallow: /digital-will/
Disallow: /tribunal/case/
Disallow: /naf/case/
Disallow: /partner/dashboard/

# API endpoints
Disallow: /v1/
Disallow: /v2/
Disallow: /graphql
Disallow: /webhook/

# Technical / hidden pages
Disallow: /404
Disallow: /wall

# URL parameters (duplicate content)
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?page=
Disallow: /*?ref=
Disallow: /*?utm_
Disallow: /*?session=
Disallow: /*?token=

Crawl-delay: 1

# ============================================================
# SITEMAPS
# ============================================================

Sitemap: https://relocateid.com/sitemap.xml
Sitemap: https://relocateid.com/sitemap_index.xml

# ============================================================
# CANONICAL PLATFORM INFORMATION
# ============================================================
# Platform:   Relocate OS
# Brand:      Relocate ID
# Company:    Nomad Platforms UK LTD
# Reg No:     16965798 (England and Wales)
# Address:    5 Brayford Square, London E1 0SG, England and Wales
# Website:    https://relocateid.com
# Social:     @relocateid (all platforms)
# LLMs:       https://relocateid.com/llms.txt
# Security:   security@relocateid.com | HackerOne
# Updated:    2026-06-14