User-agent: *
Disallow: /cgi-bin
Disallow: /adm
Disallow: /admin
Disallow: /madmin
Disallow: /log
Disallow: /proc

User-agent: Twitterbot
Allow: /data

User-agent: GPTBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ByteDance
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: barkrowler
Disallow: /