# generic

User-agent: *
Disallow: /articles/
Disallow: /cgi-bin/

# htdig can access articles tree
User-agent: htdig
Disallow: