User-agent: * # applies to all robots Allow: / # allow indexing of all pages Disallow: /website.nsf/ClickThru # disallow crawling into ClickThru agent