user-agent: * allow: /sites/js/*.js* allow: /sites/css/*.css* allow: /sites/images/*.jpg* allow: /sites/images/*.png* allow: /sites/images/*.gif* allow: /sites/images/*.ico* disallow: /*.html disallow: /sites/docs disallow: /pages