User-agent: * Allow: / Allow: /assets/ # Block the raw data folder (where your CSVs/GeoJSONs/YMLs live) Disallow: /data/ Disallow: /_data/ # Technically redundant but safe Disallow: /*.csv$ Disallow: /*.geojson$ Disallow: /*.yml$ # Reference your sitemap Sitemap: https://divs.au/sitemap.xml