# All robots will spider the domain
User-agent: *

# Disallow directories
Disallow: /assets/downloads/
Disallow: /datastore/
Disallow: /assets/weekly_graphs/
Disallow: /assets/graphs/
Disallow: /data-providers/
Disallow: /forecasting/locations

# Disallow interactive data sections
# to stop bots hammering the databases
Disallow: /data/data_selector
Disallow: /data/exceedence
Disallow: /data/data-availability
Disallow: /data/DAQI-regional-data
Disallow: /data/non-auto-data
Disallow: /data/gis-mapping
Disallow: /data/openair
Disallow: /data/laqm-background-maps
Disallow: /data/ozone-data
Disallow: /data/uv-data
Disallow: /data/uv-index-graphs