Sitemap: https://www.dlx.co.uk/sitemap.xml

# Crawlers Setup
User-agent: Googlebot
Disallow:

User-agent: Googlebot-Image
Disallow:

User-agent: *

# Directories (platform/system)
Disallow: /app/
Disallow: /bin/
Disallow: /directory/
Disallow: /downloader/
Disallow: /install/
Disallow: /js/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /pkginfo/
Disallow: /private/
Disallow: /setup/
Disallow: /skin/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/

# Paths (clean URLs)
Disallow: /admin/
Disallow: */api/
Disallow: */catalog/category/view/
Disallow: */catalog/product/view/
Disallow: */catalogsearch
Disallow: */checkout
Disallow: */customer/
Disallow: */newsletter
Disallow: */order/
Disallow: */poll/
Disallow: */report/
Disallow: */review/
Disallow: */rma/
Disallow: */sendfriend
Disallow: */wishlist

# Files
Disallow: /cron
Disallow: /cron.php

# Do not index pages that are sorted or filtered.
Disallow: */cat/*
Disallow: /*/pricematch/*
Disallow: /*/colour/*
Disallow: /*/show/*
Disallow: /*/size/*
Disallow: /*/price/*
Disallow: */sort-by/*
Disallow: /*?*product_list_dir=
Disallow: /*?*product_list_limit=
Disallow: /*?*product_list_mode=
Disallow: /*?*product_list_order=

# Clean URLs - Suffixes
Disallow: /*.php
Disallow: /*.sh$
#Disallow: /*?

# CVS, SVN directory and dump files
Disallow: /*.CSV$
Disallow: /*.csv$
Disallow: /*.gitignore$
Disallow: /*.sample$
Disallow: /*.sql$
Disallow: /*.zip$

# Parameters (layered nav facets & misc)
Disallow: /*&carrier=*
Disallow: /*&cat=*
Disallow: /*&color_filter=*
Disallow: /*&display_size=*
Disallow: /*&name=*
Disallow: /*&network=*
Disallow: /*&os=*
Disallow: /*&price=*
Disallow: /*&timestamp=*
Disallow: /*&top_seller=*
Disallow: /*?SID=*
Disallow: /*?___SID=*
Disallow: /*?___store=*
Disallow: /*?carrier=*
Disallow: /*?cat=*
Disallow: /*?color_filter=*
Disallow: /*?display_size=*
Disallow: /*?name=*
Disallow: /*?network=*
Disallow: /*?os=*
Disallow: /*?PageSpeed=noscript
Disallow: /*?price=*
Disallow: /*?timestamp=*
Disallow: /*?top_seller=*

# Additional DLX layered-nav params commonly used
Disallow: /*?size_legacy=*
Disallow: /*?base_color=*
Disallow: /*?manufacturer=*
Disallow: /*?insulation_type=*
Disallow: /*?gender=*
Disallow: /*?p=2&size_legacy=*

Disallow: *?size_legacy=*
Disallow: *?base_color=*
Disallow: *?manufacturer=*
Disallow: *?insulation_type=*
Disallow: *?gender=*
Disallow: *?p=2&size_legacy=*

# Block filters for product categories
Disallow: /*/new_product_type/*
Disallow: /*/product_subtype/*
Disallow: /*?category_id=*

# Tracking/affiliate & misc
Disallow: /*?adnetwork=*
Disallow: /&adnetwork=
Disallow: /?affc=
Disallow: /&affc=
Disallow: /*?cto_pld=*
Disallow: /&cto_pld=
Disallow: /*?ved=*
Disallow: /*?fbclid=*
Disallow: /*?search=sf-*

# Allow pagination
Allow: /*?p=

# Crawl-Delay (not honoured by Google; fine for others)
Crawl-Delay: 10

# Bad bots
User-agent: 008
Disallow: /

User-agent: AhrefsBot
Crawl-Delay: 20

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: Nutch
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Seekbot
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: Sogou inst spider
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: Yandexbot
Disallow: /

User-agent: adsbot
Disallow: /

User-agent: baiduspider
Disallow: /

User-agent: baiduspider-image
Disallow: /

User-agent: baiduspider-mobile
Disallow: /

User-agent: baiduspider-news
Disallow: /

User-agent: baiduspider-video
Disallow: /

User-agent: barkrowler
Disallow: /

User-agent: botify
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: idealo-bot
Disallow: /

User-agent: megaindex.com
Disallow: /

User-agent: newspaper/0.2.8
Disallow: /