# wardow.com - Taschen & Accessoires online

## GENERAL SETTINGS

# Google Image Crawler Setup - having crawler-specific sections makes it ignore generic e.g *
User-agent: Googlebot-Image
Disallow:

## Enable robots.txt rules for all crawlers
User-agent: *

# Sitemap Files
Sitemap: https://www.wardow.com/media/sitemap.xml
Sitemap: https://www.wardow.com/media/sitemap_en.xml
Sitemap: https://www.wardow.com/media/sitemap_fr.xml
Sitemap: https://www.wardow.com/media/sitemap_uk.xml
Sitemap: https://www.wardow.com/media/sitemap_it.xml
Sitemap: https://www.wardow.com/media/sitemap_es.xml
Sitemap: https://www.wardow.com/media/sitemap_pl.xml
Sitemap: https://www.wardow.com/media/sitemap_se.xml
Sitemap: https://www.wardow.com/media/sitemap_dk.xml
Sitemap: https://www.wardow.com/media/sitemap_no.xml
Sitemap: https://www.wardow.com/media/sitemap_cz.xml
Sitemap: https://www.wardow.com/media/sitemap_fi.xml
Sitemap: https://www.wardow.com/media/sitemap_nl.xml
Sitemap: https://www.wardow.com/media/sitemap_ch.xml

## DEVELOPMENT RELATED SETTINGS

## Do not crawl development files and folders: CVS, svn directories and dump files
Disallow: CVS
Disallow: .svn
Disallow: .idea
Disallow: .sql
Disallow: .tgz

## GENERAL MAGENTO SETTINGS

## Do not crawl common Magento technical folders
Disallow: /app/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
Disallow: /js/
Disallow: /lib/
Disallow: /pkginfo/
Disallow: /shell/
Disallow: /skin/
Disallow: /var/
Disallow: /report/

Disallow: /media/captcha/
Disallow: /media/customer/
Disallow: /media/dhl/
Disallow: /media/downloadable/
Disallow: /media/import/
Disallow: /media/productsfeed
Disallow: /media/sales/
Disallow: /media/tmp/
Disallow: /media/wysiwyg/
Disallow: /media/xmlconnect/

## Do not crawl common Magento files
Disallow: /api.php
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /get.php
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /README.txt
Disallow: /RELEASE_NOTES.txt

## MAGENTO SEO IMPROVEMENTS

## Do not crawl sub category pages that are sorted or filtered.
Allow: /*?p=
Allow: /*?utm*
Disallow: /*?
Disallow: /*?dir*
Disallow: /*?limit*
Disallow: /*?mode*
Disallow: /*?cat=*
Disallow: /*?q=*

## Do not crawl 2-nd home page copy (example.com/index.php/). Uncomment it only if you activated Magento SEO URLs.
Disallow: /index.php/

## Do not crawl links with session IDs
Disallow: /*?SID=
Disallow: /*.php$
Disallow: /*?p=*&

## Do not crawl checkout and user account pages
Disallow: */index.php/
Disallow: */catalog/product/gallery/
Disallow: */checkout/
Disallow: */onestepcheckout/
Disallow: */control/
Disallow: */contacts/
Disallow: */customize/
Disallow: */customer/
Disallow: */customer/account/
Disallow: */customer/account/login/
Disallow: */newsletter/
Disallow: */poll/
Disallow: */sendfriend/
Disallow: */tag/
Disallow: */wishlist/

## Do not crawl search pages and not-SEO optimized catalog links
Disallow: */catalogsearch/
Disallow: */catalog/product_compare/
Disallow: */catalog/category/view/
Disallow: */catalog/product/view/
Disallow: */pfullscreen/
Disallow: */quicklook/

## Do not crawl availability check ##
Disallow: */productalert/add/stock/product_id/

## SERVER SETTINGS

## Do not crawl common server technical folders and files
Disallow: /cgi-bin/
Disallow: /cleanup.php
Disallow: /apc.php
Disallow: /memcache.php
Disallow: /phpinfo.php

#some crawler specific settings
User-agent: adidxbot
Crawl-delay: 0

User-Agent: bingbot
Crawl-delay: 0