## 
## robots.txt for Magento Community and Enterprise

User-agent: *

# Sitemap (use HTTPS if site supports it)
Sitemap: https://www.nationalcycle.com/sitemap/sitemap.xml

# Crawl pacing (Google ignores but others may honor)
Crawl-delay: 10


############################
# CORE MAGENTO BLOCKS
############################
Disallow: /admin/
Disallow: /app/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
Disallow: /lib/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /shell/
Disallow: /var/
Disallow: /tmp/
Disallow: /update/


############################
# CUSTOMER / CHECKOUT AREAS
############################
Disallow: /catalogsearch/
Disallow: /checkout/
Disallow: /customer/
Disallow: /wishlist/
Disallow: /review/
Disallow: /tag/
Disallow: /sendfriend/


############################
# MAJOR MAGENTO 1.x CRAWL PROBLEMS
############################

# Session IDs (big Magento 1 issue)
Disallow: /*?SID=

# Sorting / filtering / layered nav
Disallow: /*?*dir=
Disallow: /*?*order=
Disallow: /*?*limit=
Disallow: /*?*mode=

Disallow: /*?*price=
Disallow: /*?*color=
Disallow: /*?*mount=
Disallow: /*?*material=
Disallow: /*?*product_type=
Disallow: /*?*thickness=
Disallow: /*?*windscreen=


# Pagination duplicates
Disallow: /*?*p=

# Catch-all for parameter URLs (optional but recommended)
Disallow: /*?*&*

############################
# DUPLICATE URL PATHS
############################

# Duplicate category/product access paths
Disallow: /catalog/product/view/
Disallow: /catalog/category/view/

# Optional if SEO URLs enabled
Disallow: /index.php/

############################
# SERVER / MISC
############################
Disallow: /cgi-bin/
Disallow: /.git/
Disallow: /.gitignore/
Disallow: /*.sql$
Disallow: /*.tgz$

############################
# DON'T REWRITE/REDIRECT
############################
Disallow: /vehicles.html


############################
# ALLOW IMPORTANT CONTENT
############################
Allow: /media/catalog/
Allow: /vehicles/motorcycle-1/