User-agent: *
Crawl-delay: 1

User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: WhatsApp
User-agent: TwitterBot
User-agent: Applebot


Disallow: /*?


Allow: /*?page=
Allow: /*?p=
Allow: /*?product_list_limit=
Allow: /*?product_list_order=
Allow: /*?product_list_dir=
Allow: /*?utm_source=
Allow: /*?utm_medium=
Allow: /*?utm_campaign=
Allow: /*?utm_content=
Allow: /*?utm_term=
Allow: /*?gclid=
Allow: /*?fbclid=
Allow: /graphql?

Disallow: /*SID=
Disallow: /*sid=
Disallow: /*PHPSESSID=

Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /pkginfo/
Disallow: /setup/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/
Disallow: /generated/

Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /catalog/seo_sitemap/
Disallow: /catalogsearch/
Disallow: /checkout/
Disallow: /customer/
Disallow: /downloadable/
Disallow: /multishipping/
Disallow: /newsletter/
Disallow: /paypal/
Disallow: /productalert/
Disallow: /review/product/
Disallow: /sales/
Disallow: /sendfriend/
Disallow: /tag/
Disallow: /vault/
Disallow: /wishlist/

Disallow: /recaptcha/
Disallow: /customer/section/
Disallow: /customer/ajax/
Disallow: /page_cache/

Disallow: /rest/
Disallow: /soap/
Disallow: /ajax/
Disallow: /ajaxcart/
Disallow: /quickview/

Disallow: /catalogsearch/
Disallow: /advanced_search/
Disallow: /search/
Disallow: /mfproductsearch/

Disallow: /customer/account/
Disallow: /customer/address/
Disallow: /customer/wishlist/
Disallow: /oauth/
Disallow: /pslogin/
Disallow: /subscription/

Disallow: /admin/
Disallow: /admin_*
Disallow: /backend/
Disallow: /control/
Disallow: /customize/
Disallow: /downloader/

Disallow: /*.php$
Disallow: /*.phtml$
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /RELEASE_NOTES.txt
Disallow: /STATUS.txt
Disallow: /SECURITY.md
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /package.json
Disallow: /Gruntfile.js
Disallow: /gulpfile.js
Disallow: /.htaccess
Disallow: /.htpasswd
Disallow: /.git
Disallow: /.gitignore
Disallow: /.gitattributes


Disallow: /404/
Disallow: /no-route/
Disallow: /enable-cookies/
Disallow: /privacy-policy-cookie-restriction-mode/
Disallow: /service-unavailable/
Disallow: /maintenance/


Disallow: /media/customer/
Disallow: /media/downloadable/
Disallow: /media/import/
Disallow: /media/tmp/
Allow: /media/catalog/
Allow: /media/wysiwyg/
Allow: /static/


Disallow: /report/
Disallow: /var/report/
Disallow: /var/log/
Disallow: /var/export/
Disallow: /var/import/
Disallow: /var/session/
Disallow: /var/tmp/
Disallow: /var/cache/


Disallow: /push_notification/
Disallow: /mfcmsdr/
Disallow: /repo/


User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: DotBot
User-agent: MJ12bot
User-agent: BlexBot
User-agent: SEOkicks-Robot
Crawl-delay: 10

User-agent: PetalBot
User-agent: CCBot
Disallow: /

Sitemap: https://bfs.lv/pub/sitemap/bfs/sitemap_lv.xml
Sitemap: https://bfs.lv/pub/sitemap/bfs/sitemap_en.xml

# Block Japanese SEO spam paths
Disallow: /any-mall/
Disallow: /lv/any-mall/
Disallow: /en/any-mall/