Site Search

Find sitemap.xml, robots.txt, ads.txt, app-ads.txt and sellers.json - Real-Time Search

Not http https www subdomain

ukdigital.co.uk robots.txt

ukdigital.co.uk
Site: ukdigital.co.uk

The robots.txt file is a file at the root a domain indicates parts of the site should not be accessed by search engine crawlers.

(*) Thumbnail Screenshots by Thumbshots



# robots.txt M2 - July 2019
# Google Image Crawler Setup
#User-agent: Googlebot-Image
#Disallow: /
# Crawlers Setup
User-agent: *
#
# Directories
Disallow: */404/*
Disallow: */admin/*
Disallow: */app/*
Disallow: */bin/*
Disallow: */cgi-bin/*
Disallow: */dev/*
Disallow: */downloader/*
Disallow: */errors/*
Disallow: */export/*
Disallow: */import/*
Disallow: */itactica_quickview/*
Disallow: */itactica_quickview/product/view/id/*
Disallow: */includes/*
#Disallow: /js/*
#Disallow: /lib/*
#Disallow: /media/*
Disallow: */media/captcha/*
#Disallow: /media/catalog/*
Disallow: */media/customer/*
Disallow: */media/dhl/*
Disallow: */media/downloadable/*
Disallow: */media/pdf/*
Disallow: */media/sales/*
Disallow: */media/tmp/*
#Disallow: /media/wysiwyg/*
Disallow: */myphp/*
Disallow: */phpserver/
Disallow: */pkginfo/*
Disallow: */pub/*
Disallow: */report/*
Disallow: */setup/*
Disallow: */scripts/*
Disallow: */shell/*
#Disallow: */skin/*
Disallow: */staff/*
Disallow: */stats/*
Disallow: */temp/*
Disallow: */tmp/*
Disallow: */update/*
Disallow: */upload/*
Disallow: */vendor/*
Disallow: */var/*
Disallow: */xmlconnect/*
#
# Paths (clean URLs)
Disallow: /index.php/*
Disallow: /emailbt.php
Disallow: /emailbt1.php
Disallow: /emailbt2.php
Disallow: */account/*
Disallow: */catalog/product_compare/*
#Disallow: */catalog/category/view/*
#Disallow: */catalog/product/view/*
#Disallow: */catalog/product/gallery/*
Disallow: */cart/*
Disallow: */catalogsearch/*
Disallow: */checkout/*
Disallow: */checkouthere/*
Disallow: */contacts/*
Disallow: */control/*
Disallow: */currency/switch/*
Disallow: */customer/*
Disallow: */customer/account/*
Disallow: */customer/account/forgot*
Disallow: */customer/account/login/*
Disallow: */customer/account/login/referer/*
Disallow: */customize/*
Disallow: /itactica_quickview*
Disallow: /itactica-productlabel*
Disallow: */form_key/*
Disallow: */newsletter/*
Disallow: */onestepcheckout/*
Disallow: */order/history*
Disallow: */sales/order/*
Disallow: /sales/order/history/
Disallow: */poll/*
Disallow: */popup/*
Disallow: */login*
#added for M2
Disallow: /review/
Disallow: */review/product/*
Disallow: */referer/*
Disallow: */sendfriend/*
Disallow: */sociallogin/*
Disallow: */stocknotification/*
Disallow: */popup/*
Disallow: */tag/*
Disallow: */wishlist/*
Disallow: */extended-reviews/*
Disallow: */add/id/*
#
# Files
Disallow: *AGREEMENT.*
Disallow: */composer.*
Disallow: /CONTRIBUTING.*
Disallow: /CONTRIBUTOR_*
Disallow: */COPYING.*
Disallow: */Gruntfile.*
Disallow: */gruntfile.*
Disallow: */RELEASE*
Disallow: *NOTES.*
Disallow: *LICENSE*
Disallow: *license.*
Disallow: */package.*
Disallow: */nginx.*
Disallow: *.conf*
Disallow: *.sample*
Disallow: *.access*
Disallow: /*.backup*
Disallow: /*.cvs
Disallow: /*.CVS
Disallow: /*.exe$
Disallow: /*.ini$
Disallow: /*.sh$
Disallow: /*.Svn$
Disallow: /*.svn$
Disallow: /*.Idea$
Disallow: /*.idea$
Disallow: /*.Sql$
Disallow: /*.sql*
Disallow: /*.Tgz$
Disallow: /*.tgz$
Disallow: /*.Zip$
Disallow: /*.zip$
#
# Paths (no clean URLs)
#Disallow: /*.js$
#Disallow: /*.css$
#Disallow: /*?
Disallow: /*.php$
Disallow: /*?SID=
Disallow: /*?___SID=
# Dont crawl these filters 
Disallow: /*&order=*
Disallow: /*?dir=*
Disallow: /*?limit=*
Disallow: /*?mode*
Disallow: /*?cat=*
Disallow: /*?q=*
Disallow: /*?filter=*
Disallow: /shopby/*
# Do not index pages that are sorted or filtered.
Disallow: /*?*product_list_mode=*
Disallow: /*?*product_list_order=*
Disallow: /*?*product_list_limit=*
Disallow: /*?*product_list_dir=*
# Dont crawl these category filters
#Disallow: */filter/*
Disallow: */lens_type/*
Disallow: */lens_coverage/*
Disallow: */lens_use/*
Disallow: */lens_type/*
Disallow: */filter_size/*
Disallow: */price/*
Disallow: */camera_viewfinder/*
#Disallow: */camera_type/*
Disallow: */color/*
#Disallow: */filter/cat/*
Disallow: */flash_fitting/*
Disallow: */flash_wireless/*
Disallow: */tripod_load/*
Disallow: */tripod_weight/*
Disallow: */tripod_height/*
Disallow: */tripod_use/*
Disallow: */tripod_type/*
Disallow: */tripod_lock_type/*
Disallow: */no,yes*
Disallow: */no/*
Disallow: */yes/*
Disallow: */filter/tripod_head/*
Disallow: */tripod_head/other*
Disallow: */tripod_head/ball-head*
Disallow: */tripod_head/video-head*
Disallow: */tripod_lock_type/*
Disallow: */tripod_arca/*
Disallow: */filter/tripod_monopod/*
Disallow: */tripod_monopod/yes*
Disallow: */flash_wireless/*
Disallow: */flash_fitting/*
##

User-agent: 008
Disallow: /

User-agent: 8LEGS
Disallow: /

User-agent: adidxbot
Disallow: /

User-agent: adscanner
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: Alexibot
Disallow: /

User-agent: Aqua_Products
Disallow: /

User-agent: asterias
Disallow: /

User-agent: AURESYS
Disallow: /

User-agent: b2w/0.1
Disallow: /

User-agent: BackDoorBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: BecomeBot
Disallow: /

User-agent: Black Hole
Disallow: /

User-agent: BlackWidow
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: BlowFish
Disallow: /

User-agent: Bookmark search tool
Disallow: /

User-agent: BotALot
Disallow: /

User-agent: BotRightHere
Disallow: /

User-agent: BUbiNG
Disallow: /

User-agent: BuiltBotTough
Disallow: /

User-agent: Bullseye
Disallow: /

User-agent: Bullseye/1.0
Disallow: /

User-agent: BunnySlippers
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Cegbfeieh
Disallow: /

User-agent: CipaCrawler
Disallow: /

User-agent: CheeseBot
Disallow: /

User-agent: CherryPicker
Disallow: /

User-agent: ChinaClaw
Disallow: /

User-agent: Cliqzbot
Disallow: /

User-agent: coccocbot-image
Disallow: /

User-agent: coccocbot-web
Disallow: /

User-agent: Companybook
Disallow: /

User-agent: Copernic
Disallow: /

User-agent: CopyRightCheck
Disallow: /

User-agent: cosmos
Disallow: /

User-agent: Crescent
Disallow: /

User-agent: Custo
Disallow: /

User-agent: DAUM
Disallow: /

User-agent: DISCo
Disallow: /

User-agent: DittoSpyder
Disallow: /

User-agent: DomainCrawler
Disallow: /

User-agent: Dotbot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: Download\ Demon
Disallow: /

User-agent: duggmirror
Disallow: /

User-agent: dumbot
Disallow: /

User-agent: eCatch
Disallow: /

User-agent: ECCP/1.0
Disallow: /

User-agent: EirGrabber
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: EnigmaBot
Disallow: /

User-agent: Enterprise_Search
Disallow: /

User-agent: EroCrawler
Disallow: /

User-agent: es
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: Exabot/3.0
Disallow: /

User-agent: Express\ WebPictures
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: EyeNetIE
Disallow: /

User-agent: Ezooms
Disallow: /

User-agent: FairAd Client
Disallow: /

User-agent: Flaming AttackBot
Disallow: /

User-agent: FlashGet
Disallow: /

User-agent: Foobot
Disallow: /

User-agent: Freecrawl
Disallow: /

User-agent: FreeFind
Disallow: /

User-agent: FrontPage
Disallow: /

User-agent: Gaisbot
Disallow: /

User-agent: GarlikCrawler
Disallow: /

User-agent: GetRight
Disallow: /

User-agent: GetRight/4.2
Disallow: /

User-agent: GetWeb!
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: Go!Zilla
Disallow: /

User-agent: Go-Ahead-Got-It
Disallow: /

User-agent: GoScraper
Disallow: /

User-agent: GrabNet
Disallow: /

User-agent: Grafula
Disallow: /

User-agent: grub
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: Harvest/1.5
Disallow: /

User-agent: Hatena Antenna
Disallow: /

User-agent: Heritrix
Disallow: /

User-agent: hloader
Disallow: /

User-agent: HMView
Disallow: /

User-agent: HTTP Weazel
Disallow: /

User-agent: httplib
Disallow: /

User-agent: HTTrack Website Copier
Disallow: /

User-agent: Huaweisymantecspider
Disallow: /

User-agent: humanlinks
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: Image Stripper
Disallow: /

User-agent: Image Sucker
Disallow: /

User-agent: IndustryIndexBot
Disallow: /

User-agent: Indy\ Library
Disallow: /

User-agent: InfoNaviRobot
Disallow: /

User-agent: INGRID/0.1
Disallow: /

User-agent: InterGET
Disallow: /

User-agent: Internet Ninja
Disallow: /

User-agent: Iron33/1.0.2
Disallow: /

User-agent: JennyBot
Disallow: /

User-agent: Jetbot
Disallow: /

User-agent: JetCar
Disallow: /

User-agent: JobdiggerSpider
Disallow: /

User-agent: JOC Web Spider
Disallow: /

User-agent: JubiiRobot
Disallow: /

User-agent: jumpstation
Disallow: /

User-agent: Kenjin Spider
Disallow: /

User-agent: Keyword Density/0.9
Disallow: /

User-agent: KomodiaBot
Disallow: /

User-agent: larbin
Disallow: /

User-agent: Leech
Disallow: /

User-agent: LeechFTP
Disallow: /

User-agent: LexiBot
Disallow: /

User-agent: libWeb/clsHTTP
Disallow: /

User-agent: linkdexbot
Disallow: /

User-agent: LinkextractorPro
Disallow: /

User-agent: linklooker
Disallow: /

User-agent: LinkScan/8.1a Unix
Disallow: /

User-agent: LinkWalker
Disallow: /

User-agent: LNSpiderguy
Disallow: /

User-agent: ltx71
Disallow: /

User-agent: lwp-trivial
Disallow: /

User-agent: Mail.RU
Disallow: /

User-agent: Mass Downloader
Disallow: /

User-agent: Mata Hari
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: Microsoft URL Control
Disallow: /

User-agent: MIDown\ tool
Disallow: /

User-agent: MIIxpc
Disallow: /

User-agent: Mister PiX
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: moget
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: naver
Disallow: /

User-agent: Navroad
Disallow: /

User-agent: NearSite
Disallow: /

User-agent: Net\ Vampire
Disallow: /

User-agent: NetAnts
Disallow: /

User-agent: netEstate
Disallow: /

User-agent: NetMechanic
Disallow: /

User-agent: NetSpider
Disallow: /

User-agent: NetZIP
Disallow: /

User-agent: NICErsPRO
Disallow: /

User-agent: Nutch
Disallow: /

User-agent: oBot
Disallow: /

User-agent: Octopus
Disallow: /

User-agent: Offline Commander
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Offline Explorer Pro
Disallow: /

User-agent: Offline Navigator
Disallow: /

User-agent: Openbot
Disallow: /

User-agent: Openfind
Disallow: /

User-agent: Oracle Ultra Search
Disallow: /

User-agent: PageGrabber
Disallow: /

User-agent: Papa Foto
Disallow: /

User-agent: pavuk
Disallow: /

User-agent: pcBrowser
Disallow: /

User-agent: Perl LWP
Disallow: /

User-agent: PerMan
Disallow: /

User-agent: Photon/1.0
Disallow: /

User-agent: pricebot
Disallow: /

User-agent: ProPowerBot/2.14
Disallow: /

User-agent: ProWebWalker
Disallow: /

User-agent: psbot
Disallow: /

User-agent: Python-urllib
Disallow: /

User-agent: quanta-probe
Disallow: /

User-agent: QueryN Metasearch
Disallow: /

User-agent: Qwantify
Disallow: /

User-agent: Radiation Retriever 1.1
Disallow: /

User-agent: RealDownload
Disallow: /

User-agent: ReGet
Disallow: /

User-agent: RepoMonkey
Disallow: /

User-agent: Riddler
Disallow: /

User-agent: RMA
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: SafetyNet Robot 0.1
Disallow: /

User-agent: SearchmetricsBot
Disallow: /

User-agent: searchpreview
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SEOkicks
Disallow: /

User-agent: SISTRIX
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: SmartDownload
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: sogou
Disallow: /

User-agent: sootle
Disallow: /

User-agent: SpankBot
Disallow: /

User-agent: spanner
Disallow: /

User-agent: spbot
Disallow: /

User-agent: Stanford
Disallow: /

User-agent: Stanford Comp Sci
Disallow: /

User-agent: SuperBot
Disallow: /

User-agent: SuperHTTP
Disallow: /

User-agent: Surfbot
Disallow: /

User-agent: suzuran
Disallow: /

User-agent: Szukacz/1.4
Disallow: /

User-agent: tAkeOut
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: Telesoft
Disallow: /

User-agent: The Intraformant
Disallow: /

User-agent: TheNomad
Disallow: /

User-agent: TightTwatBot
Disallow: /

User-agent: Titan
Disallow: /

User-agent: toCrawl/UrlDispatcher
Disallow: /

User-agent: True_Robot
Disallow: /

User-agent: turingos
Disallow: /

User-agent: Turnitin
Disallow: /

User-agent: TwengaBot-2.0
Disallow: /

User-agent: UptimeRobot
Disallow: /

User-agent: Uptimebot
Disallow: /

User-agent: URL Control
Disallow: /

User-agent: URL_Spider_Pro
Disallow: /

User-agent: URLy Warning
Disallow: /

User-agent: Vagabondo
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: VCI
Disallow: /

User-agent: VoidEYE
Disallow: /

User-agent: Web Downloader
Disallow: /

User-agent: Web Image Collector
Disallow: /

User-agent: Web Sucker
Disallow: /

User-agent: WebAuto
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebEnhancer
Disallow: /

User-agent: WebFetch
Disallow: /

User-agent: WebGo\ IS
Disallow: /

User-agent: WebLeacher
Disallow: /

User-agent: WebmasterWorldForumBot
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebSauger
Disallow: /

User-agent: Website eXtractor
Disallow: /

User-agent: Website Quester
Disallow: /

User-agent: WebSnake
Disallow: /

User-agent: Webster Pro
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebVac
Disallow: /

User-agent: WebWhacker
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: WebZip
Disallow: /

User-agent: Wget
Disallow: /

User-agent: WiseGuys
Disallow: /

User-agent: woobot
Disallow: /

User-agent: WWW-Collector-E
Disallow: /

User-agent: WWWOFFLE
Disallow: /

User-agent: Xaldon_WebSpider
Disallow: /

User-agent: Xenu's
Disallow: /

User-agent: Xombot
Disallow: /

User-agent: YaK
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: yoozBot
Disallow: /

User-agent: YoudaoBot
Disallow: /

User-agent: Zeus
Disallow: /

User-agent: Zing-BottaBot
Disallow: /

User-agent: ZyBorg
Disallow: /

Sitemap: https://www.ukdigital.co.uk/sitemap/ukdigital/sitemap.xml
Sitemap: https://www.ukdigital.co.uk/sitemap/ukdigital/sitemap-blog.xml
Sitemap: https://www.ukoptics.co.uk/sitemap/ukoptics/sitemap.xml
Sitemap: https://www.novo-photo.com/sitemap/novo/sitemap.xml
Sitemap: https://www.laowalens.co.uk/sitemap/laowa/sitemap.xml
              
Sunday, 22 September 2019