Find sitemap.xml, robots.txt, ads.txt, app-ads.txt and sellers.json - Real-Time Search
The robots.txt file is a file at the root a domain indicates parts of the site should not be accessed by search engine crawlers.
Sitemap: http://www.networktechinc.com/sitemap.gz
User-agent: *
Disallow: /logs
Disallow: /enviro-outdoor-p.html
Disallow: /enviro-rems-p.html
Disallow: /hdmicat-p.html
Disallow: /srvsw-term-ssh-p.html
Disallow: /usbkvm-p.html
Disallow: /cbm.html
Disallow: /cn2.html
Disallow: /nw2.html
Disallow: /mc.html
Disallow: /scn.html
Disallow: /scn2.html
Disallow: /scn3.html
Disallow: /pa2.html
Disallow: /ds/
Disallow: /dvi-optical-mm-extender/
Disallow: /dvi32-p/
Disallow: /dvimatrix/
Disallow: /ev/
# User actions like adding to cart, signing up for alerts, checkout, etc.
Disallow: /cgi-bin/keemux/process.html
Disallow: /cgi-bin/keemux/ord/
Disallow: /cgi-bin/keemux/stock-alert.html
Disallow: /cgi-bin/keemux/order.html
Disallow: /cgi-bin/keemux/order.html*
Disallow: /cgi-bin/keemux/order
Disallow: /cgi-bin/keemux/order*
Disallow: /eshop/order
Disallow: /cgi-bin/keemux/order?
Disallow: /cgi-bin/keemux/order?*
Disallow: /order
# Zoom Flash object and individual images
Disallow: /zoom.swf
Disallow: /images/zoom/
User-agent: Applebot
Disallow: /cgi-bin/keemux
Disallow: /cgi-bin/keemux/
User-agent: The Knowledge AI
Disallow: /
User-agent: SemrushBot
Disallow: /cgi-bin/keemux
Disallow: /cgi-bin/keemux/
