-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathrobots.txt
66 lines (50 loc) · 963 Bytes
/
robots.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
User-agent: *
Disallow:
Disallow:
Disallow:
User-agent: MJ12bot
Disallow:
User-agent: AhrefsBot
Disallow:
User-agent: BLEXBot
Disallow:
# Block SISTRIX
User-agent: SISTRIX Crawler
Disallow:
User-agent: sistrix
Disallow:
User-agent: 007ac9
Disallow:
User-agent: 007ac9 Crawler
Disallow:
# Block Uptime robot
User-agent: UptimeRobot/2.0
Disallow:
# Block Ezooms Robot
User-agent: Ezooms Robot
Disallow:
# Block Perl LWP
User-agent: Perl LWP
Disallow:
# Block netEstate NE Crawler (+http://www.website-datenbank.de/)
User-agent: netEstate NE Crawler (+http://www.website-datenbank.de/)
Disallow:
# Block WiseGuys Robot
User-agent: WiseGuys Robot
Disallow:
# Block Turnitin Robot
User-agent: Turnitin Robot
Disallow:
# Block Heritrix
User-agent: Heritrix
Disallow:
# Block pricepi
User-agent: pimonster
Disallow:
User-agent: Pimonster
Disallow:
User-agent: SurdotlyBot
Disallow:
User-agent: ZoominfoBot
Disallow:
Sitemap: http://wzj.life/sitemap.xml