Meta crawlers

Bloody hell, ate up 850GB+ bandwidth in 3 days. what the fuck!! & wtf is this crawler: Mozilla/5.0 (compatible; SeznamBot/4.0; +https://o-seznam.cz/napoveda/vyhledavani/en/seznambot-crawler/)
This commit is contained in:
FbIN Support 2026-05-12 18:59:35 +05:30
commit 4af84898c7

118
bots.txt
View file

@ -126,6 +126,47 @@
14.176.221.56
14.182.193.138
16.54.83.227
17.0.0.0/21
17.0.64.0/18
17.0.71.0/24
17.0.99.0/24
17.0.109.0/24
17.0.117.0/24
17.0.120.0/24
17.0.128.0/18
17.0.176.0/20
17.0.184.0/23
17.0.224.0/24
17.1.0.0/19
17.1.64.0/18
17.1.108.0/24
17.1.128.0/18
17.1.184.0/24
17.1.185.0/24
17.1.193.0/24
17.1.197.0/24
17.4.10.0/23
17.4.13.0/24
17.4.44.0/23
17.4.46.0/24
17.4.49.0/24
17.4.8.0/23
17.7.64.0/24
17.7.68.0/24
17.7.72.0/24
17.7.74.0/23
17.7.80.0/24
17.8.64.0/23
17.8.66.0/23
17.8.68.0/22
17.8.76.0/23
17.8.80.0/23
17.8.86.0/23
17.8.152.0/21
17.8.164.0/22
17.8.192.0/20
17.8.208.0/20
17.8.224.0/20
17.22.237.27
17.22.237.29
17.22.237.67
@ -155,6 +196,15 @@
17.22.253.186
17.22.253.217
17.22.253.223
17.36.0.0/15
17.38.0.0/15
17.40.0.0/15
17.102.0.0/15
17.104.0.0/15
17.106.0.0/15
17.108.0.0/15
17.110.0.0/15
17.114.0.0/15
17.241.75.18
17.241.75.28
17.241.75.41
@ -1106,6 +1156,22 @@
54.232.33.171
54.235.116.66
54.235.191.179
57.0.0.0/8
57.141.0.0/16
57.141.0.0/17
57.141.0.0/18
57.141.0.0/19
57.141.0.0/20
57.141.0.0/21
57.141.0.0/22
57.141.0.0/23
57.141.0.0/24
57.141.0.0/25
57.141.0.0/26
57.141.0.0/27
57.141.0.0/28
57.141.0.0/29
57.141.0.0/30
57.141.0.1
57.141.0.2
57.141.0.3
@ -1182,6 +1248,13 @@
57.141.8.38
57.141.8.65
57.141.16.0/24
57.141.20.0/24
57.141.20.0/25
57.141.20.0/26
57.141.20.0/27
57.141.20.0/28
57.141.20.0/29
57.141.20.0/30
57.141.20.5
57.141.20.10
57.141.20.16
@ -1192,6 +1265,18 @@
57.141.20.49
64.23.166.186
64.23.173.155
66.249.64.0/19
66.249.64.0/20
66.249.64.0/21
66.249.64.0/22
66.249.64.0/23
66.249.64.0/24
66.249.64.0/25
66.249.64.0/26
66.249.64.0/27
66.249.64.0/28
66.249.64.0/29
66.249.64.0/30
66.249.65.33
66.249.65.34
66.249.65.35
@ -1331,6 +1416,25 @@
74.7.243.32
74.7.244.30
74.248.121.178
77.75.72.0/21
77.75.72.0/22
77.75.72.0/23
77.75.72.0/24
77.75.72.0/25
77.75.72.0/26
77.75.72.0/27
77.75.72.0/28
77.75.72.0/29
77.75.72.0/30
77.75.76.0/23
77.75.76.0/24
77.75.76.0/25
77.75.76.0/26
77.75.76.0/27
77.75.76.0/28
77.75.76.0/29
77.75.76.0/30
77.75.77.17
77.75.77.32
77.75.78.160
77.75.78.166
@ -1346,6 +1450,7 @@
83.99.206.159
84.33.224.35
86.38.236.59
89.167.32.97
89.249.193.0/24
92.222.0.0/16
92.222.104.192
@ -5964,6 +6069,19 @@
172.120.119.47
172.121.159.164
173.0.10.60
173.252.64.0/18
173.252.64.0/19
173.252.64.0/20
173.252.64.0/21
173.252.64.0/22
173.252.64.0/23
173.252.64.0/24
173.252.64.0/25
173.252.64.0/26
173.252.64.0/27
173.252.64.0/28
173.252.64.0/29
173.252.64.0/30
173.252.69.1
173.252.69.2
173.252.69.3