Commit c2549849 authored by Holger Levsen's avatar Holger Levsen

block some bots for good

Signed-off-by: Holger Levsen's avatarHolger Levsen <holger@layer-acht.org>
parent b0fe38dd
...@@ -20,36 +20,61 @@ ...@@ -20,36 +20,61 @@
ServerAdmin holger@layer-acht.org ServerAdmin holger@layer-acht.org
#block bad bots with a 403 #block bad bots with a 403
SetEnvIfNoCase User-Agent "facebookexternalhit" bad_bot SetEnvIfNoCase User-Agent "AhrefsBot" bad_bot
SetEnvIfNoCase User-Agent "Twitterbot" bad_bot
SetEnvIfNoCase User-Agent "Baiduspider" bad_bot SetEnvIfNoCase User-Agent "Baiduspider" bad_bot
SetEnvIfNoCase User-Agent "MetaURI" bad_bot SetEnvIfNoCase User-Agent "Exabot" bad_bot
SetEnvIfNoCase User-Agent "mediawords" bad_bot SetEnvIfNoCase User-Agent "Experibot" bad_bot
SetEnvIfNoCase User-Agent "facebookexternalhit" bad_bot
SetEnvIfNoCase User-Agent "FlipboardProxy" bad_bot SetEnvIfNoCase User-Agent "FlipboardProxy" bad_bot
SetEnvIfNoCase User-Agent "ltx71" bad_bot
SetEnvIfNoCase User-Agent "mediawords" bad_bot
SetEnvIfNoCase User-Agent "MetaURI" bad_bot
SetEnvIfNoCase User-Agent "SemrushBot" bad_bot SetEnvIfNoCase User-Agent "SemrushBot" bad_bot
SetEnvIfNoCase User-Agent "Sogou web spider" bad_bot SetEnvIfNoCase User-Agent "Sogou web spider" bad_bot
SetEnvIfNoCase User-Agent "Twitterbot" bad_bot
SetEnvIfNoCase User-Agent "yacybot" bad_bot
#end block bad bots #end block bad bots
<Directory /> <Directory />
Options FollowSymLinks Options FollowSymLinks
AllowOverride None AllowOverride None
<Limit GET POST HEAD>
Order Allow,Deny
Allow from all
Deny from env=bad_bot
</Limit>
</Directory> </Directory>
<Directory /var/www/> <Directory /var/www/>
Options Indexes FollowSymLinks MultiViews Options Indexes FollowSymLinks MultiViews
AllowOverride None AllowOverride None
Require all granted Require all granted
AddType text/plain .log AddType text/plain .log
<Limit GET POST HEAD>
Order Allow,Deny
Allow from all
Deny from env=bad_bot
</Limit>
</Directory> </Directory>
<Directory /var/lib/jenkins/userContent> <Directory /var/lib/jenkins/userContent>
Options Indexes FollowSymLinks MultiViews Options Indexes FollowSymLinks MultiViews
AllowOverride None AllowOverride None
Require all granted Require all granted
AddType text/plain .log AddType text/plain .log
<Limit GET POST HEAD>
Order Allow,Deny
Allow from all
Deny from env=bad_bot
</Limit>
</Directory> </Directory>
<Directory /srv/reproducible-builds.org/lfs> <Directory /srv/reproducible-builds.org/lfs>
Options Indexes FollowSymlinks MultiViews Options Indexes FollowSymlinks MultiViews
AllowOverride None AllowOverride None
Require all granted Require all granted
<Limit GET POST HEAD>
Order Allow,Deny
Allow from all
Deny from env=bad_bot
</Limit>
</Directory> </Directory>
<FilesMatch "\.gz$"> <FilesMatch "\.gz$">
...@@ -80,11 +105,6 @@ Use https-redirect www.diffoscope.org ...@@ -80,11 +105,6 @@ Use https-redirect www.diffoscope.org
Use common-directives tests.reproducible-builds.org Use common-directives tests.reproducible-builds.org
RedirectMatch permanent "^/((?!debian/repository/debian/).*)" https://tests.reproducible-builds.org/$1 RedirectMatch permanent "^/((?!debian/repository/debian/).*)" https://tests.reproducible-builds.org/$1
DocumentRoot /var/lib/jenkins/userContent/reproducible DocumentRoot /var/lib/jenkins/userContent/reproducible
<Limit GET POST HEAD>
Order Allow,Deny
Allow from all
Deny from env=bad_bot
</Limit>
</VirtualHost> </VirtualHost>
<VirtualHost *:443> <VirtualHost *:443>
...@@ -107,11 +127,6 @@ Use https-redirect www.diffoscope.org ...@@ -107,11 +127,6 @@ Use https-redirect www.diffoscope.org
DocumentRoot /var/www DocumentRoot /var/www
AddDefaultCharset utf-8 AddDefaultCharset utf-8
<Limit GET POST HEAD>
Order Allow,Deny
Allow from all
Deny from env=bad_bot
</Limit>
Alias /userContent /var/lib/jenkins/userContent Alias /userContent /var/lib/jenkins/userContent
Alias /robots.txt /var/lib/jenkins/userContent/robots.txt Alias /robots.txt /var/lib/jenkins/userContent/robots.txt
...@@ -172,16 +187,15 @@ Use https-redirect www.diffoscope.org ...@@ -172,16 +187,15 @@ Use https-redirect www.diffoscope.org
DocumentRoot /var/lib/jenkins/userContent/reproducible DocumentRoot /var/lib/jenkins/userContent/reproducible
AddDefaultCharset utf-8 AddDefaultCharset utf-8
<Limit GET POST HEAD>
Order Allow,Deny
Allow from all
Deny from env=bad_bot
</Limit>
Include reproduciblemap.conf Include reproduciblemap.conf
<Directory /var/lib/jenkins/userContent/reproducible/debian/artifacts> <Directory /var/lib/jenkins/userContent/reproducible/debian/artifacts>
HeaderName .HEADER.html HeaderName .HEADER.html
<Limit GET POST HEAD>
Order Allow,Deny
Allow from all
Deny from env=bad_bot
</Limit>
</Directory> </Directory>
# for watching service logfiles # for watching service logfiles
...@@ -190,6 +204,11 @@ Use https-redirect www.diffoscope.org ...@@ -190,6 +204,11 @@ Use https-redirect www.diffoscope.org
AllowOverride None AllowOverride None
Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch
Require all granted Require all granted
<Limit GET POST HEAD>
Order Allow,Deny
Allow from all
Deny from env=bad_bot
</Limit>
</Directory> </Directory>
# Use the sso.debian.org CA to validate client certificates # Use the sso.debian.org CA to validate client certificates
# Keep these files up to date with update-debsso-ca # Keep these files up to date with update-debsso-ca
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment