Commit b0fe38dd authored by Holger Levsen's avatar Holger Levsen
Browse files

another stab at blocking some bad bots 



Signed-off-by: default avatarHolger Levsen <holger@layer-acht.org>
parent acc1d7a1

hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf

+28 −0
Original line number Diff line number Diff line
@@ -19,6 +19,17 @@ 
	ServerName $name

	ServerAdmin holger@layer-acht.org



	#block bad bots with a 403

	SetEnvIfNoCase User-Agent "facebookexternalhit" bad_bot

	SetEnvIfNoCase User-Agent "Twitterbot" bad_bot

	SetEnvIfNoCase User-Agent "Baiduspider" bad_bot

	SetEnvIfNoCase User-Agent "MetaURI" bad_bot

	SetEnvIfNoCase User-Agent "mediawords" bad_bot

	SetEnvIfNoCase User-Agent "FlipboardProxy" bad_bot

	SetEnvIfNoCase User-Agent "SemrushBot" bad_bot

	SetEnvIfNoCase User-Agent "Sogou web spider" bad_bot

	#end block bad bots



	<Directory />

		Options FollowSymLinks

		AllowOverride None
@@ -69,6 +80,11 @@ Use https-redirect www.diffoscope.org 
	Use common-directives tests.reproducible-builds.org

	RedirectMatch permanent "^/((?!debian/repository/debian/).*)" https://tests.reproducible-builds.org/$1

	DocumentRoot /var/lib/jenkins/userContent/reproducible

	<Limit GET POST HEAD>

	  Order Allow,Deny

	  Allow from all

	  Deny from env=bad_bot

	</Limit>

</VirtualHost>



<VirtualHost *:443>
@@ -91,6 +107,12 @@ Use https-redirect www.diffoscope.org 
	DocumentRoot /var/www

	AddDefaultCharset utf-8



	<Limit GET POST HEAD>

	  Order Allow,Deny

	  Allow from all

	  Deny from env=bad_bot

	</Limit>



	Alias /userContent /var/lib/jenkins/userContent

	Alias /robots.txt /var/lib/jenkins/userContent/robots.txt
@@ -150,6 +172,12 @@ Use https-redirect www.diffoscope.org 
	DocumentRoot /var/lib/jenkins/userContent/reproducible

	AddDefaultCharset utf-8



	<Limit GET POST HEAD>

	  Order Allow,Deny

	  Allow from all

	  Deny from env=bad_bot

	</Limit>



	Include reproduciblemap.conf



	<Directory /var/lib/jenkins/userContent/reproducible/debian/artifacts>