Commit c2549849 authored by Holger Levsen's avatar Holger Levsen
Browse files

block some bots for good 



Signed-off-by: default avatarHolger Levsen <holger@layer-acht.org>
parent b0fe38dd

hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf

+39 −20
Original line number Diff line number Diff line
@@ -20,36 +20,61 @@ 
	ServerAdmin holger@layer-acht.org



	#block bad bots with a 403

	SetEnvIfNoCase User-Agent "facebookexternalhit" bad_bot

	SetEnvIfNoCase User-Agent "Twitterbot" bad_bot

	SetEnvIfNoCase User-Agent "AhrefsBot" bad_bot

	SetEnvIfNoCase User-Agent "Baiduspider" bad_bot

	SetEnvIfNoCase User-Agent "MetaURI" bad_bot

	SetEnvIfNoCase User-Agent "mediawords" bad_bot

	SetEnvIfNoCase User-Agent "Exabot" bad_bot

	SetEnvIfNoCase User-Agent "Experibot" bad_bot

	SetEnvIfNoCase User-Agent "facebookexternalhit" bad_bot

	SetEnvIfNoCase User-Agent "FlipboardProxy" bad_bot

	SetEnvIfNoCase User-Agent "ltx71" bad_bot

	SetEnvIfNoCase User-Agent "mediawords" bad_bot

	SetEnvIfNoCase User-Agent "MetaURI" bad_bot

	SetEnvIfNoCase User-Agent "SemrushBot" bad_bot

	SetEnvIfNoCase User-Agent "Sogou web spider" bad_bot

	SetEnvIfNoCase User-Agent "Twitterbot" bad_bot

	SetEnvIfNoCase User-Agent "yacybot" bad_bot

	#end block bad bots



	<Directory />

		Options FollowSymLinks

		AllowOverride None

		<Limit GET POST HEAD>

		  Order Allow,Deny

		  Allow from all

		  Deny from env=bad_bot

		</Limit>

	</Directory>

	<Directory /var/www/>

		Options Indexes FollowSymLinks MultiViews

		AllowOverride None

		Require all granted

		AddType text/plain .log

		<Limit GET POST HEAD>

		  Order Allow,Deny

		  Allow from all

		  Deny from env=bad_bot

		</Limit>

	</Directory>

	<Directory /var/lib/jenkins/userContent>

		Options Indexes FollowSymLinks MultiViews

		AllowOverride None

		Require all granted

		AddType text/plain .log

		<Limit GET POST HEAD>

		  Order Allow,Deny

		  Allow from all

		  Deny from env=bad_bot

		</Limit>

	</Directory>

	<Directory /srv/reproducible-builds.org/lfs>

		Options Indexes FollowSymlinks MultiViews

		AllowOverride None

		Require all granted

		<Limit GET POST HEAD>

		  Order Allow,Deny

		  Allow from all

		  Deny from env=bad_bot

		</Limit>

	</Directory>



	<FilesMatch "\.gz$">
@@ -80,11 +105,6 @@ Use https-redirect www.diffoscope.org 
	Use common-directives tests.reproducible-builds.org

	RedirectMatch permanent "^/((?!debian/repository/debian/).*)" https://tests.reproducible-builds.org/$1

	DocumentRoot /var/lib/jenkins/userContent/reproducible

	<Limit GET POST HEAD>

	  Order Allow,Deny

	  Allow from all

	  Deny from env=bad_bot

	</Limit>

</VirtualHost>



<VirtualHost *:443>
@@ -107,11 +127,6 @@ Use https-redirect www.diffoscope.org 
	DocumentRoot /var/www

	AddDefaultCharset utf-8



	<Limit GET POST HEAD>

	  Order Allow,Deny

	  Allow from all

	  Deny from env=bad_bot

	</Limit>



	Alias /userContent /var/lib/jenkins/userContent

	Alias /robots.txt /var/lib/jenkins/userContent/robots.txt
@@ -172,16 +187,15 @@ Use https-redirect www.diffoscope.org 
	DocumentRoot /var/lib/jenkins/userContent/reproducible

	AddDefaultCharset utf-8



	Include reproduciblemap.conf



	<Directory /var/lib/jenkins/userContent/reproducible/debian/artifacts>

		HeaderName .HEADER.html

		<Limit GET POST HEAD>

		  Order Allow,Deny

		  Allow from all

		  Deny from env=bad_bot

		</Limit>



	Include reproduciblemap.conf



	<Directory /var/lib/jenkins/userContent/reproducible/debian/artifacts>

		HeaderName .HEADER.html

	</Directory>



	# for watching service logfiles
@@ -190,6 +204,11 @@ Use https-redirect www.diffoscope.org 
	    AllowOverride None

	    Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch

	    Require all granted

		<Limit GET POST HEAD>

		  Order Allow,Deny

		  Allow from all

		  Deny from env=bad_bot

		</Limit>

	</Directory>

	# Use the sso.debian.org CA to validate client certificates

	# Keep these files up to date with update-debsso-ca