Files
fedora-infra_ansible/roles/download/templates/httpd/dl.fedoraproject.org.conf
Kevin Fenzi fefbc356f0 download: block infinite crawler looping
The /pub/alt/virtio-win directory had in it some 11 year old things.
One was a readme noting that it moved 11 years ago and nothing was
still here. The others were links to .

The scrapers, being as dumb as posts followed all those links over and
over again to the tune of millions per day.
I removed the links, but of course they were still trying, so
lets be a bit more aggressive and just 403 them all.

Signed-off-by: Kevin Fenzi <kevin@scrye.com>
2026-02-12 16:54:16 -08:00

75 lines
2.2 KiB
Plaintext

<VirtualHost *:80>
ServerName dl.fedoraproject.org
ServerAlias archive.fedoraproject.org archives.fedoraproject.org secondary.fedoraproject.org download-ib01.fedoraproject.org download-cc-rdu01.fedoraproject.org
ServerAdmin webmaster@fedoraproject.org
TraceEnable Off
#
# redirect everyone to use https
#
RewriteEngine on
RewriteCond %{SERVER_PORT} !^443$
RewriteRule ^/(.*) https://%{HTTP_HOST}/$1 [L,R]
Include "conf.d/dl.fedoraproject.org/*.conf"
</VirtualHost>
<VirtualHost *:443>
ServerName dl.fedoraproject.org
ServerAlias archive.fedoraproject.org archives.fedoraproject.org secondary.fedoraproject.org download-ib01.fedoraproject.org download-cc-rdu01.fedoraproject.org
ServerAdmin webmaster@fedoraproject.org
SSLEngine on
SSLCertificateFile /etc/pki/tls/certs/{{ wildcard_crt_file }}
SSLCertificateKeyFile /etc/pki/tls/private/{{ wildcard_key_file }}
SSLCertificateChainFile /etc/pki/tls/certs/{{ wildcard_int_file }}
SSLHonorCipherOrder On
# https://fedorahosted.org/fedora-infrastructure/ticket/4101#comment:14
# If you change the protocols or cipher suites, you should probably update
# modules/squid/files/squid.conf too, to keep it in sync.
SSLProtocol {{ ssl_protocols }}
SSLCipherSuite {{ ssl_ciphers }}
RewriteEngine On
RewriteCond %{REQUEST_URI} ^/pub/alt/virtio-win/.*$
RewriteRule .* - [F]
# proxy all requests to anubis after ssl termination
RequestHeader set "X-Real-Ip" expr=%{REMOTE_ADDR}
RequestHeader set X-Forwarded-Proto "https"
RequestHeader set "X-Http-Version" "%{SERVER_PROTOCOL}s"
ProxyPreserveHost On
ProxyRequests Off
ProxyVia Off
ProxyPass / http://[::1]:8923/
ProxyPassReverse / http://[::1]:8923/
</VirtualHost>
# Once anubis has processed the request it sends it to port 3923
Listen 3923 http
<VirtualHost *:3923>
ServerName dl.fedoraproject.org
ServerAlias archive.fedoraproject.org archives.fedoraproject.org secondary.fedoraproject.org download-ib01.fedoraproject.org download-cc-rdu01.fedoraproject.org
ServerAdmin webmaster@fedoraproject.org
RemoteIPHeader X-Real-Ip
RemoteIPInternalProxy ::1
Include "conf.d/dl.fedoraproject.org/*.conf"
</VirtualHost>
ServerLimit 3200
MaxRequestWorkers 3200
MaxRequestsPerChild 10000