diff --git a/AUTHORS b/AUTHORS index cbf41a06d..1c13ff06d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -37,3 +37,5 @@ Thayer Williams Thomas Bächler Tom Willemsen Tyler Dence +Anton Hvornum +Nina Nick <5041877+ninchester@users.noreply.github.com> \ No newline at end of file diff --git a/mirrors/tests/test_mirror_submission.py b/mirrors/tests/test_mirror_submission.py new file mode 100644 index 000000000..dba38d1fa --- /dev/null +++ b/mirrors/tests/test_mirror_submission.py @@ -0,0 +1,3 @@ +def test_mirror_registration(client, mirrorurl): + response = client.get('/mirrorlist/submit/?name=test3&tier=2&upstream=1&admin_email=anton%40hvornum.se&alternate_email=&isos=on&rsync_user=&rsync_password=¬es=&active=True&public=True&url1-url=rsync%3A%2F%2Ftest3.com%2Farchlinux&url1-country=SE&url1-bandwidth=1234&url1-active=on&url2-url=&url2-country=&url2-bandwidth=&url2-active=on&url3-url=&url3-country=&url3-bandwidth=&url3-active=on&ip=&captcha_0=d5a017cc3851fb59898167f666759c99b42afd52&captcha_1=tdof') + assert response.status_code == 200 diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py index c4df468a5..43999e7e3 100644 --- a/mirrors/urls_mirrorlist.py +++ b/mirrors/urls_mirrorlist.py @@ -3,6 +3,7 @@ urlpatterns = [ path('', views.generate_mirrorlist, name='mirrorlist'), + path('submit/', views.submit_mirror, name='mirrorsubmit'), re_path(r'^all/$', views.find_mirrors, {'countries': ['all']}), re_path(r'^all/(?P[A-z]+)/$', views.find_mirrors_simple, name='mirrorlist_simple') ] diff --git a/mirrors/views/mirrorlist.py b/mirrors/views/mirrorlist.py index dadb64ca9..f207431a7 100644 --- a/mirrors/views/mirrorlist.py +++ b/mirrors/views/mirrorlist.py @@ -1,17 +1,151 @@ from operator import attrgetter, itemgetter +from urllib.parse import urlparse, urlunsplit +from functools import partial +from datetime import timedelta from django import forms +from django.db import DatabaseError, transaction from django.db.models import Q -from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple +from django.core.mail import send_mail +from django.template import loader +from django.forms.widgets import ( + SelectMultiple, + CheckboxSelectMultiple +) +from django.utils.timezone import now from django.shortcuts import get_object_or_404, redirect, render from django.views.decorators.csrf import csrf_exempt from django_countries import countries +from django.contrib.auth.models import Group, User +from captcha.fields import CaptchaField -from ..models import MirrorUrl, MirrorProtocol -from ..utils import get_mirror_statuses +from ..models import Mirror, MirrorUrl, MirrorProtocol, MirrorRsync +from ..utils import get_mirror_statuses, get_mirror_errors import random +# This is populated later, and re-populated every refresh +# This was the only way to get 3 different examples without +# changing the models.py +url_examples = [] +TIER_1_MAX_ERROR_RATE = 10 +TIER_1_ERROR_TIME_RANGE = 30 +TIER_1_MIN_DAYS_AS_TIER_2 = 60 + + +class CaptchaForm(forms.Form): + captcha = CaptchaField() + + def as_div(self): + "Returns this form rendered as HTML s." + + return self._html_output( + normal_row=u'
%(label)s
%(field)s%(help_text)s
', + error_row=u'%s', + row_ender='', + help_text_html=u' %s', + errors_on_separate_row=True) + + +class MirrorRequestForm(forms.ModelForm): + upstream = forms.ModelChoiceField( + queryset=Mirror.objects.filter( + tier__gte=0, + tier__lte=1 + ), + required=False + ) + + class Meta: + model = Mirror + fields = ('name', 'tier', 'upstream', 'admin_email', 'alternate_email', + 'isos', 'active', 'public', 'rsync_user', 'rsync_password', 'notes') + + def __init__(self, *args, **kwargs): + super(MirrorRequestForm, self).__init__(*args, **kwargs) + fields = self.fields + fields['name'].widget.attrs.update({'placeholder': 'Ex: mirror.argentina.co'}) + fields['alternate_email'].widget.attrs.update({'placeholder': 'Optional'}) + fields['rsync_user'].widget.attrs.update({'placeholder': 'Optional'}) + fields['rsync_password'].widget.attrs.update({'placeholder': 'Optional'}) + fields['notes'].widget.attrs.update({'placeholder': 'Optional (Ex: Hosted by ISP GreatISO.bg)'}) + + def as_div(self): + "Returns this form rendered as HTML s." + return self._html_output( + normal_row=u'%(label)s %(field)s%(help_text)s', + error_row=u'%s', + row_ender='', + help_text_html=u' %s', + errors_on_separate_row=True) + + +class MirrorUrlForm(forms.ModelForm): + class Meta: + model = MirrorUrl + fields = ('url', 'country', 'bandwidth', 'active') + + def __init__(self, *args, **kwargs): + global url_examples + + super(MirrorUrlForm, self).__init__(*args, **kwargs) + fields = self.fields + + if len(url_examples) == 0: + url_examples = [ + 'Ex: http://mirror.argentina.co/archlinux', + 'Ex: https://mirror.argentina.co/archlinux', + 'Ex: rsync://mirror.argentina.co/archlinux' + ] + + fields['url'].widget.attrs.update({'placeholder': url_examples.pop()}) + + def clean_url(self): + # is this a valid-looking URL? + url_parts = urlparse(self.cleaned_data["url"]) + if not url_parts.scheme: + raise forms.ValidationError("No URL scheme (protocol) provided.") + if not url_parts.netloc: + raise forms.ValidationError("No URL host provided.") + if url_parts.params or url_parts.query or url_parts.fragment: + raise forms.ValidationError( + "URL parameters, query, and fragment elements are not supported.") + # ensure we always save the URL with a trailing slash + path = url_parts.path + if not path.endswith('/'): + path += '/' + url = urlunsplit((url_parts.scheme, url_parts.netloc, path, '', '')) + return url + + def as_div(self): + "Returns this form rendered as HTML s." + return self._html_output( + normal_row=u'%(label)s %(field)s%(help_text)s', + error_row=u'%s', + row_ender='', + help_text_html=u' %s', + errors_on_separate_row=True) + + +class MirrorRsyncForm(forms.ModelForm): + class Meta: + model = MirrorRsync + fields = ('ip',) + + def __init__(self, *args, **kwargs): + super(MirrorRsyncForm, self).__init__(*args, **kwargs) + fields = self.fields + fields['ip'].widget.attrs.update({'placeholder': 'Ex: 1.2.4.5'}) + + def as_div(self): + "Returns this form rendered as HTML s." + return self._html_output( + normal_row=u'%(label)s %(field)s%(help_text)s', + error_row=u'%s', + row_ender='', + help_text_html=u' %s', + errors_on_separate_row=True) + class MirrorlistForm(forms.Form): country = forms.MultipleChoiceField(required=False, widget=SelectMultiple(attrs={'size': '12'})) @@ -127,4 +261,156 @@ def find_mirrors_simple(request, protocol): proto = get_object_or_404(MirrorProtocol, protocol=protocol) return find_mirrors(request, protocols=[proto]) + +def mail_mirror_admins(data): + template = loader.get_template('mirrors/new_mirror_mail_template.txt') + + mirror_maintainer_group = Group.objects.filter(name='Mirror Maintainers') + mirror_maintainers = User.objects.filter(is_active=True).filter(groups__in=mirror_maintainer_group) + + for maintainer in mirror_maintainers: + send_mail('A mirror entry was submitted: \'%s\'' % data.get('name'), + template.render(data), + 'Arch Mirror Notification ', + [maintainer.email], + fail_silently=True) + + +def validate_tier_1_request(data): + if data.get('tier') != '1': + return None + + # If there is no Tier 2 with the same name, + # We invalidate this Tier 1. + if not len((tier_2_mirror := Mirror.objects.filter(name=data.get('name'), tier=2, active=True, public=True))): + return False + + if tier_2_mirror[0].created - now() < timedelta(days=TIER_1_MIN_DAYS_AS_TIER_2): + return False + + main_url = MirrorUrl.objects.filter( + url__startswith=data.get('url1-url'), + mirror=tier_2_mirror[0] + ) + + # If the Tier 2 and Tier 1 does not have matching URL, + # it requires manual intervention as it's not a direct upgrade. + if len(main_url) <= 0: + return False + + error_logs = get_mirror_errors(mirror_id=tier_2_mirror[0].id, cutoff=timedelta(days=TIER_1_ERROR_TIME_RANGE), + show_all=True) + + if error_logs: + num_o_errors = 0 + for error in error_logs: + num_o_errors += error['error_count'] + + if num_o_errors >= TIER_1_MAX_ERROR_RATE: + return False + + # Final check, is the mirror old enough to qualify for Tier 1? + print(tier_2_mirror[0].created) + + return tier_2_mirror + + +def submit_mirror(request): + if request.method == 'POST' or len(request.GET) > 0: + data = request.POST if request.method == 'POST' else request.GET + + captcha_form = CaptchaForm(data=data) + + # data is immutable, need to be copied and modified to enforce + # active and public is False. + tmp = data.copy() + tmp['active'] = False + tmp['public'] = False + data = tmp + + mirror_form = MirrorRequestForm(data=data) + + mirror_url1_form = MirrorUrlForm(data=data, prefix="url1") + if data.get('url2-url') != '': + mirror_url2_form = MirrorUrlForm(data=data, prefix="url2") + else: + mirror_url2_form = MirrorUrlForm(prefix="url2") + if data.get('url3-url') != '': + mirror_url3_form = MirrorUrlForm(data=data, prefix="url3") + else: + mirror_url3_form = MirrorUrlForm(prefix="url3") + + rsync_form = MirrorRsyncForm(data=data) + + mirror_url2_form.fields['url'].required = False + mirror_url3_form.fields['url'].required = False + rsync_form.fields['ip'].required = False + + if data.get('tier') == '1': + if existing_mirror := validate_tier_1_request(data): + existing_mirror.update(tier=1) + else: + return render( + request, + 'mirrors/mirror_submit_error_upgrading.html', + { + 'TIER_1_ERROR_TIME_RANGE': TIER_1_ERROR_TIME_RANGE, + 'TIER_1_MAX_ERROR_RATE': TIER_1_MAX_ERROR_RATE, + } + ) + else: + if captcha_form.is_valid() and mirror_form.is_valid() and mirror_url1_form.is_valid(): + try: + with transaction.atomic(): + transaction.on_commit(partial(mail_mirror_admins, data)) + + mirror = mirror_form.save() + mirror_url1 = mirror_url1_form.save(commit=False) + mirror_url1.mirror = mirror + mirror_url1.save() + + if data.get('url2-url') != '' and mirror_url2_form.is_valid(): + mirror_url2 = mirror_url2_form.save(commit=False) + mirror_url2.mirror = mirror + mirror_url2.save() + if data.get('url3-url') != '' and mirror_url3_form.is_valid(): + mirror_url3 = mirror_url3_form.save(commit=False) + mirror_url3.mirror = mirror + mirror_url3.save() + + if data.get('ip') != '' and rsync_form.is_valid(): + rsync = rsync_form.save(commit=False) + rsync.mirror = mirror + rsync.save() + + except DatabaseError as error: + print(error) + + else: + captcha_form = CaptchaForm() + mirror_form = MirrorRequestForm() + mirror_url1_form = MirrorUrlForm(prefix="url1") + mirror_url2_form = MirrorUrlForm(prefix="url2") + mirror_url3_form = MirrorUrlForm(prefix="url3") + rsync_form = MirrorRsyncForm() + + mirror_form.fields['active'].widget = forms.HiddenInput() + mirror_form.fields['public'].widget = forms.HiddenInput() + mirror_url2_form.fields['url'].required = False + mirror_url3_form.fields['url'].required = False + rsync_form.fields['ip'].required = False + + return render( + request, + 'mirrors/mirror_submit.html', + { + 'captcha': captcha_form, + 'mirror_form': mirror_form, + 'mirror_url1_form': mirror_url1_form, + 'mirror_url2_form': mirror_url2_form, + 'mirror_url3_form': mirror_url3_form, + 'rsync_form': rsync_form + } + ) + # vim: set ts=4 sw=4 et: diff --git a/requirements.txt b/requirements.txt index 663c5ca2f..d1cf1d2b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ Markdown==3.3.7 bencode.py==4.0.0 django-countries==7.3.2 django-extensions==3.1.3 +django-simple-captcha==0.5.17 jsmin==3.0.1 pgpdump==1.5 parse==1.19.0 @@ -15,4 +16,4 @@ feedparser==6.0.10 bleach==5.0.0 requests==2.25.1 xtarfile==0.1.0 -zstandard==0.17.0 +zstandard==0.17.0 \ No newline at end of file diff --git a/settings.py b/settings.py index 3488a22e7..16dc01958 100644 --- a/settings.py +++ b/settings.py @@ -135,6 +135,7 @@ 'public', 'releng', 'visualize', + 'captcha' ) # Logging configuration for not getting overspammed diff --git a/sitestatic/archweb.css b/sitestatic/archweb.css index aa2c62089..f07a1cdb8 100644 --- a/sitestatic/archweb.css +++ b/sitestatic/archweb.css @@ -72,6 +72,15 @@ code { padding: 0.15em 0.25em; } +.error { + font-family: monospace, monospace; + padding: 0.15em 0.25em; + overflow: auto; + color: #a94442; + background: #f2dede; + border: 1px solid #ebccd1; +} + pre { font-family: monospace, monospace; border: 1px solid #bdb; @@ -1197,3 +1206,12 @@ ul.signoff-list { .pgp-key-ids { display: inline-block; } + +.captcha { + display: flex; +} + +.captcha-input { + display: flex; + flex-direction: column; +} \ No newline at end of file diff --git a/templates/mirrors/mirror_submit.html b/templates/mirrors/mirror_submit.html new file mode 100644 index 000000000..13a6eb0a8 --- /dev/null +++ b/templates/mirrors/mirror_submit.html @@ -0,0 +1,43 @@ +{% extends "base.html" %} +{% load package_extras %} +{% block title %}Arch Linux - Submit New Mirror{% endblock %} + +{% block content %} +
+ +

Mirror Request

+ +

This page is meant as a replacement of the old way of registring as a Tier 1 or Tier 2 mirror. Previously this was done through https://bugs.archlinux.org and would require manual intervention from the mirror maintainer(s). This process is now semi-automated and various checks against your mirror will be performed when submitting via the below form.

+ +

Available mirrors

+ +

Below are direct links to the two different tiers you will need to be acquainted with.

+ +

+ +

Mirror information

+ + {% if mirror_form.is_valid or mirror_url1_form.is_valid %} + + Your request have successfully been submitted and we will mark the mirror as public and active once we have manually verified the mirror. + + {% else %} +

Before you can submit a Tier 1 request the mirror in question must first be a registered Tier 2 for a certain amount of time with proven reliablity. Once the submitted information is verified by the Arch Linux mirror maintainers the mirror will be visible under the appropriate tier list above.

+ +
+ {{ mirror_form.as_div }} + {{ mirror_url1_form.as_div }} + {{ mirror_url2_form.as_div }} + {{ mirror_url3_form.as_div }} +

If you are registring a Tier 1 mirror, you need to supply the static IP to the machine that will be using rsync towards the Tier 0 mirror. This is so we can allow it to sync. + {{ rsync_form.as_div }} + {{ captcha.as_div }} +

+
+ {% endif %} +
+{% endblock %} + diff --git a/templates/mirrors/mirror_submit_error_upgrading.html b/templates/mirrors/mirror_submit_error_upgrading.html new file mode 100644 index 000000000..0876969aa --- /dev/null +++ b/templates/mirrors/mirror_submit_error_upgrading.html @@ -0,0 +1,32 @@ +{% extends "base.html" %} +{% load package_extras %} +{% block title %}Arch Linux - Submit New Mirror{% endblock %} + +{% block content %} +
+ +

Mirror Request

+ +

This page is meant as a replacement of the old way of registring as a Tier 1 or Tier 2 mirror. Previously this was done through https://bugs.archlinux.org and would require manual intervention from the mirror maintainer(s). This process is now semi-automated and various checks against your mirror will be performed when submitting via the below form.

+ +

Available mirrors

+ +

Below are direct links to the two different tiers you will need to be acquainted with.

+ +

+ +

Mirror information

+ + 
+    
+    The mirror does not meet the minimum requirements to be upgraded to a Tier 1 mirror.
+    Please make sure it has been a reliable Tier 2 mirror with at most {{ TIER_1_MAX_ERROR_RATE }} errors over the last {{ TIER_1_ERROR_TIME_RANGE }} days.
+
+ + Please read the requirements on https://wiki.archlinux.org/title/DeveloperWiki:NewMirrors#Tier_1_requirements +
+{% endblock %} + diff --git a/templates/mirrors/new_mirror_mail_template.txt b/templates/mirrors/new_mirror_mail_template.txt new file mode 100644 index 000000000..9b474ca42 --- /dev/null +++ b/templates/mirrors/new_mirror_mail_template.txt @@ -0,0 +1,9 @@ +{% autoescape off %}A new mirror has been requested called "{{ name }}". As the mirror administrator, check in on https://archlinux.org/mirrors/{{ name }}/ after a few minutes and check: + + * Is the Completion % more than 98% for Tier 2 and 100% for Tier 1? + * Does it have HTTP or HTTPS protocols? + +If so, go to https://archlinux.org/admin/mirrors/, find the new mirror and mark it as Active and Public! +If the mirror hasn't synced in a couple of days, email the mirror admin asking if we can assist in any way. + +{% endautoescape %} diff --git a/urls.py b/urls.py index f5edec5cb..e8f5d6b4c 100644 --- a/urls.py +++ b/urls.py @@ -110,6 +110,11 @@ path('logout/', auth_views.LogoutView.as_view(template_name='registration/logout.html'), name='logout'), ]) +# Captcha +urlpatterns.extend([ + path('captcha/', include('captcha.urls')), +]) + # django-toolbar if settings.DEBUG_TOOLBAR: # pragma: no cover import debug_toolbar