ci: merge main to release (#7777)

ci: merge main to release

.github/workflows/build.yml

@@ -47,6 +47,10 @@ on:
         required: true
         type: boolean
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   # -----------------------------------------------------------------
   # PREPARE

README.md

@@ -8,7 +8,7 @@
 [![Python Version](https://img.shields.io/badge/python-3.9-blue?logo=python&logoColor=white)](#prerequisites)
 [![Django Version](https://img.shields.io/badge/django-4.x-51be95?logo=django&logoColor=white)](#prerequisites)
 [![Node Version](https://img.shields.io/badge/node.js-16.x-green?logo=node.js&logoColor=white)](#prerequisites)
-[![MariaDB Version](https://img.shields.io/badge/postgres-14-blue?logo=postgresql&logoColor=white)](#prerequisites)
+[![MariaDB Version](https://img.shields.io/badge/postgres-16-blue?logo=postgresql&logoColor=white)](#prerequisites)
 
 ##### The day-to-day front-end to the IETF database for people who work on IETF standards.
 

ietf/api/views.py

@@ -429,16 +429,19 @@ def directauth(request):
                 data = None
 
         if raw_data is None or data is None:
+            log.log("Request body is either missing or invalid")
             return HttpResponse(json.dumps(dict(result="failure",reason="invalid post")), content_type='application/json')
 
         authtoken = data.get('authtoken', None)
         username = data.get('username', None)
         password = data.get('password', None)
 
         if any([item is None for item in (authtoken, username, password)]):
+            log.log("One or more mandatory fields are missing: authtoken, username, password")
             return HttpResponse(json.dumps(dict(result="failure",reason="invalid post")), content_type='application/json')
 
         if not is_valid_token("ietf.api.views.directauth", authtoken):
+            log.log("Auth token provided is invalid")
             return HttpResponse(json.dumps(dict(result="failure",reason="invalid authtoken")), content_type='application/json')
 
         user_query = User.objects.filter(username__iexact=username)
@@ -449,18 +452,20 @@ def directauth(request):
 
 
         # Note well that we are using user.username, not what was passed to the API.
-        if user_query.count() == 1 and authenticate(username = user_query.first().username, password = password):
+        user_count = user_query.count()
+        if user_count == 1 and authenticate(username = user_query.first().username, password = password):
             user = user_query.get()
             if user_query.filter(person__isnull=True).count() == 1: # Can't inspect user.person direclty here
-                log.log(f"Direct auth of personless user {user.pk}:{user.username}")
+                log.log(f"Direct auth success (personless user): {user.pk}:{user.username}")
             else:
-                log.log(f"Direct auth: {user.pk}:{user.person.plain_name()}")
+                log.log(f"Direct auth success: {user.pk}:{user.person.plain_name()}")
             return HttpResponse(json.dumps(dict(result="success")), content_type='application/json')
 
-        log.log(f"Direct auth failure: {username}")
+        log.log(f"Direct auth failure: {username} ({user_count} user(s) found)")
         return HttpResponse(json.dumps(dict(result="failure", reason="authentication failed")), content_type='application/json') 
 
     else:
+        log.log(f"Request must be POST: {request.method} received")
         return HttpResponse(status=405)
 
 

ietf/doc/admin.py

@@ -142,6 +142,13 @@ class DocumentActionHolderAdmin(admin.ModelAdmin):
 
 # events
 
+class DeletedEventAdmin(admin.ModelAdmin):
+    list_display = ['id', 'content_type', 'json', 'by', 'time']
+    list_filter = ['time']
+    raw_id_fields = ['content_type', 'by']
+admin.site.register(DeletedEvent, DeletedEventAdmin)
+
+
 class DocEventAdmin(admin.ModelAdmin):
     def event_type(self, obj):
         return str(obj.type)
@@ -159,39 +166,42 @@ def short_desc(self, obj):
 admin.site.register(StateDocEvent, DocEventAdmin)
 admin.site.register(ConsensusDocEvent, DocEventAdmin)
 admin.site.register(BallotDocEvent, DocEventAdmin)
+admin.site.register(IRSGBallotDocEvent, DocEventAdmin)
 admin.site.register(WriteupDocEvent, DocEventAdmin)
 admin.site.register(LastCallDocEvent, DocEventAdmin)
 admin.site.register(TelechatDocEvent, DocEventAdmin)
-admin.site.register(ReviewRequestDocEvent, DocEventAdmin)
-admin.site.register(ReviewAssignmentDocEvent, DocEventAdmin)
 admin.site.register(InitialReviewDocEvent, DocEventAdmin)
-admin.site.register(AddedMessageEvent, DocEventAdmin)
-admin.site.register(SubmissionDocEvent, DocEventAdmin)
 admin.site.register(EditedAuthorsDocEvent, DocEventAdmin)
 admin.site.register(IanaExpertDocEvent, DocEventAdmin)
 
-class DeletedEventAdmin(admin.ModelAdmin):
-    list_display = ['id', 'content_type', 'json', 'by', 'time']
-    list_filter = ['time']
-    raw_id_fields = ['content_type', 'by']
-admin.site.register(DeletedEvent, DeletedEventAdmin)
-
 class BallotPositionDocEventAdmin(DocEventAdmin):
-    raw_id_fields = ["doc", "by", "balloter", "ballot"]
+    raw_id_fields = DocEventAdmin.raw_id_fields + ["balloter", "ballot"]
 admin.site.register(BallotPositionDocEvent, BallotPositionDocEventAdmin)
-
-class IRSGBallotDocEventAdmin(DocEventAdmin):
-    raw_id_fields = ["doc", "by"]
-admin.site.register(IRSGBallotDocEvent, IRSGBallotDocEventAdmin)
 
 class BofreqEditorDocEventAdmin(DocEventAdmin):
-    raw_id_fields = ["doc", "by", "editors" ]
+    raw_id_fields = DocEventAdmin.raw_id_fields + ["editors"]
 admin.site.register(BofreqEditorDocEvent, BofreqEditorDocEventAdmin)
 
 class BofreqResponsibleDocEventAdmin(DocEventAdmin):
-    raw_id_fields = ["doc", "by", "responsible" ]
+    raw_id_fields = DocEventAdmin.raw_id_fields + ["responsible"]
 admin.site.register(BofreqResponsibleDocEvent, BofreqResponsibleDocEventAdmin)
 
+class ReviewRequestDocEventAdmin(DocEventAdmin):
+    raw_id_fields = DocEventAdmin.raw_id_fields + ["review_request"]
+admin.site.register(ReviewRequestDocEvent, ReviewRequestDocEventAdmin)
+
+class ReviewAssignmentDocEventAdmin(DocEventAdmin):
+    raw_id_fields = DocEventAdmin.raw_id_fields + ["review_assignment"]
+admin.site.register(ReviewAssignmentDocEvent, ReviewAssignmentDocEventAdmin)
+
+class AddedMessageEventAdmin(DocEventAdmin):
+    raw_id_fields = DocEventAdmin.raw_id_fields + ["message"]
+admin.site.register(AddedMessageEvent, AddedMessageEventAdmin)
+
+class SubmissionDocEventAdmin(DocEventAdmin):
+    raw_id_fields = DocEventAdmin.raw_id_fields + ["submission"]
+admin.site.register(SubmissionDocEvent, SubmissionDocEventAdmin)
+
 class DocumentUrlAdmin(admin.ModelAdmin):
     list_display = ['id', 'doc', 'tag', 'url', 'desc', ]
     search_fields = ['doc__name', 'url', ]

ietf/doc/views_doc.py

@@ -265,6 +265,8 @@ def document_main(request, name, rev=None, document_html=False):
         can_change_stream = bool(can_edit or roles)
 
         file_urls, found_types = build_file_urls(doc)
+        if not request.user.is_authenticated:
+            file_urls = [fu for fu in file_urls if fu[0] != "pdfized"]
         content = doc.text_or_error() # pyflakes:ignore
         content = markup_txt.markup(maybe_split(content, split=split_content))
 
@@ -406,6 +408,8 @@ def document_main(request, name, rev=None, document_html=False):
         latest_revision = None
 
         file_urls, found_types = build_file_urls(doc)
+        if not request.user.is_authenticated:
+            file_urls = [fu for fu in file_urls if fu[0] != "pdfized"]
         content = doc.text_or_error() # pyflakes:ignore
         content = markup_txt.markup(maybe_split(content, split=split_content))
 
@@ -1039,6 +1043,8 @@ def document_html(request, name, rev=None):
         document_html=True,
     )
 
+
+@login_required
 def document_pdfized(request, name, rev=None, ext=None):
 
     found = fuzzy_find_documents(name, rev)

ietf/iesg/tests.py

@@ -52,6 +52,15 @@ def test_feed(self):
         self.assertContains(r, draft.name)
         self.assertContains(r, escape(pos.balloter.plain_name()))
 
+        # Mark draft as replaced
+        draft.set_state(State.objects.get(type="draft", slug="repl"))
+
+        r = self.client.get(urlreverse("ietf.iesg.views.discusses"))
+        self.assertEqual(r.status_code, 200)
+
+        self.assertNotContains(r, draft.name)
+        self.assertNotContains(r, escape(pos.balloter.plain_name()))
+
     def test_milestones_needing_review(self):
         draft = WgDraftFactory()
         RoleFactory(name_id='ad',group=draft.group,person=Person.objects.get(user__username='ad'))

ietf/iesg/views.py

@@ -483,6 +483,7 @@ def discusses(request):
                                             models.Q(states__type__in=("statchg", "conflrev"),
                                                      states__slug__in=("iesgeval", "defer")),
                                             docevent__ballotpositiondocevent__pos__blocking=True)
+    possible_docs = possible_docs.exclude(states__in=State.objects.filter(type="draft", slug="repl"))
     possible_docs = possible_docs.select_related("stream", "group", "ad").distinct()
 
     docs = []

ietf/meeting/tests_views.py

@@ -6246,6 +6246,12 @@ def test_upload_minutes_agenda(self):
             q = PyQuery(r.content)
             self.assertTrue(q('form input[type="checkbox"]'))
 
+            # test not submitting a file
+            r = self.client.post(url, dict(submission_method="upload"))
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            self.assertTrue(q("form .is-invalid"))
+
             test_file = BytesIO(b'this is some text for a test')
             test_file.name = "not_really.json"
             r = self.client.post(url,dict(submission_method="upload",file=test_file))

ietf/meeting/views.py

@@ -2790,7 +2790,8 @@ def clean_content(self):
     def clean_file(self):
         submission_method = self.cleaned_data.get("submission_method")
         if submission_method == "upload":
-            return super().clean_file()
+            if self.cleaned_data.get("file", None) is not None:
+                return super().clean_file()
         return None
 
     def clean(self):

ietf/name/fixtures/names.json

@@ -3464,7 +3464,7 @@
       "parent_types": [],
       "req_subm_approval": true,
       "role_order": "[\n    \"chair\",\n    \"delegate\"\n]",
-      "session_purposes": "[\n    \"officehours\"\n]",
+      "session_purposes": "[\n    \"officehours\",\n    \"regular\"\n]",
       "show_on_agenda": true
     },
     "model": "group.groupfeatures",

ietf/person/urls.py

@@ -2,7 +2,7 @@
 from ietf.utils.urls import url
 
 urlpatterns = [
-    url(r'^merge/$', views.merge),
+    url(r'^merge/?$', views.merge),
     url(r'^search/(?P<model_name>(person|email))/$', views.ajax_select2_search),
     url(r'^(?P<personid>[0-9]+)/email.json$', ajax.person_email_json),
     url(r'^(?P<email_or_name>[^/]+)$', views.profile),

ietf/secr/announcement/forms.py

@@ -42,8 +42,11 @@ def get_from_choices(user):
     nomcom_choices = get_nomcom_choices(user)
     if nomcom_choices:
         addresses = list(addresses) + nomcom_choices
-
-    return list(zip(addresses, addresses))
+
+    choices = list(zip(addresses, addresses))
+    if len(choices) > 1:
+        choices.insert(0, ('', '(Choose an option)'))
+    return choices
 
 
 def get_nomcom_choices(user):

ietf/secr/announcement/tests.py

@@ -48,7 +48,7 @@ def test_main_announce_from(self):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
-        self.assertEqual(len(q('#id_frm option')),3)
+        self.assertEqual(len(q('#id_frm option')),4)
 
         # IAB Chair
         self.client.login(username="iab-chair", password="iab-chair+password")

ietf/static/css/ietf.scss

@@ -1188,3 +1188,20 @@ blockquote {
     padding-left: 1rem;
     border-left: solid 1px var(--bs-body-color);
 }
+
+.overflow-shadows {
+    transition: box-shadow 0.5s;
+}
+
+.overflow-shadows--both {
+    box-shadow: inset 0px 21px 18px -20px var(--bs-body-color),
+                inset 0px -21px 18px -20px var(--bs-body-color);
+}
+
+.overflow-shadows--top-only {
+    box-shadow: inset 0px 21px 18px -20px var(--bs-body-color);
+}
+
+.overflow-shadows--bottom-only {
+    box-shadow: inset 0px -21px 18px -20px var(--bs-body-color);
+}

ietf/static/js/ietf.js

@@ -91,6 +91,27 @@ $(document)
         //     });
     });
 
+function overflowShadows(el) {
+    function handleScroll(){
+        const canScrollUp = el.scrollTop > 0
+        const canScrollDown = el.offsetHeight + el.scrollTop < el.scrollHeight
+        el.classList.toggle("overflow-shadows--both", canScrollUp && canScrollDown)
+        el.classList.toggle("overflow-shadows--top-only", canScrollUp && !canScrollDown)
+        el.classList.toggle("overflow-shadows--bottom-only", !canScrollUp && canScrollDown)
+    }
+
+    el.addEventListener("scroll", handleScroll, {passive: true})
+    handleScroll()
+
+    const observer = new IntersectionObserver(handleScroll)
+    observer.observe(el) // el won't have scrollTop etc when hidden, so we need to recalculate when it's revealed
+
+    return () => {
+        el.removeEventListener("scroll", handleScroll)
+        observer.unobserve(el)
+    }
+}
+
 $(document)
     .ready(function () {
         // load data for the menu
@@ -108,7 +129,7 @@ $(document)
                     }
                     attachTo.find(".dropdown-menu")
                         .remove();
-                    var menu = ['<ul class="dropdown-menu ms-n1 mt-n1">'];
+                    var menu = ['<ul class="dropdown-menu ms-n1 mt-n1 overflow-shadows">'];
                     var groups = data[parentId];
                     var gtype = "";
                     for (var i = 0; i < groups.length; ++i) {
@@ -127,6 +148,8 @@ $(document)
                         attachTo.closest(".dropdown-menu");
                     }
                     attachTo.append(menu.join(""));
+
+                    attachTo.find(".overflow-shadows").each(function(){ overflowShadows(this)})
                 }
             }
         });

ietf/static/js/list.js

@@ -5,8 +5,10 @@ import {
 function text_sort(a, b, options) {
 
     function prep(e, options) {
-        return $($.parseHTML(e.values()[options.valueName]))
-            .text()
+        const el = $($.parseHTML(e.values()[options.valueName]));
+        const cell_el = e.elm.querySelector(`.${options.valueName}`)
+        const sort_by_number = cell_el?.getAttribute('data-sort-number')
+        return sort_by_number ?? el.text()
             .trim()
             .replaceAll(/\s+/g, ' ');
     }

ietf/templates/doc/review/complete_review.html

@@ -66,7 +66,7 @@ <h1>
         </p>
         <p>
             If you enter the review below, the review will be sent
-            to {% for addr in to %}{{ addr|linkify }}{% if not forloop.last %}, {% endif %}{% endfor %}{% if review_cc %}, with a CC to {% for addr in cc %}{{ addr|linkify }}{% if not forloop.last %}, {% endif %}{% endfor %}{% endif %}.
+            to {% for addr in review_to %}{{ addr|linkify }}{% if not forloop.last %}, {% endif %}{% endfor %}{% if review_cc %}, with a CC to {% for addr in review_cc %}{{ addr|linkify }}{% if not forloop.last %}, {% endif %}{% endfor %}{% endif %}.
         </p>
     {% elif assignment %}
         <p>

ietf/templates/group/concluded_groups.html

@@ -40,8 +40,8 @@ <h2 class="mt-5" id="{{ label|slugify }}">{{ label }}</h2>
                         <tr>
                             <th scope="col" data-sort="group">Group</th>
                             <th scope="col" data-sort="name">Name</th>
-                            <th scope="col" data-sort="date">Start</th>
-                            <th scope="col" data-sort="date">Concluded</th>
+                            <th scope="col" data-sort="date-start">Start</th>
+                            <th scope="col" data-sort="date-concluded">Concluded</th>
                         </tr>
                     </thead>
                     <tbody>
@@ -51,8 +51,8 @@ <h2 class="mt-5" id="{{ label|slugify }}">{{ label }}</h2>
                                     <a href="{{ g.about_url }}">{{ g.acronym }}</a>
                                 </td>
                                 <td>{{ g.name }}</td>
-                                <td>{{ g.start_date|date:"Y-m" }}</td>
-                                <td>{{ g.conclude_date|date:"Y-m" }}</td>
+                                <td data-sort-number="{{ g.start_date|date:"U" }}">{{ g.start_date|date:"Y-m" }}</td>
+                                <td data-sort-number="{{ g.conclude_date|date:"U" }}">{{ g.conclude_date|date:"Y-m" }}</td>
                             </tr>
                         {% endfor %}
                     </tbody>

ietf/templates/group/milestones.html

@@ -49,7 +49,7 @@ <h2 class="my-3">
                     <td>{{ milestone.desc|urlize_ietf_docs }}</td>
                     <td>
                         {% for d in milestone.docs.all %}
-                            <a href="{% url "ietf.doc.views_doc.document_main" name=d.name %}">{{ d.name }}</a>
+                            <a href="{% url "ietf.doc.views_doc.document_main" name=d.name %}">{% if d.became_rfc %}{{ d.became_rfc }} (was {% endif %}{{ d.name }}{% if d.became_rfc %}){% endif %}</a>
                             <br>
                         {% endfor %}
                     </td>

ietf/templates/person/profile.html

@@ -106,7 +106,7 @@ <h2 class="mt-5" id="rfcs-{{ forloop.counter }}">
                             <td>
                                 <a class="text-nowrap" href="{{ doc.get_absolute_url }}">RFC {{ doc.rfc_number }}</a>
                             </td>
-                            <td>{{ doc.pub_date|date:"b Y"|title }}</td>
+                            <td data-sort-number="{{ doc.pub_date|date:"U" }}">{{ doc.pub_date|date:"b Y"|title }}</td>
                             <td>{{ doc.title|urlize_ietf_docs }}</td>
                             <td class="text-end">
                                 {% with doc.referenced_by_rfcs_as_rfc_or_draft.count as refbycount %}

ietf/utils/jsonlogger.py

@@ -24,3 +24,11 @@ def add_fields(self, log_record, record, message_dict):
         log_record.setdefault("user_agent", record.args["a"])
         log_record.setdefault("len_bytes", record.args["B"])
         log_record.setdefault("duration_ms", record.args["M"])
+        log_record.setdefault("host", record.args["{host}i"])
+        log_record.setdefault("x_request_start", record.args["{x-request-start}i"])
+        log_record.setdefault("x_real_ip", record.args["{x-real-ip}i"])
+        log_record.setdefault("x_forwarded_for", record.args["{x-forwarded-for}i"])
+        log_record.setdefault("x_forwarded_proto", record.args["{x-forwarded-proto}i"])
+        log_record.setdefault("cf_connecting_ip", record.args["{cf-connecting-ip}i"])
+        log_record.setdefault("cf_connecting_ipv6", record.args["{cf-connecting-ipv6}i"])
+        log_record.setdefault("cf_ray", record.args["{cf-ray}i"])