diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 7af27e7d13..a923bf693f 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20241127T2054 +FROM ghcr.io/ietf-tools/datatracker-app-base:20241212T1741 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index e4b05ed700..b5d33714f2 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20241127T2054 +20241212T1741 diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py index cabbee0b1e..6666a0d37d 100644 --- a/dev/build/gunicorn.conf.py +++ b/dev/build/gunicorn.conf.py @@ -64,18 +64,21 @@ def _describe_request(req): start and end of handling a request. E.g., do not include a timestamp. """ client_ip = "-" + asn = "-" cf_ray = "-" for header, value in req.headers: header = header.lower() if header == "cf-connecting-ip": client_ip = value + elif header == "x-ip-src-asnum": + asn = value elif header == "cf-ray": cf_ray = value if req.query: path = f"{req.path}?{req.query}" else: path = req.path - return f"{req.method} {path} (client_ip={client_ip}, cf_ray={cf_ray})" + return f"{req.method} {path} (client_ip={client_ip}, asn={asn}, cf_ray={cf_ray})" def pre_request(worker, req): diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index ae698e20b6..07bf0a7511 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -40,7 +40,6 @@ SUBMIT_YANG_CATALOG_MODEL_DIR = '/assets/ietf-ftp/yang/catalogmod/' SUBMIT_YANG_DRAFT_MODEL_DIR = '/assets/ietf-ftp/yang/draftmod/' -SUBMIT_YANG_INVAL_MODEL_DIR = '/assets/ietf-ftp/yang/invalmod/' SUBMIT_YANG_IANA_MODEL_DIR = '/assets/ietf-ftp/yang/ianamod/' SUBMIT_YANG_RFC_MODEL_DIR = '/assets/ietf-ftp/yang/rfcmod/' diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py index 774c7797cf..6bcee46b61 100644 --- a/dev/diff/settings_local.py +++ b/dev/diff/settings_local.py @@ -37,7 +37,6 @@ SUBMIT_YANG_CATALOG_MODEL_DIR = '/assets/ietf-ftp/yang/catalogmod/' SUBMIT_YANG_DRAFT_MODEL_DIR = '/assets/ietf-ftp/yang/draftmod/' -SUBMIT_YANG_INVAL_MODEL_DIR = '/assets/ietf-ftp/yang/invalmod/' SUBMIT_YANG_IANA_MODEL_DIR = '/assets/ietf-ftp/yang/ianamod/' SUBMIT_YANG_RFC_MODEL_DIR = '/assets/ietf-ftp/yang/rfcmod/' diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py index 20941359d4..afadb3760b 100644 --- a/dev/tests/settings_local.py +++ b/dev/tests/settings_local.py @@ -36,7 +36,6 @@ SUBMIT_YANG_CATALOG_MODEL_DIR = '/assets/ietf-ftp/yang/catalogmod/' SUBMIT_YANG_DRAFT_MODEL_DIR = '/assets/ietf-ftp/yang/draftmod/' -SUBMIT_YANG_INVAL_MODEL_DIR = '/assets/ietf-ftp/yang/invalmod/' SUBMIT_YANG_IANA_MODEL_DIR = '/assets/ietf-ftp/yang/ianamod/' SUBMIT_YANG_RFC_MODEL_DIR = '/assets/ietf-ftp/yang/rfcmod/' diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 5d9859c19b..a1c19c80cf 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -26,7 +26,6 @@ SUBMIT_YANG_CATALOG_MODEL_DIR = '/assets/ietf-ftp/yang/catalogmod/' SUBMIT_YANG_DRAFT_MODEL_DIR = '/assets/ietf-ftp/yang/draftmod/' -SUBMIT_YANG_INVAL_MODEL_DIR = '/assets/ietf-ftp/yang/invalmod/' SUBMIT_YANG_IANA_MODEL_DIR = '/assets/ietf-ftp/yang/ianamod/' SUBMIT_YANG_RFC_MODEL_DIR = '/assets/ietf-ftp/yang/rfcmod/' diff --git a/ietf/settings.py b/ietf/settings.py index cf8abe9f4d..7c3dc7fa16 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -809,8 +809,8 @@ def skip_unreadable_post(record): SESSION_REQUEST_FROM_EMAIL = 'IETF Meeting Session Request Tool ' SECRETARIAT_SUPPORT_EMAIL = "support@ietf.org" -SECRETARIAT_ACTION_EMAIL = "ietf-action@ietf.org" -SECRETARIAT_INFO_EMAIL = "ietf-info@ietf.org" +SECRETARIAT_ACTION_EMAIL = SECRETARIAT_SUPPORT_EMAIL +SECRETARIAT_INFO_EMAIL = SECRETARIAT_SUPPORT_EMAIL # Put real password in settings_local.py IANA_SYNC_PASSWORD = "secret" diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index f0e8a19c4a..47027277be 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -13,22 +13,16 @@ import debug # pyflakes:ignore from django.urls import reverse as urlreverse -from django.utils import timezone from ietf.utils.test_utils import login_testing_unauthorized, TestCase import ietf.stats.views -from ietf.submit.models import Submission -from ietf.doc.factories import WgDraftFactory, WgRfcFactory -from ietf.doc.models import Document, State, RelatedDocument, NewRevisionDocEvent, DocumentAuthor + from ietf.group.factories import RoleFactory -from ietf.meeting.factories import MeetingFactory, AttendedFactory +from ietf.meeting.factories import MeetingFactory from ietf.person.factories import PersonFactory -from ietf.person.models import Person, Email -from ietf.name.models import FormalLanguageName, DocRelationshipName, CountryName from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory -from ietf.stats.models import MeetingRegistration, CountryAlias -from ietf.stats.factories import MeetingRegistrationFactory +from ietf.stats.models import MeetingRegistration from ietf.stats.tasks import fetch_meeting_attendance_task from ietf.stats.utils import get_meeting_registration_data, FetchStats, fetch_attendance_from_meetings from ietf.utils.timezone import date_today @@ -41,121 +35,14 @@ def test_stats_index(self): self.assertEqual(r.status_code, 200) def test_document_stats(self): - WgRfcFactory() - draft = WgDraftFactory() - DocumentAuthor.objects.create( - document=draft, - person=Person.objects.get(email__address="aread@example.org"), - email=Email.objects.get(address="aread@example.org"), - country="Germany", - affiliation="IETF", - order=1 - ) - - # create some data for the statistics - Submission.objects.create( - authors=[ { "name": "Some Body", "email": "somebody@example.com", "affiliation": "Some Inc.", "country": "US" }], - pages=30, - rev=draft.rev, - words=4000, - draft=draft, - file_types=".txt", - state_id="posted", - ) - - draft.formal_languages.add(FormalLanguageName.objects.get(slug="xml")) - Document.objects.filter(pk=draft.pk).update(words=4000) - # move it back so it shows up in the yearly summaries - NewRevisionDocEvent.objects.filter(doc=draft, rev=draft.rev).update( - time=timezone.now() - datetime.timedelta(days=500)) - - referencing_draft = Document.objects.create( - name="draft-ietf-mars-referencing", - type_id="draft", - title="Referencing", - stream_id="ietf", - abstract="Test", - rev="00", - pages=2, - words=100 - ) - referencing_draft.set_state(State.objects.get(used=True, type="draft", slug="active")) - RelatedDocument.objects.create( - source=referencing_draft, - target=draft, - relationship=DocRelationshipName.objects.get(slug="refinfo") - ) - NewRevisionDocEvent.objects.create( - type="new_revision", - by=Person.objects.get(name="(System)"), - doc=referencing_draft, - desc="New revision available", - rev=referencing_draft.rev, - time=timezone.now() - datetime.timedelta(days=1000) - ) + r = self.client.get(urlreverse("ietf.stats.views.document_stats")) + self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index")) - # check redirect - url = urlreverse(ietf.stats.views.document_stats) - - authors_url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": "authors" }) - - r = self.client.get(url) - self.assertEqual(r.status_code, 302) - self.assertTrue(authors_url in r["Location"]) - - # check various stats types - for stats_type in ["authors", "pages", "words", "format", "formlang", - "author/documents", "author/affiliation", "author/country", - "author/continent", "author/citations", "author/hindex", - "yearly/affiliation", "yearly/country", "yearly/continent"]: - for document_type in ["", "rfc", "draft"]: - for time_choice in ["", "5y"]: - url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": stats_type }) - r = self.client.get(url, { - "type": document_type, - "time": time_choice, - }) - self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) - self.assertTrue(q('#chart')) - if not stats_type.startswith("yearly"): - self.assertTrue(q('table.stats-data')) - def test_meeting_stats(self): - # create some data for the statistics - meeting = MeetingFactory(type_id='ietf', date=date_today(), number="96") - MeetingRegistrationFactory(first_name='John', last_name='Smith', country_code='US', email="john.smith@example.us", meeting=meeting, attended=True) - CountryAlias.objects.get_or_create(alias="US", country=CountryName.objects.get(slug="US")) - p = MeetingRegistrationFactory(first_name='Jaume', last_name='Guillaume', country_code='FR', email="jaume.guillaume@example.fr", meeting=meeting, attended=False).person - CountryAlias.objects.get_or_create(alias="FR", country=CountryName.objects.get(slug="FR")) - AttendedFactory(session__meeting=meeting,person=p) - # check redirect - url = urlreverse(ietf.stats.views.meeting_stats) - - authors_url = urlreverse(ietf.stats.views.meeting_stats, kwargs={ "stats_type": "overview" }) - - r = self.client.get(url) - self.assertEqual(r.status_code, 302) - self.assertTrue(authors_url in r["Location"]) - - # check various stats types - for stats_type in ["overview", "country", "continent"]: - url = urlreverse(ietf.stats.views.meeting_stats, kwargs={ "stats_type": stats_type }) - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) - self.assertTrue(q('#chart')) - if stats_type == "overview": - self.assertTrue(q('table.stats-data')) + r = self.client.get(urlreverse("ietf.stats.views.meeting_stats")) + self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index")) - for stats_type in ["country", "continent"]: - url = urlreverse(ietf.stats.views.meeting_stats, kwargs={ "stats_type": stats_type, "num": meeting.number }) - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) - self.assertTrue(q('#chart')) - self.assertTrue(q('table.stats-data')) def test_known_country_list(self): # check redirect diff --git a/ietf/stats/views.py b/ietf/stats/views.py index ea73d9f4fc..504d84e86d 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -2,25 +2,18 @@ # -*- coding: utf-8 -*- -import os import calendar import datetime -import email.utils import itertools import json import dateutil.relativedelta from collections import defaultdict -from django.conf import settings from django.contrib.auth.decorators import login_required -from django.core.cache import cache -from django.db.models import Count, Q, Subquery, OuterRef from django.http import HttpResponseRedirect -from django.shortcuts import get_object_or_404, render +from django.shortcuts import render from django.urls import reverse as urlreverse -from django.utils import timezone -from django.utils.safestring import mark_safe -from django.utils.text import slugify + import debug # pyflakes:ignore @@ -29,18 +22,12 @@ ReviewAssignmentData, sum_period_review_assignment_stats, sum_raw_review_assignment_aggregations) -from ietf.submit.models import Submission from ietf.group.models import Role, Group from ietf.person.models import Person -from ietf.name.models import ReviewResultName, CountryName, DocRelationshipName, ReviewAssignmentStateName -from ietf.person.name import plain_name -from ietf.doc.models import Document, RelatedDocument, State, DocEvent -from ietf.meeting.models import Meeting -from ietf.stats.models import MeetingRegistration, CountryAlias -from ietf.stats.utils import get_aliased_affiliations, get_aliased_countries, compute_hirsch_index +from ietf.name.models import ReviewResultName, CountryName, ReviewAssignmentStateName from ietf.ietfauth.utils import has_role from ietf.utils.response import permission_denied -from ietf.utils.timezone import date_today, DEADLINE_TZINFO, RPC_TZINFO +from ietf.utils.timezone import date_today, DEADLINE_TZINFO def stats_index(request): @@ -135,632 +122,8 @@ def add_labeled_top_series_from_bins(chart_data, bins, limit): }) def document_stats(request, stats_type=None): - def build_document_stats_url(stats_type_override=Ellipsis, get_overrides=None): - if get_overrides is None: - get_overrides={} - kwargs = { - "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override, - } - - return urlreverse(document_stats, kwargs={ k: v for k, v in kwargs.items() if v is not None }) + generate_query_string(request.GET, get_overrides) - - # the length limitation is to keep the key shorter than memcached's limit - # of 250 after django has added the key_prefix and key_version parameters - cache_key = ("stats:document_stats:%s:%s" % (stats_type, slugify(request.META.get('QUERY_STRING',''))))[:228] - data = cache.get(cache_key) - if not data: - names_limit = settings.STATS_NAMES_LIMIT - # statistics types - possible_document_stats_types = add_url_to_choices([ - ("authors", "Number of authors"), - ("pages", "Pages"), - ("words", "Words"), - ("format", "Format"), - ("formlang", "Formal languages"), - ], lambda slug: build_document_stats_url(stats_type_override=slug)) - - possible_author_stats_types = add_url_to_choices([ - ("author/documents", "Number of documents"), - ("author/affiliation", "Affiliation"), - ("author/country", "Country"), - ("author/continent", "Continent"), - ("author/citations", "Citations"), - ("author/hindex", "h-index"), - ], lambda slug: build_document_stats_url(stats_type_override=slug)) - - possible_yearly_stats_types = add_url_to_choices([ - ("yearly/affiliation", "Affiliation"), - ("yearly/country", "Country"), - ("yearly/continent", "Continent"), - ], lambda slug: build_document_stats_url(stats_type_override=slug)) - - - if not stats_type: - return HttpResponseRedirect(build_document_stats_url(stats_type_override=possible_document_stats_types[0][0])) - - - possible_document_types = add_url_to_choices([ - ("", "All"), - ("rfc", "RFCs"), - ("draft", "Internet-Drafts"), - ], lambda slug: build_document_stats_url(get_overrides={ "type": slug })) - - document_type = get_choice(request, "type", possible_document_types) or "" - - - possible_time_choices = add_url_to_choices([ - ("", "All time"), - ("5y", "Past 5 years"), - ], lambda slug: build_document_stats_url(get_overrides={ "time": slug })) - - time_choice = request.GET.get("time") or "" - - from_time = None - if "y" in time_choice: - try: - y = int(time_choice.rstrip("y")) - from_time = timezone.now() - dateutil.relativedelta.relativedelta(years=y) - except ValueError: - pass - - chart_data = [] - table_data = [] - stats_title = "" - template_name = stats_type.replace("/", "_") - bin_size = 1 - alias_data = [] - eu_countries = None - - - if any(stats_type == t[0] for t in possible_document_stats_types): - # filter documents - document_filters = Q(type__in=["draft","rfc"]) # TODO - review lots of "rfc is a draft" assumptions below - - rfc_state = State.objects.get(type="rfc", slug="published") - if document_type == "rfc": - document_filters &= Q(states=rfc_state) - elif document_type == "draft": - document_filters &= ~Q(states=rfc_state) - - if from_time: - # this is actually faster than joining in the database, - # despite the round-trip back and forth - docs_within_time_constraint = list(Document.objects.filter( - type="draft", - docevent__time__gte=from_time, - docevent__type__in=["published_rfc", "new_revision"], - ).values_list("pk",flat=True)) - - document_filters &= Q(pk__in=docs_within_time_constraint) - - document_qs = Document.objects.filter(document_filters) - - if document_type == "rfc": - doc_label = "RFC" - elif document_type == "draft": - doc_label = "draft" - else: - doc_label = "document" - - total_docs = document_qs.values_list("name").distinct().count() - - if stats_type == "authors": - stats_title = "Number of authors for each {}".format(doc_label) - - bins = defaultdict(set) - - for name, author_count in document_qs.values_list("name").annotate(Count("documentauthor")).values_list("name","documentauthor__count"): - bins[author_count or 0].add(name) - - series_data = [] - for author_count, names in sorted(bins.items(), key=lambda t: t[0]): - percentage = len(names) * 100.0 / (total_docs or 1) - series_data.append((author_count, percentage)) - table_data.append((author_count, percentage, len(names), list(names)[:names_limit])) - - chart_data.append({ "data": series_data }) - - elif stats_type == "pages": - stats_title = "Number of pages for each {}".format(doc_label) - - bins = defaultdict(set) - - for name, pages in document_qs.values_list("name", "pages"): - bins[pages or 0].add(name) - - series_data = [] - for pages, names in sorted(bins.items(), key=lambda t: t[0]): - percentage = len(names) * 100.0 / (total_docs or 1) - if pages is not None: - series_data.append((pages, len(names))) - table_data.append((pages, percentage, len(names), list(names)[:names_limit])) - - chart_data.append({ "data": series_data }) - - elif stats_type == "words": - stats_title = "Number of words for each {}".format(doc_label) - - bin_size = 500 - - bins = defaultdict(set) - - for name, words in document_qs.values_list("name", "words"): - bins[put_into_bin(words, bin_size)].add(name) - - series_data = [] - for (value, words), names in sorted(bins.items(), key=lambda t: t[0][0]): - percentage = len(names) * 100.0 / (total_docs or 1) - if words is not None: - series_data.append((value, len(names))) - - table_data.append((words, percentage, len(names), list(names)[:names_limit])) - - chart_data.append({ "data": series_data }) - - elif stats_type == "format": - stats_title = "Submission formats for each {}".format(doc_label) - - bins = defaultdict(set) - - # on new documents, we should have a Submission row with the file types - submission_types = {} - - for doc_name, file_types in Submission.objects.values_list("draft", "file_types").order_by("submission_date", "id"): - submission_types[doc_name] = file_types - - doc_names_with_missing_types = {} - for doc_name, doc_type, rev in document_qs.values_list("name", "type_id", "rev"): - types = submission_types.get(doc_name) - if types: - for dot_ext in types.split(","): - bins[dot_ext.lstrip(".").upper()].add(doc_name) - - else: - - if doc_type == "rfc": - filename = doc_name - else: - filename = doc_name + "-" + rev - - doc_names_with_missing_types[filename] = doc_name - - # look up the remaining documents on disk - for filename in itertools.chain(os.listdir(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR), os.listdir(settings.RFC_PATH)): - t = filename.split(".", 1) - if len(t) != 2: - continue - - basename, ext = t - ext = ext.lower() - if not any(ext==allowlisted_ext for allowlisted_ext in settings.DOCUMENT_FORMAT_ALLOWLIST): - continue - - name = doc_names_with_missing_types.get(basename) - - if name: - bins[ext.upper()].add(name) - - series_data = [] - for fmt, names in sorted(bins.items(), key=lambda t: t[0]): - percentage = len(names) * 100.0 / (total_docs or 1) - series_data.append((fmt, len(names))) - - table_data.append((fmt, percentage, len(names), list(names)[:names_limit])) - - chart_data.append({ "data": series_data }) - - elif stats_type == "formlang": - stats_title = "Formal languages used for each {}".format(doc_label) - - bins = defaultdict(set) - - for name, formal_language_name in document_qs.values_list("name", "formal_languages__name"): - bins[formal_language_name or ""].add(name) - - series_data = [] - for formal_language, names in sorted(bins.items(), key=lambda t: t[0]): - percentage = len(names) * 100.0 / (total_docs or 1) - if formal_language is not None: - series_data.append((formal_language, len(names))) - table_data.append((formal_language, percentage, len(names), list(names)[:names_limit])) - - chart_data.append({ "data": series_data }) - - elif any(stats_type == t[0] for t in possible_author_stats_types): - person_filters = Q(documentauthor__document__type="draft") - - # filter persons - rfc_state = State.objects.get(type="rfc", slug="published") - if document_type == "rfc": - person_filters &= Q(documentauthor__document__states=rfc_state) - elif document_type == "draft": - person_filters &= ~Q(documentauthor__document__states=rfc_state) - - if from_time: - # this is actually faster than joining in the database, - # despite the round-trip back and forth - docs_within_time_constraint = set(Document.objects.filter( - type="draft", - docevent__time__gte=from_time, - docevent__type__in=["published_rfc", "new_revision"], - ).values_list("pk")) - - person_filters &= Q(documentauthor__document__in=docs_within_time_constraint) - - person_qs = Person.objects.filter(person_filters) - - if document_type == "rfc": - doc_label = "RFC" - elif document_type == "draft": - doc_label = "draft" - else: - doc_label = "document" - - if stats_type == "author/documents": - stats_title = "Number of {}s per author".format(doc_label) - - bins = defaultdict(set) - - person_qs = Person.objects.filter(person_filters) - - for name, document_count in person_qs.values_list("name").annotate(Count("documentauthor")): - bins[document_count or 0].add(name) - - total_persons = count_bins(bins) - - series_data = [] - for document_count, names in sorted(bins.items(), key=lambda t: t[0]): - percentage = len(names) * 100.0 / (total_persons or 1) - series_data.append((document_count, percentage)) - plain_names = sorted([ plain_name(n) for n in names ]) - table_data.append((document_count, percentage, len(plain_names), list(plain_names)[:names_limit])) - - chart_data.append({ "data": series_data }) - - elif stats_type == "author/affiliation": - stats_title = "Number of {} authors per affiliation".format(doc_label) - - bins = defaultdict(set) - - person_qs = Person.objects.filter(person_filters) - - # Since people don't write the affiliation names in the - # same way, and we don't want to go back and edit them - # either, we transform them here. - - name_affiliation_set = { - (name, affiliation) - for name, affiliation in person_qs.values_list("name", "documentauthor__affiliation") - } - - aliases = get_aliased_affiliations(affiliation for _, affiliation in name_affiliation_set) - - for name, affiliation in name_affiliation_set: - bins[aliases.get(affiliation, affiliation)].add(name) - - prune_unknown_bin_with_known(bins) - total_persons = count_bins(bins) - - series_data = [] - for affiliation, names in sorted(bins.items(), key=lambda t: t[0].lower()): - percentage = len(names) * 100.0 / (total_persons or 1) - if affiliation: - series_data.append((affiliation, len(names))) - plain_names = sorted([ plain_name(n) for n in names ]) - table_data.append((affiliation, percentage, len(plain_names), list(plain_names)[:names_limit])) - - series_data.sort(key=lambda t: t[1], reverse=True) - series_data = series_data[:30] - - chart_data.append({ "data": series_data }) - - for alias, name in sorted(aliases.items(), key=lambda t: t[1]): - alias_data.append((name, alias)) - - elif stats_type == "author/country": - stats_title = "Number of {} authors per country".format(doc_label) - - bins = defaultdict(set) - - person_qs = Person.objects.filter(person_filters) - - # Since people don't write the country names in the - # same way, and we don't want to go back and edit them - # either, we transform them here. - - name_country_set = { - (name, country) - for name, country in person_qs.values_list("name", "documentauthor__country") - } - - aliases = get_aliased_countries(country for _, country in name_country_set) - - countries = { c.name: c for c in CountryName.objects.all() } - eu_name = "EU" - eu_countries = { c for c in countries.values() if c.in_eu } - - for name, country in name_country_set: - country_name = aliases.get(country, country) - bins[country_name].add(name) - - c = countries.get(country_name) - if c and c.in_eu: - bins[eu_name].add(name) - - prune_unknown_bin_with_known(bins) - total_persons = count_bins(bins) + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) - series_data = [] - for country, names in sorted(bins.items(), key=lambda t: t[0].lower()): - percentage = len(names) * 100.0 / (total_persons or 1) - if country: - series_data.append((country, len(names))) - plain_names = sorted([ plain_name(n) for n in names ]) - table_data.append((country, percentage, len(plain_names), list(plain_names)[:names_limit])) - - series_data.sort(key=lambda t: t[1], reverse=True) - series_data = series_data[:30] - - chart_data.append({ "data": series_data }) - - for alias, country_name in aliases.items(): - alias_data.append((country_name, alias, countries.get(country_name))) - - alias_data.sort() - - elif stats_type == "author/continent": - stats_title = "Number of {} authors per continent".format(doc_label) - - bins = defaultdict(set) - - person_qs = Person.objects.filter(person_filters) - - name_country_set = { - (name, country) - for name, country in person_qs.values_list("name", "documentauthor__country") - } - - aliases = get_aliased_countries(country for _, country in name_country_set) - - country_to_continent = dict(CountryName.objects.values_list("name", "continent__name")) - - for name, country in name_country_set: - country_name = aliases.get(country, country) - continent_name = country_to_continent.get(country_name, "") - bins[continent_name].add(name) - - prune_unknown_bin_with_known(bins) - total_persons = count_bins(bins) - - series_data = [] - for continent, names in sorted(bins.items(), key=lambda t: t[0].lower()): - percentage = len(names) * 100.0 / (total_persons or 1) - if continent: - series_data.append((continent, len(names))) - plain_names = sorted([ plain_name(n) for n in names ]) - table_data.append((continent, percentage, len(plain_names), list(plain_names)[:names_limit])) - - series_data.sort(key=lambda t: t[1], reverse=True) - - chart_data.append({ "data": series_data }) - - elif stats_type == "author/citations": - stats_title = "Number of citations of {}s written by author".format(doc_label) - - bins = defaultdict(set) - - cite_relationships = list(DocRelationshipName.objects.filter(slug__in=['refnorm', 'refinfo', 'refunk', 'refold'])) - person_filters &= Q(documentauthor__document__relateddocument__relationship__in=cite_relationships) - - person_qs = Person.objects.filter(person_filters) - - for name, citations in person_qs.values_list("name").annotate(Count("documentauthor__document__relateddocument")): - bins[citations or 0].add(name) - - total_persons = count_bins(bins) - - series_data = [] - for citations, names in sorted(bins.items(), key=lambda t: t[0], reverse=True): - percentage = len(names) * 100.0 / (total_persons or 1) - series_data.append((citations, percentage)) - plain_names = sorted([ plain_name(n) for n in names ]) - table_data.append((citations, percentage, len(plain_names), list(plain_names)[:names_limit])) - - chart_data.append({ "data": sorted(series_data, key=lambda t: t[0]) }) - - elif stats_type == "author/hindex": - stats_title = "h-index for {}s written by author".format(doc_label) - - bins = defaultdict(set) - - cite_relationships = list(DocRelationshipName.objects.filter(slug__in=['refnorm', 'refinfo', 'refunk', 'refold'])) - person_filters &= Q(documentauthor__document__relateddocument__relationship__in=cite_relationships) - - person_qs = Person.objects.filter(person_filters) - - values = person_qs.values_list("name", "documentauthor__document").annotate(Count("documentauthor__document__relateddocument")) - for name, ts in itertools.groupby(values.order_by("name"), key=lambda t: t[0]): - h_index = compute_hirsch_index([citations for _, document, citations in ts]) - bins[h_index or 0].add(name) - - total_persons = count_bins(bins) - - series_data = [] - for citations, names in sorted(bins.items(), key=lambda t: t[0], reverse=True): - percentage = len(names) * 100.0 / (total_persons or 1) - series_data.append((citations, percentage)) - plain_names = sorted([ plain_name(n) for n in names ]) - table_data.append((citations, percentage, len(plain_names), list(plain_names)[:names_limit])) - - chart_data.append({ "data": sorted(series_data, key=lambda t: t[0]) }) - - elif any(stats_type == t[0] for t in possible_yearly_stats_types): - - # filter persons - rfc_state = State.objects.get(type="rfc", slug="published") - if document_type == "rfc": - person_filters = Q(documentauthor__document__type="rfc") - person_filters &= Q(documentauthor__document__states=rfc_state) - elif document_type == "draft": - person_filters = Q(documentauthor__document__type="draft") - person_filters &= ~Q(documentauthor__document__states=rfc_state) - else: - person_filters = Q(documentauthor__document__type="rfc") - person_filters |= Q(documentauthor__document__type="draft") - - doc_years = defaultdict(set) - - draftevent_qs = DocEvent.objects.filter( - doc__type="draft", - type = "new_revision", - ).values_list("doc","time").order_by("doc") - - for doc_id, time in draftevent_qs.iterator(): - # RPC_TZINFO is used to match the timezone handling in Document.pub_date() - doc_years[doc_id].add(time.astimezone(RPC_TZINFO).year) - - rfcevent_qs = ( - DocEvent.objects.filter(doc__type="rfc", type="published_rfc") - .annotate( - draft=Subquery( - RelatedDocument.objects.filter( - target=OuterRef("doc__pk"), relationship_id="became_rfc" - ).values_list("source", flat=True)[:1] - ) - ) - .values_list("doc", "time") - .order_by("doc") - ) - - for doc_id, time in rfcevent_qs.iterator(): - doc_years[doc_id].add(time.astimezone(RPC_TZINFO).year) - - person_qs = Person.objects.filter(person_filters) - - if document_type == "rfc": - doc_label = "RFC" - elif document_type == "draft": - doc_label = "draft" - else: - doc_label = "document" - - template_name = "yearly" - - years_from = from_time.year if from_time else 1 - years_to = timezone.now().year - 1 - - - if stats_type == "yearly/affiliation": - stats_title = "Number of {} authors per affiliation over the years".format(doc_label) - - person_qs = Person.objects.filter(person_filters) - - name_affiliation_doc_set = { - (name, affiliation, doc) - for name, affiliation, doc in person_qs.values_list("name", "documentauthor__affiliation", "documentauthor__document") - } - - aliases = get_aliased_affiliations(affiliation for _, affiliation, _ in name_affiliation_doc_set) - - bins = defaultdict(set) - for name, affiliation, doc in name_affiliation_doc_set: - a = aliases.get(affiliation, affiliation) - if a: - years = doc_years.get(doc) - if years: - for year in years: - if years_from <= year <= years_to: - bins[(year, a)].add(name) - - add_labeled_top_series_from_bins(chart_data, bins, limit=8) - - elif stats_type == "yearly/country": - stats_title = "Number of {} authors per country over the years".format(doc_label) - - person_qs = Person.objects.filter(person_filters) - - name_country_doc_set = { - (name, country, doc) - for name, country, doc in person_qs.values_list("name", "documentauthor__country", "documentauthor__document") - } - - aliases = get_aliased_countries(country for _, country, _ in name_country_doc_set) - - countries = { c.name: c for c in CountryName.objects.all() } - eu_name = "EU" - eu_countries = { c for c in countries.values() if c.in_eu } - - bins = defaultdict(set) - - for name, country, doc in name_country_doc_set: - country_name = aliases.get(country, country) - c = countries.get(country_name) - - years = doc_years.get(doc) - if country_name and years: - for year in years: - if years_from <= year <= years_to: - bins[(year, country_name)].add(name) - - if c and c.in_eu: - bins[(year, eu_name)].add(name) - - add_labeled_top_series_from_bins(chart_data, bins, limit=8) - - - elif stats_type == "yearly/continent": - stats_title = "Number of {} authors per continent".format(doc_label) - - person_qs = Person.objects.filter(person_filters) - - name_country_doc_set = { - (name, country, doc) - for name, country, doc in person_qs.values_list("name", "documentauthor__country", "documentauthor__document") - } - - aliases = get_aliased_countries(country for _, country, _ in name_country_doc_set) - - country_to_continent = dict(CountryName.objects.values_list("name", "continent__name")) - - bins = defaultdict(set) - - for name, country, doc in name_country_doc_set: - country_name = aliases.get(country, country) - continent_name = country_to_continent.get(country_name, "") - - if continent_name: - years = doc_years.get(doc) - if years: - for year in years: - if years_from <= year <= years_to: - bins[(year, continent_name)].add(name) - - add_labeled_top_series_from_bins(chart_data, bins, limit=8) - - data = { - "chart_data": mark_safe(json.dumps(chart_data)), - "table_data": table_data, - "stats_title": stats_title, - "possible_document_stats_types": possible_document_stats_types, - "possible_author_stats_types": possible_author_stats_types, - "possible_yearly_stats_types": possible_yearly_stats_types, - "stats_type": stats_type, - "possible_document_types": possible_document_types, - "document_type": document_type, - "possible_time_choices": possible_time_choices, - "time_choice": time_choice, - "doc_label": doc_label, - "bin_size": bin_size, - "show_aliases_url": build_document_stats_url(get_overrides={ "showaliases": "1" }), - "hide_aliases_url": build_document_stats_url(get_overrides={ "showaliases": None }), - "alias_data": alias_data, - "eu_countries": sorted(eu_countries or [], key=lambda c: c.name), - "content_template": "stats/document_stats_{}.html".format(template_name), - } - # Logs are full of these, but nobody is using them - # log("Cache miss for '%s'. Data size: %sk" % (cache_key, len(str(data))/1000)) - cache.set(cache_key, data, 24*60*60) - return render(request, "stats/document_stats.html", data) def known_countries_list(request, stats_type=None, acronym=None): countries = CountryName.objects.prefetch_related("countryalias_set") @@ -774,252 +137,7 @@ def known_countries_list(request, stats_type=None, acronym=None): }) def meeting_stats(request, num=None, stats_type=None): - meeting = None - if num is not None: - meeting = get_object_or_404(Meeting, number=num, type="ietf") - - def build_meeting_stats_url(number=None, stats_type_override=Ellipsis, get_overrides=None): - if get_overrides is None: - get_overrides = {} - kwargs = { - "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override, - } - - if number is not None: - kwargs["num"] = number - - return urlreverse(meeting_stats, kwargs={ k: v for k, v in kwargs.items() if v is not None }) + generate_query_string(request.GET, get_overrides) - - cache_key = ("stats:meeting_stats:%s:%s:%s" % (num, stats_type, slugify(request.META.get('QUERY_STRING',''))))[:228] - data = cache.get(cache_key) - if not data: - names_limit = settings.STATS_NAMES_LIMIT - # statistics types - if meeting: - possible_stats_types = add_url_to_choices([ - ("country", "Country"), - ("continent", "Continent"), - ], lambda slug: build_meeting_stats_url(number=meeting.number, stats_type_override=slug)) - else: - possible_stats_types = add_url_to_choices([ - ("overview", "Overview"), - ("country", "Country"), - ("continent", "Continent"), - ], lambda slug: build_meeting_stats_url(number=None, stats_type_override=slug)) - - if not stats_type: - return HttpResponseRedirect(build_meeting_stats_url(number=num, stats_type_override=possible_stats_types[0][0])) - - chart_data = [] - piechart_data = [] - table_data = [] - stats_title = "" - template_name = stats_type - bin_size = 1 - eu_countries = None - - def get_country_mapping(attendees): - return { - alias.alias: alias.country - for alias in CountryAlias.objects.filter(alias__in=set(r.country_code for r in attendees)).select_related("country", "country__continent") - if alias.alias.isupper() - } - - def reg_name(r): - return email.utils.formataddr(((r.first_name + " " + r.last_name).strip(), r.email)) - - if meeting and any(stats_type == t[0] for t in possible_stats_types): - attendees = MeetingRegistration.objects.filter( - meeting=meeting, - reg_type__in=['onsite', 'remote'] - ).filter( - Q( attended=True) | Q( checkedin=True ) - ) - - if stats_type == "country": - stats_title = "Number of attendees for {} {} per country".format(meeting.type.name, meeting.number) - - bins = defaultdict(set) - - country_mapping = get_country_mapping(attendees) - - eu_name = "EU" - eu_countries = set(CountryName.objects.filter(in_eu=True)) - - for r in attendees: - name = reg_name(r) - c = country_mapping.get(r.country_code) - bins[c.name if c else ""].add(name) - - if c and c.in_eu: - bins[eu_name].add(name) - - prune_unknown_bin_with_known(bins) - total_attendees = count_bins(bins) - - series_data = [] - for country, names in sorted(bins.items(), key=lambda t: t[0].lower()): - percentage = len(names) * 100.0 / (total_attendees or 1) - if country: - series_data.append((country, len(names))) - table_data.append((country, percentage, len(names), list(names)[:names_limit])) - - if country and country != eu_name: - piechart_data.append({ "name": country, "y": percentage }) - - series_data.sort(key=lambda t: t[1], reverse=True) - series_data = series_data[:20] - - piechart_data.sort(key=lambda d: d["y"], reverse=True) - pie_cut_off = 8 - piechart_data = piechart_data[:pie_cut_off] + [{ "name": "Other", "y": sum(d["y"] for d in piechart_data[pie_cut_off:])}] - - chart_data.append({ "data": series_data }) - - elif stats_type == "continent": - stats_title = "Number of attendees for {} {} per continent".format(meeting.type.name, meeting.number) - - bins = defaultdict(set) - - country_mapping = get_country_mapping(attendees) - - for r in attendees: - name = reg_name(r) - c = country_mapping.get(r.country_code) - bins[c.continent.name if c else ""].add(name) - - prune_unknown_bin_with_known(bins) - total_attendees = count_bins(bins) - - series_data = [] - for continent, names in sorted(bins.items(), key=lambda t: t[0].lower()): - percentage = len(names) * 100.0 / (total_attendees or 1) - if continent: - series_data.append((continent, len(names))) - table_data.append((continent, percentage, len(names), list(names)[:names_limit])) - - series_data.sort(key=lambda t: t[1], reverse=True) - - chart_data.append({ "data": series_data }) - - - elif not meeting and any(stats_type == t[0] for t in possible_stats_types): - template_name = "overview" - - attendees = MeetingRegistration.objects.filter( - meeting__type="ietf", - attended=True, - reg_type__in=['onsite', 'remote'] - ).filter( - Q( attended=True) | Q( checkedin=True ) - ).select_related('meeting') - - if stats_type == "overview": - stats_title = "Number of attendees per meeting" - - continents = {} - - meetings = Meeting.objects.filter(type='ietf', date__lte=date_today()).order_by('number') - for m in meetings: - country = CountryName.objects.get(slug=m.country) - continents[country.continent.name] = country.continent.name - - bins = defaultdict(set) - - for r in attendees: - meeting_number = int(r.meeting.number) - name = reg_name(r) - bins[meeting_number].add(name) - - series_data = {} - for continent in list(continents.keys()): - series_data[continent] = [] - - for m in meetings: - country = CountryName.objects.get(slug=m.country) - url = build_meeting_stats_url(number=m.number, - stats_type_override="country") - for continent in list(continents.keys()): - if continent == country.continent.name: - d = { - "name": "IETF {} - {}, {}".format(int(m.number), m.city, country), - "x": int(m.number), - "y": m.attendees, - "date": m.date.strftime("%d %b %Y"), - "url": url, - } - else: - d = { - "x": int(m.number), - "y": 0, - } - series_data[continent].append(d) - table_data.append((m, url, - m.attendees, country)) - - for continent in list(continents.keys()): -# series_data[continent].sort(key=lambda t: t[0]["x"]) - chart_data.append( { "name": continent, - "data": series_data[continent] }) - - table_data.sort(key=lambda t: int(t[0].number), reverse=True) - - elif stats_type == "country": - stats_title = "Number of attendees per country across meetings" - - country_mapping = get_country_mapping(attendees) - - eu_name = "EU" - eu_countries = set(CountryName.objects.filter(in_eu=True)) - - bins = defaultdict(set) - - for r in attendees: - meeting_number = int(r.meeting.number) - name = reg_name(r) - c = country_mapping.get(r.country_code) - - if c: - bins[(meeting_number, c.name)].add(name) - if c.in_eu: - bins[(meeting_number, eu_name)].add(name) - - add_labeled_top_series_from_bins(chart_data, bins, limit=8) - - - elif stats_type == "continent": - stats_title = "Number of attendees per continent across meetings" - - country_mapping = get_country_mapping(attendees) - - bins = defaultdict(set) - - for r in attendees: - meeting_number = int(r.meeting.number) - name = reg_name(r) - c = country_mapping.get(r.country_code) - - if c: - bins[(meeting_number, c.continent.name)].add(name) - - add_labeled_top_series_from_bins(chart_data, bins, limit=8) - data = { - "chart_data": mark_safe(json.dumps(chart_data)), - "piechart_data": mark_safe(json.dumps(piechart_data)), - "table_data": table_data, - "stats_title": stats_title, - "possible_stats_types": possible_stats_types, - "stats_type": stats_type, - "bin_size": bin_size, - "meeting": meeting, - "eu_countries": sorted(eu_countries or [], key=lambda c: c.name), - "content_template": "stats/meeting_stats_{}.html".format(template_name), - } - # Logs are full of these, but nobody is using them... - # log("Cache miss for '%s'. Data size: %sk" % (cache_key, len(str(data))/1000)) - cache.set(cache_key, data, 24*60*60) - # - return render(request, "stats/meeting_stats.html", data) + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) @login_required diff --git a/ietf/templates/stats/document_stats.html b/ietf/templates/stats/document_stats.html deleted file mode 100644 index 4e66bed37e..0000000000 --- a/ietf/templates/stats/document_stats.html +++ /dev/null @@ -1,86 +0,0 @@ -{% extends "base.html" %} -{% load origin %} -{% load ietf_filters static %} -{% block title %}{{ stats_title }}{% endblock %} -{% block pagehead %} - - -{% endblock %} -{% block content %} - {% origin %} -

Internet-Draft and RFC statistics

-
- -
- {% for slug, label, url in possible_document_stats_types %} - {{ label }} - {% endfor %} -
-
-
- -
- {% for slug, label, url in possible_author_stats_types %} - {{ label }} - {% endfor %} -
-
-
- -
- {% for slug, label, url in possible_yearly_stats_types %} - {{ label }} - {% endfor %} -
-
-

Options

-
- -
- {% for slug, label, url in possible_document_types %} - {{ label }} - {% endfor %} -
-
-
- -
- {% for slug, label, url in possible_time_choices %} - {{ label }} - {% endfor %} -
-
-
- Please Note: The author information in the datatracker about RFCs - with numbers lower than about 1300 and Internet-Drafts from before 2001 is - unreliable and in many cases absent. For this reason, statistics on these - pages does not show correct author stats for corpus selections that involve such - documents. -
- {% include content_template %} -{% endblock %} -{% block js %} - - - -{% endblock %} \ No newline at end of file diff --git a/ietf/templates/stats/document_stats_author_affiliation.html b/ietf/templates/stats/document_stats_author_affiliation.html deleted file mode 100644 index 9c798cb924..0000000000 --- a/ietf/templates/stats/document_stats_author_affiliation.html +++ /dev/null @@ -1,113 +0,0 @@ -{% load origin %} -{% origin %} -
- -

Data

- - - - - - - - - {% if table_data %} - - {% for affiliation, percentage, count, names in table_data %} - - - - - - {% endfor %} - - {% endif %} -
AffiliationPercentage of authorsAuthors
{{ affiliation|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
-

- The statistics are based entirely on the author affiliation - provided with each Internet-Draft. Since this may vary across documents, an - author may be counted with more than one affiliation, making the - total sum more than 100%. -

-

Affiliation Aliases

-

- In generating the above statistics, some heuristics have been - applied to determine the affiliations of each author. -

-{% if request.GET.showaliases %} -

- Hide generated aliases -

- {% if request.user.is_staff %} -

- Note: since you're an admin, you can - add an extra known alias - or see the - existing known aliases - and - generally ignored endings. -

- {% endif %} - {% if alias_data %} - - - - - - - - {% if alias_data %} - - {% for name, alias in alias_data %} - - - - - {% endfor %} - - {% endif %} -
AffiliationAlias
{{ name|default:"(unknown)" }}{{ alias }}
- {% endif %} -{% else %} -

- Show generated aliases -

-{% endif %} \ No newline at end of file diff --git a/ietf/templates/stats/document_stats_author_citations.html b/ietf/templates/stats/document_stats_author_citations.html deleted file mode 100644 index ae89335fae..0000000000 --- a/ietf/templates/stats/document_stats_author_citations.html +++ /dev/null @@ -1,72 +0,0 @@ -{% load origin %}{% origin %} -
- - - -

Data

- - - - - - - - - - {% if table_data %} - - {% for citations, percentage, count, names in table_data %} - - - - - - {% endfor %} - - {% endif %} -
CitationsPercentage of authorsAuthors
{{ citations }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" with content_limit=10 %}
- -

Note that the citation counts do not exclude self-references.

diff --git a/ietf/templates/stats/document_stats_author_continent.html b/ietf/templates/stats/document_stats_author_continent.html deleted file mode 100644 index 5554ac341e..0000000000 --- a/ietf/templates/stats/document_stats_author_continent.html +++ /dev/null @@ -1,69 +0,0 @@ -{% load origin %} -{% origin %} -
- -

Data

- - - - - - - - - {% if table_data %} - - {% for continent, percentage, count, names in table_data %} - - - - - - {% endfor %} - - {% endif %} -
ContinentPercentage of authorsAuthors
{{ continent|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
-

- The statistics are based entirely on the author addresses provided - with each Internet-Draft. Since this varies across documents, a traveling - author may be counted in more than country, making the total sum - more than 100%. -

\ No newline at end of file diff --git a/ietf/templates/stats/document_stats_author_country.html b/ietf/templates/stats/document_stats_author_country.html deleted file mode 100644 index 72299cc397..0000000000 --- a/ietf/templates/stats/document_stats_author_country.html +++ /dev/null @@ -1,136 +0,0 @@ -{% load origin %} -{% origin %} -
- -

Data

- - - - - - - - - {% if table_data %} - - {% for country, percentage, count, names in table_data %} - - - - - - {% endfor %} - - {% endif %} -
CountryPercentage of authorsAuthors
{{ country|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
-

- The statistics are based entirely on the author addresses provided - with each Internet-Draft. Since this varies across documents, a traveling - author may be counted in more than country, making the total sum - more than 100%. -

-

- In case no country information is found for an author in the time - period, the author is counted as (unknown). -

-

- EU (European Union) is not a country, but has been added for reference, as the sum of - all current EU member countries: - {% for c in eu_countries %} - {{ c.name }}{% if not forloop.last %},{% endif %} - {% endfor %} - . -

-

Country Aliases

-

- In generating the above statistics, some heuristics have been - applied to figure out which country each author is from. -

-{% if request.GET.showaliases %} -

- Hide generated aliases -

- {% if request.user.is_staff %} -

- Note: since you're an admin, some extra links are visible. You - can either correct a document author entry directly in case the - information is obviously missing or add an alias if an unknown - country name - is being used. -

- {% endif %} - {% if alias_data %} - - - - - - {% if alias_data %} - - {% for name, alias, country in alias_data %} - - - - - {% endfor %} - - {% endif %} -
CountryAlias
- {% if country and request.user.is_staff %} - {{ name|default:"(unknown)" }} - {% else %} - {{ name|default:"(unknown)" }} - {% endif %} - - {{ alias }} - {% if request.user.is_staff and name != "EU" %} - - Matching authors - - {% endif %} -
- {% endif %} -{% else %} -

- Show generated aliases -

-{% endif %} \ No newline at end of file diff --git a/ietf/templates/stats/document_stats_author_documents.html b/ietf/templates/stats/document_stats_author_documents.html deleted file mode 100644 index 28e33e6737..0000000000 --- a/ietf/templates/stats/document_stats_author_documents.html +++ /dev/null @@ -1,69 +0,0 @@ -{% load origin %} -{% origin %} -
- -

Data

- - - - - - - - - {% if table_data %} - - {% for document_count, percentage, count, names in table_data %} - - - - - - {% endfor %} - - {% endif %} -
DocumentsPercentage of authorsAuthors
{{ document_count }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" with content_limit=10 %}
diff --git a/ietf/templates/stats/document_stats_author_hindex.html b/ietf/templates/stats/document_stats_author_hindex.html deleted file mode 100644 index ab3215d355..0000000000 --- a/ietf/templates/stats/document_stats_author_hindex.html +++ /dev/null @@ -1,83 +0,0 @@ -{% load origin %} -{% origin %} -
- -

Data

- - - - - - - - - {% if table_data %} - - {% for h_index, percentage, count, names in table_data %} - - - - - - {% endfor %} - - {% endif %} -
h-indexPercentage of authorsAuthors
{{ h_index }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" with content_limit=25 %}
-

- Hirsch index or h-index is a - - measure of the - productivity and impact of the publications of an author - . - An - author with an h-index of 5 has had 5 publications each cited at - least 5 times - to increase the index to 6, the 5 publications plus - 1 more would have to have been cited at least 6 times, each. Thus a - high h-index requires many highly-cited publications. -

-

- Note that the h-index calculations do not exclude self-references. -

diff --git a/ietf/templates/stats/document_stats_authors.html b/ietf/templates/stats/document_stats_authors.html deleted file mode 100644 index 5c1bbbdf4c..0000000000 --- a/ietf/templates/stats/document_stats_authors.html +++ /dev/null @@ -1,68 +0,0 @@ -{% load origin %} -{% origin %} -
- -

Data

- - - - - - - - - {% if table_data %} - - {% for author_count, percentage, count, names in table_data %} - - - - - - {% endfor %} - - {% endif %} -
AuthorsPercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ author_count }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/document_stats_format.html b/ietf/templates/stats/document_stats_format.html deleted file mode 100644 index 32c25fe378..0000000000 --- a/ietf/templates/stats/document_stats_format.html +++ /dev/null @@ -1,63 +0,0 @@ -{% load origin %} -{% origin %} -
- -

Data

- - - - - - - - - {% if table_data %} - - {% for pages, percentage, count, names in table_data %} - - - - - - {% endfor %} - - {% endif %} -
FormatPercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ pages }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
\ No newline at end of file diff --git a/ietf/templates/stats/document_stats_formlang.html b/ietf/templates/stats/document_stats_formlang.html deleted file mode 100644 index 217d79e3ef..0000000000 --- a/ietf/templates/stats/document_stats_formlang.html +++ /dev/null @@ -1,63 +0,0 @@ -{% load origin %} -{% origin %} -
- -

Data

- - - - - - - - - {% if table_data %} - - {% for formal_language, percentage, count, names in table_data %} - - - - - - {% endfor %} - - {% endif %} -
Formal languagePercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ formal_language }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
\ No newline at end of file diff --git a/ietf/templates/stats/document_stats_pages.html b/ietf/templates/stats/document_stats_pages.html deleted file mode 100644 index 73231b0e90..0000000000 --- a/ietf/templates/stats/document_stats_pages.html +++ /dev/null @@ -1,62 +0,0 @@ -{% load origin %} -{% origin %} -
- -

Data

- - - - - - - - - {% if table_data %} - - {% for pages, percentage, count, names in table_data %} - - - - - - {% endfor %} - - {% endif %} -
PagesPercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ pages }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/document_stats_words.html b/ietf/templates/stats/document_stats_words.html deleted file mode 100644 index 4e8c15e937..0000000000 --- a/ietf/templates/stats/document_stats_words.html +++ /dev/null @@ -1,62 +0,0 @@ -{% load origin %} -{% origin %} -
- -

Data

- - - - - - - - - {% if table_data %} - - {% for pages, percentage, count, names in table_data %} - - - - - - {% endfor %} - - {% endif %} -
WordsPercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ pages }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/document_stats_yearly.html b/ietf/templates/stats/document_stats_yearly.html deleted file mode 100644 index b819255ced..0000000000 --- a/ietf/templates/stats/document_stats_yearly.html +++ /dev/null @@ -1,52 +0,0 @@ -{% load origin %} -{% origin %} -
- \ No newline at end of file diff --git a/ietf/templates/stats/includes/number_with_details_cell.html b/ietf/templates/stats/includes/number_with_details_cell.html deleted file mode 100644 index a5e88113ca..0000000000 --- a/ietf/templates/stats/includes/number_with_details_cell.html +++ /dev/null @@ -1,15 +0,0 @@ -{% load person_filters %} -{% if content_limit and count <= content_limit %} - {% for n in names %} - {% with n|person_by_name as person %} - {% if person %} - {% person_link person %} - {% else %} - {{ n }} - {% endif %} -
- {% endwith %} - {% endfor %} -{% else %} - {{ count }} -{% endif %} \ No newline at end of file diff --git a/ietf/templates/stats/index.html b/ietf/templates/stats/index.html index 1c5026013c..2000168525 100644 --- a/ietf/templates/stats/index.html +++ b/ietf/templates/stats/index.html @@ -10,15 +10,12 @@

Statistics on...

+

+ Statistics on meetings and authorship are not currently available. +

{% endblock %} \ No newline at end of file diff --git a/ietf/templates/stats/meeting_stats.html b/ietf/templates/stats/meeting_stats.html deleted file mode 100644 index 606caffde0..0000000000 --- a/ietf/templates/stats/meeting_stats.html +++ /dev/null @@ -1,35 +0,0 @@ -{% extends "base.html" %} -{% load origin %} -{% load ietf_filters static django_bootstrap5 %} -{% block title %}{{ stats_title }}{% endblock %} -{% block pagehead %} - - -{% endblock %} -{% block content %} - {% origin %} -

Meeting Statistics

- {% if meeting %} -

- « Back to overview -

- {% endif %} -
- -
- {% for slug, label, url in possible_stats_types %} - {{ label }} - {% endfor %} -
-
-
{% include content_template %}
-{% endblock %} -{% block js %} - - - -{% endblock %} \ No newline at end of file diff --git a/ietf/templates/stats/meeting_stats_continent.html b/ietf/templates/stats/meeting_stats_continent.html deleted file mode 100644 index 42ca03a409..0000000000 --- a/ietf/templates/stats/meeting_stats_continent.html +++ /dev/null @@ -1,61 +0,0 @@ -{% load origin %} -{% origin %} -
- -

Data

- - - - - - - - - - {% for continent, percentage, count, names in table_data %} - - - - - - {% endfor %} - -
ContinentPercentage of attendeesAttendees
{{ continent|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/meeting_stats_country.html b/ietf/templates/stats/meeting_stats_country.html deleted file mode 100644 index cebbad3c9f..0000000000 --- a/ietf/templates/stats/meeting_stats_country.html +++ /dev/null @@ -1,97 +0,0 @@ -{% load origin %} -{% origin %} -
- -
- -

Data

- - - - - - - - - - {% for country, percentage, count, names in table_data %} - - - - - - {% endfor %} - -
CountryPercentage of attendeesAttendees
{{ country|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
-

- EU (European Union) is not a country, but has been added for reference, as the sum of - all current EU member countries: - {% for c in eu_countries %} - {{ c.name }}{% if not forloop.last %},{% endif %} - {% endfor %} - . -

\ No newline at end of file diff --git a/ietf/templates/stats/meeting_stats_overview.html b/ietf/templates/stats/meeting_stats_overview.html deleted file mode 100644 index 1136e458b8..0000000000 --- a/ietf/templates/stats/meeting_stats_overview.html +++ /dev/null @@ -1,160 +0,0 @@ -{% load origin %} -{% origin %} -
- -{% if table_data %} -

Data

- - - - - - - - - - - - - {% for meeting, url, count, country in table_data %} - - {% if meeting.get_number > 71 %} - - - - - - - {% else %} - - - - - - - {% endif %} - - {% endfor %} - -
MeetingDateCityCountryContinentAttendees
- {{ meeting.number }} - {{ meeting.date }} - {{ meeting.city }} - {{ country.name }}{{ country.continent }}{% include "stats/includes/number_with_details_cell.html" %}{{ meeting.number }}{{ meeting.date }}{{ meeting.city }}{{ country.name }}{{ country.continent }}{% include "stats/includes/number_with_details_cell.html" %}
-{% endif %} diff --git a/ietf/utils/jsonlogger.py b/ietf/utils/jsonlogger.py index b02cd7af2b..589132977d 100644 --- a/ietf/utils/jsonlogger.py +++ b/ietf/utils/jsonlogger.py @@ -1,9 +1,9 @@ # Copyright The IETF Trust 2024, All Rights Reserved -from pythonjsonlogger import jsonlogger +from pythonjsonlogger.json import JsonFormatter import time -class DatatrackerJsonFormatter(jsonlogger.JsonFormatter): +class DatatrackerJsonFormatter(JsonFormatter): converter = time.gmtime # use UTC default_msec_format = "%s.%03d" # '.' instead of ',' @@ -29,6 +29,6 @@ def add_fields(self, log_record, record, message_dict): log_record.setdefault("x_forwarded_for", record.args["{x-forwarded-for}i"]) log_record.setdefault("x_forwarded_proto", record.args["{x-forwarded-proto}i"]) log_record.setdefault("cf_connecting_ip", record.args["{cf-connecting-ip}i"]) - log_record.setdefault("cf_connecting_ipv6", record.args["{cf-connecting-ipv6}i"]) log_record.setdefault("cf_ray", record.args["{cf-ray}i"]) + log_record.setdefault("asn", record.args["{x-ip-src-asnum}i"]) log_record.setdefault("is_authenticated", record.args["{x-datatracker-is-authenticated}o"]) diff --git a/k8s/nginx-logging.conf b/k8s/nginx-logging.conf index 3c4ade4614..673d7a29ab 100644 --- a/k8s/nginx-logging.conf +++ b/k8s/nginx-logging.conf @@ -1,4 +1,6 @@ -# Define JSON log format - must be loaded before config that references it +# Define JSON log format - must be loaded before config that references it. +# Note that each line is fully enclosed in single quotes. Commas in arrays are +# intentionally inside the single quotes. log_format ietfjson escape=json '{' '"time":"$${keepempty}time_iso8601",' @@ -15,6 +17,6 @@ log_format ietfjson escape=json '"x_forwarded_for":"$${keepempty}http_x_forwarded_for",' '"x_forwarded_proto":"$${keepempty}http_x_forwarded_proto",' '"cf_connecting_ip":"$${keepempty}http_cf_connecting_ip",' - '"cf_connecting_ipv6":"$${keepempty}http_cf_connecting_ipv6",' - '"cf_ray":"$${keepempty}http_cf_ray"' + '"cf_ray":"$${keepempty}http_cf_ray",' + '"asn":"$${keepempty}http_x_ip_src_asnum"' '}'; diff --git a/requirements.txt b/requirements.txt index f974113d8f..ec5fc60b5f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ django-csp>=3.7 django-cors-headers>=3.11.0 django-debug-toolbar>=3.2.4 django-markup>=1.5 # Limited use - need to reconcile against direct use of markdown -django-oidc-provider>=0.8.1 # 0.8 dropped Django 2 support +django-oidc-provider==0.8.2 # 0.8.3 changes logout flow and claim return django-referrer-policy>=1.0 django-simple-history>=3.0.0 django-stubs>=4.2.7,<5 # The django-stubs version used determines the the mypy version indicated below @@ -59,7 +59,7 @@ pyopenssl>=22.0.0 # Used by urllib3.contrib, which is used by PyQuery but not pyquery>=1.4.3 python-dateutil>=2.8.2 types-python-dateutil>=2.8.2 -python-json-logger>=2.0.7 +python-json-logger>=3.1.0 python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form failures pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=1.6 # from TastyPie