+ Is the datatracker server running?
+Using VS Code, open the Run and Debug tab on the left and click the ‣ symbol (Run Server) to start the server.
+Otherwise, run the command ietf/manage.py runserver 0.0.0.0:8001 from the terminal.
For more information, check out the Datatracker Development in Docker guide.
+ + diff --git a/docker/configs/nginx-proxy.conf b/docker/configs/nginx-proxy.conf new file mode 100644 index 0000000000..02f5208caa --- /dev/null +++ b/docker/configs/nginx-proxy.conf @@ -0,0 +1,24 @@ +server { + listen 8000 default_server; + listen [::]:8000 default_server; + + root /var/www/html; + index index.html index.htm index.nginx-debian.html; + + server_name _; + + location /_static/ { + proxy_pass http://localhost:80/; + } + + location / { + error_page 502 /502.html; + proxy_pass http://localhost:8001/; + proxy_set_header Host localhost:8000; + } + + location /502.html { + root /var/www/html; + internal; + } +} diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 4f0d6a2496..fc3052ff98 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -56,4 +56,5 @@ DE_GFM_BINARY = '/usr/local/bin/de-gfm' -STATIC_IETF_ORG = "http://localhost:8001" +STATIC_IETF_ORG = "/_static" +STATIC_IETF_ORG_INTERNAL = "http://localhost:80" diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh index 61d7ef851d..dd8404b10b 100755 --- a/docker/scripts/app-init.sh +++ b/docker/scripts/app-init.sh @@ -21,6 +21,11 @@ sudo chown dev:dev "/assets" echo "Fix chromedriver /dev/shm permissions..." sudo chmod 1777 /dev/shm +# Run nginx + +echo "Starting nginx..." +sudo nginx + # Build node packages that requrie native compilation echo "Compiling native node packages..." yarn rebuild @@ -35,37 +40,27 @@ cp $WORKSPACEDIR/docker/configs/settings_postgresqldb.py $WORKSPACEDIR/ietf/sett if [ ! -f "$WORKSPACEDIR/ietf/settings_local.py" ]; then echo "Setting up a default settings_local.py ..." - cp $WORKSPACEDIR/docker/configs/settings_local.py $WORKSPACEDIR/ietf/settings_local.py - else - echo "Using existing ietf/settings_local.py file" - if ! cmp -s $WORKSPACEDIR/docker/configs/settings_local.py $WORKSPACEDIR/ietf/settings_local.py; then - echo "NOTE: Differences detected compared to docker/configs/settings_local.py!" - echo "We'll assume you made these deliberately." - fi + echo "Renaming existing ietf/settings_local.py to ietf/settings_local.py.bak" + mv -f $WORKSPACEDIR/ietf/settings_local.py $WORKSPACEDIR/ietf/settings_local.py.bak fi +cp $WORKSPACEDIR/docker/configs/settings_local.py $WORKSPACEDIR/ietf/settings_local.py if [ ! -f "$WORKSPACEDIR/ietf/settings_local_debug.py" ]; then echo "Setting up a default settings_local_debug.py ..." - cp $WORKSPACEDIR/docker/configs/settings_local_debug.py $WORKSPACEDIR/ietf/settings_local_debug.py else - echo "Using existing ietf/settings_local_debug.py file" - if ! cmp -s $WORKSPACEDIR/docker/configs/settings_local_debug.py $WORKSPACEDIR/ietf/settings_local_debug.py; then - echo "NOTE: Differences detected compared to docker/configs/settings_local_debug.py!" - echo "We'll assume you made these deliberately." - fi + echo "Renaming existing ietf/settings_local_debug.py to ietf/settings_local_debug.py.bak" + mv -f $WORKSPACEDIR/ietf/settings_local_debug.py $WORKSPACEDIR/ietf/settings_local_debug.py.bak fi +cp $WORKSPACEDIR/docker/configs/settings_local_debug.py $WORKSPACEDIR/ietf/settings_local_debug.py if [ ! -f "$WORKSPACEDIR/ietf/settings_local_vite.py" ]; then echo "Setting up a default settings_local_vite.py ..." - cp $WORKSPACEDIR/docker/configs/settings_local_vite.py $WORKSPACEDIR/ietf/settings_local_vite.py else - echo "Using existing ietf/settings_local_vite.py file" - if ! cmp -s $WORKSPACEDIR/docker/configs/settings_local_vite.py $WORKSPACEDIR/ietf/settings_local_vite.py; then - echo "NOTE: Differences detected compared to docker/configs/settings_local_vite.py!" - echo "We'll assume you made these deliberately." - fi + echo "Renaming existing ietf/settings_local_vite.py to ietf/settings_local_vite.py.bak" + mv -f $WORKSPACEDIR/ietf/settings_local_vite.py $WORKSPACEDIR/ietf/settings_local_vite.py.bak fi +cp $WORKSPACEDIR/docker/configs/settings_local_vite.py $WORKSPACEDIR/ietf/settings_local_vite.py # Create data directories @@ -99,14 +94,13 @@ echo "Running initial checks..." /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --fake-initial --settings=settings_local -echo "-----------------------------------------------------------------" -echo "Done!" -echo "-----------------------------------------------------------------" - if [ -z "$EDITOR_VSCODE" ]; then CODE=0 python -m smtpd -n -c DebuggingServer localhost:2025 & if [ -z "$*" ]; then + echo "-----------------------------------------------------------------" + echo "Ready!" + echo "-----------------------------------------------------------------" echo echo "You can execute arbitrary commands now, e.g.," echo diff --git a/docker/scripts/app-setup-nginx.sh b/docker/scripts/app-setup-nginx.sh new file mode 100644 index 0000000000..cdeb2ea4cb --- /dev/null +++ b/docker/scripts/app-setup-nginx.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +apt-get update -y +apt-get install -y nginx diff --git a/docker/scripts/app-start.sh b/docker/scripts/app-start.sh new file mode 100644 index 0000000000..c3369bab93 --- /dev/null +++ b/docker/scripts/app-start.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +sudo service rsyslog start &>/dev/null + +# Run nginx + +echo "Starting nginx..." +pidof nginx >/dev/null && echo "nginx is already running [ OK ]" || sudo nginx + +# Run memcached + +echo "Starting memcached..." +pidof memcached >/dev/null && echo "memcached is already running [ OK ]" || /usr/bin/memcached -u dev -d + +echo "-----------------------------------------------------------------" +echo "Ready!" +echo "-----------------------------------------------------------------" diff --git a/ietf/.gitignore b/ietf/.gitignore index 2483dbda77..af738db98c 100644 --- a/ietf/.gitignore +++ b/ietf/.gitignore @@ -1,8 +1,8 @@ /*.pyc /settings_local.py -/settings_local.py.bak /settings_local_debug.py /settings_local_vite.py /settings_mysqldb.py /settings_postgresqldb.py +/settings_*.py.bak /ietfdb.sql.gz diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 85b01c3d02..b570391f9e 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -627,7 +627,7 @@ def pdfized(self): stylesheets.append(finders.find("ietf/css/document_html_txt.css")) else: text = self.htmlized() - stylesheets.append(f'{settings.STATIC_IETF_ORG}/fonts/noto-sans-mono/import.css') + stylesheets.append(f'{settings.STATIC_IETF_ORG_INTERNAL}/fonts/noto-sans-mono/import.css') cache = caches["pdfized"] cache_key = name.split(".")[0] diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 5bd1c85172..c500e702eb 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -65,6 +65,7 @@ from ietf.person.models import Person from ietf.person.utils import get_active_ads from ietf.utils.draft_search import normalize_draftname +from ietf.utils.log import log from ietf.doc.utils_search import prepare_document_table @@ -222,12 +223,14 @@ def search(request): return HttpResponseBadRequest("form not valid: %s" % form.errors) cache_key = get_search_cache_key(get_params) - results = cache.get(cache_key) - if not results: + cached_val = cache.get(cache_key) + if cached_val: + [results, meta] = cached_val + else: results = retrieve_search_results(form) - cache.set(cache_key, results) - - results, meta = prepare_document_table(request, results, get_params) + results, meta = prepare_document_table(request, results, get_params) + cache.set(cache_key, [results, meta]) # for settings.CACHE_MIDDLEWARE_SECONDS + log(f"Search results computed for {get_params}") meta['searching'] = True else: form = SearchForm() @@ -846,11 +849,12 @@ def index_all_drafts(request): return render(request, 'doc/index_all_drafts.html', { "categories": categories }) def index_active_drafts(request): + slowcache = caches['slowpages'] cache_key = 'doc:index_active_drafts' - groups = cache.get(cache_key) + groups = slowcache.get(cache_key) if not groups: groups = active_drafts_index_by_group() - cache.set(cache_key, groups, 15*60) + slowcache.set(cache_key, groups, 15*60) return render(request, "doc/index_active_drafts.html", { 'groups': groups }) def ajax_select2_search_docs(request, model_name, doc_type): diff --git a/ietf/settings.py b/ietf/settings.py index 7d07f12207..9f6284003f 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -178,7 +178,10 @@ 'django.contrib.staticfiles.finders.AppDirectoriesFinder', ) +# Client-side static.ietf.org URL STATIC_IETF_ORG = "https://static.ietf.org" +# Server-side static.ietf.org URL (used in pdfized) +STATIC_IETF_ORG_INTERNAL = STATIC_IETF_ORG WSGI_APPLICATION = "ietf.wsgi.application" @@ -726,13 +729,13 @@ def skip_unreadable_post(record): # This setting is possibly overridden further down, after the import of settings_local CACHES = { 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', + 'BACKEND': 'ietf.utils.cache.LenientMemcacheCache', 'LOCATION': '127.0.0.1:11211', 'VERSION': __version__, 'KEY_PREFIX': 'ietf:dt', }, 'sessions': { - 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', + 'BACKEND': 'ietf.utils.cache.LenientMemcacheCache', 'LOCATION': '127.0.0.1:11211', # No release-specific VERSION setting. 'KEY_PREFIX': 'ietf:dt', @@ -1239,7 +1242,7 @@ def skip_unreadable_post(record): CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', - #'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', + #'BACKEND': 'ietf.utils.cache.LenientMemcacheCache', #'LOCATION': '127.0.0.1:11211', #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', 'VERSION': __version__, @@ -1292,6 +1295,6 @@ def skip_unreadable_post(record): # Cannot have this set to True if we're using http: from the dev-server: CSRF_COOKIE_SECURE = False CSRF_COOKIE_SAMESITE = 'Lax' + CSRF_TRUSTED_ORIGINS = ['http://localhost:8000'] SESSION_COOKIE_SECURE = False SESSION_COOKIE_SAMESITE = 'Lax' - diff --git a/ietf/templates/doc/document_info.html b/ietf/templates/doc/document_info.html index 392c5f124b..c2bcdc42b2 100644 --- a/ietf/templates/doc/document_info.html +++ b/ietf/templates/doc/document_info.html @@ -100,7 +100,7 @@ {% endif %}diff --git a/ietf/templates/doc/opengraph.html b/ietf/templates/doc/opengraph.html index 239b081af4..f4527966a9 100644 --- a/ietf/templates/doc/opengraph.html +++ b/ietf/templates/doc/opengraph.html @@ -36,7 +36,7 @@ {% else %}{# TODO: We need a card image for individual I-Ds. #} {% endif %} -{% for author in doc.documentauthor_set.all %} -{% endfor %} +{% if doc.pk %}{% for author in doc.documentauthor_set.all %} +{% endfor %}{% endif %} {% if published %}{% endif %} {% if expires %}{% endif %} \ No newline at end of file diff --git a/ietf/templates/release/about.html b/ietf/templates/release/about.html index 9d30bdb509..23c2c9205c 100644 --- a/ietf/templates/release/about.html +++ b/ietf/templates/release/about.html @@ -23,6 +23,7 @@
All timestamps in the database are now stored as UTC. Values reported through the API for several diff --git a/ietf/utils/cache.py b/ietf/utils/cache.py new file mode 100644 index 0000000000..76d1dae54d --- /dev/null +++ b/ietf/utils/cache.py @@ -0,0 +1,19 @@ +# Copyright The IETF Trust 2023, All Rights Reserved +# -*- coding: utf-8 -*- + +from django.core.cache.backends.memcached import PyMemcacheCache +from pymemcache.exceptions import MemcacheServerError + +from .log import log + + +class LenientMemcacheCache(PyMemcacheCache): + """PyMemcacheCache backend that tolerates failed inserts due to object size""" + def set(self, key, value, timeout=None, version=None): + try: + super().set(key, value, timeout, version) + except MemcacheServerError as err: + if "object too large for cache" in str(err): + log(f"Memcache failed to cache large object for {key}") + else: + raise diff --git a/requirements.txt b/requirements.txt index f52ab3a099..81f72ce60f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,6 @@ docutils>=0.18.1 # Used only by dbtemplates for RestructuredText types-docutils>=0.18.1 factory-boy>=3.2.1 github3.py>=3.2.0 -google-i18n-address<3 # for xml2rfc gunicorn>=20.1.0 hashids>=1.3.1 html2text>=2020.1.16 # Used only to clean comment field of secr/sreq