From 50d4dc61fe9a6a8bb78e4af71f45b41081b396a5 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Mon, 28 Oct 2024 22:10:40 -0400 Subject: [PATCH 1/8] ci: Update build-base-app.yml with image versioning --- .github/workflows/build-base-app.yml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 3995b4b49b..3cea907fdd 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -14,12 +14,17 @@ jobs: publish: runs-on: ubuntu-latest permissions: - contents: read + contents: write packages: write steps: - uses: actions/checkout@v4 + - name: Set Version + run: | + printf -v CURDATE '%(%Y%m%d%H%M)' -1 + echo "IMGVERSION=$CURDATE" >> $GITHUB_OUTPUT + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -42,4 +47,17 @@ jobs: file: docker/base.Dockerfile platforms: linux/amd64,linux/arm64 push: true - tags: ghcr.io/ietf-tools/datatracker-app-base:latest + tags: | + ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }} + ghcr.io/ietf-tools/datatracker-app-base:latest + + - name: Update version references + run: | + sed -i "1s/.*/FROM ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }}/" dev/build/Dockerfile + + - name: Commit CHANGELOG.md + uses: stefanzweifel/git-auto-commit-action@v5 + with: + branch: main + commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}' + file_pattern: dev/build/Dockerfile From 88dfeff3f765b79d6086362cbdd250e889fe0815 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Mon, 28 Oct 2024 22:40:12 -0400 Subject: [PATCH 2/8] ci: read target base image from TARGET_BASE file --- .github/workflows/build-base-app.yml | 5 +++-- .github/workflows/build.yml | 10 +++++++++- .github/workflows/tests.yml | 9 +++++++-- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 3cea907fdd..f5bcc49033 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -23,7 +23,7 @@ jobs: - name: Set Version run: | printf -v CURDATE '%(%Y%m%d%H%M)' -1 - echo "IMGVERSION=$CURDATE" >> $GITHUB_OUTPUT + echo "IMGVERSION=$CURDATE" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -54,10 +54,11 @@ jobs: - name: Update version references run: | sed -i "1s/.*/FROM ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }}/" dev/build/Dockerfile + echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE - name: Commit CHANGELOG.md uses: stefanzweifel/git-auto-commit-action@v5 with: branch: main commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}' - file_pattern: dev/build/Dockerfile + file_pattern: dev/build/Dockerfile dev/build/TARGET_BASE diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 756600cc1e..6041fe474b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -63,6 +63,7 @@ jobs: pkg_version: ${{ steps.buildvars.outputs.pkg_version }} from_tag: ${{ steps.semver.outputs.nextStrict }} to_tag: ${{ steps.semver.outputs.current }} + base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} steps: - uses: actions/checkout@v4 @@ -122,6 +123,11 @@ jobs: echo "::notice::Non-production build ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME" fi + - name: Get Base Image Target Version + id: baseimgversion + run: | + echo "base_image_version=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT + # ----------------------------------------------------------------- # TESTS # ----------------------------------------------------------------- @@ -133,6 +139,7 @@ jobs: needs: [prepare] with: ignoreLowerCoverage: ${{ github.event.inputs.ignoreLowerCoverage == 'true' }} + targetBaseVersion: ${{ needs.prepare.outputs.base_image_version }} # ----------------------------------------------------------------- # RELEASE @@ -200,6 +207,7 @@ jobs: SKIP_TESTS: ${{ github.event.inputs.skiptests }} DEBIAN_FRONTEND: noninteractive BROWSERSLIST_IGNORE_OLD_DATA: 1 + TARGETBASE: ${{ needs.prepare.outputs.base_image_version }} with: host: ${{ steps.azlaunch.outputs.ipaddr }} port: 22 @@ -310,7 +318,7 @@ jobs: echo "==========================================================================" echo "Collecting statics..." echo "==========================================================================" - sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:latest sh dev/build/collectstatics.sh + sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:$TARGETBASE sh dev/build/collectstatics.sh echo "Pushing statics..." cd static aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 64db03db09..7da7ec897f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,12 +8,17 @@ on: default: false required: true type: boolean + targetBaseVersion: + description: 'Target Base Image Version' + default: latest + required: true + type: string jobs: tests-python: name: Python Tests runs-on: ubuntu-latest - container: ghcr.io/ietf-tools/datatracker-app-base:latest + container: ghcr.io/ietf-tools/datatracker-app-base:${{ github.event.inputs.targetBaseVersion }} services: db: @@ -114,7 +119,7 @@ jobs: tests-playwright-legacy: name: Playwright Legacy Tests runs-on: ubuntu-latest - container: ghcr.io/ietf-tools/datatracker-app-base:latest + container: ghcr.io/ietf-tools/datatracker-app-base:${{ github.event.inputs.targetBaseVersion }} strategy: fail-fast: false matrix: From d1cffe7a145f61d366404574ab0aeac82eb148a9 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Mon, 28 Oct 2024 22:45:07 -0400 Subject: [PATCH 3/8] ci: fix base image version command --- .github/workflows/build-base-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index f5bcc49033..0c3d07a378 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -22,7 +22,7 @@ jobs: - name: Set Version run: | - printf -v CURDATE '%(%Y%m%d%H%M)' -1 + printf -v CURDATE '%(%Y%m%dT%H%M)T' -1 echo "IMGVERSION=$CURDATE" >> $GITHUB_ENV - name: Set up QEMU From 96060fcf6f863e4bcb117db6cd95a9ef5f918f1a Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Mon, 28 Oct 2024 22:59:29 -0400 Subject: [PATCH 4/8] ci: fix build-base-app workflow --- .github/workflows/build-base-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 0c3d07a378..11958dda72 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -53,7 +53,7 @@ jobs: - name: Update version references run: | - sed -i "1s/.*/FROM ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }}/" dev/build/Dockerfile + sed -i "1s/.*/FROM ghcr.io\/ietf-tools\/datatracker-app-base:${{ env.IMGVERSION }}/" dev/build/Dockerfile echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE - name: Commit CHANGELOG.md From 8a4d02026853a8118e4472e766447bf8bcae8952 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Mon, 28 Oct 2024 23:27:30 -0400 Subject: [PATCH 5/8] ci: use build account to push base image changes --- .github/workflows/build-base-app.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 11958dda72..609b641ef9 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -19,6 +19,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + token: ${{ secrets.GH_COMMON_TOKEN }} - name: Set Version run: | From b926178e62e030db126f7aa291ca86e921d20af1 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 29 Oct 2024 11:18:31 -0500 Subject: [PATCH 6/8] fix: quicker calculation of status from draft text (#8111) * fix: quicker calculation of status from draft text * chore: remove unused import * fix: only read a small prefix of draft text when needed --- ietf/doc/models.py | 15 +++++++++++---- ietf/doc/views_doc.py | 11 +++++------ ietf/utils/draft.py | 18 ++++++++++++++++++ 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 639e6ca857..077502db11 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -530,7 +530,7 @@ def replaces(self): def replaced_by(self): return set([ r.document for r in self.related_that("replaces") ]) - def text(self): + def text(self, size = -1): path = self.get_file_name() root, ext = os.path.splitext(path) txtpath = root+'.txt' @@ -538,14 +538,21 @@ def text(self): path = txtpath try: with io.open(path, 'rb') as file: - raw = file.read() + raw = file.read(size) except IOError: return None + text = None try: text = raw.decode('utf-8') except UnicodeDecodeError: - text = raw.decode('latin-1') - # + for back in range(1,4): + try: + text = raw[:-back].decode('utf-8') + break + except UnicodeDecodeError: + pass + if text is None: + text = raw.decode('latin-1') return text def text_or_error(self): diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 915dcebde5..50c60aefc5 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -84,7 +84,7 @@ from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs, review_requests_to_list_for_docs from ietf.review.utils import no_review_from_teams_on_doc from ietf.utils import markup_txt, log, markdown -from ietf.utils.draft import PlaintextDraft +from ietf.utils.draft import get_status_from_draft_text from ietf.utils.meetecho import MeetechoAPIError, SlidesManager from ietf.utils.response import permission_denied from ietf.utils.text import maybe_split @@ -2261,12 +2261,11 @@ def idnits2_state(request, name, rev=None): elif doc.intended_std_level: doc.deststatus = doc.intended_std_level.name else: - text = doc.text() + # 10000 is a conservative prefix on number of utf-8 encoded bytes to + # cover at least the first 10 lines of characters + text = doc.text(size=10000) if text: - parsed_draft = PlaintextDraft( - text=doc.text(), source=name, name_from_source=False - ) - doc.deststatus = parsed_draft.get_status() + doc.deststatus = get_status_from_draft_text(text) else: doc.deststatus = "Unknown" return render( diff --git a/ietf/utils/draft.py b/ietf/utils/draft.py index a1e79760ea..50add5abba 100755 --- a/ietf/utils/draft.py +++ b/ietf/utils/draft.py @@ -131,6 +131,24 @@ def acronym_match(s, l): #_debug(" s:%s; l:%s => %s; %s" % (s, l, acronym, s==acronym)) return s == acronym +def get_status_from_draft_text(text): + + # Take prefix to shortcut work over very large drafts + # 5000 is conservatively much more than a full page of characters and we + # only want the first 10 lines. + text = text.strip()[:5000] # Take prefix to shortcut work over very large drafts + text = re.sub(".\x08", "", text) # Get rid of inkribbon backspace-emphasis + text = text.replace("\r\n", "\n") # Convert DOS to unix + text = text.replace("\r", "\n") # Convert MAC to unix + lines = text.split("\n")[:10] + status = None + for line in lines: + status_match = re.search(r"^\s*Intended [Ss]tatus:\s*(.*?) ", line) + if status_match: + status = status_match.group(1) + break + return status + class Draft: """Base class for drafts From 0d43663e9bb99b45ee779b469f4c38ca14cfb208 Mon Sep 17 00:00:00 2001 From: NGPixel Date: Tue, 29 Oct 2024 16:43:21 +0000 Subject: [PATCH 7/8] ci: update base image target version to 20241029T1632 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 dev/build/TARGET_BASE diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index bede86fac9..728b37156c 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:2024102800 +FROM ghcr.io/ietf-tools/datatracker-app-base:20241029T1632 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE new file mode 100644 index 0000000000..2f8aac20f9 --- /dev/null +++ b/dev/build/TARGET_BASE @@ -0,0 +1 @@ +20241029T1632 From 5f685f45fb5f94f714a9c7a650d111d2e2bc16eb Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Tue, 29 Oct 2024 12:52:18 -0400 Subject: [PATCH 8/8] ci: fix tests.yml workflow --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7da7ec897f..4e0b31fb14 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,7 +18,7 @@ jobs: tests-python: name: Python Tests runs-on: ubuntu-latest - container: ghcr.io/ietf-tools/datatracker-app-base:${{ github.event.inputs.targetBaseVersion }} + container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }} services: db: @@ -48,7 +48,7 @@ jobs: exit 1 fi echo "Running tests..." - if [[ "x${{ github.event.inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then + if [[ "x${{ inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then echo "Lower coverage failures will be ignored." HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test --ignore-lower-coverage else @@ -119,7 +119,7 @@ jobs: tests-playwright-legacy: name: Playwright Legacy Tests runs-on: ubuntu-latest - container: ghcr.io/ietf-tools/datatracker-app-base:${{ github.event.inputs.targetBaseVersion }} + container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }} strategy: fail-fast: false matrix: