ReDATA #41
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ReDATA | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 11 8,22 * *' # every 8th and 22nd of the month at 4 AM (11 AM UTC) | |
| env: | |
| OUTDIR: 'redata' | |
| SITE: 'https://api.figshare.com/v2/articles?institution=797' | |
| NOTIFY: '@zoidy' #enter GitHub IDs in the form @username. Separate multiple entries by commas or spaces | |
| jobs: | |
| runlinkcheck: | |
| name: Linkcheck | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write # for committing | |
| issues: write # for creating issues | |
| outputs: | |
| out_sha: ${{ steps.commit-action.outputs.commit_hash }} | |
| out_sha_short: ${{ steps.get_short_sha.outputs.shortsha }} | |
| out_path: ${{ env.OUTDIR }}/${{ env.OUTFILE }} | |
| issue: ${{ steps.create-issue.outputs.number }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v3 | |
| - name: create issue | |
| id: create-issue | |
| uses: JasonEtco/create-an-issue@v2 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| runid: ${{ github.run_id }} | |
| workflow: ${{ github.workflow }} | |
| result: run Linkcheck fail | |
| result_link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| with: | |
| filename: .github/workflows/report_issue_template.md | |
| - name: install Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| cache: 'pip' | |
| - name: install linkchecker | |
| run: pip install -r requirements.txt | |
| - name: set output filename | |
| run: | |
| echo "OUTFILE=$(date "+%Y.%m.%d-%H.%M.%S").html" >> "$GITHUB_ENV" | |
| - name: fetch data from figshare | |
| run: | | |
| python .github/workflows/json_to_html_redata.py ${{ env.SITE }} data.html | |
| split -d --verbose -b1M --additional-suffix=.html data.html data | |
| - name: Archive source html | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: data.html | |
| - name: run check | |
| continue-on-error: true | |
| shell: bash {0} #avoid exiting this step on non-zero return values | |
| run: | | |
| rm data.html | |
| for f in data*.html; do linkchecker -f linkcheckerrc-${{ env.OUTDIR }} -F html/tmpreport$(date +%s).html "${f}"; done; | |
| rm data*.html | |
| - name: process report | |
| run: | | |
| cat $(find -name "tmpreport*.html" | head -n1) | head -n25 > reports/${{ env.OUTDIR }}/${{ env.OUTFILE }} | |
| for f in tmpreport*.html; do tail -n+26 ${f} | head -n-1 >> reports/${{ env.OUTDIR }}/${{ env.OUTFILE }}; done; | |
| echo "</small></body></html>" >> reports/${{ env.OUTDIR }}/${{ env.OUTFILE }} | |
| rm tmpreport*.html | |
| - name: commit output report | |
| id: commit-action | |
| uses: stefanzweifel/git-auto-commit-action@v4 | |
| with: | |
| commit_author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> | |
| commit_message: 📝 Create report for ${{ github.workflow }} [skip ci] | |
| - name: short sha | |
| id: get_short_sha | |
| run: | |
| echo "shortsha=$(cut -b 1-7 - <<< ${{ steps.commit-action.outputs.commit_hash }})" >> "$GITHUB_OUTPUT" | |
| - name: update issue | |
| uses: JasonEtco/create-an-issue@v2 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| runid: ${{ github.run_id }} | |
| workflow: ${{ github.workflow }} | |
| result: Run Linkcheck success, generate Create Directory Index fail | |
| site: ${{ env.SITE }} | |
| result_link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| short_sha: ${{ steps.get_short_sha.outputs.shortsha }} | |
| commit_link: ${{ github.server_url }}/${{ github.repository }}/commit/${{ steps.commit-action.outputs.commit_hash }} | |
| with: | |
| filename: .github/workflows/report_issue_template.md | |
| update_existing: true | |
| pages-directory-listing: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| issues: write # for editing issues | |
| name: Create Directory Listings Index | |
| needs: runlinkcheck | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.runlinkcheck.outputs.out_sha }} | |
| - name: Generate Directory Listings | |
| uses: jayanta525/github-pages-directory-listing@20409ce0808d816bcbc3090ed7a6a784cf6ed9df | |
| with: | |
| FOLDER: 'reports' #directory to generate index | |
| - name: Upload artifact | |
| uses: actions/upload-pages-artifact@v1 | |
| with: | |
| path: 'reports' # upload generated folder | |
| - name: update issue | |
| uses: JasonEtco/create-an-issue@v2 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| runid: ${{ github.run_id }} | |
| workflow: ${{ github.workflow }} | |
| result: Create Directory Index success, deploy fail | |
| site: ${{ env.SITE }} | |
| result_link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| short_sha: ${{ needs.runlinkcheck.outputs.out_sha_short }} | |
| commit_link: ${{ github.server_url }}/${{ github.repository }}/commit/${{ needs.runlinkcheck.outputs.out_sha }} | |
| with: | |
| filename: .github/workflows/report_issue_template.md | |
| update_existing: true | |
| deploy: #ensure GH pages has been enabled w/GitHub Actions as a source on the repo or this will fail | |
| needs: [runlinkcheck, pages-directory-listing] | |
| name: Deploy Listings Index Page | |
| permissions: | |
| pages: write # to deploy to Pages | |
| id-token: write # to verify the deployment originates from an appropriate source | |
| issues: write # for updating the issue | |
| environment: | |
| name: github-pages | |
| runs-on: ubuntu-latest | |
| outputs: | |
| page_url: ${{ steps.deployment.outputs.page_url }} | |
| steps: | |
| - name: Deploy to GitHub Pages | |
| id: deployment | |
| uses: actions/deploy-pages@v2 | |
| - name: close issue | |
| uses: peter-evans/close-issue@v3 | |
| with: | |
| issue-number: ${{ needs.runlinkcheck.outputs.issue }} | |
| on-success: | |
| # Update the assignees in a separate job to avoid a notification when the issue is closed. | |
| # The only notification will be when the issue is assigned | |
| name: Update issue on success | |
| runs-on: ubuntu-latest | |
| needs: [runlinkcheck, pages-directory-listing, deploy] | |
| permissions: | |
| issues: write | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v3 | |
| - name: update issue | |
| uses: JasonEtco/create-an-issue@v2 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| runid: ${{ github.run_id }} | |
| workflow: ${{ github.workflow }} | |
| result: Deploy page successful | |
| site: ${{ env.SITE }} | |
| result_link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| short_sha: ${{ needs.runlinkcheck.outputs.out_sha_short }} | |
| commit_link: ${{ github.server_url }}/${{ github.repository }}/commit/${{ needs.runlinkcheck.outputs.out_sha }} | |
| report_link: ${{ needs.deploy.outputs.page_url }}/${{ needs.runlinkcheck.outputs.out_path }} | |
| assignees: ${{ env.NOTIFY }} | |
| with: | |
| filename: .github/workflows/report_issue_template.md | |
| update_existing: true | |
| search_existing: closed | |
| on-error: | |
| name: Update issue on error | |
| runs-on: ubuntu-latest | |
| needs: [runlinkcheck, pages-directory-listing, deploy] | |
| if: ${{ always() && contains(needs.*.result, 'failure') }} # https://stackoverflow.com/a/74562058 | |
| permissions: | |
| issues: write | |
| steps: | |
| - name: Update issue | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: 'update-issue' | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| issue-number: ${{ needs.runlinkcheck.outputs.issue }} | |
| state: 'closed' | |
| body: '${{ env.NOTIFY }}' | |
| update-mode: 'append' |