validate-docs.yml

name: Validate documentation
on:
  schedule:
    # * is a special character in YAML so you have to quote this string
    - cron: '0 10 * * MON'
jobs:
  check_404s:
    name: Check for 404 errors in docs
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@master
      - uses: actions/setup-node@v1
        with:
          node-version: '12.13.0'
      - name: Install python3
        uses: actions/setup-python@v1
        with:
          python-version: '3.x'
          architecture: 'x64'
      - name: Scrape docs for 404s
        run: |
          pip install scrapy
          scrapy runspider \
          -o errors.csv \
          -a site="https://botpress.com/docs" \
          -a domain="botpress.com" \
          .github/workflows/code/validate-docs/spider.py \
          -s AUTOTHROTTLE_ENABLED=true \
          -s CONCURRENT_REQUESTS_PER_DOMAIN=32 \
          -s HTTPCACHE_ENABLED=False \
          -s HTTPERROR_ALLOW_ALL=True \
          -s LOG_LEVEL=INFO
      - name: Check scrape results
        run: |
          ERRORS_COUNT=$(cat errors.csv | wc -l)
          echo "Found $ERRORS_COUNT errors"
          cat errors.csv
          if [ "$ERRORS_COUNT" != "0" ]; then
            exit 1
          fi