Merge pull request #1 from fail2ban/master

Merge master.

.codespellrc

@@ -0,0 +1,12 @@
+[codespell]
+# THANKS - names
+skip = .git,*.pdf,*.svg,venv,.codespellrc,THANKS,*test*.log,logs
+check-hidden = true
+# Ignore all acronyms etc as plenty e.g. in fail2ban/server/strptime.py
+# Try to identify incomplete words which are part of a regex, hence having [] at the beginning
+# Ignore all urls as something with :// in it
+# Ignore all lines with codespell-ignore in them for pragma annotation
+ignore-regex = (\b([A-Z][A-Z][A-Z]+|gir\.st)\b)|\[[a-zA-Z]+\][a-z]+\b|[a-z]+://\S+|.*codespell-ignore.*
+# some oddly named variables, some names, etc
+# wee -- comes in regex etc for weeks
+ignore-words-list = theis,timere,alls,wee,wight,ans,re-use

.github/workflows/codespell.yml

@@ -0,0 +1,22 @@
+---
+name: Codespell
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+permissions:
+  contents: read
+
+jobs:
+  codespell:
+    name: Check for spelling errors
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Codespell
+        uses: codespell-project/actions-codespell@v2

.github/workflows/main.yml

@@ -22,15 +22,15 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11.0-beta.3', pypy2, pypy3]
+        python-version: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12', '3.13.0-alpha.2', pypy3.10]
       fail-fast: false
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -51,24 +51,32 @@ jobs:
 
       - name: Install dependencies
         run: |
-          if [[ "$F2B_PY" = 3 ]]; then python -m pip install --upgrade pip || echo "can't upgrade pip"; fi
-          if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then
-            #pip install 2to3
-            sudo apt-get -y install 2to3
-          fi
+          #if [[ "$F2B_PY" = 3 ]]; then python -m pip install --upgrade pip || echo "can't upgrade pip"; fi
           #sudo apt-get -y install python${F2B_PY/2/}-pyinotify || echo 'inotify not available'
           python -m pip install pyinotify || echo 'inotify not available'
+          sudo apt-get -y install sqlite3 || echo 'sqlite3 not available'
           #sudo apt-get -y install python${F2B_PY/2/}-systemd || echo 'systemd not available'
           sudo apt-get -y install libsystemd-dev || echo 'systemd dependencies seems to be unavailable'
           python -m pip install systemd-python || echo 'systemd not available'
-          #readline if available as module:
+          # readline if available as module:
           python -c 'import readline' 2> /dev/null || python -m pip install readline || echo 'readline not available'
+          # asyncore/asynchat:
+          if dpkg --compare-versions "$F2B_PYV" ge 3.12; then
+            #sudo apt-get -y install python${F2B_PY/2/}-setuptools || echo 'setuptools not unavailable'
+            python -m pip install setuptools || echo "can't install setuptools"
+            # don't install async* modules, we need to cover bundled-in libraries:
+            #python -m pip install pyasynchat || echo "can't install pyasynchat";
+            #python -m pip install pyasyncore || echo "can't install pyasyncore";
+          fi
+          # aiosmtpd in test_smtp (for 3.10+, no need to test it everywhere):
+          if dpkg --compare-versions "$F2B_PYV" ge 3.10; then
+            #sudo apt-get -y install python${F2B_PY/2/}-aiosmtpd || echo 'aiosmtpd not available'
+            python -m pip install aiosmtpd || echo 'aiosmtpd not available'
+          fi
 
       - name: Before scripts
         run: |
           cd "$GITHUB_WORKSPACE"
-          # Manually execute 2to3 for now
-          if [[ "$F2B_PY" = 3 ]]; then echo "2to3 ..." && ./fail2ban-2to3; fi
           _debug() { echo -n "$1 "; err=$("${@:2}" 2>&1) && echo 'OK' || echo -e "FAIL\n$err"; }
           # (debug) output current preferred encoding:
           _debug 'Encodings:' python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))'
@@ -80,14 +88,8 @@ jobs:
 
       - name: Test suite
         run: |
-          if [[ "$F2B_PY" = 2 ]]; then
-            python setup.py test
-          elif dpkg --compare-versions "$F2B_PYV" lt 3.10; then 
-            python bin/fail2ban-testcases --verbosity=2
-          else
-            echo "Skip systemd backend since systemd-python module must be fixed for python >= v.3.10 in GHA ..."
-            python bin/fail2ban-testcases --verbosity=2 -i "[sS]ystemd|[jJ]ournal"
-          fi
+          #python setup.py test
+          python bin/fail2ban-testcases --verbosity=2
 
       #- name: Test suite (debug some systemd tests only)
          #run: python bin/fail2ban-testcases --verbosity=2 "[sS]ystemd|[jJ]ournal"

.gitignore

@@ -10,3 +10,4 @@ htmlcov
 __pycache__
 .vagrant/
 .idea/
+.venv/

ChangeLog

@@ -12,12 +12,29 @@ ver. 1.0.3-dev-1 (20??/??/??) - development nightly edition
 
 ### Fixes
 * circumvent SEGFAULT in a python's socket module by getaddrinfo with disabled IPv6 (gh-3438)
+* `action.d/cloudflare-token.conf` - fixes gh-3479, url-encode args by unban
+* `action.d/*ipset*`: make `maxelem` ipset option configurable through banaction arguments (gh-3564)
 
 ### New Features and Enhancements
+* `fail2ban-regex` extended to load settings from jail (by simple name it'd prefer jail to the filter now, gh-2655);
+  to load the settings from filter one could use:
+```diff
+- fail2ban-regex ... sshd           ; # jail
++ fail2ban-regex ... sshd.conf      ; # filter
+# or:
++ fail2ban-regex ... filter.d/sshd  ; # filter
+```
 * better auto-detection for IPv6 support (`allowipv6 = auto` by default), trying to check sysctl net.ipv6.conf.all.disable_ipv6
   (value read from `/proc/sys/net/ipv6/conf/all/disable_ipv6`) if available, otherwise seeks over local IPv6 from network interfaces
   if available for platform and uses DNS to find local IPv6 as a fallback only
 * improve `ignoreself` by considering all local addresses from network interfaces additionally to IPs from hostnames (gh-3132)
+* `action.d/mikrotik.conf` - new action for mikrotik routerOS, adds and removes entries from address lists on the router (gh-2860)
+* `action.d/smtp.py` - added optional support for TLS connections via the `ssl` arg.
+* `filter.d/exim.conf` - fixed "dropped: too many ..." regex, also matching unrecognized commands now (gh-3502)
+* `filter.d/nginx-forbidden.conf` - new filter to ban forbidden locations, e. g. using `deny` directive (gh-2226)
+* `filter.d/sshd.conf`:
+  - avoid double counting for "maximum authentication attempts exceeded" (gh-3502)
+  - message "Disconnecting ... Too many authentication failures" is not a failure anymore
 
 
 ver. 1.0.2 (2022/11/09) - finally-war-game-test-tape-not-a-nuclear-alarm
@@ -66,7 +83,7 @@ ver. 1.0.1 (2022/09/27) - energy-equals-mass-times-the-speed-of-light-squared
 * [stability] solves race condition with uncontrolled growth of failure list (jail with too many matches,
   that did not cause ban), behavior changed to ban ASAP, gh-2945
 * fixes search for the best datepattern - e. g. if line is too short, boundaries check for previously known
-  unprecise pattern may fail on incomplete lines (logging break-off, no flush, etc), gh-3020
+  imprecise pattern may fail on incomplete lines (logging break-off, no flush, etc), gh-3020
 * [stability, performance] backend `systemd`:
   - fixes error "local variable 'line' referenced before assignment", introduced in 55d7d9e2, gh-3097
   - don't update database too often (every 10 ticks or ~ 10 seconds in production)
@@ -404,7 +421,7 @@ filter = flt[logtype=short]
 * `filter.d/znc-adminlog.conf`: new filter for ZNC (IRC bouncer); requires the adminlog module to be loaded
 
 ### Enhancements
-* introduced new options: `dbmaxmatches` (fail2ban.conf) and `maxmatches` (jail.conf) to contol
+* introduced new options: `dbmaxmatches` (fail2ban.conf) and `maxmatches` (jail.conf) to control
   how many matches per ticket fail2ban can hold in memory and store in database (gh-2402, gh-2118);
 * fail2ban.conf: introduced new section `[Thread]` and option `stacksize` to configure default size
   of the stack for threads running in fail2ban (gh-2356), it could be set in `fail2ban.local` to
@@ -514,7 +531,7 @@ ver. 0.10.3 (2018/04/04) - the-time-is-always-right-to-do-what-is-right
   - fixed root login refused regex (optional port before preauth, gh-2080);
   - avoid banning of legitimate users when pam_unix used in combination with other password method, so
     bypass pam_unix failures if accepted available for this user gh-2070;
-  - amend to gh-1263 with better handling of multiple attempts (failures for different user-names recognized immediatelly);
+  - amend to gh-1263 with better handling of multiple attempts (failures for different user-names recognized immediately);
   - mode `ddos` (and `aggressive`) extended to catch `Connection closed by ... [preauth]`, so in DDOS mode
     it counts failure on closing connection within preauth-stage (gh-2085);
 * `action.d/abuseipdb.conf`: fixed curl cypher errors and comment quote-issue (gh-2044, gh-2101);
@@ -844,7 +861,7 @@ ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc
   sane environment in error case of `actioncheck`.
 * Reporting via abuseipdb.com:
   - Bans can now be reported to abuseipdb
-  - Catagories must be set in the config
+  - Categories must be set in the config
   - Relevant log lines included in report
 
 ### Enhancements
@@ -981,7 +998,7 @@ releases.
     - Rewritten without end-anchor ($), because of potential vulnerability on very long URLs.
 * filter.d/apache-badbots.conf - extended to recognize Jorgee Vulnerability Scanner (gh-1882)
 * filter.d/asterisk.conf
-    - fixed failregex AMI Asterisk authentification failed (see gh-1302)
+    - fixed failregex AMI Asterisk authentication failed (see gh-1302)
     - removed invalid (vulnerable) regex blocking IPs using forign data (from header "from")
       thus not the IP-address that really originates the request (see gh-1927)
     - fixed failregex for the SQL-injection attempts with single-quotes in connect-string (see gh-2011)
@@ -1281,7 +1298,7 @@ ver. 0.9.3 (2015/08/01) - lets-all-stay-friends
 * `filter.d/roundcube-auth.conf`
      - Updated regex to work with 'errors' log (1.0.5 and 1.1.1)
      - Added regex to work with 'userlogins' log
-* `action.d/sendmail*.conf` - use LC_ALL (superseeding LC_TIME) to override
+* `action.d/sendmail*.conf` - use LC_ALL (superseding LC_TIME) to override
   locale on systems with customized LC_ALL
 * performance fix: minimizes connection overhead, close socket only at
   communication end (gh-1099)
@@ -1451,7 +1468,7 @@ ver. 0.9.1 (2014/10/29) - better, faster, stronger
 * Ignored IPs are no longer banned when being restored from persistent
   database
 * Manually unbanned IPs are now removed from persistent database, such they
-  wont be banned again when Fail2Ban is restarted
+  won't be banned again when Fail2Ban is restarted
 * Pass "bantime" parameter to the actions in default jail's action
   definition(s)
 * `filters.d/sieve.conf` - fixed typo in _daemon.  Thanks Jisoo Park
@@ -1742,7 +1759,7 @@ those filters were used.
       all platforms to ensure permissions are the same before and after a ban.
       Closes gh-266. hostsdeny supports daemon_list now too.
     * `action.d/bsd-ipfw` - action option unused. Change blocktype to port unreach
-      instead of deny for consistancy.
+      instead of deny for consistency.
     * `filter.d/dovecot` - added to support different dovecot failure
       "..disallowed plaintext auth". Closes Debian bug #709324
     * `filter.d/roundcube-auth` - timezone offset can be positive or negative
@@ -1932,7 +1949,7 @@ fail2ban-users mailing list and IRC.
 ### New Features
 - Yaroslav Halchenko
     * [9ba27353] Add support for `jail.d/{confilefile}` and `fail2ban.d/{configfile}`
-      to provide additional flexibility to system adminstrators. Thanks to
+      to provide additional flexibility to system administrators. Thanks to
       beilber for the idea. Closes gh-114.
     * [3ce53e87] Add exim filter.
 - Erwan Ben Souiden
@@ -2083,7 +2100,7 @@ ver. 0.8.7 (2012/07/31) - stable
     * [47c03a2] files/nagios - spelling/grammar fixes
     * [b083038] updated Free Software Foundation's address
     * [9092a63] changed TLDs to invalid domains, in accordance with RFC 2606
-    * [642d9af,3282f86] reformated printing of jail's name to be consistent
+    * [642d9af,3282f86] reformatted printing of jail's name to be consistent
       with init's info messages
     * [3282f86] uniform use of capitalized Jail in the messages
 - Leonardo Chiquitto
@@ -2428,7 +2445,7 @@ ver. 0.6.1 (2006/03/16) - stable
 - Fixed crash when time format does not match data
 - Propagated patch from Debian to fix fail2ban search path addition to the path
   search list: now it is added first. Thanks to Nick Craig-Wood
-- Added SMTP authentification for mail notification. Thanks to Markus Hoffmann
+- Added SMTP authentication for mail notification. Thanks to Markus Hoffmann
 - Removed debug mode as it is confusing for people
 - Added parsing of timestamp in TAI64N format (#1275325). Thanks to Mark
   Edgington
@@ -2461,7 +2478,7 @@ ver. 0.5.5 (2005/10/26) - beta
     further adjusted by upstream author).
   * Added -f command line parameter for [findtime].
   * Added a cleanup of firewall rules on emergency shutdown when unknown
-    exception is catched.
+    exception is caught.
   * Fail2ban should not crash now if a wrong file name is specified in config.
   * reordered code a bit so that log targets are setup right after background
     and then only loglevel (verbose, debug) is processed, so the warning could

FILTERS

@@ -129,7 +129,7 @@ Date/Time
 ---------
 
 At the moment, Fail2Ban depends on log lines to have time stamps.  That is why
-before starting to develop failregex, check if your log line format known to
+before starting to develop failregex, check if your log line format is known to
 Fail2Ban.  Copy the time component from the log line and append an IP address to
 test with following command::