feat: switch from edi-energy.de and scraping to bdew-mako.de and a real API; drop support for Python 3.9

.github/workflows/unittests.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v4

.gitignore

@@ -2,7 +2,7 @@
 __pycache__/
 *.py[cod]
 *$py.class
-
+foo
 # C extensions
 *.so
 

README.md

@@ -8,11 +8,11 @@
 ![Black status badge](https://github.com/Hochfrequenz/edi_energy_scraper/workflows/Black/badge.svg)
 ![PyPi Status Badge](https://img.shields.io/pypi/v/edi_energy_scraper)
 
-The Python package `edi_energy_scraper` provides easy to use methods to mirror the website edi-energy.de.
+The Python package `edi_energy_scraper` provides easy to use methods to mirror the free documents on bdew-mako.de.
 
 ### Rationale / Why?
 
-If you'd like to be informed about new regulations or data formats being published on edi-energy.de you can either
+If you'd like to be informed about new regulations or data formats being published on bdew-mako.de you can either
 
 - visit the site every day and hope that you see the changes if this is your favourite hobby,
 - or automate the task.
@@ -44,7 +44,8 @@ Then import it and start the download:
 
 ```python
 import asyncio
-from edi_energy_scraper import EdiEnergyScraper
+from edi_energy_scraper.scraper import EdiEnergyScraper
+
 
 # add the following lines to enable debug logging to stdout (CLI)
 # import logging

mwe.py

@@ -3,12 +3,15 @@
 """
 
 import asyncio
+from pathlib import Path
 
-from edi_energy_scraper import EdiEnergyScraper
+from edi_energy_scraper.scraper import EdiEnergyScraper
+
+my_target_dir = Path(__file__).parent / "foo"
 
 
 async def mirror():
-    scraper = EdiEnergyScraper(path_to_mirror_directory="edi_energy_de")
+    scraper = EdiEnergyScraper(path_to_mirror_directory=my_target_dir)
     await scraper.mirror()
 
 

pyproject.toml

@@ -14,7 +14,7 @@ build-backend = "hatchling.build"
 name = "edi_energy_scraper"
 description = "a scraper to mirror edi-energy.de"
 license = { text = "MIT" }
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 authors = [
     { name = "Hochfrequenz Unternehmensberatung GmbH", email = "info+github@hochfrequenz.de" },
 ]
@@ -27,19 +27,19 @@ classifiers = [
     "Operating System :: OS Independent",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
-    "beautifulsoup4>=4.11.1",
     "aiohttp>=3.8.4",
-    "aiohttp-requests>=0.2.2",
     "pypdf>=3.4.1",
     "efoli>=1.4.0",
     "pytz>=2022.7.1",
+    "pydantic>=2",
+    "pytz>=2024.2",
+    "more_itertools"
 ]
 dynamic = ["readme", "version"]
 
@@ -66,8 +66,8 @@ tests = [
     "freezegun==1.5.1",
     "pytest==8.3.4",
     "pytest-asyncio==0.25.0",
-    "pytest-datafiles==3.0.0",
-    "pytest-mock==3.14.0"
+    "pytest-mock==3.14.0",
+    "syrupy==4.8.0"
 ]
 type_check = [
     "mypy==1.14.1",
@@ -115,8 +115,8 @@ max-line-length = 120
 # even if they have no @pytest.mark.asyncio marker.
 # https://github.com/pytest-dev/pytest-asyncio#auto-mode
 asyncio_mode = "auto"
-markers = ["datafiles"]
 
+markers = ["snapshot: mark a test as a snapshot test"]
 # the following lines are needed if you would like to build a python package
 # and you want to use semantic versioning
 # [build-system]

requirements.txt

@@ -4,22 +4,14 @@
 #
 #    pip-compile pyproject.toml
 #
-aiohttp[speedups]==3.9.5
-    # via
-    #   aiohttp-requests
-    #   edi_energy_scraper (pyproject.toml)
-aiohttp-requests==0.2.4
+aiohttp==3.9.5
     # via edi_energy_scraper (pyproject.toml)
 aiosignal==1.3.1
     # via aiohttp
+annotated-types==0.7.0
+    # via pydantic
 attrs==24.2.0
     # via aiohttp
-beautifulsoup4==4.12.3
-    # via edi_energy_scraper (pyproject.toml)
-brotli==1.1.0
-    # via aiohttp
-coworker==2.0.1
-    # via aiohttp-requests
 efoli==1.4.0
     # via edi_energy_scraper (pyproject.toml)
 frozenlist==1.5.0
@@ -28,17 +20,27 @@ frozenlist==1.5.0
     #   aiosignal
 idna==3.10
     # via yarl
+more-itertools==10.5.0
+    # via edi_energy_scraper (pyproject.toml)
 multidict==6.1.0
     # via
     #   aiohttp
     #   yarl
+propcache==0.2.1
+    # via yarl
+pydantic==2.10.4
+    # via edi_energy_scraper (pyproject.toml)
+pydantic-core==2.27.2
+    # via pydantic
 pypdf==5.1.0
     # via edi_energy_scraper (pyproject.toml)
 pytz==2024.2
     # via
     #   edi_energy_scraper (pyproject.toml)
     #   efoli
-soupsieve==2.6
-    # via beautifulsoup4
+typing-extensions==4.12.2
+    # via
+    #   pydantic
+    #   pydantic-core
 yarl==1.18.3
     # via aiohttp