From bf10b720fd7e09bb51de251746594afc87617c8b Mon Sep 17 00:00:00 2001 From: Bernard Grymonpon Date: Mon, 7 Oct 2024 19:21:11 +0200 Subject: [PATCH 1/4] an attempt at fixing the 15m vs 1h resolutions (for now) --- custom_components/entsoe/api_client.py | 102 +++--- custom_components/entsoe/test/__init__.py | 0 .../test/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 199 bytes .../test_api_client.cpython-312.pyc | Bin 0 -> 11344 bytes .../entsoe/test/datasets/BE_60M.xml | 129 +++++++ .../entsoe/test/datasets/BE_60M_15M_mix.xml | 331 ++++++++++++++++++ .../entsoe/test/test_api_client.py | 134 +++++++ 7 files changed, 650 insertions(+), 46 deletions(-) create mode 100644 custom_components/entsoe/test/__init__.py create mode 100644 custom_components/entsoe/test/__pycache__/__init__.cpython-312.pyc create mode 100644 custom_components/entsoe/test/__pycache__/test_api_client.cpython-312.pyc create mode 100644 custom_components/entsoe/test/datasets/BE_60M.xml create mode 100644 custom_components/entsoe/test/datasets/BE_60M_15M_mix.xml create mode 100644 custom_components/entsoe/test/test_api_client.py diff --git a/custom_components/entsoe/api_client.py b/custom_components/entsoe/api_client.py index 0495d59..facc429 100644 --- a/custom_components/entsoe/api_client.py +++ b/custom_components/entsoe/api_client.py @@ -70,52 +70,7 @@ def query_day_ahead_prices( if response.status_code == 200: try: - root = self._remove_namespace(ET.fromstring(response.content)) - _LOGGER.debug(f"content: {root}") - series = {} - - # Extract TimeSeries data - for timeseries in root.findall(".//TimeSeries"): - for period in timeseries.findall(".//Period"): - resolution = period.find(".//resolution").text - - if resolution != "PT60M": - continue - - response_start = period.find(".//timeInterval/start").text - start_time = ( - datetime.strptime(response_start, "%Y-%m-%dT%H:%MZ") - .replace(tzinfo=pytz.UTC) - .astimezone() - ) - - response_end = period.find(".//timeInterval/end").text - end_time = ( - datetime.strptime(response_end, "%Y-%m-%dT%H:%MZ") - .replace(tzinfo=pytz.UTC) - .astimezone() - ) - - _LOGGER.debug(f"Period found is from {start_time} till {end_time}") - - for point in period.findall(".//Point"): - position = point.find(".//position").text - price = point.find(".//price.amount").text - hour = int(position) - 1 - series[start_time + timedelta(hours=hour)] = float(price) - - # Now fill in any missing hours - current_time = start_time - last_price = series[current_time] - - while current_time < end_time: # upto excluding! the endtime - if current_time in series: - last_price = series[current_time] # Update to the current price - else: - _LOGGER.debug(f"Extending the price {last_price} of the previous hour to {current_time}") - series[current_time] = last_price # Fill with the last known price - current_time += timedelta(hours=1) - + series = self.parse_price_document(response.content) return dict(sorted(series.items())) except Exception as exc: @@ -125,6 +80,61 @@ def query_day_ahead_prices( print(f"Failed to retrieve data: {response.status_code}") return None + def parse_price_document( + self, document: str + ) -> str: + root = self._remove_namespace(ET.fromstring(document)) + _LOGGER.debug(f"content: {root}") + series = {} + + # Extract TimeSeries data + for timeseries in root.findall(".//TimeSeries"): + for period in timeseries.findall(".//Period"): + resolution = period.find(".//resolution").text + + if resolution == "PT60M" or resolution == "PT1H": + delta_unit = timedelta(hours = 1) + elif resolution == "PT15M": + # we bluntly assume we have only one datapoint per hour + # todo - allow to have an aggregation function + delta_unit = timedelta(minutes=15) + else: + raise(f"Unknown resolution passed: '%s'", resolution) + + response_start = period.find(".//timeInterval/start").text + start_time = ( + datetime.strptime(response_start, "%Y-%m-%dT%H:%MZ") + .replace(tzinfo=pytz.UTC) + .astimezone() + ) + + response_end = period.find(".//timeInterval/end").text + end_time = ( + datetime.strptime(response_end, "%Y-%m-%dT%H:%MZ") + .replace(tzinfo=pytz.UTC) + .astimezone() + ) + + _LOGGER.debug(f"Period found is from {start_time} till {end_time}") + + for point in period.findall(".//Point"): + position = point.find(".//position").text + price = point.find(".//price.amount").text + hour = int(position) - 1 + series[start_time + (int(position) - 1) * delta_unit] = float(price) + + # Now fill in any missing hours + current_time = start_time + last_price = series[current_time] + + while current_time < end_time: # upto excluding! the endtime + if current_time in series: + last_price = series[current_time] # Update to the current price + else: + _LOGGER.debug(f"Extending the price {last_price} of the previous hour to {current_time}") + series[current_time] = last_price # Fill with the last known price + current_time += timedelta(hours=1) + return series class Area(enum.Enum): """ diff --git a/custom_components/entsoe/test/__init__.py b/custom_components/entsoe/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/custom_components/entsoe/test/__pycache__/__init__.cpython-312.pyc b/custom_components/entsoe/test/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..17a89f2ac2786dcd026204e83d4748765218ab15 GIT binary patch literal 199 zcmZ8aOA5k344ttGBJ?0G+FU?9f(ya*&`+#IXHq6pq$lwVp2Z`$^aRq4bY;e+50bo3 zLLO#Q)lwYE KJJDMEMA#Qi3OEt~ literal 0 HcmV?d00001 diff --git a/custom_components/entsoe/test/__pycache__/test_api_client.cpython-312.pyc b/custom_components/entsoe/test/__pycache__/test_api_client.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e9f3b4da8456da162fc9cb65286b3cdde4c1e79b GIT binary patch literal 11344 zcmb`Ne^3z(7t@(EO*2k9oi%Eb@vpx3cK3Vi z+wHxy(-ptB_dcK9xA)z*$K$*Er^3Q~gqOiWS9GBYq0h)0W69B19z4V=*HHvTgeer# zUWJfA-#bDMNtnzvHp#qB=4IB6HOXOX78pG98gbkZw0FjBq~<;LA1H(hs4dE%as zM?enLhax%WQ6yJFlS?!EFxGB3wY5;bR1k3=Z$L<1>Oj~nH;l~d4SiKWbHbe9^V}ov zhv%J5O@5z{_68FvB`Nhx#iT?EzxPDKDJd0;OXHb6Af$JP9&Ev_7jfbwM3NP996*v& zaz%t~xIL0{2~7wQ$9ZokC*nMhLb(waj_ylnRtR}UkS{k~GAb#lKyrL0j(gi1mX%l{ zn$8`kosa9;q5YB$$^QpkIQ&q!j#5A)f|0*dNQBmu3yXBQESy5G79wPPZrrhBc-}J+ zJ}xyLm*#vfne140zeuSA(>_PqrASi~={!*!p9(9AC@KzY)t-Gk|;N3MYdGF+3f^U6NDKG(9Ji;X7Ypld2|qocOaL&IGjLVJPztV^N$;Ya>8qLRC*r&&a9o!$Mj|;}Hy2zt z*Y$v#$940;E#SIda0|Ka6W|tc-6h}_bKRxjmT=utaF=o2GH{o3-Ewd%xNar5PjcNV zaI3lQ3UF(l+GUdCjs1u{lP!itcCQtmOrJaLN={3O zv`d!4k#rHBeMoXD5F1YgPt1g;w7E{{G@frs;^ptiPFCKHcz0lCP;F$rH=@%b9NNy;Lj7;mA>;>iM@sT3A} zxOBrpVWWzKysBRn-zh1*-hQ?H+LlX!Z=ASA%|&^Iw>?$2JXLRu{^anRhi_MR-m31r z8Cj_AQBg&&kbl=(dfQuf%Ukzh{niC<$IYPXB+Y-fH0O}!hq{{bCO^L~Prc>fpy!QW_5=Zc)Skx!Th)j^lpBl`9TRlt@eme;h;0!ZO zkjW9#W%3DQE%VAukh8ny337JVJVDOxnkUHFUGoGvyK9~xXLrpLwXX1m$>eEa4&G(m%+VA#ye*= za|!Ir9P|5NzQQrDfcYxN`~jG+#ax_{Tke$;ddcne;rUv-lAGAlHYCPlv!wJkpDDey zWlC?+KYxIiUkrAcu~E#4;)2MdhN|AhxII&Pi~1-JFxV)GXj|Xi!C<416s2XqquUq^ zeQWg=78`}6DE)iu5A4p3LXy_^Z+9{~HwsBnT88>ZH-n8rQj`|4e%{4kqmUG(RjSWf zY!s5Bv<&rbvw3SMziCH~mN6KzMoWM5=YPNGl(dhw7+Ev7Lb87JCRyI!VaAZGjdbZ} zx`brWdZ4h2!H_Ic1KoFQFM}aj|HLJ`3xgvJhGc1Q_}#& z82Z+(Jkzy{LXnhy5B$2-my7jAFw-yjM1SmU>aw5 zYseTqthx}>FJw&nLKtT-WQ-p5FUJ@R8KZ@M@hF3hjNJ>dA1X7x={M+)nNtDdTV8^< zlu7kgCQl#8PV<3 zVHF9hNmxU|S`zS+3`q8ou#SZFBy1p|k%T4^{3JA!&_V*Kf6E(5*hE4b3GF11Gb)qk z738fXbdc~A3FKJ?xs!zLB<#S!=aIX}`c53u?s#}M5Sy5gyGau%-pM^AkW#rEAR&l@ zVycRlEGl~AUw?7(>u-K7r#fzkQhUWIS=d+&I>|FKv; zEZCRDX}fJ*TqAacagEqj#x-JB8rO(jZCoRE#c_>Ts^dYmL?f2+ctBmI5lekMs4mxt zU4dL9b`^4sSW4spwOS*V8hKEy(TH7DJ5lfjoaDTN%)>sPY>ROF7Scy*~ z>#StGMmAVUqehyn#IKQND{0Y4tCeij$QCQvs*w&Wc}gSOtfW&T+pT1WM!Ezm+9_zP z+lKY%EMUWeI_tGzeLCy6uwHeS&IT+j@YSHsc3W6b9n#sb4cnu$y*6y0&i31|5uJ@% zShspWX9q1Tpgyg$XDlqJ9@1IJ!g|%iIy+)vLG@Xk9ksAtRn*yYHY}{OF$?Qf$8{F5 zu%IgGY{G^`bv9+g;yO#%u%ynWt?a84I+LwTRdj}*Q?uN5oYC3yHteL%W^LG<&gN~{ z3pzVxVLj?;oxNybLG_Hzx*Y6r2GpGn{W`L+pxUFefDH@ktk;J1>8#(v0_rZE4Om!E z9n{%w8#bh~VH>tbXM1hfKAr8iu-^M4IvcgH|LJDrVQqJh(6E5-N9M8jBST(i+M5$9 zbK_?Yb7d(tBPWJ^d1;R*Cc<$^6w~>l7*9rKrpUTi6vujGrbN*v$mDrCg*+_TMepLv{^@VcIj3h^5G^C+$?`l(ZwMq+QeD z)UkAKcp5*75J|hkV+w63)*g3k$~@$#J%LH)Q)v&GBlX}1cCHTInTwSb49N^uDh?guNPe{dMjsP z$(p~QwV$JPpP=f8t~o&{c%$KV<@#Hd>mTALVub?rc_(stFU^1CthnPUKHGnB=-kkS LJt}Ujq(|^?E!@vc literal 0 HcmV?d00001 diff --git a/custom_components/entsoe/test/datasets/BE_60M.xml b/custom_components/entsoe/test/datasets/BE_60M.xml new file mode 100644 index 0000000..8828789 --- /dev/null +++ b/custom_components/entsoe/test/datasets/BE_60M.xml @@ -0,0 +1,129 @@ + + + c3891474d3014008b9c6036658010b88 + 1 + A44 + 10X1001A1001A450 + A32 + 10X1001A1001A450 + A33 + 2024-10-07T14:34:54Z + + 2024-10-07T22:00Z + 2024-10-08T22:00Z + + + 1 + A01 + A62 + 10YBE----------2 + 10YBE----------2 + A01 + EUR + MWH + A03 + + + 2024-10-07T22:00Z + 2024-10-08T22:00Z + + PT60M + + 1 + 64.98 + + + 2 + 57.86 + + + 3 + 53.73 + + + 4 + 47.52 + + + 5 + 47.05 + + + 6 + 56.89 + + + 7 + 77.77 + + + 8 + 88.24 + + + 9 + 100 + + + 10 + 84.92 + + + 11 + 74.6 + + + 12 + 68.82 + + + 13 + 60.56 + + + 14 + 63.86 + + + 15 + 68.1 + + + 16 + 68.37 + + + 17 + 76.35 + + + 18 + 54.04 + + + 19 + 98.97 + + + 20 + 115.47 + + + 21 + 86.85 + + + 22 + 69.59 + + + 23 + 57.42 + + + 24 + 50 + + + + diff --git a/custom_components/entsoe/test/datasets/BE_60M_15M_mix.xml b/custom_components/entsoe/test/datasets/BE_60M_15M_mix.xml new file mode 100644 index 0000000..f668ced --- /dev/null +++ b/custom_components/entsoe/test/datasets/BE_60M_15M_mix.xml @@ -0,0 +1,331 @@ + + + 64e2af3a87c2404cbea80edc067a1b6f + 1 + A44 + 10X1001A1001A450 + A32 + 10X1001A1001A450 + A33 + 2024-10-07T14:36:40Z + + 2024-10-05T22:00Z + 2024-10-06T22:00Z + + + 1 + A01 + A62 + 10YBE----------2 + 10YBE----------2 + A01 + EUR + MWH + A03 + + + 2024-10-05T22:00Z + 2024-10-06T22:00Z + + PT15M + + 1 + 55.35 + + + 5 + 44.22 + + + 9 + 40.32 + + + 13 + 31.86 + + + 17 + 28.37 + + + 21 + 28.71 + + + 25 + 31.75 + + + 29 + 35.47 + + + 33 + 37.8 + + + 37 + 33.31 + + + 41 + 33.79 + + + 45 + 16.68 + + + 49 + 5.25 + + + 53 + -0.01 + + + 61 + 0.2 + + + 65 + 48.4 + + + 69 + 50.01 + + + 73 + 65.63 + + + 77 + 77.18 + + + 81 + 81.92 + + + 85 + 64.36 + + + 89 + 60.79 + + + 93 + 52.33 + + + + + 2024-10-06T22:00Z + 2024-10-07T22:00Z + + PT15M + + 1 + 34.58 + + + 5 + 35.34 + + + 9 + 33.25 + + + 13 + 29.48 + + + 17 + 31.88 + + + 21 + 41.35 + + + 25 + 57.14 + + + 29 + 91.84 + + + 33 + 108.32 + + + 37 + 91.8 + + + 41 + 66.05 + + + 45 + 60.21 + + + 49 + 56.02 + + + 53 + 43.29 + + + 57 + 55 + + + 61 + 57.6 + + + 65 + 81.16 + + + 69 + 104.54 + + + 73 + 159.2 + + + 77 + 149.41 + + + 81 + 121.49 + + + 85 + 90 + + + 89 + 90.44 + + + 93 + 77.18 + + + + + 2024-10-07T22:00Z + 2024-10-08T22:00Z + + PT60M + + 1 + 64.98 + + + 2 + 57.86 + + + 3 + 53.73 + + + 4 + 47.52 + + + 5 + 47.05 + + + 6 + 56.89 + + + 7 + 77.77 + + + 8 + 88.24 + + + 9 + 100 + + + 10 + 84.92 + + + 11 + 74.6 + + + 12 + 68.82 + + + 13 + 60.56 + + + 14 + 63.86 + + + 15 + 68.1 + + + 16 + 68.37 + + + 17 + 76.35 + + + 18 + 54.04 + + + 19 + 98.97 + + + 20 + 115.47 + + + 21 + 86.85 + + + 22 + 69.59 + + + 23 + 57.42 + + + 24 + 50 + + + + diff --git a/custom_components/entsoe/test/test_api_client.py b/custom_components/entsoe/test/test_api_client.py new file mode 100644 index 0000000..3eee94c --- /dev/null +++ b/custom_components/entsoe/test/test_api_client.py @@ -0,0 +1,134 @@ +import unittest + +import sys +import os +sys.path.append(os.path.abspath('../')) + +from entsoe.api_client import EntsoeClient +from datetime import datetime + +class TestDocumentParsing(unittest.TestCase): + client: EntsoeClient + + def setUp(self) -> None: + self.client = EntsoeClient("fake-key") + return super().setUp() + + def test_be_60m(self): + with open("test/datasets/BE_60M.xml") as f: + data = f.read() + + self.assertDictEqual(self.client.parse_price_document(data), { + datetime.fromisoformat("2024-10-07T22:00:00Z"): 64.98, + datetime.fromisoformat("2024-10-07T23:00:00Z"): 57.86, + datetime.fromisoformat("2024-10-08T00:00:00Z"): 53.73, + datetime.fromisoformat("2024-10-08T01:00:00Z"): 47.52, + datetime.fromisoformat("2024-10-08T02:00:00Z"): 47.05, + datetime.fromisoformat("2024-10-08T03:00:00Z"): 56.89, + datetime.fromisoformat("2024-10-08T04:00:00Z"): 77.77, + datetime.fromisoformat("2024-10-08T05:00:00Z"): 88.24, + datetime.fromisoformat("2024-10-08T06:00:00Z"): 100, + datetime.fromisoformat("2024-10-08T07:00:00Z"): 84.92, + datetime.fromisoformat("2024-10-08T08:00:00Z"): 74.6, + datetime.fromisoformat("2024-10-08T09:00:00Z"): 68.82, + datetime.fromisoformat("2024-10-08T10:00:00Z"): 60.56, + datetime.fromisoformat("2024-10-08T11:00:00Z"): 63.86, + datetime.fromisoformat("2024-10-08T12:00:00Z"): 68.1, + datetime.fromisoformat("2024-10-08T13:00:00Z"): 68.37, + datetime.fromisoformat("2024-10-08T14:00:00Z"): 76.35, + datetime.fromisoformat("2024-10-08T15:00:00Z"): 54.04, + datetime.fromisoformat("2024-10-08T16:00:00Z"): 98.97, + datetime.fromisoformat("2024-10-08T17:00:00Z"): 115.47, + datetime.fromisoformat("2024-10-08T18:00:00Z"): 86.85, + datetime.fromisoformat("2024-10-08T19:00:00Z"): 69.59, + datetime.fromisoformat("2024-10-08T20:00:00Z"): 57.42, + datetime.fromisoformat("2024-10-08T21:00:00Z"): 50, + }) + + def test_be_60m_15m_mix(self): + with open("test/datasets/BE_60M_15M_mix.xml") as f: + data = f.read() + + self.maxDiff = None + self.assertDictEqual(self.client.parse_price_document(data), { + # part 1 - 15M resolution + datetime.fromisoformat("2024-10-05T22:00:00Z"): 55.35, + datetime.fromisoformat("2024-10-05T23:00:00Z"): 44.22, + datetime.fromisoformat("2024-10-06T00:00:00Z"): 40.32, + datetime.fromisoformat("2024-10-06T01:00:00Z"): 31.86, + datetime.fromisoformat("2024-10-06T02:00:00Z"): 28.37, + datetime.fromisoformat("2024-10-06T03:00:00Z"): 28.71, + datetime.fromisoformat("2024-10-06T04:00:00Z"): 31.75, + datetime.fromisoformat("2024-10-06T05:00:00Z"): 35.47, + datetime.fromisoformat("2024-10-06T06:00:00Z"): 37.8, + datetime.fromisoformat("2024-10-06T07:00:00Z"): 33.31, + datetime.fromisoformat("2024-10-06T08:00:00Z"): 33.79, + datetime.fromisoformat("2024-10-06T09:00:00Z"): 16.68, + datetime.fromisoformat("2024-10-06T10:00:00Z"): 5.25, + datetime.fromisoformat("2024-10-06T11:00:00Z"): -0.01, + datetime.fromisoformat("2024-10-06T12:00:00Z"): -0.01, # repeated value, not present in the dataset! + datetime.fromisoformat("2024-10-06T13:00:00Z"): 0.2, + datetime.fromisoformat("2024-10-06T14:00:00Z"): 48.4, + datetime.fromisoformat("2024-10-06T15:00:00Z"): 50.01, + datetime.fromisoformat("2024-10-06T16:00:00Z"): 65.63, + datetime.fromisoformat("2024-10-06T17:00:00Z"): 77.18, + datetime.fromisoformat("2024-10-06T18:00:00Z"): 81.92, + datetime.fromisoformat("2024-10-06T19:00:00Z"): 64.36, + datetime.fromisoformat("2024-10-06T20:00:00Z"): 60.79, + datetime.fromisoformat("2024-10-06T21:00:00Z"): 52.33, + + # part 2 - 15M resolution + datetime.fromisoformat("2024-10-06T22:00:00Z"): 34.58, + datetime.fromisoformat("2024-10-06T23:00:00Z"): 35.34, + datetime.fromisoformat("2024-10-07T00:00:00Z"): 33.25, + datetime.fromisoformat("2024-10-07T01:00:00Z"): 29.48, + datetime.fromisoformat("2024-10-07T02:00:00Z"): 31.88, + datetime.fromisoformat("2024-10-07T03:00:00Z"): 41.35, + datetime.fromisoformat("2024-10-07T04:00:00Z"): 57.14, + datetime.fromisoformat("2024-10-07T05:00:00Z"): 91.84, + datetime.fromisoformat("2024-10-07T06:00:00Z"): 108.32, + datetime.fromisoformat("2024-10-07T07:00:00Z"): 91.8, + datetime.fromisoformat("2024-10-07T08:00:00Z"): 66.05, + datetime.fromisoformat("2024-10-07T09:00:00Z"): 60.21, + datetime.fromisoformat("2024-10-07T10:00:00Z"): 56.02, + datetime.fromisoformat("2024-10-07T11:00:00Z"): 43.29, + datetime.fromisoformat("2024-10-07T12:00:00Z"): 55, + datetime.fromisoformat("2024-10-07T13:00:00Z"): 57.6, + datetime.fromisoformat("2024-10-07T14:00:00Z"): 81.16, + datetime.fromisoformat("2024-10-07T15:00:00Z"): 104.54, + datetime.fromisoformat("2024-10-07T16:00:00Z"): 159.2, + datetime.fromisoformat("2024-10-07T17:00:00Z"): 149.41, + datetime.fromisoformat("2024-10-07T18:00:00Z"): 121.49, + datetime.fromisoformat("2024-10-07T19:00:00Z"): 90, + datetime.fromisoformat("2024-10-07T20:00:00Z"): 90.44, + datetime.fromisoformat("2024-10-07T21:00:00Z"): 77.18, + + # part 3 - 60M resolution + datetime.fromisoformat("2024-10-07T22:00:00Z"): 64.98, + datetime.fromisoformat("2024-10-07T23:00:00Z"): 57.86, + datetime.fromisoformat("2024-10-08T00:00:00Z"): 53.73, + datetime.fromisoformat("2024-10-08T01:00:00Z"): 47.52, + datetime.fromisoformat("2024-10-08T02:00:00Z"): 47.05, + datetime.fromisoformat("2024-10-08T03:00:00Z"): 56.89, + datetime.fromisoformat("2024-10-08T04:00:00Z"): 77.77, + datetime.fromisoformat("2024-10-08T05:00:00Z"): 88.24, + datetime.fromisoformat("2024-10-08T06:00:00Z"): 100, + datetime.fromisoformat("2024-10-08T07:00:00Z"): 84.92, + datetime.fromisoformat("2024-10-08T08:00:00Z"): 74.6, + datetime.fromisoformat("2024-10-08T09:00:00Z"): 68.82, + datetime.fromisoformat("2024-10-08T10:00:00Z"): 60.56, + datetime.fromisoformat("2024-10-08T11:00:00Z"): 63.86, + datetime.fromisoformat("2024-10-08T12:00:00Z"): 68.1, + datetime.fromisoformat("2024-10-08T13:00:00Z"): 68.37, + datetime.fromisoformat("2024-10-08T14:00:00Z"): 76.35, + datetime.fromisoformat("2024-10-08T15:00:00Z"): 54.04, + datetime.fromisoformat("2024-10-08T16:00:00Z"): 98.97, + datetime.fromisoformat("2024-10-08T17:00:00Z"): 115.47, + datetime.fromisoformat("2024-10-08T18:00:00Z"): 86.85, + datetime.fromisoformat("2024-10-08T19:00:00Z"): 69.59, + datetime.fromisoformat("2024-10-08T20:00:00Z"): 57.42, + datetime.fromisoformat("2024-10-08T21:00:00Z"): 50, + }) + +if __name__ == '__main__': + unittest.main() From fcf4a8686d219fe4335308e28e00e5737ea81614 Mon Sep 17 00:00:00 2001 From: Bernard Grymonpon Date: Mon, 7 Oct 2024 19:24:59 +0200 Subject: [PATCH 2/4] silly cache --- .../test/__pycache__/__init__.cpython-312.pyc | Bin 199 -> 0 bytes .../__pycache__/test_api_client.cpython-312.pyc | Bin 11344 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 custom_components/entsoe/test/__pycache__/__init__.cpython-312.pyc delete mode 100644 custom_components/entsoe/test/__pycache__/test_api_client.cpython-312.pyc diff --git a/custom_components/entsoe/test/__pycache__/__init__.cpython-312.pyc b/custom_components/entsoe/test/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 17a89f2ac2786dcd026204e83d4748765218ab15..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 199 zcmZ8aOA5k344ttGBJ?0G+FU?9f(ya*&`+#IXHq6pq$lwVp2Z`$^aRq4bY;e+50bo3 zLLO#Q)lwYE KJJDMEMA#Qi3OEt~ diff --git a/custom_components/entsoe/test/__pycache__/test_api_client.cpython-312.pyc b/custom_components/entsoe/test/__pycache__/test_api_client.cpython-312.pyc deleted file mode 100644 index e9f3b4da8456da162fc9cb65286b3cdde4c1e79b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11344 zcmb`Ne^3z(7t@(EO*2k9oi%Eb@vpx3cK3Vi z+wHxy(-ptB_dcK9xA)z*$K$*Er^3Q~gqOiWS9GBYq0h)0W69B19z4V=*HHvTgeer# zUWJfA-#bDMNtnzvHp#qB=4IB6HOXOX78pG98gbkZw0FjBq~<;LA1H(hs4dE%as zM?enLhax%WQ6yJFlS?!EFxGB3wY5;bR1k3=Z$L<1>Oj~nH;l~d4SiKWbHbe9^V}ov zhv%J5O@5z{_68FvB`Nhx#iT?EzxPDKDJd0;OXHb6Af$JP9&Ev_7jfbwM3NP996*v& zaz%t~xIL0{2~7wQ$9ZokC*nMhLb(waj_ylnRtR}UkS{k~GAb#lKyrL0j(gi1mX%l{ zn$8`kosa9;q5YB$$^QpkIQ&q!j#5A)f|0*dNQBmu3yXBQESy5G79wPPZrrhBc-}J+ zJ}xyLm*#vfne140zeuSA(>_PqrASi~={!*!p9(9AC@KzY)t-Gk|;N3MYdGF+3f^U6NDKG(9Ji;X7Ypld2|qocOaL&IGjLVJPztV^N$;Ya>8qLRC*r&&a9o!$Mj|;}Hy2zt z*Y$v#$940;E#SIda0|Ka6W|tc-6h}_bKRxjmT=utaF=o2GH{o3-Ewd%xNar5PjcNV zaI3lQ3UF(l+GUdCjs1u{lP!itcCQtmOrJaLN={3O zv`d!4k#rHBeMoXD5F1YgPt1g;w7E{{G@frs;^ptiPFCKHcz0lCP;F$rH=@%b9NNy;Lj7;mA>;>iM@sT3A} zxOBrpVWWzKysBRn-zh1*-hQ?H+LlX!Z=ASA%|&^Iw>?$2JXLRu{^anRhi_MR-m31r z8Cj_AQBg&&kbl=(dfQuf%Ukzh{niC<$IYPXB+Y-fH0O}!hq{{bCO^L~Prc>fpy!QW_5=Zc)Skx!Th)j^lpBl`9TRlt@eme;h;0!ZO zkjW9#W%3DQE%VAukh8ny337JVJVDOxnkUHFUGoGvyK9~xXLrpLwXX1m$>eEa4&G(m%+VA#ye*= za|!Ir9P|5NzQQrDfcYxN`~jG+#ax_{Tke$;ddcne;rUv-lAGAlHYCPlv!wJkpDDey zWlC?+KYxIiUkrAcu~E#4;)2MdhN|AhxII&Pi~1-JFxV)GXj|Xi!C<416s2XqquUq^ zeQWg=78`}6DE)iu5A4p3LXy_^Z+9{~HwsBnT88>ZH-n8rQj`|4e%{4kqmUG(RjSWf zY!s5Bv<&rbvw3SMziCH~mN6KzMoWM5=YPNGl(dhw7+Ev7Lb87JCRyI!VaAZGjdbZ} zx`brWdZ4h2!H_Ic1KoFQFM}aj|HLJ`3xgvJhGc1Q_}#& z82Z+(Jkzy{LXnhy5B$2-my7jAFw-yjM1SmU>aw5 zYseTqthx}>FJw&nLKtT-WQ-p5FUJ@R8KZ@M@hF3hjNJ>dA1X7x={M+)nNtDdTV8^< zlu7kgCQl#8PV<3 zVHF9hNmxU|S`zS+3`q8ou#SZFBy1p|k%T4^{3JA!&_V*Kf6E(5*hE4b3GF11Gb)qk z738fXbdc~A3FKJ?xs!zLB<#S!=aIX}`c53u?s#}M5Sy5gyGau%-pM^AkW#rEAR&l@ zVycRlEGl~AUw?7(>u-K7r#fzkQhUWIS=d+&I>|FKv; zEZCRDX}fJ*TqAacagEqj#x-JB8rO(jZCoRE#c_>Ts^dYmL?f2+ctBmI5lekMs4mxt zU4dL9b`^4sSW4spwOS*V8hKEy(TH7DJ5lfjoaDTN%)>sPY>ROF7Scy*~ z>#StGMmAVUqehyn#IKQND{0Y4tCeij$QCQvs*w&Wc}gSOtfW&T+pT1WM!Ezm+9_zP z+lKY%EMUWeI_tGzeLCy6uwHeS&IT+j@YSHsc3W6b9n#sb4cnu$y*6y0&i31|5uJ@% zShspWX9q1Tpgyg$XDlqJ9@1IJ!g|%iIy+)vLG@Xk9ksAtRn*yYHY}{OF$?Qf$8{F5 zu%IgGY{G^`bv9+g;yO#%u%ynWt?a84I+LwTRdj}*Q?uN5oYC3yHteL%W^LG<&gN~{ z3pzVxVLj?;oxNybLG_Hzx*Y6r2GpGn{W`L+pxUFefDH@ktk;J1>8#(v0_rZE4Om!E z9n{%w8#bh~VH>tbXM1hfKAr8iu-^M4IvcgH|LJDrVQqJh(6E5-N9M8jBST(i+M5$9 zbK_?Yb7d(tBPWJ^d1;R*Cc<$^6w~>l7*9rKrpUTi6vujGrbN*v$mDrCg*+_TMepLv{^@VcIj3h^5G^C+$?`l(ZwMq+QeD z)UkAKcp5*75J|hkV+w63)*g3k$~@$#J%LH)Q)v&GBlX}1cCHTInTwSb49N^uDh?guNPe{dMjsP z$(p~QwV$JPpP=f8t~o&{c%$KV<@#Hd>mTALVub?rc_(stFU^1CthnPUKHGnB=-kkS LJt}Ujq(|^?E!@vc From 2b32daf4b6485c3506811f3a9879c5482619ac26 Mon Sep 17 00:00:00 2001 From: Bernard Grymonpon Date: Mon, 7 Oct 2024 20:53:57 +0200 Subject: [PATCH 3/4] fix 15m overlapping data for DE case --- custom_components/entsoe/api_client.py | 19 ++++++++++++++++--- .../entsoe/test/test_api_client.py | 9 +++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/custom_components/entsoe/api_client.py b/custom_components/entsoe/api_client.py index facc429..68dc739 100644 --- a/custom_components/entsoe/api_client.py +++ b/custom_components/entsoe/api_client.py @@ -83,9 +83,14 @@ def query_day_ahead_prices( def parse_price_document( self, document: str ) -> str: + + # note: this function prefers hourly data over 15m interval data. + # if we don't get a hourly datapoint, but we have a 15m datapoint which matches, we use it + root = self._remove_namespace(ET.fromstring(document)) _LOGGER.debug(f"content: {root}") series = {} + series_datasource = {} # Extract TimeSeries data for timeseries in root.findall(".//TimeSeries"): @@ -93,6 +98,7 @@ def parse_price_document( resolution = period.find(".//resolution").text if resolution == "PT60M" or resolution == "PT1H": + resolution = "PT60M" delta_unit = timedelta(hours = 1) elif resolution == "PT15M": # we bluntly assume we have only one datapoint per hour @@ -118,10 +124,16 @@ def parse_price_document( _LOGGER.debug(f"Period found is from {start_time} till {end_time}") for point in period.findall(".//Point"): - position = point.find(".//position").text + position = int(point.find(".//position").text) + if resolution == "PT15M" and (position - 1) % 4: # not interested in non-full-hour-datapoints + continue price = point.find(".//price.amount").text - hour = int(position) - 1 - series[start_time + (int(position) - 1) * delta_unit] = float(price) + key = start_time + (position - 1) * delta_unit + + # only write if we don't have a 60m data (so, either it's empty, or it has 15m data, which we overwrite) + if not key in series_datasource or series_datasource[key] != "PT60M": + series[key] = float(price) + series_datasource[key] = resolution # Now fill in any missing hours current_time = start_time @@ -134,6 +146,7 @@ def parse_price_document( _LOGGER.debug(f"Extending the price {last_price} of the previous hour to {current_time}") series[current_time] = last_price # Fill with the last known price current_time += timedelta(hours=1) + return series class Area(enum.Enum): diff --git a/custom_components/entsoe/test/test_api_client.py b/custom_components/entsoe/test/test_api_client.py index 3eee94c..2a71c8a 100644 --- a/custom_components/entsoe/test/test_api_client.py +++ b/custom_components/entsoe/test/test_api_client.py @@ -130,5 +130,14 @@ def test_be_60m_15m_mix(self): datetime.fromisoformat("2024-10-08T21:00:00Z"): 50, }) + def test_de_60m_15m_overlap(self): + with open("test/datasets/DE_60M_15M_overlap.xml") as f: + data = f.read() + + self.assertEqual(self.client.parse_price_document(data)[datetime.fromisoformat("2024-10-05T22:00:00Z")], 67.04) + self.assertEqual(self.client.parse_price_document(data)[datetime.fromisoformat("2024-10-06T22:00:00Z")], 34.58) + self.assertEqual(self.client.parse_price_document(data)[datetime.fromisoformat("2024-10-07T21:00:00Z")], 79.12) + + if __name__ == '__main__': unittest.main() From 706e0601ec671a56b12864d21ce1290e533d4002 Mon Sep 17 00:00:00 2001 From: Bernard Grymonpon Date: Wed, 9 Oct 2024 07:20:50 +0200 Subject: [PATCH 4/4] add missing DE testdata --- .../test/datasets/DE_60M_15M_overlap.xml | 1015 +++++++++++++++++ 1 file changed, 1015 insertions(+) create mode 100644 custom_components/entsoe/test/datasets/DE_60M_15M_overlap.xml diff --git a/custom_components/entsoe/test/datasets/DE_60M_15M_overlap.xml b/custom_components/entsoe/test/datasets/DE_60M_15M_overlap.xml new file mode 100644 index 0000000..0435931 --- /dev/null +++ b/custom_components/entsoe/test/datasets/DE_60M_15M_overlap.xml @@ -0,0 +1,1015 @@ + + + 2d27edd953004059aa96e104fd2d6aee + 1 + A44 + 10X1001A1001A450 + A32 + 10X1001A1001A450 + A33 + 2024-10-07T18:08:18Z + + 2024-10-05T22:00Z + 2024-10-06T22:00Z + + + 1 + A01 + A62 + 10Y1001A1001A82H + 10Y1001A1001A82H + A01 + EUR + MWH + 1 + A03 + + + 2024-10-05T22:00Z + 2024-10-06T22:00Z + + PT60M + + 1 + 67.04 + + + 2 + 63.97 + + + 3 + 62.83 + + + 4 + 63.35 + + + 5 + 62.71 + + + 6 + 63.97 + + + 7 + 63.41 + + + 8 + 72.81 + + + 9 + 77.2 + + + 10 + 66.06 + + + 11 + 35.28 + + + 12 + 16.68 + + + 13 + 5.25 + + + 14 + -0.01 + + + 16 + 0.2 + + + 17 + 59.6 + + + 18 + 90.94 + + + 19 + 106.3 + + + 20 + 97.22 + + + 21 + 72.98 + + + 22 + 59.37 + + + 23 + 58.69 + + + 24 + 51.71 + + + + + 2024-10-06T22:00Z + 2024-10-07T22:00Z + + PT60M + + 1 + 34.58 + + + 2 + 35.34 + + + 3 + 33.25 + + + 4 + 30.15 + + + 5 + 36.09 + + + 6 + 46.73 + + + 7 + 67.59 + + + 8 + 100.92 + + + 9 + 108.32 + + + 10 + 91.86 + + + 11 + 66.09 + + + 12 + 60.22 + + + 13 + 54.11 + + + 14 + 43.29 + + + 15 + 55 + + + 16 + 67.01 + + + 17 + 97.9 + + + 18 + 120.71 + + + 19 + 237.65 + + + 20 + 229.53 + + + 21 + 121.98 + + + 22 + 99.93 + + + 23 + 91.91 + + + 24 + 79.12 + + + + + 2 + A01 + A62 + 10Y1001A1001A82H + 10Y1001A1001A82H + A01 + EUR + MWH + 2 + A03 + + + 2024-10-05T22:00Z + 2024-10-06T22:00Z + + PT15M + + 1 + 98.1 + + + 2 + 89.5 + + + 3 + 77.21 + + + 4 + 40.09 + + + 5 + 87.2 + + + 6 + 80.1 + + + 7 + 75.3 + + + 8 + 51.34 + + + 9 + 76.8 + + + 10 + 70.8 + + + 11 + 69.1 + + + 12 + 67.3 + + + 13 + 72.4 + + + 14 + 70.3 + + + 15 + 65.17 + + + 16 + 70.4 + + + 17 + 72.88 + + + 18 + 67.1 + + + 19 + 67.7 + + + 20 + 66.7 + + + 21 + 71.8 + + + 22 + 68.2 + + + 24 + 63.76 + + + 25 + 82 + + + 26 + 72.9 + + + 27 + 73.65 + + + 28 + 45.1 + + + 29 + 74.91 + + + 30 + 79.91 + + + 31 + 79.9 + + + 32 + 41.29 + + + 33 + 107.8 + + + 34 + 86.6 + + + 35 + 55.09 + + + 36 + 15.09 + + + 37 + 104.9 + + + 38 + 74.9 + + + 39 + 54.91 + + + 40 + 5.85 + + + 41 + 79.8 + + + 42 + 52.9 + + + 43 + 28.43 + + + 44 + -19.91 + + + 45 + 59.1 + + + 46 + 41.3 + + + 47 + 0.49 + + + 48 + -29.9 + + + 49 + 44.91 + + + 50 + 30.3 + + + 51 + -14.27 + + + 52 + -29.91 + + + 53 + 24.92 + + + 54 + 20.2 + + + 55 + -13.21 + + + 56 + -14.9 + + + 57 + -24.9 + + + 58 + -19.9 + + + 59 + 18.18 + + + 60 + 36.9 + + + 61 + -29.91 + + + 62 + -14.9 + + + 63 + 12.13 + + + 64 + 67.9 + + + 65 + -19.91 + + + 66 + 27.33 + + + 67 + 74.1 + + + 68 + 101.8 + + + 69 + 30.72 + + + 70 + 74.6 + + + 71 + 103.4 + + + 72 + 144.92 + + + 73 + 78.5 + + + 74 + 102.35 + + + 75 + 119.94 + + + 76 + 129.95 + + + 77 + 114.57 + + + 78 + 114.91 + + + 79 + 109.91 + + + 80 + 113.6 + + + 81 + 114.91 + + + 82 + 109.9 + + + 83 + 67.77 + + + 84 + 86.6 + + + 85 + 104.92 + + + 86 + 99.91 + + + 87 + 84.07 + + + 88 + 35.1 + + + 89 + 94.92 + + + 90 + 89.91 + + + 91 + 79.9 + + + 92 + 30.24 + + + 93 + 89.92 + + + 94 + 79.9 + + + 95 + 55.86 + + + 96 + 25.09 + + + + + 2024-10-06T22:00Z + 2024-10-07T22:00Z + + PT15M + + 1 + 79.91 + + + 2 + 70.92 + + + 3 + 65.1 + + + 4 + 15.09 + + + 5 + 64.91 + + + 7 + 63.1 + + + 8 + 29.93 + + + 9 + 64.91 + + + 10 + 61 + + + 11 + 59.4 + + + 12 + 13.01 + + + 13 + 59.91 + + + 14 + 16.36 + + + 15 + 59.6 + + + 16 + 60.9 + + + 17 + 49.9 + + + 18 + 34.93 + + + 19 + 62.1 + + + 20 + 74.8 + + + 21 + -4.93 + + + 22 + 57.6 + + + 23 + 84.92 + + + 24 + 130.19 + + + 25 + 51.2 + + + 26 + 84.03 + + + 27 + 107.7 + + + 28 + 130.3 + + + 29 + 89 + + + 30 + 109.92 + + + 31 + 124.3 + + + 32 + 133 + + + 33 + 137.5 + + + 34 + 122.38 + + + 35 + 119.91 + + + 36 + 109.9 + + + 37 + 124.2 + + + 38 + 108 + + + 39 + 86.5 + + + 40 + 72 + + + 41 + 113 + + + 42 + 90.7 + + + 43 + 70.33 + + + 44 + 47.2 + + + 45 + 91.8 + + + 46 + 62.9 + + + 47 + 52.9 + + + 48 + 41.04 + + + 49 + 66.1 + + + 50 + 55.28 + + + 51 + 47.9 + + + 52 + 45 + + + 53 + 63.5 + + + 54 + 51.83 + + + 55 + 39 + + + 56 + 40.08 + + + 57 + 42.5 + + + 58 + 49.41 + + + 59 + 40.1 + + + 60 + 67.3 + + + 61 + 30.09 + + + 62 + 40.09 + + + 63 + 85.76 + + + 64 + 85.1 + + + 65 + 62.7 + + + 66 + 90.8 + + + 67 + 90.33 + + + 68 + 115.1 + + + 69 + 80.19 + + + 70 + 100.8 + + + 71 + 121.9 + + + 72 + 148.2 + + + 73 + 117.5 + + + 74 + 134.77 + + + 75 + 153.1 + + + 76 + 164.96 + + + 77 + 164.95 + + + 78 + 159.21 + + + 79 + 159.92 + + + 80 + 159.91 + + + 81 + 189.94 + + + 82 + 132.78 + + + 83 + 95.07 + + + 84 + 75.06 + + + 85 + 149.98 + + + 86 + 116.9 + + + 87 + 101.8 + + + 88 + 63.85 + + + 89 + 134.98 + + + 90 + 115.8 + + + 91 + 94.1 + + + 92 + 43.74 + + + 93 + 109.7 + + + 94 + 94.3 + + + 95 + 82 + + + 96 + 70.39 + + + +