diff --git a/.dockerignore b/.dockerignore index 5b92fc37c9..79a606b2af 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,6 +7,9 @@ flexmeasures.log* # Ignore locally made docs flexmeasures/ui/static/documentation +# Checked out separately by Docker CI for host-side client integration tests +flexmeasures-client + # Ignore all forecasting artifacts (models, predictions, etc.) flexmeasures/data/models/forecasting/**/artifacts/ flexmeasures/data/models/forecasting/**/*.pkl diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index f4554d2682..14819701f9 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -28,6 +28,17 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + - name: Checkout flexmeasures-client + uses: actions/checkout@v4 + with: + repository: FlexMeasures/flexmeasures-client + ref: main + path: flexmeasures-client + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.10.9" + enable-cache: true - name: Build Docker Image run: docker build -t flexmeasures:latest -f Dockerfile . - name: Generate random secret key @@ -48,6 +59,10 @@ jobs: - name: Add toy user run: docker exec --env-file .env fm-container flexmeasures add toy-account --kind battery --shell-vars | grep '^FM_TOY_' >> $GITHUB_ENV + - name: Run data-ingestion tutorial + env: + FLEXMEASURES_CLIENT_PROJECT: ${{ github.workspace }}/flexmeasures-client + run: ./documentation/tut/scripts/run-data-ingestion-in-docker.sh fm-container - name: Generate prices dummy data run: ci/generate-dummy-price.sh - name: Copy prices dummy data diff --git a/.github/workflows/docker-qa.yml b/.github/workflows/docker-qa.yml index f171a4b2db..231d00edcf 100644 --- a/.github/workflows/docker-qa.yml +++ b/.github/workflows/docker-qa.yml @@ -84,6 +84,11 @@ jobs: echo "::endgroup::" done + - name: Run data-ingestion tutorial + env: + FLEXMEASURES_CLIENT_PROJECT: ${{ github.workspace }}/flexmeasures-client + run: ./documentation/tut/scripts/run-data-ingestion-in-docker.sh + - name: Create HEMS admin account and user run: | OUTPUT=$(docker compose exec -T server flexmeasures add account --name "HEMS Admin Org") diff --git a/documentation/changelog.rst b/documentation/changelog.rst index e5d3180401..83d32bffdc 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -72,6 +72,8 @@ New features Infrastructure / Support ---------------------- + +* Add a hands-on data-ingestion tutorial with executable FlexMeasures Client examples for Excel and CSV uploads and export scripts [see `PR #2376 `_] * Add a ``FLEXMEASURES_SENTRY_DAILY_RATE_LIMIT`` setting for spreading a host's Sentry error allowance across the month with a fail-open daily Redis counter, and send the startup error about the database schema not being at the Alembic head revision to Sentry at most once per UTC calendar day per pair of current and expected revisions (it is still logged in full on every start) [see `PR #2366 `_] * Shrink the scheduler's mixed-integer program for one-way devices: where a device can only consume or only produce, its power-sign binaries and their big-M constraints are dropped, as simultaneous consumption and production is already ruled out by the power bounds [see `PR #2412 `_] * ``uv run poe clean-db`` now works on macOS as well, takes its arguments as ``--db-name my-db --db-user my-user``, reads your answers to its prompts, and handles names containing a dash [see `PR #2408 `_] @@ -105,6 +107,7 @@ Infrastructure / Support Bugfixes ----------- +* Include the Excel reader in default installations so XLSX sensor-data uploads work outside test environments [see `PR #2376 `_] * In a multi-device flex-model, a device without a stock (e.g. a converter port or curtailable generator) silently disabled constraint validation for all devices after it; validation now covers every device, and also newly checks that each device's power bounds do not contradict each other, so a contradictory hard bound fails with a clear per-time-step message instead of a bare solver infeasibility [see `PR #2252 `_] * The scheduler now rejects a commitment that no constraint would bind — a stock commitment naming no device or known stock group, or a commodity commitment for a commodity that no commitment maps devices to — instead of silently dropping it from the problem, or letting a favourably priced deviation make the problem unbounded; the error names the commitment [see `PR #2410 `_ and `PR #2413 `_] * Show icons for more asset types in the UI's asset structure view, which previously fell back to a question mark: the ``wind``, ``process`` and ``heat-storage`` types that FlexMeasures seeds by default, and EV infrastructure under its various names (such as ``one-way_evse``, ``two-way_evse``, ``evse``, ``charging_station`` and ``charging_hub``) and building services equipment (``hvac``, ``ahu``, ``dhw``, ``heatpump``, ``chiller``, ``lighting`` and ``other-loads``). Asset type names are now matched ignoring case and separators, so an asset type named ``charge-point`` gets the same icon as ``chargepoint`` [see `PR #2391 `_] diff --git a/documentation/dev/scripting.rst b/documentation/dev/scripting.rst index 85739a80ae..aaf9e7d474 100644 --- a/documentation/dev/scripting.rst +++ b/documentation/dev/scripting.rst @@ -13,6 +13,7 @@ Scripting via the FlexMeasures-Client The most universal way to script FlexMeasures is via `the FlexMeasures Client `_. Actually, this is scripting via the API, as the client is not much more than a wrapper around the FlexMeasures server API. +For a hands-on example which uploads Excel or CSV data and posts values from an export script, see :ref:`tut_posting_data`. Let's look at two examples, to give an impression. The first one creates a sensor: diff --git a/documentation/getting-started.rst b/documentation/getting-started.rst index 6f2aaf23ef..e2893571c6 100644 --- a/documentation/getting-started.rst +++ b/documentation/getting-started.rst @@ -29,6 +29,7 @@ Find an optimized schedule for your flexible asset, like a battery, with standar 2. Automate ^^^^^^^^^^^^^^^^^^^ +Turn spreadsheet or export-script data into a repeatable pipeline with :ref:`tut_posting_data`. Get the prices from an open API, for instance `ENTSO-E `_ (using a plugin like `flexmeasures-entsoe `_), and run the scheduler regularly in a cron job. Do more, like automating forecasts, too. And for scale, copy your successfully-tested asset programmatically (or in the UI), to set up new customers quickly, e.g. as you are servicing the same kind of site often. 3. Integrate @@ -89,4 +90,3 @@ Core developers ^^^^^^^^^^^^^^^^ You want to help develop FlexMeasures, e.g. to fix a bug. We provide a getting-started guide to becoming a developer at :ref:`developing`. - diff --git a/documentation/host/installation.rst b/documentation/host/installation.rst index f1fbccc31e..bc509e744c 100644 --- a/documentation/host/installation.rst +++ b/documentation/host/installation.rst @@ -312,9 +312,10 @@ Below are some additional steps you might consider. Add time series data (beliefs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are three ways to add data: +For a complete walkthrough from a first UI upload to an automated pipeline, see :ref:`tut_posting_data`. +The main ingestion routes are the FlexMeasures Client and the API; hosts can also import files directly with the CLI. -First, you can load in data from a file (CSV or Excel) via the ``flexmeasures`` :ref:`cli`: +To load data from a CSV or Excel file on the server, use the ``flexmeasures`` :ref:`cli`: .. code-block:: bash @@ -323,11 +324,11 @@ First, you can load in data from a file (CSV or Excel) via the ``flexmeasures`` This assumes you have a file `my-data.csv` with measurements, which was exported from some legacy database, and that the data is about our sensor with ID 1. This command has many options, so do use its ``--help`` function. For instance, to add data as forecasts, use the ``--beliefcol`` parameter, to say precisely when these forecasts were made. Or add ``--horizon`` for rolling forecasts if they all share the same horizon. -Second, you can use the `POST /api/v3_0/sensors//data <../api/v3_0.html#post--api-v3_0-sensors-id-data>`_ endpoint in the FlexMeasures API to send meter data. +For automated pipelines running elsewhere, use the `FlexMeasures Client `_ or call the `POST /api/v3_0/sensors//data <../api/v3_0.html#post--api-v3_0-sensors-id-data>`_ and file-upload endpoints directly. You can also use the API to send forecast data. Similar to the ``add beliefs`` commands, you would use here the fields ``prior`` (to denote time of knowledge of data) or ``horizon`` (for rolling forecast data with equal horizon). Consult the documentation at :ref:`posting_sensor_data`. -Finally, you can tell FlexMeasures to compute forecasts based on existing meter data with the ``flexmeasures add forecasts`` command, here is an example: +After ingesting meter data, you can tell FlexMeasures to compute forecasts with the ``flexmeasures add forecasts`` command, here is an example: .. code-block:: bash diff --git a/documentation/index.rst b/documentation/index.rst index ee1155e994..0ba736b7c5 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -184,12 +184,12 @@ In :ref:`getting_started`, we have some helpful tips how to dive into this docum tut/toy-example-from-scratch tut/toy-example-expanded tut/toy-example-multiasset-curtailment - tut/toy-example-group-constraints tut/flex-model-v2g tut/multi-feed-storage tut/multi-commodity tut/toy-example-process tut/toy-example-reporter + tut/toy-example-group-constraints tut/posting_data tut/forecasting_scheduling tut/building_uis diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index d01fd67cf5..680f9401d7 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -148,9 +148,8 @@ It usually involves a linear program that combines a state of energy flexibility There are two ways to queue a scheduling job: First, we can add a scheduling job to the queue via the API. -We already learned about the `[POST] /schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-id-schedules-trigger>`_ endpoint in :ref:`posting_flex_states`, where we saw how to post a flexibility state (in this case, the state of charge of a battery at a certain point in time). - -Here, we extend that (storage) example with an additional target value, representing a desired future state of charge. +The `[POST] /schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-id-schedules-trigger>`_ endpoint accepts a flexibility state, such as the state of charge of a battery at a certain point in time. +This storage example also includes a target value representing a desired future state of charge. .. code-block:: json :emphasize-lines: 6-11 @@ -292,7 +291,7 @@ This example requests a prognosis for 24 hours, with a rolling horizon of 6 hour Getting schedules (control signals) ----------------------- -We saw above how FlexMeasures can create optimised schedules with control signals for flexible devices (see :ref:`posting_flex_states`). You can access the schedules via the `[GET] /schedules/ <../api/v3_0.html#get--api-v3_0-sensors-id-schedules-uuid>`_ endpoint. The URL then looks like this: +After FlexMeasures creates an optimised schedule with control signals for flexible devices, you can access it via the `[GET] /schedules/ <../api/v3_0.html#get--api-v3_0-sensors-id-schedules-uuid>`_ endpoint. The URL then looks like this: .. code-block:: html diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index 92de1afe7f..2677a72c0b 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -1,231 +1,322 @@ .. _tut_posting_data: -Posting data -============ +Ingesting data from files and scripts +====================================== -The platform FlexMeasures strives on the data you feed it. Let's demonstrate how you can get data into FlexMeasures using the API. This is where FlexMeasures gets connected to your system as a smart backend and helps you build smart energy services. +FlexMeasures turns time-series data into forecasts, reports and optimized schedules. +This tutorial shows how to build the first part of that journey: an automated data-ingestion pipeline. +In this tutorial, we build a complete script which you can run from the command line. -We will show how to use the API endpoints for POSTing data. -You can call these at regular intervals (through scheduled scripts in your system, for example), so that FlexMeasures always has recent data to work with. -Of course, these endpoints can also be used to load historic data into FlexMeasures, so that the forecasting models have access to enough data history. - -.. note:: For the purposes of forecasting and scheduling, it is often advisable to use a less fine-grained resolution than most metering services keep. For example, while such services might measure every ten seconds, FlexMeasures will usually do its job no less effective if you feed it data with a resolution of five minutes. This will also make the data integration much easier. Keep in mind that many data sources like weather forecasting or markets can have data resolutions of an hour, anyway. +You will upload a CSV or Excel file with the `FlexMeasures Client `_, +verify the stored values, and then adapt the example to values produced by an export script. +The same API accepts meter readings, prices, weather data, state of charge and other numeric time series. .. contents:: Table of contents :local: - :depth: 1 + :depth: 2 + +Choosing an ingestion route +--------------------------- + +Most production pipelines should use the FlexMeasures Client or the API. +The UI upload is useful for a quick first success and for checking whether a file is accepted before automating it. + +=============================== ================================================ +Situation Recommended route +=============================== ================================================ +Values available in Python FlexMeasures Client +CSV or Excel on another system FlexMeasures Client or file-upload API +Another programming language REST API +Quickly validate a file Sensor UI +One-off import on the server FlexMeasures CLI +Reusable third-party connector FlexMeasures plugin +=============================== ================================================ Prerequisites --------------- +------------- + +You need: + +- a running FlexMeasures server; +- the hostname and login details of a user allowed to record data; +- the ID of an existing sensor; +- the sensor's unit, event resolution and timezone; and +- Python 3.10 or newer. + +A sensor is the contract for a time series: it tells FlexMeasures what the values mean, +which unit they use, how long each event lasts and which timezone applies. +Ask the administrator of your FlexMeasures organisation for these details, +or find the sensor in the UI. +If you host FlexMeasures yourself, :ref:`getting_started` and :ref:`cli` explain how to create the required structure. + +For a recurring pipeline, consider a dedicated integration user. +FlexMeasures records the authenticated user as the data source, making the pipeline's provenance easy to recognize. + +Install version 0.9.4 or newer of the client: + +.. code-block:: console + + $ pip install "flexmeasures-client>=0.9.4" + +Store connection details outside the script, for example as environment variables: + +.. code-block:: console + + $ export FLEXMEASURES_HOST="company.flexmeasures.io" + $ export FLEXMEASURES_EMAIL="data-pipeline@example.com" + $ export FLEXMEASURES_SENSOR_ID="16" + +When you run the complete script, it securely prompts for the password without recording it in your shell history. +For an unattended pipeline, inject ``FLEXMEASURES_PASSWORD`` from your deployment platform's secret manager. + +Preparing a file +---------------- + +The simplest input has two columns: an event start and a numeric value. +For example, a 15-minute power time series may look like this: + +.. code-block:: text -- FlexMeasures needs some structural meta data for data to be understood. For example, for adding weather data we need to define a weather sensor, and what kind of weather sensors there are. You also need a user account. If you host FlexMeasures yourself, you need to add this info first. Head over to :ref:`getting_started`, where these steps are covered, study our :ref:`cli` or look into plugins which do this like `flexmeasures-entsoe `_ or `flexmeasures-weather `_. -- You should be familiar with where to find your API endpoints (see :ref:`api_versions`) and how to authenticate against the API (see :ref:`api_auth`). + event_start,event_value + 2026-07-30T08:00:00+02:00,4.2 + 2026-07-30T08:15:00+02:00,4.8 + 2026-07-30T08:30:00+02:00,5.1 + 2026-07-30T08:45:00+02:00,4.6 -.. note:: For deeper explanations of the data and the meta fields we'll send here, You can always read the :ref:`api_introduction`, to the FlexMeasures API, e.g. :ref:`signs`, :ref:`frequency_and_resolution`, :ref:`prognoses` and :ref:`units`. +Save this as ``meter-readings.csv``, or put the same two columns in ``meter-readings.xlsx``. +CSV, XLSX, XLS and XLSM files are supported. +Use timezone-aware ISO 8601 timestamps when possible. +If timestamps have no UTC offset, FlexMeasures interprets them in the sensor's timezone. +Explicit offsets avoid ambiguity around daylight-saving-time transitions. +Aligning timestamps and frequency with the sensor's event resolution also makes the pipeline easier to reason about, +although FlexMeasures can resample compatible resolutions. + +Optionally validate the file in the UI +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Before automating the pipeline, you can test the file on the sensor page: + +1. Open the relevant sensor. +2. Expand **Upload data** in the left side panel. +3. Choose the file and its unit. +4. Select **Measured instantly** if each value was known when its event ended. +5. Upload the file and inspect the chart. + +The panel also offers an example Excel file. +If the UI accepts your file but the automated upload does not, +focus troubleshooting on the client configuration, credentials and network connection rather than the file format. + +.. A future screenshot belongs here. It should show the sensor page with the + "Upload data" panel expanded, including the file selector, "Measured + instantly" checkbox, unit selector and upload button. + +Uploading the file with the FlexMeasures Client +----------------------------------------------- + +The following call works for both CSV and Excel files: + +.. literalinclude:: scripts/run-data-ingestion.py + :language: python + :start-after: # Start file upload example + :end-before: # End file upload example + :dedent: 8 + +``belief_time_measured_instantly=True`` records each value as known when its event ended. +Leave it at the default ``False`` when the values only became known at upload time. + +The file upload currently assumes that values use the sensor's unit. +The UI and raw API additionally let you specify a different compatible input unit for conversion. + +On a server with an ingestion worker, the request may return ``202 Accepted`` while the file is processed in the background. +Client 0.9.4 accepts both synchronous and queued responses. +Do not assume that accepted data is immediately available; verify it or follow the returned job URL before starting dependent work. .. _posting_sensor_data: -Posting sensor data -------------------- - -Sensor data (both observations and forecasts) can be posted to `POST /sensors//data <../api/v3_0.html#post--api-v3_0-sensors-id-data>`_. -This endpoint represents the basic method of getting time series data into FlexMeasures via API. -It is agnostic to the type of sensor and can be used to POST data for both physical and economical events that have happened in the past or will happen in the future. -Some examples: - -- readings from electricity and gas meters -- readings from temperature and pressure sensors -- state of charge of a battery -- estimated availability of parking spots -- price forecasts - -The exact URL will depend on your domain name, and will look approximately like this: - -.. code-block:: html - - [POST] https://company.flexmeasures.io/api/v3_0/sensors/16/data - -This example "PostSensorDataRequest" message posts prices for hourly intervals between midnight and midnight the next day -for the Korean Power Exchange (KPX) day-ahead auction, registered under sensor 16. -The ``prior`` indicates that the prices were published at 3pm on December 31st 2014 (i.e. the clearing time of the KPX day-ahead market, which is at 3 PM on the previous day ― see below for a deeper explanation). - -.. code-block:: json - - { - "type": "PostSensorDataRequest", - "values": [ - 52.37, - 51.14, - 49.09, - 48.35, - 48.47, - 49.98, - 58.7, - 67.76, - 69.21, - 70.26, - 70.46, - 70, - 70.7, - 70.41, - 70, - 64.53, - 65.92, - 69.72, - 70.51, - 75.49, - 70.35, - 70.01, - 66.98, - 58.61 - ], - "start": "2015-01-01T00:00:00+09:00", - "duration": "PT24H", - "prior": "2014-12-31T15:00:00+09:00", - "unit": "KRW/kWh" - } +Posting values from an export script +------------------------------------ -Note how the resolution of the data comes out at 60 minutes when you divide the duration by the number of data points. -If this resolution does not match the sensor's resolution, FlexMeasures will try to upsample the data to make the match or, if that is not possible, complain. -Likewise, if the data unit does not match the sensor’s unit, FlexMeasures will attempt to convert the data or, if that is not possible, complain. +Often a script has already fetched or computed the values, so writing an intermediate file is unnecessary. +Post an equally spaced sequence directly: +.. literalinclude:: scripts/run-data-ingestion.py + :language: python + :start-after: # Start export script example + :end-before: # End export script example + :dedent: 8 -Being explicit when posting power data -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The number of values and the duration determine their frequency. +For example, four values over ``PT1H`` represent four 15-minute events. +The duration covers the complete events, so it is one hour rather than the 45-minute difference between the first and last timestamps. -For power data, USEF[1] specifies separate message types for observations and forecasts. -Correspondingly, we allow the following message types to be used with the `POST /sensors/16/data <../api/v3_0.html#post--api-v3_0-sensors-id-data>`_ endpoint: +For data collected dynamically, the surrounding pipeline could look like this: -.. code-block:: json +.. code-block:: python - { - "type": "PostMeterDataRequest" - } + import asyncio + import getpass + import os -.. code-block:: json + from flexmeasures_client import FlexMeasuresClient - { - "type": "PostPrognosisRequest" - } -For these message types, FlexMeasures validates whether the data unit is suitable for communicating power data. -Additionally, we validate whether meter data lies in the past, and prognoses lie in the future. + async def main(): + email = os.environ["FLEXMEASURES_EMAIL"] + client = FlexMeasuresClient( + host=os.environ["FLEXMEASURES_HOST"], + ssl=True, + email=email, + password=os.getenv("FLEXMEASURES_PASSWORD") + or getpass.getpass(f"FlexMeasures password for {email}: "), + ) + try: + values = export_latest_meter_values() # Your database or vendor API + await client.post_sensor_data( + sensor_id=int(os.environ["FLEXMEASURES_SENSOR_ID"]), + start="2026-07-30T08:00:00+02:00", + duration="PT1H", + values=values, + unit="kW", + ) + finally: + await client.close() -Single value, single sensor -^^^^^^^^^^^^^^^^^^^^^^^^^^^ -A single average power value for a 15-minute time interval for a single sensor, posted 5 minutes after realisation. + asyncio.run(main()) -.. code-block:: json +The client expects a hostname without ``https://``; set ``ssl=True`` for HTTPS. +The vendor-specific placeholder ``export_latest_meter_values()`` must return a ``list[float]``, ordered from the oldest interval to the newest. - { - "type": "PostSensorDataRequest", - "value": 220, - "start": "2015-01-01T00:00:00+00:00", - "duration": "PT0H15M", - "horizon": "-PT5M", - "unit": "MW" - } +Verifying the result +-------------------- -Multiple values, single sensor -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Reading values back is not required for ingestion. +We do it here for the sake of the tutorial, to demonstrate that the pipeline stored the expected data. +Read back the same interval: -Multiple values (indicating a univariate timeseries) for 15-minute time intervals for a single sensor, posted 5 minutes after each realisation. +.. code-block:: python -.. code-block:: json + sensor_data = await client.get_sensor_data( + sensor_id=sensor_id, + start="2026-07-30T08:00:00+02:00", + duration="PT1H", + resolution="PT15M", + unit="kW", + ) + assert sensor_data["values"] == [4.2, 4.8, 5.1, 4.6] - { - "type": "PostSensorDataRequest", - "values": [ - 220, - 210, - 200 - ], - "start": "2015-01-01T00:00:00+00:00", - "duration": "PT0H45M", - "horizon": "-PT5M", - "unit": "MW" - } +You can also inspect the sensor chart in the UI. +Verifying the values, unit and interval catches mistakes that a successful HTTP response alone cannot. +Running the executable tutorial +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. _observations_vs_forecasts +The complete script uploads an example Excel file, +posts a second interval as in-memory values and verifies both results: -Observations vs forecasts: The time of knowledge -------------------------------------------------- +.. code-block:: console -To correctly tell FlexMeasures when a meter reading or forecast was known is crucial, as it determines which data is being used to compute schedules or to make other forecasts. + $ uv run --no-project --with "flexmeasures-client>=0.9.4" \ + python documentation/tut/scripts/run-data-ingestion.py \ + --sensor-id 16 --unit EUR/MWh --resolution PT1H -Usually, the time of posting is assumed to be the time when the data was known. But you can also explicitly tell FlexMeasures what these times are. This either works with one fixed time (for the whole set of data being sent) or with a horizon (which applies to each data point separately). +FlexMeasures maintainers can run the corresponding Docker QA wrapper against the development stack: -E.g. to post a forecast rather than an observation after the fact, simply set the ``prior`` to the moment at which the forecasts were made, e.g. at "2015-01-01T16:30:00+09:00". Assuming your data starts at 5.00pm, this denotes that the data are forecasts, made half an hour before realisation. +.. code-block:: console -Alternatively, to indicate that each individual observation was made directly after the end of its 15-minute interval (i.e. at 3.15pm, 3.30pm and so on), set a ``horizon`` to "PT0H" instead of a ``prior``. + $ ./documentation/tut/scripts/run-data-ingestion-in-docker.sh -Finally, delays in reading out sensor data can be simulated by setting the ``horizon`` field to a negative value. -For example, a horizon of "-PT1H" would denote that each temperature reading was observed one hour after the fact (i.e. at 4.15pm, 4.30pm and so on). +The same runner is used in CI so that the client examples in this tutorial remain executable. -See :ref:`prognoses` for more information regarding the ``prior`` and ``horizon`` fields. +Making the pipeline reliable +---------------------------- -A good example for the use of the ``prior`` field are markets, which have clearing times. -For example, at the KPX day-ahead auction this is every day at 3pm. -This point in time (i.e. when contracts are signed) determines the difference between an ex-post observation and an ex-ante forecast. +For a recurring pipeline: -Another example for the ``prior`` field is running simulations with FlexMeasures. It gives you control over the timing so that you could run a month in the past as if it happened right now. +- store the last successfully ingested timestamp or derive the next window from the source system; +- retry transient connection failures with bounded backoff; +- log the sensor ID, interval, number of values and ingestion job ID; +- split large histories into bounded requests (the server limit defaults to 3 MiB per request); +- wait for queued ingestion before triggering work which needs the new data; and +- alert when the source has stopped producing data or verification fails. +Reposting identical data is safe: unchanged beliefs are skipped. +Changing a value with the same sensor, source, event and recording time is rejected by default rather than silently overwritten. -.. _posting_flex_states: +Common problems +--------------- + +``401 Unauthorized`` + Check the email, password or access token. -Posting flexibility states -------------------------------- +``403 Forbidden`` + The user is authenticated but lacks permission to record data on this sensor. -There is one more crucial kind of data that FlexMeasures needs to know about: What are the current states of flexible devices? -For example, a battery has a certain state of charge, which is relevant to describe the flexibility that the battery currently has. -In our terminology, this is called the "flex model" and you can read more at :ref:`describing_flexibility`. +``413 Payload Too Large`` + Split the file or values into smaller time windows. -Owners of such devices can post the flex model along with triggering the creation of a new schedule, to one of two endpoints: +``422 Unprocessable Entity`` + Inspect the response for an incompatible unit or resolution, invalid timestamps or non-numeric values. -1. `[POST] /assets//schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-id-schedules-trigger>`_ - for scheduling multiple devices -2. `[POST] /sensors//schedules/trigger <../api/v3_0.html#post--api-v3_0-sensors-id-schedules-trigger>`_ - for scheduling a single device (which can also be done with the first endpoint) +Unexpected timestamps + Check timezone offsets and whether the sensor floors timestamps to its event resolution. -The URL might look like this: +Missing values + JSON value lists may contain ``null`` to preserve spacing. File uploads may contain gaps if FlexMeasures can still infer a regular frequency. -.. code-block:: html +Without the Python client +------------------------- - https://company.flexmeasures.io/api/v3_0/assets/10/schedules/trigger +The client wraps the FlexMeasures API, so other languages can call the same endpoints. +For example, upload a file with an access token: -The following example triggers a schedule for a power sensor (with ID 15) of a battery asset (with ID 10), asking to take into account the battery's current state of charge. -From this, FlexMeasures derives the energy flexibility this battery has in the next 48 hours and computes an optimal charging schedule. -The endpoint also allows to limit the flexibility range and also to set target values. +.. code-block:: console -.. code-block:: json + $ curl --fail-with-body \ + -H "Authorization: ${FLEXMEASURES_ACCESS_TOKEN}" \ + -F "uploaded-files=@meter-readings.xlsx" \ + -F "belief-time-measured-instantly=true" \ + "https://${FLEXMEASURES_HOST}/api/v3_0/sensors/${FLEXMEASURES_SENSOR_ID}/data/upload" - { - "start": "2015-06-02T10:00:00+00:00", - "flex-model": [ - { - "sensor": 15, - "soc-at-start": "12.1 kWh" - } - ] - } +Or post values as JSON: -.. note:: More details on supported flex models can be found in :ref:`flex_models_and_schedulers`. +.. code-block:: console -.. note:: - Flexibility states posted in trigger messages are only stored temporarily to describe the scheduling job. - To record a more complete history of the flexibility state, set up separate sensors and post data to them using `[POST] /sensors/data <../api/v3_0.html#post--api-v3_0-sensors-data>`_ (see :ref:`posting_sensor_data`). - Then reference those sensors in your flex model. - For example, say you use sensor 82 to record the power-to-heat efficiency of a heating system, then use this sensor reference in your flex model: + $ curl --fail-with-body \ + -H "Authorization: ${FLEXMEASURES_ACCESS_TOKEN}" \ + -H "Content-Type: application/json" \ + --data '{ + "values": [4.2, 4.8, 5.1, 4.6], + "start": "2026-07-30T08:00:00+02:00", + "duration": "PT1H", + "unit": "kW" + }' \ + "https://${FLEXMEASURES_HOST}/api/v3_0/sensors/${FLEXMEASURES_SENSOR_ID}/data" - .. code-block:: json +See :ref:`api_auth` for obtaining an access token and :ref:`v3_0` for the complete endpoint reference. - { - "charging-efficiency": {"sensor": 82} - } +.. _observations_vs_forecasts: +Measurements, forecasts and time of knowledge +---------------------------------------------- -In :ref:`how_queue_scheduling`, we'll cover what happens when FlexMeasures is triggered to create a new schedule, and how those schedules can be retrieved via the API, so they can be used to steer assets. +FlexMeasures stores not only an event value but also when that value became known. +This prevents a forecast or simulation from accidentally using information which was unavailable at the time. + +If neither ``prior`` nor ``horizon`` is supplied, the API uses the request time as the time of knowledge. +Use ``prior`` for one fixed publication time, such as the issue time of a day-ahead price or weather forecast. +Use ``horizon`` when every value has the same relationship between its event time and recording time. +For example, ``PT0H`` means each measurement became known when its event ended, +while ``-PT1H`` represents a one-hour reporting delay. + +See :ref:`prognoses` for the full explanation of ``prior`` and ``horizon``. + +.. _posting_flex_states: +Next: use the ingested data +--------------------------- -[1] https://www.usef.energy/app/uploads/2020/01/USEF-Flex-Trading-Protocol-Specifications-1.01.pdf +After ingesting measurements, prices and forecasts, you can ask FlexMeasures to forecast new data or compute optimized schedules. +Current device state, such as a battery's state of charge, may be supplied in the scheduling trigger's ``flex-model``. +See :ref:`tut_forecasting_scheduling` and :ref:`describing_flexibility` for the next steps. diff --git a/documentation/tut/scripts/Readme.md b/documentation/tut/scripts/Readme.md index 84d4ca6619..af4a6a6d5c 100644 --- a/documentation/tut/scripts/Readme.md +++ b/documentation/tut/scripts/Readme.md @@ -4,21 +4,24 @@ The tutorials in the docs are for you to run step by step, command by command, so that every step clarifies more of what FlexMeasures is for, and what it can do for you. However, sometimes one might want to run through them all. -We scripted the tutorials, so they can be automated. They don't come with a guarantee. +We scripted the tutorials so they can be automated. They don't come with a guarantee. -For us, they are actually a step in [our release checklist](https://github.com/FlexMeasures/tsc/blob/main/RELEASE.md) before we upload a new version to Pypi. +For us, they are also a step in [our release checklist](https://github.com/FlexMeasures/tsc/blob/main/RELEASE.md) before we upload a new version to PyPI. -We run these tests in the docker compose stack: +We run these tests in the Docker Compose stack: docker compose build - docker compose up + docker compose up --detach --wait ./documentation/tut/scripts/run-tutorial-in-docker.sh ./documentation/tut/scripts/run-tutorial2-in-docker.sh ./documentation/tut/scripts/run-tutorial3-in-docker.sh ./documentation/tut/scripts/run-tutorial4-in-docker.sh + ./documentation/tut/scripts/run-tutorial5-in-docker.sh + ./documentation/tut/scripts/run-data-ingestion-in-docker.sh - One still needs to check the output (no errors?) and plotted data (plots like we expect?) -- These need to be run in order so the sensor IDs match (just like when you run them from the docs) -- Need to start over? `docker rm --force flexmeasures-dev-db-1`, then `down` and `up` with your compose stack.. -- We try to keep these script in sync with the tutorials. But as you can imagine, this is hard, as is keeping docs up to date in general. +- The toy tutorial runners use `flexmeasures add toy-account --shell-vars`, so they no longer depend on fixed numeric IDs. Tutorials 1-5 still run in chapter order because later tutorials use data created by earlier ones. +- The data-ingestion runner is standalone. It obtains the toy sensor ID through `--shell-vars` when needed and verifies the values it writes. +- Need to start over? Run `docker compose down --volumes`, then rebuild and start the Compose stack. +- We try to keep these scripts in sync with the tutorials. But as you can imagine, this is hard, as is keeping docs up to date in general. - At least, this might see some regular use by us. The tutorial in the docs sees more usage by new users, who sometimes tell us what they found. diff --git a/documentation/tut/scripts/run-data-ingestion-in-docker.sh b/documentation/tut/scripts/run-data-ingestion-in-docker.sh new file mode 100755 index 0000000000..4d0fc6a2ba --- /dev/null +++ b/documentation/tut/scripts/run-data-ingestion-in-docker.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -euo pipefail + +CONTAINER_NAME="${1:-$(basename "$(pwd)")-server-1}" +CLIENT_REQUIREMENT="flexmeasures-client>=0.9.4" + +echo "[TUTORIAL-RUNNER] RUNNING DATA-INGESTION TUTORIAL ..." +echo "-----------------------------------------------------" + +if ! command -v uv >/dev/null 2>&1; then + echo "The data-ingestion runner requires uv: https://docs.astral.sh/uv/" >&2 + exit 1 +fi + +curl --fail --silent --show-error \ + --retry 30 --retry-delay 1 --retry-all-errors \ + "http://localhost:5000/api/v3_0/health/ready" >/dev/null + +if [[ -z "${FM_TOY_BATTERY_SENSOR_ID:-}" ]]; then + eval "$(docker exec -i "${CONTAINER_NAME}" flexmeasures add toy-account \ + --kind battery --shell-vars | grep '^FM_TOY_')" +fi + +if [[ -n "${FLEXMEASURES_CLIENT_PROJECT:-}" ]]; then + CLIENT_COMMAND=(uv run --project "${FLEXMEASURES_CLIENT_PROJECT}") +else + CLIENT_COMMAND=(uv run --no-project --with "${CLIENT_REQUIREMENT}") +fi + +FLEXMEASURES_SENSOR_ID="${FM_TOY_BATTERY_SENSOR_ID}" \ +FLEXMEASURES_SENSOR_UNIT="kW" \ +FLEXMEASURES_SENSOR_RESOLUTION="PT1H" \ +FLEXMEASURES_EMAIL="${FLEXMEASURES_EMAIL:-toy-user@flexmeasures.io}" \ +FLEXMEASURES_PASSWORD="${FLEXMEASURES_PASSWORD:-toy-password}" \ + "${CLIENT_COMMAND[@]}" python \ + documentation/tut/scripts/run-data-ingestion.py diff --git a/documentation/tut/scripts/run-data-ingestion.py b/documentation/tut/scripts/run-data-ingestion.py new file mode 100755 index 0000000000..5441460c02 --- /dev/null +++ b/documentation/tut/scripts/run-data-ingestion.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Run and verify the data-ingestion tutorial against a FlexMeasures server.""" + +from __future__ import annotations + +import argparse +import asyncio +import getpass +import os +from pathlib import Path +from time import monotonic + +from flexmeasures_client import FlexMeasuresClient + +EXAMPLE_FILE_VALUES = [1.0, 3.0, 9.0, 8.0, 7.0, 10.0] +EXAMPLE_FILE_START = "2022-12-11T05:00:00+00:00" +EXAMPLE_FILE_DURATION = "PT6H" +EXPORT_VALUES = [4.2, 4.8, 5.1, 4.6] +EXPORT_START = "2022-12-12T05:00:00+00:00" +EXPORT_DURATION = "PT4H" + + +def parse_args() -> argparse.Namespace: + """Parse connection details and tutorial parameters.""" + + repository_root = Path(__file__).resolve().parents[3] + parser = argparse.ArgumentParser( + description="Upload example sensor data and verify that FlexMeasures stored it." + ) + parser.add_argument( + "--host", default=os.getenv("FLEXMEASURES_HOST", "localhost:5000") + ) + parser.add_argument( + "--ssl", + action=argparse.BooleanOptionalAction, + default=os.getenv("FLEXMEASURES_SSL", "false").lower() == "true", + ) + parser.add_argument( + "--email", + default=os.getenv("FLEXMEASURES_EMAIL", "toy-user@flexmeasures.io"), + ) + parser.add_argument( + "--sensor-id", + type=int, + default=os.getenv("FLEXMEASURES_SENSOR_ID"), + required=os.getenv("FLEXMEASURES_SENSOR_ID") is None, + ) + parser.add_argument( + "--unit", default=os.getenv("FLEXMEASURES_SENSOR_UNIT", "EUR/MWh") + ) + parser.add_argument( + "--resolution", default=os.getenv("FLEXMEASURES_SENSOR_RESOLUTION", "PT1H") + ) + parser.add_argument( + "--file", + type=Path, + default=repository_root + / "flexmeasures" + / "ui" + / "static" + / "examples" + / "sensors-data.xlsx", + ) + parser.add_argument("--timeout", type=float, default=60) + return parser.parse_args() + + +async def wait_for_values_and_verify( + client: FlexMeasuresClient, + *, + sensor_id: int, + start: str, + duration: str, + resolution: str, + unit: str, + expected_values: list[float], + timeout: float, +) -> None: + """Wait for synchronous or queued ingestion and verify the stored values.""" + + deadline = monotonic() + timeout + last_values: list[float] | None = None + while monotonic() < deadline: + sensor_data = await client.get_sensor_data( + sensor_id=sensor_id, + start=start, + duration=duration, + resolution=resolution, + unit=unit, + ) + last_values = sensor_data["values"] + if last_values == expected_values: + assert sensor_data["start"] == start + assert sensor_data["duration"] == duration + assert sensor_data["unit"] == unit + return + await asyncio.sleep(1) + raise AssertionError( + f"Expected {expected_values} for sensor {sensor_id}, got {last_values}." + ) + + +async def run_tutorial(args: argparse.Namespace) -> None: + """Upload a spreadsheet and exported values, then verify both intervals.""" + + if not args.file.is_file(): + raise FileNotFoundError(f"Example data file not found: {args.file}") + + scheme = "https" if args.ssl else "http" + print( + f"Preparing FlexMeasures client for {scheme}://{args.host} " + f"as {args.email} (sensor {args.sensor_id}).", + flush=True, + ) + password = os.getenv("FLEXMEASURES_PASSWORD") or getpass.getpass( + f"FlexMeasures password for {args.email}: " + ) + client = FlexMeasuresClient( + host=args.host, + ssl=args.ssl, + email=args.email, + password=password, + ) + try: + # Start file upload example + print(f"Uploading spreadsheet: {args.file}", flush=True) + await client.post_sensor_data( + sensor_id=args.sensor_id, + file_path=str(args.file), + belief_time_measured_instantly=True, + ) + # End file upload example + print("Spreadsheet upload accepted; verifying stored values ...", flush=True) + await wait_for_values_and_verify( + client, + sensor_id=args.sensor_id, + start=EXAMPLE_FILE_START, + duration=EXAMPLE_FILE_DURATION, + resolution=args.resolution, + unit=args.unit, + expected_values=EXAMPLE_FILE_VALUES, + timeout=args.timeout, + ) + print("Spreadsheet values verified.", flush=True) + + # Start export script example + print(f"Uploading {len(EXPORT_VALUES)} exported meter values ...", flush=True) + await client.post_sensor_data( + sensor_id=args.sensor_id, + start=EXPORT_START, + duration=EXPORT_DURATION, + values=EXPORT_VALUES, + unit=args.unit, + ) + # End export script example + print("Exported values accepted; verifying stored values ...", flush=True) + await wait_for_values_and_verify( + client, + sensor_id=args.sensor_id, + start=EXPORT_START, + duration=EXPORT_DURATION, + resolution=args.resolution, + unit=args.unit, + expected_values=EXPORT_VALUES, + timeout=args.timeout, + ) + print("Exported meter values verified.", flush=True) + finally: + await client.close() + + print("Data-ingestion tutorial completed successfully.", flush=True) + + +if __name__ == "__main__": + asyncio.run(run_tutorial(parse_args())) diff --git a/documentation/tut/toy-example-group-constraints.rst b/documentation/tut/toy-example-group-constraints.rst index 5c13a8252d..49486025ab 100644 --- a/documentation/tut/toy-example-group-constraints.rst +++ b/documentation/tut/toy-example-group-constraints.rst @@ -1,7 +1,7 @@ .. _tut_toy_schedule_group_constraints: -Toy example IV: Intermediate power constraints (groups) +Toy example VI: Intermediate power constraints (groups) ================================================================ So far, our flexible devices (the battery and the PV inverter) have only ever been constrained directly by the building's own grid connection capacity. diff --git a/pyproject.toml b/pyproject.toml index 6441473b79..2b72e9aa3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "py-moneyed>=3.0", "iso8601>=2.1.0", "xlrd>=2.0.2", + "openpyxl>=3.1.5", "workalendar>=17.0.0", "holidays>=0.57", "inflection>=0.5.1", @@ -266,7 +267,6 @@ test = [ # required with fakeredis, maybe because we use rq "lupa>=2.6", "pytest-mock>=3.15.1", - "openpyxl>=3.1.5", "pytest-runner>=6.0.1", ] diff --git a/uv.lock b/uv.lock index a469a3f6e0..154a248c74 100644 --- a/uv.lock +++ b/uv.lock @@ -1259,6 +1259,7 @@ dependencies = [ { name = "marshmallow-oneofschema", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "marshmallow-polyfield", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "marshmallow-sqlalchemy", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "openpyxl", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "pandas", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "pillow", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "pint", version = "0.24.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform == 'darwin') or (python_full_version < '3.11' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform == 'win32')" }, @@ -1327,7 +1328,6 @@ docs = [ test = [ { name = "fakeredis", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "lupa", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "openpyxl", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "pytest", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "pytest-cov", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "pytest-flask", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -1383,6 +1383,7 @@ requires-dist = [ { name = "marshmallow-oneofschema", specifier = ">=3.2.0" }, { name = "marshmallow-polyfield", specifier = ">=5.11" }, { name = "marshmallow-sqlalchemy", specifier = ">=0.23.1" }, + { name = "openpyxl", specifier = ">=3.1.5" }, { name = "pandas", specifier = ">=2.2.1" }, { name = "pillow", specifier = ">=12.2.0" }, { name = "pint", specifier = ">=0.19.1" }, @@ -1449,7 +1450,6 @@ docs = [ test = [ { name = "fakeredis", specifier = ">=2.33.0" }, { name = "lupa", specifier = ">=2.6" }, - { name = "openpyxl", specifier = ">=3.1.5" }, { name = "pytest", specifier = ">=9.0.2" }, { name = "pytest-cov", specifier = ">=7.0.0" }, { name = "pytest-flask", specifier = ">=1.3.0" },