hdctl

#!/usr/bin/env bash
#
###############################################################################
#                                                                             #
#                                    hdctl                                    #
#                                                                             #
#   Manage maintenance / devops tasks around hetida designer installations    #
#                                                                             #
#                                                                             #
###############################################################################
#
# LICENSE
#   Copyright © 2024 fuseki GmbH
#
#   MIT License
#   see https://github.com/hetida/hetida-designer/blob/release/LICENSE
#
# INTRODUCTION
#   hdctl is a bash command line tool for managing typical maintenance /
#   devops tasks around hetida designer installations.
#
#   hdctl can export/import components and workflows from/to the backend api
#   while taking care of authentication. A typical use case is syncing
#   components / workflows from dev instances to test or prod instances.
#
#   Another use case is running maintenance operations, e.g. as described in
#   https://github.com/hetida/hetida-designer/blob/release/docs/cleanup.md
#   from the command line only, i.e. without needing docker.
#
#   hdctl is based on bash/coreutils + curl with the additional requirement
#   of awk and jq for some tasks that involve json wrangling. This enables
#   managing hetida designer instances from locations where no docker or
#   python installation is available or installing arbitrary dependencies
#   and software is difficult due to administrative / security restrictions.
#
# INSTALLATION
#   Just copy this bash script to the environment where you want to run it
#   from. There you need
#     * modern/recent bash (version >4)
#     * GNU coreutils
#     * curl (ideally >= 7.87.0)
#     * optionally awk and jq (https://stedolan.github.io/jq/) for some
#       subcommands that involve json unpacking
#
#   Notes:
#     * hdctl should run under Git Bash on Windows. Recent versions of Git
#       for Windows include curl >= 7.87.0 and gawk. jq must be installed
#       separately. If you experience problems please open an issue.
#     * Using nix, you can start an environment with all dependencies by
#       running
#         nix-shell \
#           -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-unstable.tar.gz \
#           -p bashInteractive jq gawk coreutils curl
#       At least with the following fixed nixpkgs commit it should
#       definitely work:
#         nix-shell \
#           -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/28319deb5ab05458d9cd5c7d99e1a24ec2e8fc4b.tar.gz \
#           -p bashInteractive jq gawk coreutils curl
#
#
# USAGE
#   Run
#       hdctl usage
#   to see detailed usage instructions and descriptions of all subcommands
#   and arguments.

set -e

######################## Initialize mutable Global variables ##################
ACCESS_TOKEN=""
EXPIRATION_EPOCH=0
_JQ_AVAILABLE=true

###############################################################################
#                                                                             #
#                              Logging functions                              #
#                                                                             #
###############################################################################

_info() {
    if [[ $SILENT = false ]]; then
        echo "${@}" >&2
    fi
}

_debug() {
    if [[ $VERBOSITY == "debug" ]]; then
        echo "${@}" >&2
    fi
}

_log_error() {
    echo "${@}" >&2
}

###############################################################################
#                                                                             #
#               System environment detection helper functions                 #
#                                                                             #
###############################################################################

_detect_jq() {
    if ! command -v jq 1>/dev/null; then
        _info "WARNING: jq not available. Some parameters/options cannot be used."
        _JQ_AVAILABLE=false
    else
        _info "Successfully detected jq in PATH."
    fi
}

JQ_ENSURED=false
_ensure_jq() {
    if ! command -v jq 1>/dev/null; then
        _log_error "Called command needs jq command line tool, which was not found. Aborting."
        exit 127
    fi
    JQ_ENSURED=true
}

_ensure_jq_once() {
    if ! "$JQ_ENSURED"; then
        _ensure_jq
    fi
}

# outputs a certain value from env file
value_from_env_file() {
    local var_name="$1"
    local env_file_path="$2"

    local env_file_entry
    env_file_entry="$(grep -m 1 '^'"$var_name"= "$env_file_path")"

    local str_after_equal="${env_file_entry#*=}" # may contain " or so
    local actual_str
    eval actual_str="$str_after_equal" # to get rid of enclosing ""
    printf '%s' "$actual_str"
}

# shellcheck disable=SC2046
PYTEST_INI_CONTENT=$(
    cat <<'END_HEREDOC'
[pytest]
python_files = *.py
addopts = --doctest-modules
END_HEREDOC
)

# shellcheck disable=SC2046
HDUTILS_PY_CONTENT=$(
    cat <<'END_HEREDOC'
import datetime
import io
import json
import logging
import re
from collections import defaultdict
from enum import StrEnum
from types import NoneType, UnionType
from typing import Annotated, Any, Literal, TypedDict
from uuid import UUID

import msgspec
import numpy as np
import pandas as pd
import pytz
from hdhelpers.helpers import modify_timezone  # noqa F401  # noqa F401  # noqa F401
from plotly.graph_objects import Figure
from pydantic import (
    BaseModel,
    BeforeValidator,
    ConfigDict,
    Field,
    PlainSerializer,
    PlainValidator,
    ValidationError,
    ValidatorFunctionWrapHandler,
    WrapValidator,
    create_model,
)

logger = logging.getLogger(__name__)

MULTITSFRAME_COLUMN_NAMES = ["timestamp", "metric", "value"]
SINGLETSFRAME_COLUMN_NAMES = ["timestamp", "value"]


class ComponentException(Exception):
    """Exception to re-raise exceptions with error code raised in the component code."""

    __is_hetida_designer_exception__ = True

    def __init__(
        self,
        *args: Any,
        error_code: int | str = "",
        extra_information: dict | None = None,
        **kwargs: Any,
    ) -> None:
        if not isinstance(error_code, int | str):
            raise ValueError("The ComponentException.error_code must be int or string!")
        self.error_code = error_code
        self.extra_information = extra_information
        super().__init__(*args, **kwargs)


class ComponentInputValidationException(ComponentException):
    """In code input validation failures"""

    def __init__(
        self,
        *args: Any,
        invalid_component_inputs: list[str],
        error_code: int | str = "",
        **kwargs: Any,
    ) -> None:
        super().__init__(
            *args,
            error_code=error_code,
            extra_information={"invalid_component_inputs": invalid_component_inputs},
            **kwargs,
        )


class InsufficientPlottingData(ComponentException):
    """A plot component has insufficient data to generate a meaningful plot

    This exception class should be used when custom plots generated by hetida
    designer are integrated in other frontends. This allows the frontend to
    handle the case of e.g. no data to show in a sensible way, surpressing an
    empty plot and showing an adequate message instead.
    """


class IntentionallyAbortedExecution(ComponentException):
    """Execution of this component or workflow is intentionally aborted

    Often execution of a workflow should stop if a certain condition is not
    fulfilled. This can of course be done by raising an exception. To communicate
    this intention to 3rd party systems and make it clear that this is not an actual error
    situation it is recommended to use this exception.
    """


class PlotTargetSettings(BaseModel):
    """Settings that plot components can/should use

    Some Plotly settings like locale or the timezone of timestamps must be set
    by Python and cannot easily be set by plotly.js in a frontend.

    They can be provided to execution endpoints as part of the ExecByIdBase payload,
    are made accessible to components using the execution context.

    hdutils provides helper functions to access them at runtime.
    """

    plot_target_timezone: str | None = Field(
        None,
        description=(
            "The timezone plot components should use for datetime axes etc."
            " Usually via"
            " s.index=pd.to_datetime(s.index, utc=True).tz_convert(plot_target_timezone)"
        ),
        examples=["Europe/Berlin"],
    )
    plot_target_locale: str | None = Field(
        None,
        description=(
            "Locale to set for plots, e.g. to write weekdays in the user's language."
            " This has to be set in the config of the plotly figure dict and the plotly.js "
            "must have the associated plotly local scripts loaded."
        ),
    )
    target_darkmode: bool = Field(
        False, description="Whether plots should be layouted/configured for darkmode"
    )
    datetime_tick_format: str | None = Field(
        None, description="Tickformat to use for datetime axes", examples=["%H:%M<br>%d.%m.%Y"]
    )
    datetime_x_axes_range_start: datetime.datetime | None = Field(
        None, description="datetime range start which plots should set as x axis range"
    )

    datetime_x_axes_range_end: datetime.datetime | None = Field(
        None, description="datetime range end which plots should set as x axis range"
    )


def get_plot_target_settings() -> PlotTargetSettings:
    """Obtain plot settings from runtime execution context.

    If hetdesrun is not importable or this context field is not set,
    return default values.
    """
    try:
        from hetdesrun.runtime.context import get_runtime_exec_context

        return get_runtime_exec_context().plot_target_settings
    except ImportError:
        logger.warning("Could not load runtime exec context, import failed! Switch to defaults.")
        # return defaults if hetdesrun is not available as import
        return PlotTargetSettings()


def plot_output_needed(output_name: str) -> bool:
    """Whether a plot should be created for a PLOTLYJSON output of the current component

    No plot is needed if the execution does not run plot operators (run_pure_plot_operators
    is false) or if the output is discarded, e.g. because it is wired to the drop adapter.
    Components with plot outputs and outputs of other types can use this to skip creating
    plots and return {} instead.

    If hetdesrun is not importable, plots are needed.
    """
    try:
        from hetdesrun.models.run import ConfigurationInput
        from hetdesrun.runtime.configuration import execution_config
        from hetdesrun.runtime.context import output_is_discarded
    except ImportError:
        return True

    return execution_config.get(
        ConfigurationInput()
    ).run_pure_plot_operators and not output_is_discarded(output_name)


class WrappedModelWithCustomObjects(BaseModel):
    model: Any = None
    custom_objects: dict[str, Any]


class MetaDataWrapped(BaseModel):
    """Allows to wrap pandas object data with metadata"""

    hd_wrapped_data_object__: Literal["SERIES", "DATAFRAME"] = Field(
        ..., alias="__hd_wrapped_data_object__"
    )
    metadata__: dict[str, Any] = Field(
        ...,
        alias="__metadata__",
        description="Json serializable dictionary of metadata. Will be written"
        "to the resulting pandas object's attrs attribute.",
    )
    data__: dict | list = Field(
        ...,
        alias="__data__",
        description="The actual data which constitutes the pandas object.",
    )
    data_parsing_options__: dict = Field(
        {},
        alias="__data_parsing_options__",
        description=(
            "Additional options for parsing the provided data."
            " For example, setting orient to one of the allowed values for the respective"
            " Pandas type allows to use different json representations for the actual data."
        ),
    )

    model_config = ConfigDict(
        serialize_by_alias=True, validate_by_alias=True, validate_by_name=True
    )


def try_parse_wrapped(
    data: str | dict | list,
    hd_wrapped_data_object: Literal["SERIES", "DATAFRAME"],
) -> MetaDataWrapped:
    if isinstance(data, str):
        wrapped_data = MetaDataWrapped.model_validate_json(data)

        if wrapped_data.hd_wrapped_data_object__ != hd_wrapped_data_object:
            msg = (
                f"Unexpected hd model type: {wrapped_data.hd_wrapped_data_object__}."
                f" Expected {hd_wrapped_data_object}"
            )
            logger.warning(msg)
            raise TypeError(msg)
    else:
        wrapped_data = MetaDataWrapped.model_validate(data)

    return wrapped_data


def parse_wrapped_content(
    v: str | dict | list,
    wrapped_data_object: Literal["SERIES", "DATAFRAME"],
) -> tuple[str | dict | list, None | dict[str, Any], dict[str, Any]]:
    data_content: str | dict | list
    try:
        wrapped_object = try_parse_wrapped(v, wrapped_data_object)
        parsed_metadata = wrapped_object.metadata__
        data_content = wrapped_object.data__
        parsing_options = wrapped_object.data_parsing_options__
    except (ValidationError, TypeError) as e:
        logger.debug("Data object is not wrapped: %s", str(e))
        data_content = v
        parsed_metadata = None
        parsing_options = {}

    return data_content, parsed_metadata, parsing_options


def wrap_metadata_as_attrs(
    data_object: pd.Series | pd.DataFrame, metadata: None | dict[str, Any]
) -> pd.Series | pd.DataFrame:  # TODO: make generic: input type = output type
    if metadata is not None:
        data_object.attrs = metadata
    else:
        data_object.attrs = {}
    return data_object


def parse_pandas_data_content(
    data_content: str | dict | list, typ: Literal["series", "frame"], parsing_options: dict
) -> pd.DataFrame | pd.Series:
    try:
        if isinstance(data_content, str):
            parsed_pandas_object = pd.read_json(
                io.StringIO(data_content), typ=typ, **parsing_options
            )
        else:
            parsed_pandas_object = pd.read_json(data_content, typ=typ, **parsing_options)

    except Exception:  # noqa: BLE001
        try:
            parsed_pandas_object = pd.read_json(
                io.StringIO(json.dumps(data_content)), typ=typ, **parsing_options
            )

        except Exception as read_json_exception:  # noqa: BLE001
            raise ValueError(
                "Could not parse provided input as Pandas "
                + ("Series" if typ == "series" else "DataFrame")
            ) from read_json_exception

    return parsed_pandas_object


class DataType(StrEnum):
    """hetida designer data types

    These are the types available for component/workflow inputs/outputs.
    """

    Integer = "INT"
    Float = "FLOAT"
    String = "STRING"
    DataFrame = "DATAFRAME"
    Series = "SERIES"
    MultiTSFrame = "MULTITSFRAME"
    SingleTSFrame = "SINGLETSFRAME"
    Boolean = "BOOLEAN"
    Any = "ANY"
    PlotlyJson = "PLOTLYJSON"


def parse_pydantic_series(v: pd.Series | str | dict | list) -> pd.Series:
    """Custom pydantic Data Type for parsing Pandas Series

    Parses either a json string according to pandas.read_json
    with typ="series" and default arguments otherwise or
    a Python dict-like data structure using the constructor
    of the pandas.Series class with default arguments.

    Also allows a wrapped variant where metadata can be provided.

    Examples of valid input:
        '{"0":1.0,"1":2.1,"2":3.2}'
        {"0":1.0,"1":2.1,"2":3.2}
        [1.2, 3.5, 2.9]
        '[1.2, 3.5, 2.9]'

    """
    if isinstance(v, pd.Series):
        return v

    if not isinstance(
        v, str | dict | list
    ):  # need to check at runtime since we get objects from user code
        msg = f"Got unexpected type at runtime when parsing Series: {str(type(v))}"
        logger.error(msg)
        raise ValueError(msg)

    data_content, metadata, parsing_options = parse_wrapped_content(v, "SERIES")

    return wrap_metadata_as_attrs(
        parse_pandas_data_content(data_content, "series", parsing_options), metadata
    )


def serialize_series(s: pd.Series) -> dict[str, Any]:
    return {
        "__hd_wrapped_data_object__": "SERIES",
        "__metadata__": s.attrs,
        "__data__": json.loads(
            # double serialization/deserialization in order to serialize both NaN and NaT
            # to null
            s.to_json(
                date_format="iso",
                orient="split",
                # orient="split" serialization is the only way pandas keeps duplicate index
                #  (with possibly different values) entries for Series objects!
            )
        ),
        "__data_parsing_options__": {"orient": "split"},
    }


PydanticPandasSeries = Annotated[
    pd.Series,
    PlainSerializer(serialize_series),
    BeforeValidator(parse_pydantic_series),
]


def parse_pydantic_pandas_data_frame(v: pd.DataFrame | str | dict | list) -> pd.DataFrame:
    """Custom pydantic Data Type for parsing Pandas DataFrames

    Parses either a json string according to pandas.read_json
    with typ="frame" and default arguments otherwise or
    a Python dict-like data structure using the constructor
    of the pandas.DataFrame class with default arguments.

    Additionally a MetaDataWrapped variant of these can be parsed
    and then is equipped with the provided metadata in the `attrs`
    attribute (https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.attrs.html
    """
    if isinstance(v, pd.DataFrame):
        return v

    if not isinstance(
        v, str | dict | list
    ):  # need to check at runtime since we get objects from user code
        msg = f"Got unexpected type at runtime when parsing DataFrame: {str(type(v))}"
        logger.error(msg)
        raise ValueError(msg)

    data_content, metadata, parsing_options = parse_wrapped_content(v, "DATAFRAME")

    return wrap_metadata_as_attrs(
        parse_pandas_data_content(data_content, "frame", parsing_options), metadata
    )


def serialize_dataframe(df: pd.DataFrame) -> dict[str, Any]:
    return {
        "__hd_wrapped_data_object__": "DATAFRAME",
        "__metadata__": df.attrs,
        "__data__": json.loads(
            df.to_json(date_format="iso")  # in order to serialize both NaN and NaT to null
        ),
    }


PydanticPandasDataFrame = Annotated[
    pd.DataFrame,
    PlainSerializer(serialize_dataframe),
    BeforeValidator(parse_pydantic_pandas_data_frame),
]


def validate_multits_properties(  # noqa:PLR0912
    df: pd.DataFrame,
) -> pd.DataFrame:
    if len(df.columns) == 0:
        attrs = df.attrs
        df = pd.DataFrame(columns=MULTITSFRAME_COLUMN_NAMES)
        df.attrs = attrs

    if len(df.columns) < 3:
        raise ValueError(
            "MultiTSFrame requires at least 3 columns: metric, timestamp"
            f" and at least one additional columns. Only found {str(df.columns)}"
        )

    if not ({"metric", "timestamp"}.issubset(set(df.columns))):
        column_names_string = ", ".join(df.columns)
        raise ValueError(
            f"The column names {column_names_string} don't contain required columns"
            ' "timestamp" and "metric" for a MultiTSFrame.'
        )

    if df["metric"].isna().any():
        raise ValueError("No null values are allowed for the column 'metric' of a MulitTSFrame.")

    df["metric"] = df["metric"].astype("string")

    if df["timestamp"].isna().any():
        raise ValueError("No null values are allowed for the column 'timestamp' of a MulitTSFrame.")

    if len(df.index) == 0:
        df["timestamp"] = pd.to_datetime(df["timestamp"], utc=True)

    if not isinstance(df["timestamp"].dtype, pd.DatetimeTZDtype):
        raise ValueError(
            "Column 'timestamp' of MultiTSFrame does not have DatetimeTZDtype dtype. "
            f"Got {str(df['timestamp'].dtype)} index dtype instead."
        )

    if df["timestamp"].dt.tz not in (pytz.UTC, datetime.timezone.utc):
        raise ValueError(
            "Column 'timestamp' of MultiTSFrame does not have UTC timezone. "
            f"Got {str(df['timestamp'].dt.tz)} timezone instead."
        )

    return df.sort_values("timestamp")


def parse_pydantic_multi_timeseries_pandas_data_frame(
    v: pd.DataFrame | str | dict | list,
) -> pd.DataFrame:
    if isinstance(v, pd.DataFrame):
        return v

    if not isinstance(
        v, str | dict | list
    ):  # need to check at runtime since we get objects from user code
        msg = f"Got unexpected type at runtime when parsing MultiTsFrame: {str(type(v))}"
        logger.error(msg)
        raise ValueError(msg)

    data_content, metadata, parsing_options = parse_wrapped_content(v, "DATAFRAME")

    new_df = wrap_metadata_as_attrs(
        parse_pandas_data_content(data_content, "frame", parsing_options), metadata
    )

    return validate_multits_properties(new_df)


def serialize_multitsframe(df: pd.DataFrame) -> dict[str, Any]:
    return {
        "__hd_wrapped_data_object__": "DATAFRAME",
        "__metadata__": df.attrs,
        "__data__": json.loads(
            df.to_json(date_format="iso")
        ),  # in order to serialize both NaN and NaT to null
    }


PydanticMultiTimeseriesPandasDataFrame = Annotated[
    pd.DataFrame,
    PlainSerializer(serialize_multitsframe),
    BeforeValidator(parse_pydantic_multi_timeseries_pandas_data_frame),
]


def validate_singlets_properties(
    df: pd.DataFrame,
) -> pd.DataFrame:
    """Validate the conventions of a SingleTSFrame

    A SingleTSFrame holds a single — possibly multi-dimensional — timeseries. It has the same
    tabular representation as a MultiTSFrame, except that it has no "metric" column, since
    the metric is unique for the whole frame. Every column besides "timestamp" is a value
    dimension. This explicitly includes a column named "metric", which is treated like any
    other value column here.
    """
    if len(df.columns) == 0:
        attrs = df.attrs
        df = pd.DataFrame(columns=SINGLETSFRAME_COLUMN_NAMES)
        df.attrs = attrs

    if len(df.columns) < 2:
        raise ValueError(
            "SingleTSFrame requires at least 2 columns: timestamp"
            f" and at least one value column. Only found {str(df.columns)}"
        )

    if "timestamp" not in df.columns:
        column_names_string = ", ".join(df.columns)
        raise ValueError(
            f"The column names {column_names_string} don't contain the required column"
            ' "timestamp" for a SingleTSFrame.'
        )

    if df["timestamp"].isna().any():
        raise ValueError(
            "No null values are allowed for the column 'timestamp' of a SingleTSFrame."
        )

    if len(df.index) == 0:
        df["timestamp"] = pd.to_datetime(df["timestamp"], utc=True)

    if not isinstance(df["timestamp"].dtype, pd.DatetimeTZDtype):
        raise ValueError(
            "Column 'timestamp' of SingleTSFrame does not have DatetimeTZDtype dtype. "
            f"Got {str(df['timestamp'].dtype)} index dtype instead."
        )

    if df["timestamp"].dt.tz not in (pytz.UTC, datetime.timezone.utc):
        raise ValueError(
            "Column 'timestamp' of SingleTSFrame does not have UTC timezone. "
            f"Got {str(df['timestamp'].dt.tz)} timezone instead."
        )

    return df.sort_values("timestamp")


def parse_pydantic_single_timeseries_pandas_data_frame(
    v: pd.DataFrame | str | dict | list,
) -> pd.DataFrame:
    if isinstance(v, pd.DataFrame):
        return v

    if not isinstance(
        v, str | dict | list
    ):  # need to check at runtime since we get objects from user code
        msg = f"Got unexpected type at runtime when parsing SingleTsFrame: {str(type(v))}"
        logger.error(msg)
        raise ValueError(msg)

    data_content, metadata, parsing_options = parse_wrapped_content(v, "DATAFRAME")

    new_df = wrap_metadata_as_attrs(
        parse_pandas_data_content(data_content, "frame", parsing_options), metadata
    )

    return validate_singlets_properties(new_df)


def serialize_singletsframe(df: pd.DataFrame) -> dict[str, Any]:
    return {
        "__hd_wrapped_data_object__": "DATAFRAME",
        "__metadata__": df.attrs,
        "__data__": json.loads(
            df.to_json(date_format="iso")
        ),  # in order to serialize both NaN and NaT to null
    }


PydanticSingleTimeseriesPandasDataFrame = Annotated[
    pd.DataFrame,
    PlainSerializer(serialize_singletsframe),
    BeforeValidator(parse_pydantic_single_timeseries_pandas_data_frame),
]


# "Raw" serializer variants: Previously we parsed pandas output JSON back with json.loads to
# ensure having a json serializable object at the service boundary later and avoiding having
# 5xx errors. At that boundary it was re-encoded leading to multiple serialization loops.
# Now msgspec.Raw is used to both ensure json serializability and keep the generated bytes
# avoiding further such loops
#
# A dict containing ``msgspec.Raw`` values MUST be encoded with msgspec directly; it cannot be
# routed through pydantic's ``model_dump(mode="json")`` (which does not understand ``Raw``).


def serialize_series_raw(s: pd.Series) -> dict[str, Any]:
    return {
        "__hd_wrapped_data_object__": "SERIES",
        # __metdata__ uncoerced on purpose; the msgspec enc_hook coerces exotic values
        # (see hetdesrun.service.serialization_helpers):
        "__metadata__": s.attrs,
        "__data__": msgspec.Raw(
            s.to_json(
                date_format="iso",
                orient="split",
                # orient="split" is the only way pandas keeps duplicate index entries for Series
            ).encode()
        ),
        "__data_parsing_options__": {"orient": "split"},
    }


def serialize_dataframe_raw(df: pd.DataFrame) -> dict[str, Any]:
    return {
        "__hd_wrapped_data_object__": "DATAFRAME",
        "__metadata__": df.attrs,
        "__data__": msgspec.Raw(df.to_json(date_format="iso").encode()),
    }


# MultiTSFrames and SingleTSFrames use the same wrapper as ordinary DataFrames.
serialize_multitsframe_raw = serialize_dataframe_raw
serialize_singletsframe_raw = serialize_dataframe_raw


def parse_any(v: Any) -> Any:
    """Tries to parse Any objects somehow intelligently

    Reason is that an object may be provided by the backend either as a proper json-object directly
    in some cases (dict-like objects) or as json-encoded string (happens for example for lists).

    Sometimes, if the frontend is involved, json strings get even double-string-encoded! This is a
    known bug of frontend-backend-runtime interaction.

    Sometimes adapter implementations deliver ANY-data directly as json objects and othertimes
    string-encoded.

    As a workaround for all these cases this class tries to json-parse a string if it receives one
    and only if that does not work it yields the actual string value. If it works and the result is
    itself a string again it tries to json-decode a second time and returns the result if that
    works. Otherwise it returns the result string of the first parsing.

    This workaround is justified by the argument that the user should really use a STRING input if
    a string is expected and not an ANY input. Likewise, adapters should offer string data as STRING
    data sources and not as ANY data sources.
    """
    if isinstance(v, str):
        # try to parse string as json
        try:
            parsed_json_object = json.loads(v)
        except json.decoder.JSONDecodeError:
            logger.info(
                "Could not JSON-parse string %s in Any input."
                " Therefore treating it as actual string value",
                v[:30] + "..." if len(v) > 10 else v,
            )
            return v

        if isinstance(
            parsed_json_object, str
        ):  # sometimes it even gets double-encoded for some reasons
            try:
                parsed_json_object = json.loads(parsed_json_object)
            except json.decoder.JSONDecodeError:
                logger.info(
                    "Could not JSON-parse string %s in Any input. "
                    " Therefore treating it as actual string value",
                    parsed_json_object[:30] + "..." if len(v) > 10 else v,
                )
                return parsed_json_object

        return parsed_json_object
    return v


ParsedAny = Annotated[
    Any,
    BeforeValidator(parse_any),
]


def parse_plotly_json_dict(v: dict) -> dict:
    if isinstance(v, dict):
        # TODO: check structure more thoroughly
        return v

    raise ValueError("Plotly JSON is not a dict object.")


ParsedPlotly = Annotated[dict, PlainValidator(parse_plotly_json_dict)]


def wrap_basic_type_parsing(
    value: Any, handler: ValidatorFunctionWrapHandler, basic_type: type
) -> Any:
    pydantic_parsed_basic_type = handler(value)

    if isinstance(value, basic_type) and value == pydantic_parsed_basic_type:
        return value  # original value to preserve idempotency.
    # Avoids pydantic attribute copying as well.

    return pydantic_parsed_basic_type


ParsedFloat = Annotated[
    float,
    WrapValidator(lambda value, handler: wrap_basic_type_parsing(value, handler, basic_type=float)),
]

ParsedInt = Annotated[
    int,
    WrapValidator(lambda value, handler: wrap_basic_type_parsing(value, handler, basic_type=int)),
]

ParsedBool = Annotated[
    bool,
    WrapValidator(lambda value, handler: wrap_basic_type_parsing(value, handler, basic_type=bool)),
]

ParsedStr = Annotated[
    str,
    WrapValidator(lambda value, handler: wrap_basic_type_parsing(value, handler, basic_type=str)),
]

data_type_map: dict[DataType | None, type] = {
    DataType.Integer: ParsedInt,
    DataType.Float: ParsedFloat,
    DataType.String: ParsedStr,
    DataType.Series: PydanticPandasSeries,
    DataType.MultiTSFrame: PydanticMultiTimeseriesPandasDataFrame,
    DataType.SingleTSFrame: PydanticSingleTimeseriesPandasDataFrame,
    DataType.DataFrame: PydanticPandasDataFrame,
    DataType.Boolean: ParsedBool,
    # Any as Type is the correct way to tell pydantic how to parse an arbitrary object:
    DataType.Any: ParsedAny,
    DataType.PlotlyJson: ParsedPlotly,
    None: NoneType,
}

optional_data_type_map: dict[DataType | None, Any] = {
    DataType.Integer: ParsedInt | None,
    DataType.Float: ParsedFloat | None,
    DataType.String: ParsedStr | None,
    DataType.Series: PydanticPandasSeries | None,
    DataType.MultiTSFrame: PydanticMultiTimeseriesPandasDataFrame | None,
    DataType.SingleTSFrame: PydanticSingleTimeseriesPandasDataFrame | None,
    DataType.DataFrame: PydanticPandasDataFrame | None,
    DataType.Boolean: ParsedBool | None,
    # Any as Type is the correct way to tell pydantic how to parse an arbitrary object:
    DataType.Any: ParsedAny | None,
    DataType.PlotlyJson: ParsedPlotly | None,
    None: NoneType | None,
}


def serialize_plotly_fig(v: dict[str, Any] | Figure) -> Any:
    if isinstance(v, dict):
        return v

    # possibly quite inefficient (multiple serialisation / deserialization) but
    # guarantees that the PlotlyJSONEncoder is used and so the resulting Json
    # should be definitely compatible with the plotly javascript library:

    # Whats the difference using json.loads(json.dumps(fig_dict_obj, cls=PlotlyJSONEncoder))
    # or employing fig.to_plotly_json()
    return json.loads(v.to_json(validate=False))


serializer_funcs_by_type = {
    pd.Series: serialize_series,
    pd.DataFrame: lambda v: {
        "__hd_wrapped_data_object__": "DATAFRAME",
        "__metadata__": v.attrs,
        "__data__": json.loads(
            v.to_json(date_format="iso")  # in order to serialize both NaN and NaT to null
        ),
    },
    PydanticPandasSeries: serialize_series,
    PydanticPandasDataFrame: lambda v: {
        "__hd_wrapped_data_object__": "DATAFRAME",
        "__metadata__": v.attrs,
        "__data__": json.loads(
            v.to_json(date_format="iso")  # in order to serialize both NaN and NaT to null
        ),
    },
    PydanticMultiTimeseriesPandasDataFrame: lambda v: {
        "__hd_wrapped_data_object__": "DATAFRAME",
        "__metadata__": v.attrs,
        "__data__": json.loads(
            v.to_json(date_format="iso")
        ),  # in order to serialize both NaN and NaT to null
    },
    PydanticSingleTimeseriesPandasDataFrame: lambda v: {
        "__hd_wrapped_data_object__": "DATAFRAME",
        "__metadata__": v.attrs,
        "__data__": json.loads(
            v.to_json(date_format="iso")
        ),  # in order to serialize both NaN and NaT to null
    },
    np.ndarray: lambda v: v.tolist(),
    datetime.datetime: lambda v: v.isoformat(),
    UUID: lambda v: str(v),  # alternatively: v.hex
    Figure: serialize_plotly_fig,  # lambda v: json.loads(json.dumps(v.to_plotly_json(), cls=PlotlyJSONEncoder)),
    NoneType: lambda x: x,
    None: lambda x: x,
}


# Only the pandas payload types get the ``msgspec.Raw`` splicing treatment (they are the large ones
# and the only ones that paid the json.loads round-trip). Every other output type is left uncoerced
# and handed straight to the encoder, whose ``enc_hook`` (pydantic's ``to_jsonable_python``) coerces
# any exotic values in a single encode pass - so numpy scalars, plotly dicts, datetimes etc.
# serialize (and fail) exactly as pydantic's ``model_dump(mode="json")`` did.
raw_pandas_serializer_funcs_by_type = {
    PydanticPandasSeries: serialize_series_raw,
    PydanticPandasDataFrame: serialize_dataframe_raw,
    PydanticMultiTimeseriesPandasDataFrame: serialize_multitsframe_raw,
    PydanticSingleTimeseriesPandasDataFrame: serialize_singletsframe_raw,
}


def serialize_output_results_raw(
    output_results_by_output_name: dict[str, Any],
    output_types_by_output_name: dict[str, DataType | None],
) -> dict[str, Any]:
    """Raw-splicing counterpart of ``WorkflowExecutionInfo.serialize_output_result_dict``.

    Dispatches by declared output data type: pandas payloads become the ``msgspec.Raw`` wrapper
    (skipping the ``json.loads(df.to_json())`` round-trip), every other value is passed through
    **uncoerced**. This is deliberate: a plotly (or ANY) output is arbitrary user-component code
    output, so it must still be guarded/coerced - but that happens exactly once, inside the single
    ``enc_hook`` encode pass, instead of a separate pre-walk plus the encode.

    The returned dict therefore contains ``Raw`` (and possibly uncoerced) values and must be encoded
    with msgspec using ``enc_hook=to_jsonable_python`` - not dumped by pydantic. See
    ``hetdesrun.service.serialization_helpers.encode_workflow_execution_result``.
    """
    result = {}
    for outp_name, obj in output_results_by_output_name.items():
        raw_serializer = raw_pandas_serializer_funcs_by_type.get(
            data_type_map[output_types_by_output_name[outp_name]]
        )
        if raw_serializer is not None:
            result[outp_name] = raw_serializer(obj)
        else:
            result[outp_name] = obj
    return result


class NamedDataTypedValue(TypedDict):
    name: str
    type: DataType  # noqa: A003
    value: Any


def parse_via_pydantic(
    entries: list[NamedDataTypedValue],
    type_map: dict[DataType | None, type] | dict[DataType | None, UnionType] | None = None,
    null_str_to_None: bool = False,
) -> Any:
    """Parse data dynamically into a pydantic object

    Optionally a type_map can be specified which differs from the default data_type_map

    Returns an instantiated pydantic object if no parsing exception is thrown.

    May raise the typical exceptions of pydantic parsing.

    null_str_to_None converts value strings "null" to None before injecting into the Pydantic
    model __init__. This is necessary in order to allow entering null. E.g. consider the
    a int | None type annotation of a model field. Pydantic will happily interpret
    None, 42, "42" correctly but not try to interpret non-int-interpretable strings like
    "null" as None for values in the init method.
    """
    type_dict: dict[str, tuple[type | UnionType, "ellipsis"]] = {  # noqa: F821, UP037
        entry["name"]: (
            type_map[entry["type"]]
            if type_map is not None
            else data_type_map[entry["type"]],  # default to data_type_map
            ...,
        )
        for entry in entries
    }

    DynamicModel = create_model(
        "DynamicModel",
        **type_dict,
        __config__=ConfigDict(arbitrary_types_allowed=True, coerce_numbers_to_str=True),
    )  # type: ignore

    dyn_obj = DynamicModel(  # type: ignore
        **{
            entry["name"]: (
                entry["value"]
                if (
                    not null_str_to_None
                    or not isinstance(entry["value"], str)
                    or entry["value"] != "null"
                )
                else None
            )
            for entry in entries
        }
    )
    return dyn_obj


def parse_dict_using_data_type_dict(
    data_obj_dict: dict[str, Any],
    data_type_dict: dict[str, DataType],
    nullable: bool = False,
) -> dict[str, Any]:
    entries: list[NamedDataTypedValue] = [
        {"name": key, "type": data_type_dict[key], "value": data_obj}
        for key, data_obj in data_obj_dict.items()
    ]

    parsed_objects = dict(
        parse_via_pydantic(
            entries,
            type_map=data_type_map if nullable is False else optional_data_type_map,
            null_str_to_None=nullable,  # interpretation of "null" string values as None
        )
    )

    return parsed_objects


def parsing_not_identical(
    data_obj_dict: dict[str, Any],
    data_type_dict: dict[str, DataType],
    nullable: bool = False,
) -> dict[str, tuple[str, str]]:
    """Get keys for which parsed object is not identical to the input object

    Parsing according to Datatypes should be idempotent. So if an object is already
    correctly parsed, parsing should be the identity operation and preserve id(..).

    For a dict of objects and corresponding dict of DataTypes this function returns a dict
    containing those keys for which parsing is not the identity, and as value a tuple with first
    entry the actual received object type as string and as second entry the expected hetida designer
    DataType.

    Therefore this function can be used to determine if user produced objects have the correct
    type (i.e. the returned dict is empty!)

    Raises pydantic.ValidationError if parsing fails!
    """
    parsed_objects = parse_dict_using_data_type_dict(data_obj_dict, data_type_dict, nullable)

    not_identical = {
        key: (str(type(data_obj_dict[key])), str(data_type_dict[key]))
        for key, parsed_obj in parsed_objects.items()
        if id(parsed_obj) != id(data_obj_dict[key])
    }

    return not_identical


def parse_dynamically_from_datatypes(
    entries: list[NamedDataTypedValue], nullable: bool = False
) -> Any:
    return parse_via_pydantic(
        entries,
        type_map=data_type_map if nullable is False else optional_data_type_map,
        null_str_to_None=nullable,  # interpretation of "null" string values as None
    )


def parse_single_value_dynamically(
    name: str, value: Any, data_type: DataType, nullable: bool
) -> Any:
    return dict(
        parse_dynamically_from_datatypes(
            [{"name": name, "type": data_type, "value": value}], nullable
        )
    )[name]


def parse_value(value: Any, data_type_str: str, nullable: bool) -> Any:
    return parse_single_value_dynamically("some_value", value, DataType(data_type_str), nullable)


def parse_obj_as_type(obj: Any, target_type: type) -> Any:
    """Parse a type, e.g. Pydantic Custom Data Type

    Unfortunately defining a Pydantic Custom Data Type using Annotated and a BeforeValidator
    will only invoke this BeforeValidator if it is a field of another model. If you call
    it directly on an input object like MyPydanticCustomDataTypeWithBeforeValidator("test"),
    the BeforeValidator will never be called.

    This function is a workaround to this problem
    """

    DynamicModel = create_model(
        "DynamicModel",
        value=(target_type, ...),
        __config__=ConfigDict(arbitrary_types_allowed=True, coerce_numbers_to_str=True),
    )

    return DynamicModel(value=obj).value  # type: ignore


def parse_default_value(component_info: dict, input_name: str) -> Any:
    """Parse default value from COMPONENT_INFO dict

    Used in component main function header to parse a default
    value from the COMPONENT_INFO dict for an input.
    """
    inp = component_info["inputs"][input_name]
    return parse_value(inp["default_value"], inp["data_type"], True)


def keep_auto_ticks_for_overlaying_axes(fig_dict_obj: dict[str, Any]) -> None:
    """Keep automatic tick placement for overlaying axes of a plotly json dict

    Since plotly.js 4, overlaying axes (e.g. the additional y axes of plots with multiple
    y axes) place their ticks at the tick positions of the overlaid axis by default
    (tickmode "sync"), typically resulting in tick labels at uneven values.

    This sets tickmode "auto" for overlaying axes which do not configure their tick
    placement themselves, keeping the behaviour of earlier plotly.js versions.
    """
    for key, axis in fig_dict_obj.get("layout", {}).items():
        if (
            re.fullmatch(r"[xy]axis\d*", key) is not None
            and isinstance(axis, dict)
            and axis.get("overlaying") not in (None, "free")
            and not ({"tickmode", "tickvals", "dtick"} & axis.keys())
        ):
            axis["tickmode"] = "auto"


def plotly_fig_to_json_dict(
    fig: Figure, add_config_settings: bool = True, update_x_axes_tickformat: bool = True
) -> Any:
    """Turn Plotly figure into a Python dict-like object

    This function can be used in visualization components to obtain the
    correct plotly json-like object from a Plotly Figure object.

    Overlaying axes keep their automatic tick placement (see
    keep_auto_ticks_for_overlaying_axes) and the "send to cloud" modebar button
    is disabled in the config, unless explicitly configured.

    See visualization components from the accompanying base components for
    examples on usage.
    """

    plot_target_settings = get_plot_target_settings()

    if update_x_axes_tickformat and plot_target_settings.datetime_tick_format is not None:
        fig.update_xaxes(tickformat=plot_target_settings.datetime_tick_format)

    fig_dict_obj = serialize_plotly_fig(fig)

    keep_auto_ticks_for_overlaying_axes(fig_dict_obj)

    if "config" not in fig_dict_obj:
        fig_dict_obj["config"] = {}

    # plotly.js >= 4 shows this button by default. It uploads the plot including
    # its data to Plotly Cloud.
    fig_dict_obj["config"].setdefault("showSendToCloud", False)

    if add_config_settings and plot_target_settings.plot_target_locale is not None:
        fig_dict_obj["config"]["locale"] = plot_target_settings.plot_target_locale

    return fig_dict_obj


def extract_sub_value(data: dict[str, Any], json_path: list[str]) -> Any | None:
    """Extract a sub value of a nested dict structure

    Returns either the found value, or None if it could not be found.
    """
    current: Any | None = data
    for key in json_path:
        if isinstance(current, dict):
            current = current.get(key)
            if current is None:
                return None
        else:
            return None
    return current


def extract_sub_mapping(
    dict_of_metadata: dict[str | int, dict[str, Any]], json_path: list[str]
) -> dict[str | int, Any]:
    return {
        main_key: extract_sub_value(sub_dict, json_path)
        for main_key, sub_dict in dict_of_metadata.items()
    }


def add_columns_from_metric_metadata(
    multitsframe: pd.DataFrame,
    keys: list[str | list[str]] | None = None,
    replace_metric_col_with_key: str | list[str] | None = None,
    default_value: Any = np.nan,
) -> pd.DataFrame:
    """Add columns (or replace metric column) from attrs metadata in a multitsframe

    Expects a hetida designer multitsframe with metadata according to hetida designer
    metadata conventions.

    For replacing the metric col using replace_metric_col_with_key this expects a
    value for each metric to be found in the metadata. If that is not the case it keeps
    the existing metric entry!

    For adding columns, a default_value can be specified that is used when no entry for
    a key is found in the metadata for the corresponding metric.

    Modifies in-place, but also returns the modified multitsframe.

    keys is a list of json-path-like lists. For each entry a column is added, where
    in the found metric metdata the field at subpath defined by this entry is assigned.
    For convenience a str entry is interpreted as json path of length 1.

    E.g. if your attrs looks like
        "by_metric": {
            "a": {
                "structured_metadata": {
                    "metric": {  # metadata associated to the current metric. These are
                        "name": "AAAAA",
                        "display_name": "some A",
                        "short_display_name": "a",
                        "description": "some descriptions",
                        "external_id": "AA",
                        "channel_id": "abc123-...",
                    }
                }
            },
            "b": {
                "structured_metadata": {
                    "metric": {  # metadata associated to the current metric. These are
                        "name": "BBBBB",
                        "display_name": "some B",
                        "short_display_name": "b",
                        "description": "some descriptions",
                        "external_id": "BB",
                        "channel_id": "abc123-...",
                    }
                }
            },
        }

    then you can specify to add an external_id column and an additional channel id column via
        add_columns_from_metric_metadata(
            multitsframe, keys=[
                ["structured_metadata", "metric", "external_id"],
                ["structured_metadata", "metric", "channel_id"]
            ]
        )

    Similarly replace_metric_col_with_key is a json-path-like list or interpreted as length 1 such
    path if a single str.
    """

    metrics_metadata = extract_sub_value(multitsframe.attrs, ["by_metric"])

    if metrics_metadata is None or all(
        (x is None or len(x) == 0)
        for x in metrics_metadata.values()  # nothin contained in the values
    ):
        metrics_metadata = extract_sub_value(multitsframe.attrs, ["metric_metadata"])

    if metrics_metadata is None:
        logger.warning("Did not found metric metadata for extracting data into columns.")
        return multitsframe

    if replace_metric_col_with_key is not None:
        key_list = (
            [replace_metric_col_with_key]
            if isinstance(replace_metric_col_with_key, str)
            else replace_metric_col_with_key
        )

        multitsframe["metric"] = multitsframe["metric"].replace(
            extract_sub_mapping(metrics_metadata, key_list)  # type: ignore
        )
    if keys is not None:
        for key in keys:
            key_list = [key] if isinstance(key, str) else key

            mapping_dict = extract_sub_mapping(metrics_metadata, key_list)  # type: ignore

            mapping_dict = defaultdict(lambda: default_value, mapping_dict)

            multitsframe[key_list[-1]] = multitsframe["metric"].map(mapping_dict)

    return multitsframe

END_HEREDOC
)

###############################################################################
#                                                                             #
#                                Usage / help                                 #
#                                                                             #
###############################################################################

usage() {
    if [ -n "${1-}" ]; then
        _log_error "--> ERROR: $1"
    fi

    echo "BASIC USAGE: Run"
    echo "  $0 SUBCOMMAND <parameters>"
    echo "  $0 sync SYNCCOMMAND INSTANCE"
    echo
    echo "EXAMPLES"
    echo
    echo "  SYNC:"
    echo "    Init a new hetida designer instance against which you want to sync"
    echo
    echo "       $0 sync init my-hd-instance"
    echo
    echo "    After following the instructions and setting configuration options"
    echo "    and credentials in the generated files:"
    echo
    echo "       $0 sync pull my-hd-instance    # sync to local dir"
    echo "       $0 sync push my-hd-instance    # push local dir to hd instance"
    echo
    echo "    Sync of the local file content of my-hd-instance to/from another configured"
    echo "    instance named my-other-hd-instance can be done via"
    echo
    echo "       $0 sync push my-other-hd-instance from my-hd-instance"
    echo "       $0 sync pull my-other-hd-instance to my-hd-instance"
    echo
    echo "    It is recommended to use git for versioning of local files and you"
    echo "    should commit before and after each sync operation."
    echo
    echo "  FETCHING:"
    echo "    Write json list of all components from Physics category to stdout"
    echo
    echo "        $0 "'fetch --url-append "?category=Physics&type=COMPONENT"'
    echo
    echo "    Notes: * In these examples configuration like backend api url, auth etc is assumed to"
    echo "             come from ./hdctl.env file or HDCTL_* environment variables."
    echo "           * available query parameters of the backend api are documented in the openapi"
    echo "             documentation of the /api/transformations GET / PUT endpoints and the"
    echo "             /api/maintenance/* endpoints"
    echo "           * --url-append does not urlencode automatically."
    echo
    echo "    With recent curl (>= 7.87.0, from 2023 or later) you can also propagate query params"
    echo "    via --url-query which automatically url encodes them as follows (note the space char"
    echo "    in category):"
    echo
    echo "        $0 "'fetch -- --url-query "category=Anomaly Detection" --url-query "type=COMPONENT"'
    echo
    echo "    Export json list of transformation revisions of category Physics into single file,"
    echo "    pretty printing the json by piping it through jq:"
    echo
    echo "        $0 fetch -- -d "'"category=Physics"'" | jq > physics_trafos.json"
    echo
    echo "    Export all non-deprecated workflows into directory structure at ./exported_trafos including"
    echo "    their dependencies. The directory and subdirectories for type and catgeory will be created"
    echo "    if necessary (This command needs awk and jq!)"
    echo
    echo "        $0 "'fetch --export-directory ./exported_trafos -- \'
    echo '          --url-query "type=WORKFLOW"\'
    echo '          --url-query "include_dependencies=true"\'
    echo '          --url-query "include_deprecated=false"'
    echo
    echo "  PUSHING:"
    echo '    Import from export directory only the workflows with name "Combine two Series'
    echo '    into DataFrame" which are released (there may be drafts or deprecated revisions)'
    echo "    together with their dependencies. Overwrite released trafo revs if necessary:"
    echo
    echo "      $0 "'push --export-trafos ./exported_trafos --\'
    echo '        --url-query "name=Combine two Series into DataFrame"\'
    echo '        --url-query "include_dependencies=true"\'
    echo '        --url-query "allow_overwrite_released=true"\'
    echo '        --url-query "state=RELEASED"'
    echo
    echo "    The response shows which trafo revs actually where imported and which where ignored."
    echo "    You may want to pipe it through jq for pretty printing."
    echo
    echo "    Import released components from json file containing list of trafo revs allowing overwriting"
    echo "    of released components:"
    echo
    echo "      $0 "'push physics_trafos.json --url-append "?type=COMPONENT&state=RELEASED&allow_overwrite_released=true"'
    echo
    echo "  MAINTENANCE:"
    echo
    echo "    Reset test wirings to stored release wirings for released and deprecated trafos:"
    echo
    echo "      $0 "'reset_test_wiring_to_release_wiring -i hddev'
    echo
    echo "    Redeploy base trafos on a remote instance hddev, e.g. after upgrading:"
    echo
    echo "      $0 "'deploy_base_trafos -i hddev --url-query "allow_overwrite_released=true"'
    echo
    echo "    Following examples assume that the maintenance secret is set either in hdctl.env"
    echo "    file (MAINTENANCE_SECRET) or as environment variable HDCTL_MAINTENANCE_SECRET."
    echo
    echo "    Deprecate all but the latest released (i.e. by release date, disregarding version tag!)"
    echo "    revision of every transformation revision group:"
    echo
    echo "      $0 deprecate_all_but_latest_per_group"
    echo
    echo "    Purge your installation: Delete everything and afterwards redeploy base trafos from"
    echo "    running backend (WARNING: irreversible data loss!):"
    echo
    echo "      $0 purge"
    echo
    echo "    Delete all drafts"
    echo
    echo "      $0 delete_drafts"
    echo
    echo "GENERAL HINTS"
    echo "Parameters can be supplied with increasing priority as follows:"
    echo " * On startup a .env file is sourced if available, by default ./hdctl.env."
    echo "   You can configure its path via HDCTL_ENV_FILE environment variable. It should"
    echo "   define variables corresponding to settings of form NAME_IN_UPPER_CASE. In case"
    echo "   of using the sync subcommands instance and credential files are used instead."
    echo " * Environment variables prefixed with HDCTL_ (HDCTL_NAME_IN_UPPER_CASE) are interpreted"
    echo "   and may override NAME_IN_UPPER_CASE from the env file."
    echo " * explicit command line parameters override all these settings"
    echo
    echo "On the command line values to  parameters are separated with space inbetween like"
    echo "    --backend-api-url http://localhost:8080/api"
    echo "or"
    echo "    -b http://localhost:8080/api"
    echo "(i.e. not --backend-api-url=http://localhost:8080/api )"
    echo
    echo "Most parameters are general and can be set independantly of subcommands. Whether they"
    echo "are used depends on the actual subcommand invoked."
    echo
    echo "Arguments after -- are gathered. If the subcommand invokes curl, they are appended to "
    echo "the curl command at the end. This allows for example to usq the --url-query curl argument"
    echo "to control filters when fetching / pushing, as you can see in the examples above."
    echo
    echo "In the following we reference settings by naming them by the variable as it"
    echo "would occur in the .env file (i.e. NAME_IN_UPPER_CASE). Recall that the corresponding"
    echo "environment variable then is HDCTL_NAME_IN_UPPER_CASE."
    echo
    echo "Subcommands typically make http requests to a hd backend api configured via"
    echo "HD_BACKEND_API_URL pointing to the /api resource of the backend service, e.g."
    echo "  http://localhost:8080/api"
    echo
    echo "Authentication is handled as follows:"
    echo "  * if DIRECT_ACCESS_TOKEN is set, it is passed as Bearer token."
    echo "  * if DIRECT_ACCESS_TOKEN is not set / empty string:"
    echo "    * if AUTH_URL is set (e.g. http://localhost/auth)"
    echo "      * if USERNAME is set / non-zero, an access token is tried to be obtained"
    echo "        with PASSWORD from the auth provider at AUTH_URL via openidconnect standard"
    echo "        using password grant credential flow."
    echo "      * if USERNAME is not set, CLIENT_ID and CLIENT_SECRET are used to obtain"
    echo "        an access token from the auth provider at AUTH_URL via openidconnect standard"
    echo "        using client credential grant flow."
    echo "    * if AUTH_URL is not set / empty string the backend is expected to be reachable"
    echo "      without authentication."
    echo
    echo "PARAMETERS"
    echo "  Mandatory arguments to long options are mandatory for short options too. Name of"
    echo "  corresponding environment variable with HDCTL_ prefix in parantheses."
    echo
    echo "  -e | --env-file PATH                     (HDCTL_ENV_FILE)"
    echo "      path to env file that will be sourced at startup. Defaults to ./hdctl.env."
    echo "      If set to non-default value it must exist."
    echo
    echo "  -i | --instance INSTANCE_NAME"
    echo "      Use config from instance files (see sync commands) instead of env file"
    echo
    echo "  -b | --backend-api-url URL              (HDCTL_HD_BACKEND_API_URL)"
    echo "      URL of the backend api resource, e.g. http://localhost:8080/api"
    echo
    echo "  --insecure-backend                      (HDCTL_VERIFY_BACKEND_API_URL)"
    echo "      do not verify https when communicating with the backend."
    echo
    echo "  --add                                   (HDCTL_SYNC_CLEAR_DIRECTORY)"
    echo "      For sync pull commands: If set the target export directory will not be cleared."
    echo "      Trafos will be overwritten / added instead. Note that the environment"
    echo "      variable is true by default and setting --add sets it to false."
    echo
    echo "  -d | --export-directory PATH            (HDCTL_EXPORT_DIRECTORY)"
    echo "      PATH to a directory where data should be exported to (fetch) or loaded from (push)."
    echo "      fetch will create the directory and distribute downloaded trafo revs into json files"
    echo "      at pathes like, using jq and awk"
    echo "          PATH/{workflows,components}/<CATEGORY>/<NAME>_<VERSION_TAG>_<ID>.json"
    echo "      If this is not set, fetch will instead output a json list of all downloaded trafo revs"
    echo "      to stdout."
    echo "      push will use the directory as source if this is set. If not, push expects a path"
    echo "      to a single json file containing an array of trafo revs as first argument instead."
    echo
    echo "  -a | --auth-url AUTH_URL                (HDCTL_AUTH_URL)"
    echo "      Openidconnect auth provider auth endpoint, e.g. https://localhost/auth"
    echo
    echo "  --insecure-auth                         (HDCTL_VERIFY_AUTH_URL)"
    echo "      do not verify https when communicating with auth provider"
    echo
    echo "  -t | --access-token TOKEN               (HDCTL_DIRECT_ACCESS_TOKEN)"
    echo "      If this is provided, it will be used as Bearer access token and no communication"
    echo "      with auth provider will happen. Make sure that TOKEN is not expired."
    echo
    echo "  -r | --realm REALM                      (HDCTL_REALM)"
    echo "      openidconnect realm name that should be used."
    echo
    echo "  -c | --client-id CLIENT_ID              (HDCTL_CLIENT_ID)"
    echo "      The client id. Necessary for both password grant and client credential grant auth flow."
    echo
    echo "  -s | --client-secret CLIENT_SECRET     (HDCTL_CLIENT_SECRET)"
    echo "      The client secret when using client credential secret auth flow"
    echo
    echo "  -m | --maintenance-secret SECRET       (HDCTL_MAINTENANCE_SECRET)"
    echo "      Maintenance actions require a maintenance secret to be configured in the backend"
    echo "      and provided whith each maintenance endpoint request."
    echo
    echo "   -p | --url-append PARAMSTRING         (HDCTL_QUERY_URL_APPEND)"
    echo "       PARAMSTRING will be appended as-is to the url in curl invocations. Can be used"
    echo '       to provide query parameters, for example --url-append "?type=COMPONENT&state=RELEASED"'
    echo
    echo "   -q | --quiet                          (HDCTL_SILENT)"
    echo "       suppress some output"
    echo
    echo "   -v | --verbose                        (HDCTL_VERBOSITY)"
    echo '       The default is "info". Setting it will switch to "debug" and send some additional debug'
    echo "       output to stderr"
    echo
    echo "    --"
    echo "       All command line arguments following -- will be gathered and for curl-invoking"
    echo "       subcommands they will be appended to the curl call."
    echo
    echo "AVAILABLE CUBCOMMANDS BY TOPIC"
    echo "  SYNC PULL / SYNC PUSH / SYNC INIT"
    echo "    Sync is the easiest way for bidirectional transfer of selections of workflows and"
    echo "    components between a local directory and a hetida designer instance. Under the hood"
    echo "    it uses the fetch and push commands described below, but allows configuration and"
    echo "    credentials to reside in local files. In combination with version control, e.g. git,"
    echo "    this enables:"
    echo "      * external version control"
    echo "      * editing locally / hybrid working and frequent switching between local work"
    echo "        and work in the hetida designer user interface"
    echo "      * scripting CD/CI and GitOps"
    echo
    echo "    The configuration and credential files for a remote hetida designer instance"
    echo '    you want to name "my-hd-instance" can be pregenerated via'
    echo
    echo "      $0 sync init my-hd-instance"
    echo
    echo "    Make sure you follow the instructions this command prints out! After that"
    echo "    you should have at least a my-hd-instance.hd-instance file and a "
    echo "    my-hd-instance.hd-creds file which are used as configuration for syncing."
    echo "    Additionally a my-hd-instance.hd-creds.stub file is present that can be checked into"
    echo "    version control together with the .hd-instance file. You should of course never"
    echo "    put the my-hd-instance.hd-creds into version control which you have provided"
    echo "    with actual passwords and secrets. Put *.hd-creds into your .gitignore instead!"
    echo
    echo "    You can now transfer transformations from/to the configured local directory via"
    echo
    echo "      $0 sync pull my-hd-instance"
    echo "      $0 sync push my-hd-instance"
    echo
    echo "    Note that pulling overwrites the local directory completely. It is strongly"
    echo "    recommended to use version control and commit before and after puling and"
    echo "    pushing."
    echo
    echo "    Note that what is pulled and pushed depends on the settings in your .hd-instance"
    echo "    file. The query url parameters of the /api/transformations GET and PUT endpoints"
    echo "    allow for fine-granular filtering and updating / expanding the exported"
    echo "    component code with documentation and a wiring."
    echo
    echo "    Sync of the local file content of my-hd-instance to/from another configured"
    echo "    instance named my-other-hd-instance can be done via"
    echo
    echo "       $0 sync push my-other-hd-instance from my-hd-instance"
    echo "       $0 sync pull my-other-hd-instance to my-hd-instance"
    echo
    echo "    This pushes / pulls with the settings of my-other-hd-instance but uses the"
    echo "    export directory configured for my-hd-instance as source / target."
    echo
    echo "    Finally you can explicitely override the directory to use, e.g.:"
    echo
    echo "       $0 sync push my-prod-instance -d /mounted/volume/dir"
    echo
    echo "  FETCH / EXPORT / DOWNLOAD"
    echo "    The fetch subcommand downloads / exports transformation revisions from the hetida"
    echo "    designer backends using curl, handling auth if necessary. If an export directory"
    echo "    is set it will use jq and awk to distribute the transformation revisions as single"
    echo "    json file each into a directory structure, separating by type and category"
    echo
    echo "      $0 fetch -d EXPORT_DIRECTORY"
    echo
    echo "    Otherwise it will output a json array to stdout"
    echo
    echo "      $0 fetch"
    echo
    echo "    Piping its output to a file creates a single json file containing all fetched"
    echo "    transformation revisions. See -d parameter for details."
    echo
    echo "    By supplying url query parameters as described in the examples at the beginning, only"
    echo "    a part of all available trafo revs can be fetched. See the openapi documentation"
    echo "    of the backend /api/transformations GET endpoint for available parameters."
    echo
    echo "  PUSH / IMPORT / UPLOAD"
    echo "    The push subcommand uploads / imports transformation revisions (in)to the hetida"
    echo "    designer backend using curl, handling auth if necessary. If an export directory"
    echo "    is set it will gather the json files there for the upload:"
    echo
    echo "      $0 push -d -d EXPORT_DIRECTORY"
    echo
    echo "    Otherwise it expects a path to a json file containing a json array of transformation"
    echo "    revisions as first argument:"
    echo
    echo "      $0 push ./exported_trafos.json"
    echo
    echo "    Similarly to the fetch subcommand, by supplying url query parameters, only"
    echo "    a part of all provided trafo revs will be imported. See the openapi documentation"
    echo "    of the backend /api/transformations PUT endpoint for available parameters."
    echo
    echo "  MAINTENANCE"
    echo "    These subcommands correspond to the hd backend api maintenance endpoints."
    echo "    All maintenance subcommands require a non-zero length alphanumeric maintenance"
    echo "    secret to be set at both server and client side:"
    echo "      * server side: configuration of hd backend service (HD_MAINTENANCE_SECRET)"
    echo "      * client side: via -m / --maintenance-secret CLI parameters or environment"
    echo "        variable (HDCTL_MAINTENANCE_SECRET or MAINTENANCE_SECRET in env file)"
    echo
    echo "    See https://github.com/hetida/hetida-designer/blob/release/docs/cleanup.md"
    echo "    or the OpenAPI docs of a started hd backend service for details on what"
    echo "    they do exactly."
    echo
    echo "    Again, query parameters can be provided as described above for fetch and push."
    echo
    echo "    Maintenance subcommands:"
    echo
    echo "      $0 reset_test_wiring_to_release_wiring"
    echo "      $0 deprecate_all_but_latest_per_group"
    echo
    echo ""
    echo "      $0 delete_drafts"
    echo "      $0 delete_unused_deprecated"
    echo "      $0 purge"
    echo "      $0 deploy_base_trafos"
    echo "      $0 autoimport"
    echo ""
    echo "    Execution examples:"
    echo ""
    echo "        ./hdctl delete_drafts -i hd-docker-compose -m test123"
    echo ""
    echo "    This applies the delete_drafts command and defining instance file"
    echo "    and maintenance secret."
    echo ""
    echo ""
    echo "        ./hdctl delete_unused_deprecated -i hd-docker-compose -m test123"
    echo "             -cutoff 2026-05-05 10:00:00Z"
    echo "             -exclude uuid1 -exclude uud2"
    echo ""
    echo "    This applies the delete_unused_deprecated command defining instance file"
    echo "    and maintenance secret and using the arguments cutoff_date and exclude."
    echo "    For the documentation of the effect of these parameter please have a look"
    echo "    at openapi.yaml/Swagger UI of the backend."
    echo ""
}

help() {
    usage
}

###############################################################################
#                                                                             #
#                           General helper functions                          #
#                                                                             #
###############################################################################

reverse_string() {
    # pipe a string into this function to reverse it!
    var="$(cat)" # read from stdin
    len="${#var}"
    i=0
    rev=""
    while ((i < len)); do rev="${var:i++:1}$rev"; done
    echo "$rev"
}

###############################################################################
#                                                                             #
#                               Token management                              #
#                                                                             #
###############################################################################

get_token_response() {
    curl_args=()
    if [[ $VERIFY_AUTH_URL = false ]]; then
        curl_args+=("--insecure")
    fi

    curl_args+=("-d" "client_id=$CLIENT_ID")

    if [[ -n "$USERNAME" ]]; then
        _debug "Using password credential grant mode."
        curl_args+=("-d" 'grant_type=password' "-d" "username=$USERNAME" "-d" "password=$PASSWORD")
    else
        _debug "Using client credential grant mode"
        curl_args+=("-d" 'grant_type=client_credentials' "-d" "client_secret=$CLIENT_SECRET")
    fi

    curl -s "${curl_args[@]}" "$AUTH_URL/realms/$REALM/protocol/openid-connect/token"
}

extract_access_token() {
    no_space_and_newline="$(echo "$1" | tr -d ' ' | tr -d '\n')"
    _debug "$no_space_and_newline"
    removed_prefix="${no_space_and_newline#*\"access_token\":\"}"
    ACCESS_TOKEN="$(echo "$removed_prefix" | cut -d '"' -f 1)"
}

extract_expiration() {
    no_space_and_newline="$(echo "$1" | tr -d ' ' | tr -d '\n')"
    _debug "$no_space_and_newline"
    removed_prefix="${no_space_and_newline#*\"expires_in\":}"
    echo "$removed_prefix" | cut -d ',' -f 1 | cut -d '}' -f 1
}

set_expiration_epoch() {
    local token_resp
    token_resp="$1"
    local expires_in
    expires_in="$(extract_expiration "$token_resp")"
    issue_epoch=$(date +%s)
    EXPIRATION_EPOCH=$(($issue_epoch + $expires_in - 10)) # 10 seconds buffer
}

get_access_token() {
    local token_resp
    token_resp="$(get_token_response)"
    set_expiration_epoch "$token_resp"
    extract_access_token "$token_resp"
}

check_expiration() {
    if (($(date +%s) > $EXPIRATION_EPOCH)); then
        return 1
    fi
    return 0
}

ensure_access_token() {
    if [[ -n "$DIRECT_ACCESS_TOKEN" ]]; then
        _debug "Using directly provided access token. Expiration unknown!"
        ACCESS_TOKEN="$DIRECT_ACCESS_TOKEN"
        return
    elif [[ -z "$AUTH_URL" ]]; then
        _debug "No auth configured (empty auth url). Not setting token."
        return
    elif [[ -z "$ACCESS_TOKEN" ]] || ! check_expiration; then
        _info "Need to get/renew access token."
        get_access_token
    else
        _debug "Access still token okay."
    fi
}

###############################################################################
#                                                                             #
#                                Maintenance                                  #
#                                                                             #
###############################################################################

##################### ~=~ maintenance helper functions ~=~ ####################

_ensure_maintenance_secret() {
    if [[ -z "$MAINTENANCE_SECRET" ]]; then
        _log_error "Setting maintenance secret required for requested subcommand! Aborting."
        exit 1
    fi
}

######################### ~=~ maintenance actions ~=~ #########################

# Reset test wirings to release wirings

#    Resets the current test wiring to a stored release wiring if present.
#    Only affects released and deprecated transformation revisions.
reset_test_wirings_to_release_wirings() {
    _setup "${@}"
    _ensure_maintenance_secret
    ensure_access_token

    curl_args=()
    if [[ $VERIFY_BACKEND_API_URL = false ]]; then
        curl_args+=("--insecure")
    fi
    curl "${curl_args[@]}" -X POST \
        -H "Authorization: Bearer $ACCESS_TOKEN" \
        -H "Content-Type: application/json" \
        -d '{"maintenance_secret":"'"$MAINTENANCE_SECRET"'"}' \
        "$HD_BACKEND_API_URL/maintenance/reset_test_wiring_to_release_wiring$QUERY_URL_APPEND" \
        "${REMAINING_ARGS[@]}"
}

# Deprecate old transformation revision and keep only latest

#     A transformation revision is considered "old" if it is released and there is a newer
#     released transformation revision (i.e. newer release timestamp) in the same group.

#     Note that this has nothing to to with version tags or any versioning scheme!
deprecate_all_but_latest_per_group() {
    _setup "${@}"
    _ensure_maintenance_secret
    ensure_access_token

    curl_args=()
    if [[ $VERIFY_BACKEND_API_URL = false ]]; then
        curl_args+=("--insecure")
    fi
    curl "${curl_args[@]}" -X POST \
        -H "Authorization: Bearer $ACCESS_TOKEN" \
        -H "Content-Type: application/json" \
        -d '{"maintenance_secret":"'"$MAINTENANCE_SECRET"'"}' \
        "$HD_BACKEND_API_URL/maintenance/deprecate_all_but_latest_per_group$QUERY_URL_APPEND" \
        "${REMAINING_ARGS[@]}"
}

# Delete all transformation revisions with state DRAFT
delete_drafts() {
    _setup "${@}"
    _ensure_maintenance_secret
    ensure_access_token

    curl_args=()
    if [[ $VERIFY_BACKEND_API_URL = false ]]; then
        curl_args+=("--insecure")
    fi
    curl "${curl_args[@]}" -X POST \
        -H "Authorization: Bearer $ACCESS_TOKEN" \
        -H "Content-Type: application/json" \
        -d '{"maintenance_secret":"'"$MAINTENANCE_SECRET"'"}' \
        "$HD_BACKEND_API_URL/maintenance/delete_drafts$QUERY_URL_APPEND" \
        "${REMAINING_ARGS[@]}"
}

# Delete all unused deprecated transformation revisions

#     "Unused" deprecated transformation revisions are those that are either not used in any workflow
#     or only in workflows that themselves are deprecated (and hence will be deleted as well).

#     This handles nesting, i.e. a deprecated transformation revision will not be deleted if it is used
#     indirectly across multiple nesting levels in a workflow which is not deprecated.

#     To avoid deleting certain transformation, use the exclude parameter by listing the uuids
#     of these transformations, e.g., &exclude=[uuid1,uuid2,...]. By default all unused deprecated
#     transformations are deleted.

#     To avoid that recently deprecated transformations are deleted, use the parameter cutoff_date
#     that enables deleting only transformations that have been disabled before the given date.
#     the parameter must be a n iso-formatted date in UTC, e.g., ?cutoff_date="2026-01-01 00:00:00Z".
#     By default all unused deprecated transformations are deleted.

#     **Warning**: This irrevocably deletes transformation revisions. We recommend to backup, e.g.
#     exporting / getting all transformation revisions before using this action!
delete_unused_deprecated() {

    # Global set up
    _setup "${@}"
    _ensure_maintenance_secret
    ensure_access_token

    curl_args=()
    if [[ $VERIFY_BACKEND_API_URL = false ]]; then
        curl_args+=("--insecure")
    fi

    # function set up
    _parse_delete_unused_deprecated_args "${@}"
    _build_body_delete_unused_deprecated_args

    # excecute CURL-command
    local cmd=(
        curl \
        "${curl_args[@]}" \
        -X POST \
        -H "Authorization: Bearer $ACCESS_TOKEN" \
        -H "Content-Type: application/json" \
        -d "$BODY" \
        "$HD_BACKEND_API_URL/maintenance/delete_unused_deprecated$QUERY_URL_APPEND" \
        "${REMAINING_ARGS[@]}"
    )

    "${cmd[@]}" # printf '%q ' "${cmd[@]}" for dry-run

}

# Purge and reinstall base components/workflows

#     This deletes all transformation revisions and afterwards deploys the base
#     components and workflows present in the running backend instance's image at the
#     default path.
purge() {
    _setup "${@}"
    _ensure_maintenance_secret
    ensure_access_token

    curl_args=()
    if [[ $VERIFY_BACKEND_API_URL = false ]]; then
        curl_args+=("--insecure")
    fi
    curl "${curl_args[@]}" -X POST \
        -H "Authorization: Bearer $ACCESS_TOKEN" \
        -H "Content-Type: application/json" \
        -d '{"maintenance_secret":"'"$MAINTENANCE_SECRET"'"}' \
        "$HD_BACKEND_API_URL/maintenance/purge$QUERY_URL_APPEND" \
        "${REMAINING_ARGS[@]}"
}

# Trigger deployment of base transformations from the running backend
# interesting query parameters are
# * allow_overwrite_released
# * update_component_code
deploy_base_trafos() {
    _setup "${@}"
    _ensure_maintenance_secret
    ensure_access_token

    curl_args=()
    if [[ $VERIFY_BACKEND_API_URL = false ]]; then
        curl_args+=("--insecure")
    fi
    curl "${curl_args[@]}" -X POST \
        -H "Authorization: Bearer $ACCESS_TOKEN" \
        -H "Content-Type: application/json" \
        -d '{"maintenance_secret":"'"$MAINTENANCE_SECRET"'"}' \
        "$HD_BACKEND_API_URL/maintenance/deploy_base_trafos$QUERY_URL_APPEND" \
        "${REMAINING_ARGS[@]}"
}

# Trigger autoimport
#
# note that each import is configured by accompagniying config json file, so
# this has no query params
autoimport() {
    _setup "${@}"
    _ensure_maintenance_secret
    ensure_access_token

    curl_args=()
    if [[ $VERIFY_BACKEND_API_URL = false ]]; then
        curl_args+=("--insecure")
    fi
    curl "${curl_args[@]}" -X POST \
        -H "Authorization: Bearer $ACCESS_TOKEN" \
        -H "Content-Type: application/json" \
        -d '{"maintenance_secret":"'"$MAINTENANCE_SECRET"'"}' \
        "$HD_BACKEND_API_URL/maintenance/autoimport$QUERY_URL_APPEND" \
        "${REMAINING_ARGS[@]}"
}

###############################################################################
#                                                                             #
#                              Import into hd                                 #
#                                                                             #
###############################################################################

_import_multi_trafo_json_file() {
    _file_path="$1"
    shift

    ensure_access_token

    # put to /transformations endpoint
    curl_args=()
    if [[ $VERIFY_BACKEND_API_URL = false ]]; then
        curl_args+=("--insecure")
    fi
    curl "${curl_args[@]}" -X PUT -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" -d @"$_file_path" "$HD_BACKEND_API_URL/transformations$QUERY_URL_APPEND" "${@}"
}

# encodes as json string, outputs result with "" around
_json_encode_str() {
    printf "%s" "$1" | jq -R -s '.'
}

_import_from_dir() {
    # load all trafos into one big json
    _all_trafos_json='[  ' # 2 extra spaces, see comma + newline removal below

    # trafos from json files
    # shellcheck disable=SC2044
    for f in $(find "$EXPORT_DIRECTORY" -name '*.json'); do
        _all_trafos_json+="$(cat "$f")"
        _all_trafos_json+=","$'\n' # add comma and newline
    done

    # from py files
    # shellcheck disable=SC2044
    for f in $(find "$EXPORT_DIRECTORY" -name '*.py'); do
        if [[ "$(basename "$f")" != "__init__.py" ]] && [[ "$(basename "$f")" != "hdutils.py" ]]; then
            _debug "handle py file $f"

            # only ensure jq if it is needed, i.e. only if py files need
            # to be handled:
            _ensure_jq_once

            local py_file_content
            py_file_content="$(cat "$f")"
            _all_trafos_json+="$(_json_encode_str "$py_file_content")"
            _all_trafos_json+=","$'\n' # add comma and newline
        fi
    done

    _all_trafos_json="${_all_trafos_json::-2}" # remove last comma and newline
    _all_trafos_json+=']'

    ensure_access_token

    local _temp_file
    _temp_file="$(mktemp)"
    echo "$_all_trafos_json" >"$_temp_file"

    curl_args=()
    if [[ $VERIFY_BACKEND_API_URL = false ]]; then
        curl_args+=("--insecure")
    fi

    curl "${curl_args[@]}" -X PUT -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" -d @"$_temp_file" "$HD_BACKEND_API_URL/transformations$QUERY_URL_APPEND" "${@}"

    rm "$_temp_file"
}

push() {
    _setup "${@}"
    if [[ -z "$EXPORT_DIRECTORY" ]]; then
        _import_multi_trafo_json_file "${REMAINING_ARGS[@]}"
    else
        _import_from_dir "${REMAINING_ARGS[@]}"
    fi
}

###############################################################################
#                                                                             #
#                              Export from hd                                 #
#                                                                             #
###############################################################################

# Write a json array of all trafos to stdout
_export_to_json() {
    ensure_access_token
    curl_args=()
    if [[ $VERIFY_BACKEND_API_URL = false ]]; then
        curl_args+=("--insecure")
    fi

    curl "${curl_args[@]}" -X GET -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" "$HD_BACKEND_API_URL/transformations$QUERY_URL_APPEND" -G "${REMAINING_ARGS[@]}"
}

### Export into dir
_slugify() {
    awk '{
              gsub(/[äÄ]/,"a");
              gsub(/[öÖ]/,"o");
              gsub(/[üÜ]/,"u");
              gsub(/[^0-9a-zA-Z -]/,"");
              gsub(/^[ \t\r\n]+/, "");
              gsub(/[ \t\r\n]+$/, "");
              gsub(/[ ]+/,"_");
              gsub(/-/,"_");
              print tolower($0);
        }'
}

_slugify_arg() {
    printf "%s" "$1" | _slugify
}

_subpath_no_slugify() {
    _id="$1"
    _name="$2"
    _category="$3"
    _type="$4"
    _version_tag="$5"
    _extension="${6:-json}"

    printf "%s" "$_type"s/"$_category"/"$_name"_"$_version_tag"_"$_id"."$_extension"
}

_output_hdutils_py_content() {
    echo "$HDUTILS_PY_CONTENT"
}

_output_pytest_ini_content() {
    echo "$PYTEST_INI_CONTENT"
}

_export_to_dir() {
    _ensure_jq
    ensure_access_token
    curl_args=()
    if [[ $VERIFY_BACKEND_API_URL = false ]]; then
        curl_args+=("--insecure")
    fi

    mkdir -p "$EXPORT_DIRECTORY"
    _output_hdutils_py_content >"$EXPORT_DIRECTORY"/hdutils.py
    _output_pytest_ini_content >"$EXPORT_DIRECTORY"/pytest.ini

    # echo curl "${curl_args[@]}" -X GET -G -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" "$HD_BACKEND_API_URL/transformations$QUERY_URL_APPEND" "${REMAINING_ARGS[@]}"
    # write each trafo revision to its respective directory

    file_no=0

    for row in $(curl "${curl_args[@]}" -X GET -G -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" "$HD_BACKEND_API_URL/transformations$QUERY_URL_APPEND" "${REMAINING_ARGS[@]}" | jq -r '.[] | @base64'); do
        file_no=$(($file_no + 1))
        start=$(date +%s%N)
        decoded_row="$(echo ${row} | base64 --decode)"

        # shellcheck disable=SC2046,SC2005
        read _id _name _category _type _version_tag < <(echo $(echo "$decoded_row" | jq -r '.id, .name, .category, .type, .version_tag' | _slugify))

        #_id="$(echo "$decoded_row" | jq -r '.id')"
        #_name="$(echo "$decoded_row" | jq -r '.name')"
        # _category="$(echo "$decoded_row" | jq -r '.category')"
        #_type="$(echo "$decoded_row" | jq -r '.type')"
        #_version_tag="$(echo "$decoded_row" | jq -r '.version_tag')"

        local extension
        if $EXPORT_COMPONENTS_AS_PY_FILES && [[ "$_type" == "component" ]]; then
            extension="py"
        else
            extension="json"
        fi
        _file_path="$EXPORT_DIRECTORY"/"$(_subpath_no_slugify "$_id" "$_name" "$_category" "$_type" "$_version_tag" "$extension")"

        echo "Extracting $_file_path"

        _debug "Writing file to $_file_path"
        mkdir -p "$(dirname "$_file_path")"

        # actual extraction into file
        if [[ "$extension" == "json" ]]; then
            echo "$decoded_row" | jq >"$_file_path"
        elif [[ "$extension" == "py" ]]; then
            touch "$EXPORT_DIRECTORY/components/__init__.py"
            touch "$(dirname "$_file_path")"/__init__.py
            echo "$decoded_row" | jq -j '.content' >"$_file_path"
        fi

        end=$(date +%s%N)

        runtime=$(((end - start) / 1000000))
        _debug $runtime
    done
    echo "Extracted $file_no files."

}

fetch() {
    _setup "${@}"
    if [[ -z "$EXPORT_DIRECTORY" ]]; then
        _export_to_json
    else
        _export_to_dir
    fi
}

###############################################################################
#                                                                             #
#                                Sync commands                                #
#                                                                             #
###############################################################################

_init_sync_instance() {
    local sync_instance="$1"
    if [[ -z "$sync_instance" ]]; then
        _log_error "Missing sync instance name. Aborting."
        exit 1
    fi

    local instance_file="${sync_instance}.hd-instance"
    local instance_creds_file="${sync_instance}.hd-creds"
    local instance_creds_file_stub="${sync_instance}.hd-creds.stub"

    if [[ -f "$instance_file" ]]; then
        _log_error "Instance file $instance_file already exists. Aborting."
        exit 1
    fi

    local instance_file_content
    instance_file_content=$(
        cat <<END_HEREDOC
HD_BACKEND_API_URL=
VERIFY_BACKEND_API_URL=true


# Auth settings. Actual credentials should be configured in the
# accompanying .hd-creds file.

AUTH_URL=
VERIFY_AUTH_URL=true
REALM=
CLIENT_ID=


EXPORT_DIRECTORY="./$sync_instance-trafos"
EXPORT_COMPONENTS_AS_PY_FILES=true


# Options controlling sync behaviour.
# for example you may add
# id=... or category=... or category_prefix=...
# url query params to filter which transformations to sync.

PULL_QUERY_URL_APPEND='?include_dependencies=true&include_deprecated=false&update_component_code=true&expand_component_code=true'
PUSH_QUERY_URL_APPEND='?include_dependencies=true&allow_overwrite_released=false'

PUSH_ADDITIONAL_ARGUMENTS=()
PULL_ADDITIONAL_ARGUMENTS=()

END_HEREDOC
    )
    echo "$instance_file_content" >"$instance_file"

    local cred_stub_file_content
    cred_stub_file_content=$(
        cat <<END_HEREDOC
MAINTENANCE_SECRET=
USERNAME=
PASSWORD=
CLIENT_SECRET=

END_HEREDOC
    )
    echo "$cred_stub_file_content" >"$instance_creds_file_stub"

    echo "Created files. Next steps:"
    echo ">> 1.) Edit settings in the instance file $instance_file accordingly."
    echo ">> 2.) Edit the credential stub file $instance_creds_file_stub to only contain entries"
    echo ">>     relevant to the authentification method used by the configured hd instance."
    echo ">>     Do not enter passwords yet!"
    echo ">> 3.) Copy the stub file to $instance_creds_file and actually enter passwords"
    echo ">> 4.) Add the cred file (containing passwords) to your gitignore!"
    echo ">> 5.) Add the instance file and the credential stub file to your git repo."
    echo ""
    echo "After that start syncing using the pull and push sync commands:"
    echo "    hdctl sync pull $sync_instance"
    echo "    hdctl sync push $sync_instance"
}

_load_instance_files() {
    local sync_instance="$1"

    if [[ -z "$sync_instance" ]]; then
        _error "No instance specified. Aborting"
        exit 1
    fi

    local instance_file="${sync_instance}.hd-instance"
    local instance_creds_file="${sync_instance}.hd-creds"

    # shellcheck disable=SC1090
    source "$instance_file"
    # shellcheck disable=SC1090
    source "$instance_creds_file"
}

sync() {
    local sync_command="$1"

    if [[ "$sync_command" != "push" ]] && [[ "$sync_command" != "pull" ]] && [[ "$sync_command" != "init" ]]; then
        _log_error "Only allowed sync actions are pull, push and init. Got $sync_command instead. Aborting"
        exit 1
    fi

    local sync_instance="$2"

    if [[ -z "$sync_instance" ]]; then
        _log_error "no hd instance specified. Aborting."
        exit 1
    fi

    shift
    shift

    if [[ "$sync_command" == "init" ]]; then
        _init_sync_instance "$sync_instance"
        return 0
    fi

    _load_instance_files "$sync_instance"

    if [[ "$sync_command" == "pull" ]] && [[ "$1" == "to" ]] && [[ -n "$2" ]]; then

        # pull from instance to local dir of another instance

        shift
        shift
        local other_instance_file="$2".hd-instance

        # overwrite the export directory global variable with the dir from the other instance
        EXPORT_DIRECTORY="$(value_from_env_file "EXPORT_DIRECTORY" "$other_instance_file")"

    elif [[ "$sync_command" == "push" ]] && [[ "$1" == "from" ]] && [[ -n "$2" ]]; then
        # push to instance from local dir of another instance
        local other_instance_file="$2".hd-instance

        # overwrite the export directory global variable with the dir from the other instance
        EXPORT_DIRECTORY="$(value_from_env_file "EXPORT_DIRECTORY" "$other_instance_file")"

        shift
        shift
    fi

    _initialize_variables

    local additional_arguments=()
    if [[ "$sync_command" == "pull" ]]; then
        if [[ -n "$PULL_QUERY_URL_APPEND" ]]; then
            QUERY_URL_APPEND="$PULL_QUERY_URL_APPEND"
        fi
        if [[ -n "$PULL_ADDITIONAL_ARGUMENTS" ]]; then
            additional_arguments=("${PULL_ADDITIONAL_ARGUMENTS[@]}")
        fi
    fi

    if [[ "$sync_command" == "push" ]]; then
        if [[ -n "$PUSH_QUERY_URL_APPEND" ]]; then
            QUERY_URL_APPEND="$PUSH_QUERY_URL_APPEND"
        fi
        if [[ -n "$PUSH_ADDITIONAL_ARGUMENTS" ]]; then
            additional_arguments=("${PUSH_ADDITIONAL_ARGUMENTS[@]}")
        fi
    fi

    _parse_global_options "${additional_arguments[@]}" "${@}"
    _validate_global_parameters

    # Additionally required validations for syncing

    if [[ -z "$EXPORT_DIRECTORY" ]]; then
        _log_error "No export directory specified. Aborting."
        exit 1
    fi

    # Now we are ready to actually run the requested sync command

    if [[ "$sync_command" == "pull" ]]; then
        # clear up directory first
        if "$SYNC_CLEAR_DIRECTORY"; then
            rm -Rf "$EXPORT_DIRECTORY"
        else
            _info ">>> Adding files mode."
        fi

        _export_to_dir
    elif [[ "$sync_command" == "push" ]]; then
        _import_from_dir
    fi

}

###############################################################################
#                                                                             #
#             Setup/handle params, environment variables, env file            #
#                                                                             #
###############################################################################

REMAINING_ARGS=()

# must be called with all command line arguments
_obtain_env_file_config() {
    ENV_FILE_PATH="${HDCTL_ENV_FILE:-"./hdctl.env"}"

    while [[ "$#" -gt 0 ]]; do case $1 in
        -e | --env-file)
            ENV_FILE_PATH="$2"
            shift
            shift
            break
            ;;
        -i | --instance)
            INSTANCE_NAME="$2"
            shift
            shift
            break
            ;;
        --)
            shift
            break
            ;;
        *)
            # ignore everything else
            shift
            ;;
        esac done
}

_source_env_file() {
    if [[ -z "$ENV_FILE_PATH" ]]; then
        _debug "Env file path explicitely unset!"
    elif [[ ! -e "$ENV_FILE_PATH" ]]; then
        if [[ "$ENV_FILE_PATH" == "./hdctl.env" ]]; then
            _debug "No env file at default path. Continue without sourcing"
        else
            _log_error "Explicit env file path set, but could not find file! Aborting."
            exit 1
        fi
    else
        _debug "Sourcing env file $ENV_FILE_PATH."
        # shellcheck disable=SC1090
        source "$ENV_FILE_PATH"
    fi
}

_initialize_variables() {
    # initialize with increasing priority
    # * from defaults
    # * from env file variables of form VARIABLE_NAME
    # * from prefixed environment variables, i.e. of form HDCTL_VARIABLE_NAME

    SILENT="${HDCTL_SILENT:-"${SILENT:-false}"}"
    VERBOSITY="${HDCTL_VERBOSITY:-"${VERBOSITY:-"info"}"}"

    ##### Content
    SYNC_CLEAR_DIRECTORY="${HDCTL_SYNC_CLEAR_DIRECTORY:-"${SYNC_CLEAR_DIRECOTORY:-true}"}"
    EXPORT_DIRECTORY="${HDCTL_EXPORT_DIRECTORY:-"${EXPORT_DIRECTORY:-""}"}"
    EXPORT_COMPONENTS_AS_PY_FILES="${HDCTL_EXPORT_COMPONENTS_AS_PY_FILES:-"${EXPORT_COMPONENTS_AS_PY_FILES:-"true"}"}"

    ##### HD backend

    HD_BACKEND_API_URL="${HDCTL_HD_BACKEND_API_URL:-"${HD_BACKEND_API_URL:-"http://localhost/api"}"}"
    VERIFY_BACKEND_API_URL="${HDCTL_VERIFY_BACKEND_API_URL:-"${VERIFY_BACKEND_API_URL:-true}"}"
    QUERY_URL_APPEND="${HDCTL_QUERY_URL_APPEND:-"${QUERY_URL_APPEND:-""}"}"

    ##### Authentication
    DIRECT_ACCESS_TOKEN="${HDCTL_DIRECT_ACCESS_TOKEN:-"${DIRECT_ACCES_TOKEN:-""}"}"
    AUTH_URL="${HDCTL_AUTH_URL:-"${AUTH_URL:-""}"}"
    USERNAME="${HDCTL_USERNAME:-"${USERNAME:-""}"}"
    PASSWORD="${HDCTL_PASSWORD:-"${PASSWORD:-""}"}"
    CLIENT_ID="${HDCTL_CLIENT_ID:-"${CLIENT_ID:-""}"}"
    CLIENT_SECRET="${HDCTL_CLIENT_SECRET:-"${CLIENT_SECRET:-""}"}"
    REALM="${HDCTL_REALM:-"${REALM:-"hetida"}"}"
    VERIFY_AUTH_URL="${HDCTL_VERIFY_AUTH_URL:-"${VERIFY_AUTH_URL:-true}"}"

    ##### Maintenance authorization
    MAINTENANCE_SECRET="${HDCTL_MAINTENANCE_SECRET:-"${MAINTENANCE_SECRET:-""}"}"
}

_parse_global_options() {

    REMAINING_ARGS=()
    while [[ "$#" -gt 0 ]]; do case $1 in
        -e | --env-file)
            ENV_FILE_PATH="$2"
            shift
            shift
            ;;
        -i | --instance)
            INSTANCE_NAME="$2"
            shift
            shift
            ;;
        -b | --backend-api-url)
            HD_BACKEND_API_URL="$2"
            shift
            shift
            ;;
        --insecure-backend)
            VERIFY_BACKEND_API_URL=false
            shift
            ;;
        --add)
            SYNC_CLEAR_DIRECTORY=false
            shift
            ;;
        -d | --export-directory)
            EXPORT_DIRECTORY="$2"
            shift
            shift
            ;;
        -a | --auth-url)
            AUTH_URL="$2"
            shift
            shift
            ;;
        --insecure-auth)
            VERIFY_AUTH_URL=false
            shift
            ;;
        -t | --access-token)
            DIRECT_ACCESS_TOKEN="$2"
            shift
            shift
            ;;
        -r | --realm)
            REALM="$2"
            shift
            shift
            ;;
        -c | --client-id)
            CLIENT_ID="$2"
            shift
            shift
            ;;
        -s | --client-secret)
            CLIENT_SECRET="$2"
            shift
            shift
            ;;
        -m | --maintenance-secret)
            MAINTENANCE_SECRET="$2"
            shift
            shift
            ;;
        -p | --url-append)
            QUERY_URL_APPEND="$2"
            shift
            shift
            ;;
        -q | --quiet)
            SILENT=true
            shift
            ;;
        -v | --verbose)
            VERBOSITY="debug"
            shift
            ;;
        --)
            shift
            break
            ;;
        *)
            REMAINING_ARGS+=("$1")
            shift
            ;;
            #Unknown parameter appended to REMAINING_ARGS for access by subcommands
        esac done
    REMAINING_ARGS=("${REMAINING_ARGS[@]}" "${@}") # concat with remaining arguments
}

# must be called with cli arguments
_params_setup() {
    _obtain_env_file_config "${@}"
    if [[ -z "$INSTANCE_NAME" ]]; then
        _source_env_file
    else
        _load_instance_files "$INSTANCE_NAME"
    fi
    _initialize_variables
    _parse_global_options "${@}"
}

_validate_global_parameters() {
    # if [[ -n "$AUTH_URL" ]] && [[ -z "$CLIENT_SECRET" ]]; then
    #     _log_error "Auth Url set but no client secret provided. Please provide a client secret"
    #     _log_error 'via -s SECRET or an empty auth url via -a "" to deactivate client credential'
    #     _log_error "auth on the command line. Aborting."
    #     exit 1
    # fi

    if [[ -n "$MAINTENANCE_SECRET" ]]; then
        if [[ ! "$MAINTENANCE_SECRET" =~ ^[a-zA-Z0-9]+$ ]]; then
            _log_error "Only numbers and alphabet letters allowed for the maintenance secret."
            exit 1
        fi
    fi
}

# _setup must be called with args
# typical first function of any subcommand
_setup() {
    _params_setup "${@}"
    _validate_global_parameters
}

_parse_delete_unused_deprecated_args() {

    CUTOFF_DATE=""
    EXCLUDE_UUIDS=()

    set -- "${REMAINING_ARGS[@]}"

    REMAINING_ARGS=()
    while [[ "$#" -gt 0 ]];
        do case "$1" in
            -exclude)
                EXCLUDE_UUIDS+=("$2")
                shift 2
                ;;
            -cutoff)
                CUTOFF_DATE="$2"
                shift 2
                ;;
            *)
            REMAINING_ARGS+=("$1")
            shift
        ;;
        esac
    done
    REMAINING_ARGS=("${REMAINING_ARGS[@]}" "${@}")
}

_build_body_delete_unused_deprecated_args() {

    BODY='{
        "maintenance_payload": {
            "maintenance_secret": "'"$MAINTENANCE_SECRET"'"
    }'

    if [[ -n "$CUTOFF_DATE" ]]; then
        BODY+=',
    "cutoff_date": "'"$CUTOFF_DATE"'"'
    fi

    if ((${#EXCLUDE_UUIDS[@]})); then
        BODY+=',
        "exclude": ['
        for i in "${!EXCLUDE_UUIDS[@]}"; do
            [[ $i -gt 0 ]] && BODY+=","
            BODY+="\"${EXCLUDE_UUIDS[$i]}\""
        done
        BODY+=']'
    fi

    BODY+="}"
}
###############################################################################
#                                                                             #
#                             Subcommand Execution                            #
#                                                                             #
###############################################################################

_fn_exists() {
    # Checks whether there exists a variable of type function with the given name.
    LC_ALL=C type "${1:-}" 2>/dev/null | grep -q 'function'
}

COMMAND=${1:-usage}
shift || true
ARGUMENTS=("${@}")

if [[ "$COMMAND" == "--help" || "$COMMAND" == "-h" ]]; then
    help
    exit 0
fi

if _fn_exists "$COMMAND"; then
    "$COMMAND" "${ARGUMENTS[@]}"
else
    usage "No subcommand $COMMAND"
fi