Get list of standard FMU results in a given ensemble

[anders-kiaer]

• As a user, I would like to know which types of standard FMU results exists in a given ensemble. This is then used to easily know what type of data is exported, and based on that what type of analysis I can do.

• As a client (e.g. webviz) we want to present to the user which modules/visualizations are relevant/valid for a given ensemble.

(Implementation wise, the hope is that this could be achieved by an efficient elastic search query with something equivalent to EXISTS in SQL - i.e. terminate search on first instance found for a given standard result)

[HansKallekleiv]

Implementation wise I think this can be solved efficiently in elastic search using aggregations and filters.
I will leave it to the Sumo guros to decide :)

from dataclasses import dataclass
import httpx
import asyncio
from sumo.wrapper import SumoClient


@dataclass
class Datatype:
    datatype: str
    realization_count: int


@dataclass
class Ensemble:
    ensemble_uuid: str
    datatypes: list[Datatype]


@dataclass
class Case:
    case_uuid: str
    ensembles: list[Ensemble]


async def get_case_info_arr(sumo_client: SumoClient) -> list[str]:
    """
    Get all cases with available result types from SUMO using aggregations and filters
    """

    # Example result filters
    filter_def_timeseries = {
        "bool": {
            "filter": [
                {"term": {"class.keyword": "table"}},
                {"term": {"data.content.keyword": "simulationtimeseries"}},
            ]
        }
    }
    filter_def_observed_surfaces = {
        "bool": {
            "filter": [
                {"term": {"class.keyword": "surface"}},
                {"term": {"data.is_observation": True}},
            ]
        }
    }
    filter_def_realization_surfaces = {
        "bool": {
            "filter": [
                {"term": {"class.keyword": "surface"}},
                {"term": {"data.is_observation": False}},
                {"exists": {"field": "fmu.realization.id"}},
            ]
        }
    }
    filter_def_grid = {"term": {"class.keyword": "cpgrid"}}
    filter_def_inplace_volumes = {
        "bool": {
            "filter": [
                {"term": {"class.keyword": "table"}},
                {"term": {"data.content.keyword": "volumes"}},
            ]
        }
    }

    payload = {
        "size": 0,
        "aggs": {
            "filter_defs": {
                "filters": {
                    "filters": {
                        "timeseries": filter_def_timeseries,
                        "observed_surfaces": filter_def_observed_surfaces,
                        "realization_surfaces": filter_def_realization_surfaces,
                        "grid": filter_def_grid,
                        "inplace": filter_def_inplace_volumes,
                    }
                },
                "aggs": {
                    "uuids": {
                        "terms": {"field": "_sumo.parent_object.keyword", "size": 65535},
                        "aggs": {
                            "ensemble_uuids": {
                                "terms": {"field": "fmu.iteration.uuid.keyword", "size": 65535},
                                "aggs": {"realizations_count": {"cardinality": {"field": "fmu.realization.id"}}},
                            },
                        },
                    }
                },
            },
        },
    }
    response = await sumo_client.post_async("/search", json=payload)
    response_dict = response.json()
    buckets_per_content = response_dict["aggregations"]["filter_defs"]["buckets"]

    cases = {}
    for datatype, aggregation in buckets_per_content.items():

        for case_bucket in aggregation["uuids"]["buckets"]:
            case_uuid = case_bucket["key"]
            if not case_uuid in cases:
                cases[case_uuid] = {"case_uuid": case_bucket["key"], "ensembles": {}}

            for ens_no, ens_bucket in enumerate(case_bucket["ensemble_uuids"]["buckets"]):
                if not ens_bucket["key"] in cases[case_uuid]["ensembles"]:
                    cases[case_uuid]["ensembles"][ens_bucket["key"]] = {"datatypes": {}}

                cases[case_uuid]["ensembles"][ens_bucket["key"]]["datatypes"][datatype] = {
                    "datatype": datatype,
                    "realization_count": ens_bucket["realizations_count"]["value"],
                }

    cases_ret_arr: list[Case] = []

    for case_uuid, case in cases.items():
        ensembles_ret_arr: list[Ensemble] = []
        for ensemble_uuid, ensemble in case["ensembles"].items():
            datatypes_ret_arr: list[Datatype] = []
            for datatype in ensemble["datatypes"].values():
                datatypes_ret_arr.append(Datatype(**datatype))
            ensembles_ret_arr.append(Ensemble(datatypes=datatypes_ret_arr, ensemble_uuid=ensemble_uuid))
        cases_ret_arr.append(Case(case_uuid=case_uuid, ensembles=ensembles_ret_arr))

    return cases_ret_arr


sumo_client = SumoClient(
    env="prod",
    interactive=True,
    timeout=httpx.Timeout(180.0),
)

cases_ret_arr = asyncio.run(get_case_info_arr(sumo_client))
import json

for case in cases_ret_arr:
    for ensemble in case.ensembles:
        print(ensemble)

# "case_uuid": "08b4c633-9082-4104-9e5d-89f67f22bfbf",
#  "ensembles": [
#   {
#    "ensemble_uuid": "0621142d-9a73-eb5b-fba9-cfc5fda52c0f",
#    "datatypes": [
#     {
#      "datatype": "grid",
#      "realization_count": 176
#     },
#     {
#      "datatype": "inplace",
#      "realization_count": 250
#     },
#     {
#      "datatype": "realization_surfaces",
#      "realization_count": 250
#     }
#    ]
#   }
#  ]
# }
# {
#  "case_uuid": "d4af65bc-363a-48dd-ae82-9acae4c536f2",
#  "ensembles": [
#   {
#    "ensemble_uuid": "80e32354-5656-07c5-e7f7-972efabdc510",
#    "datatypes": [
#     {
#      "datatype": "grid",
#      "realization_count": 172
#     },
#     {
#      "datatype": "inplace",
#      "realization_count": 172
#     },
#     {
#      "datatype": "realization_surfaces",
#      "realization_count": 197
#     },
#     {
#      "datatype": "timeseries",
#      "realization_count": 188
#     }
#    ]
#   }
#  ]
# }
# {
#  "case_uuid": "c02c958e-be82-4d8e-a3c2-c6c336a5b5c4",
#  "ensembles": [
#   {
#    "ensemble_uuid": "b9eb8654-0155-a783-7196-aaec1cba8c2a",
#    "datatypes": [
#     {
#      "datatype": "grid",
#      "realization_count": 133
#     },
#     {
#      "datatype": "inplace",
#      "realization_count": 134
#     },
#     {
#      "datatype": "realization_surfaces",
#      "realization_count": 189
#     }
#    ]
#   }
#  ]
# }

[rwiker]

fmu.sumo.explorer.Explorer has the method standard_results which returns a list of all standard_result names found in a SearchContext. It is also possible to filter on standard_result.

Example, using data in dev:

>>> exp=Explorer("dev")
>>> exp.standard_results
['structure_depth_surface', 'structure_depth_fault_lines', 'inplace_volumes', 'structure_depth_isochore', 'field_outline']
>>> cases_with_standard_results=exp.filter(standard_result=True).cases
>>> len(cases_with_standard_results)
2
>>> case = cases_with_standard_results[0]
>>> sds=case.filter(iteration="iter-0", standard_result="structure_depth_surface")
>>> len(sds)
600

[rwiker]

This is obviously not going to be much use for existing data, which does not have the data.standard_result.name property. It would be possible to retrofit this property to existing data, given a clear set of rules for how this should be done; this should be coordinated with the Sumo and fmu-dataio teams.

Alternatively, it might be possible to change the behaviour of the standard_result filter and the standard_results accessor, so that they fall back to something like the filters you suggested if there are no data.standard_result.name properties captured by the SearchContext.

Thanks for the code sample – I wasn't aware of the filter/filters aggregations.

[equinor-ruaj]

Anything you are missing in the Explorer for this @HansKallekleiv or is the current functionality sufficient?

[HansKallekleiv]

Short answer is I do not know.

We ended up using an ES query directly for retrieving case metadata for all cases in a field.
https://github.com/jorgenherje/webviz/blob/improve-ensemble-selector-dialog/backend_py/primary/primary/services/sumo_access/sumo_inspector.py#L49

If you see a way to do this with the explorer (without a query for each case) let us know.