From 5f7f120a27825dda7d23cc4c38583b80c06ec983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Herje?= Date: Tue, 18 Aug 2026 10:45:10 +0200 Subject: [PATCH 1/6] Fetch Sumo grid data using ri-cloud-api Grid geometry and grid properties can now be read from Sumo through the local ri-cloud-api service, alongside the summary data that was already there. A Sumo data source can be turned into a grid ensemble, whose cases load their geometry and their property time steps on demand. Downloaded property blobs are held in a cache bounded by total size, so a long session on a large case cannot grow without limit, and the time steps of a property are fetched in concurrent batches rather than one round trip at a time. Computing the mobile volume weighted mean is made optional, and is off for Sumo grid ensembles. It reads PORV and the saturation of every phase, so opening a case with SOIL displayed pulled down several properties nobody had asked for. Co-Authored-By: Claude Opus 5 --- .../Tools/Cloud/CMakeLists_files.cmake | 1 + .../Tools/Cloud/RiaSumoConnector.cpp | 671 ++++++++++++++++++ .../Tools/Cloud/RiaSumoConnector.h | 113 ++- .../Tools/Cloud/RiaSumoDefines.cpp | 16 + .../Application/Tools/Cloud/RiaSumoDefines.h | 9 + .../Tools/Cloud/RifReaderSumoGridProperty.cpp | 156 ++++ .../Tools/Cloud/RifReaderSumoGridProperty.h | 71 ++ .../Commands/Sumo/CMakeLists_files.cmake | 1 + .../Sumo/RicCreateSumoGridEnsembleFeature.cpp | 126 ++++ .../Sumo/RicCreateSumoGridEnsembleFeature.h | 38 + .../FileInterface/RifRoffFileTools.cpp | 143 +++- .../FileInterface/RifRoffFileTools.h | 33 +- .../ProjectDataModel/CMakeLists_files.cmake | 1 + .../Cloud/RimCloudDataSourceCollection.cpp | 8 + .../Rim3dOverlayInfoConfig.cpp | 17 + .../RimEclipseCaseEnsemble.cpp | 36 + .../ProjectDataModel/RimEclipseCaseEnsemble.h | 5 + .../RimHistogramCalculator.cpp | 28 +- .../ProjectDataModel/RimHistogramCalculator.h | 5 + .../ProjectDataModel/RimRoffCaseSumo.cpp | 370 ++++++++++ .../ProjectDataModel/RimRoffCaseSumo.h | 79 +++ .../Summary/Sumo/RimSumoDataSource.cpp | 189 ++++- .../Summary/Sumo/RimSumoDataSource.h | 27 +- 23 files changed, 2100 insertions(+), 43 deletions(-) create mode 100644 ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.cpp create mode 100644 ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.h create mode 100644 ApplicationLibCode/Commands/Sumo/RicCreateSumoGridEnsembleFeature.cpp create mode 100644 ApplicationLibCode/Commands/Sumo/RicCreateSumoGridEnsembleFeature.h create mode 100644 ApplicationLibCode/ProjectDataModel/RimRoffCaseSumo.cpp create mode 100644 ApplicationLibCode/ProjectDataModel/RimRoffCaseSumo.h diff --git a/ApplicationLibCode/Application/Tools/Cloud/CMakeLists_files.cmake b/ApplicationLibCode/Application/Tools/Cloud/CMakeLists_files.cmake index 4c05f2d8e6a..e914ae5cd37 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/CMakeLists_files.cmake +++ b/ApplicationLibCode/Application/Tools/Cloud/CMakeLists_files.cmake @@ -6,6 +6,7 @@ set(SOURCE_GROUP_SOURCE_FILES ${CMAKE_CURRENT_LIST_DIR}/RiaConnectorTools.cpp ${CMAKE_CURRENT_LIST_DIR}/RiaOsduConnector.cpp ${CMAKE_CURRENT_LIST_DIR}/RiaOAuthHttpServerReplyHandler.cpp + ${CMAKE_CURRENT_LIST_DIR}/RifReaderSumoGridProperty.cpp ) list(APPEND CODE_SOURCE_FILES ${SOURCE_GROUP_SOURCE_FILES}) diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp index 31117524cfc..43c507ac627 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp @@ -33,6 +33,8 @@ #include #include +#include + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -297,6 +299,568 @@ void RiaSumoConnector::requestRealizationIdsForEnsembleBlocking( const SumoCaseI wrapAndCallNetworkRequest( requestCallable, signalMethod ); } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::requestGridInfoForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ) +{ + m_gridInfos.clear(); + + requestTokenBlocking(); + + QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + QNetworkRequest m_networkRequest; + QString url = QString( "%1/cases/%2/ensembles/%3/grid_info_list" ).arg( server() ).arg( caseId.get() ).arg( encodedEnsembleName ); + m_networkRequest.setUrl( QUrl( url ) ); + + addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); + + auto reply = m_networkAccessManager->get( m_networkRequest ); + + connect( reply, + &QNetworkReply::finished, + [this, reply, ensembleName, caseId]() + { + if ( reply->error() == QNetworkReply::NoError ) + { + parseGridInfo( reply, caseId, ensembleName ); + } + else + { + RiaLogging::error( std::format( "Request grid info failed: '{}'", reply->errorString().toStdString() ) ); + emit gridInfoFinished(); + } + } ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::requestGridInfoForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ) +{ + auto requestCallable = [this, caseId, ensembleName] { requestGridInfoForEnsemble( caseId, ensembleName ); }; + QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::gridInfoFinished ); + wrapAndCallNetworkRequest( requestCallable, signalMethod ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::requestGridBlobIdForEnsemble( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ) +{ + requestTokenBlocking(); + + QNetworkRequest m_networkRequest; + + // Properly URL-encode the path components + QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + QString encodedGridName = QUrl::toPercentEncoding( gridName ); + + QString url = QString( "%1/cases/%2/ensembles/%3/grids/%4/realizations/%5/blob_id" ) + .arg( server() ) + .arg( caseId.get() ) + .arg( encodedEnsembleName ) + .arg( encodedGridName ) + .arg( realization ); + m_networkRequest.setUrl( QUrl( url ) ); + + addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); + + auto reply = m_networkAccessManager->get( m_networkRequest ); + + connect( reply, + &QNetworkReply::finished, + [this, reply, ensembleName, caseId, gridName]() + { + if ( reply->error() == QNetworkReply::NoError ) + { + parseBlobId( reply, caseId, ensembleName, gridName, false ); + } + else + { + RiaLogging::error( std::format( "Request grid blob ID failed: '{}'", reply->errorString().toStdString() ) ); + emit blobIdFinished(); + } + } ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::requestGridBlobIdForEnsembleBlocking( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization ) +{ + auto requestCallable = [this, caseId, ensembleName, gridName, realization] + { requestGridBlobIdForEnsemble( caseId, ensembleName, gridName, realization ); }; + QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::blobIdFinished ); + wrapAndCallNetworkRequest( requestCallable, signalMethod ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QByteArray + RiaSumoConnector::requestGridDataBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ) +{ + requestGridBlobIdForEnsembleBlocking( caseId, ensembleName, gridName, realization ); + + if ( m_blobId.empty() ) return {}; + + // The REST API returns the blob Id + auto blobId = m_blobId.back(); + + QEventLoop eventLoop; + QTimer timer; + timer.setSingleShot( true ); + QObject::connect( &timer, SIGNAL( timeout() ), &eventLoop, SLOT( quit() ) ); + QObject::connect( this, SIGNAL( parquetDownloadFinished( const QByteArray&, const QString& ) ), &eventLoop, SLOT( quit() ) ); + + requestBlobDownload( blobId ); + + timer.start( RiaSumoDefines::requestTimeoutMillis() ); + eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); + + for ( const auto& blobData : m_redirectInfo ) + { + if ( blobData.objectId == blobId ) + { + return blobData.contents; + } + } + + return {}; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::requestGridPropertyInfoForEnsemble( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization ) +{ + m_gridPropertyInfos.clear(); + + requestTokenBlocking(); + + QNetworkRequest m_networkRequest; + + QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + QString encodedGridName = QUrl::toPercentEncoding( gridName ); + + QString url = QString( "%1/cases/%2/ensembles/%3/grids/%4/realizations/%5/property_info_list" ) + .arg( server() ) + .arg( caseId.get() ) + .arg( encodedEnsembleName ) + .arg( encodedGridName ) + .arg( realization ); + m_networkRequest.setUrl( QUrl( url ) ); + + addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); + + auto reply = m_networkAccessManager->get( m_networkRequest ); + + connect( reply, + &QNetworkReply::finished, + [this, reply, caseId, ensembleName, gridName, realization]() + { + if ( reply->error() == QNetworkReply::NoError ) + { + parseGridPropertyInfo( reply, caseId, ensembleName, gridName, realization ); + } + else + { + RiaLogging::error( std::format( "Request grid property info failed: '{}'", reply->errorString().toStdString() ) ); + emit gridPropertyInfoFinished(); + } + } ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::requestGridPropertyInfoForEnsembleBlocking( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization ) +{ + auto requestCallable = [this, caseId, ensembleName, gridName, realization] + { requestGridPropertyInfoForEnsemble( caseId, ensembleName, gridName, realization ); }; + QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::gridPropertyInfoFinished ); + wrapAndCallNetworkRequest( requestCallable, signalMethod ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoConnector::requestGridPropertyBlobIdBlocking( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) +{ + auto reply = makeGridPropertyBlobIdRequest( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + + // Wait for THIS reply only. Binding the event loop to the reply, rather than to the shared blobIdFinished + // signal, is what makes the mapping correct: a still-pending reply from an earlier property's request can no + // longer satisfy this wait and hand us its blob id (which previously caused e.g. SWAT to be served the SWCR + // blob). The blob id is read straight off this reply and never routed through shared state. + QEventLoop eventLoop; + QTimer timer; + timer.setSingleShot( true ); + QObject::connect( &timer, &QTimer::timeout, &eventLoop, &QEventLoop::quit ); + QObject::connect( reply, &QNetworkReply::finished, &eventLoop, &QEventLoop::quit ); + + timer.start( RiaSumoDefines::requestTimeoutMillis() ); + eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); + + return blobIdFromReply( reply, propertyName ); +} + +//-------------------------------------------------------------------------------------------------- +/// Issue the blob id request for one grid property time step. The reply is returned unfinished, so the caller +/// decides how to wait for it: one at a time, or several at once when prefetching. +//-------------------------------------------------------------------------------------------------- +QNetworkReply* RiaSumoConnector::makeGridPropertyBlobIdRequest( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) +{ + requestTokenBlocking(); + + // Properly URL-encode the path components + QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + QString encodedGridName = QUrl::toPercentEncoding( gridName ); + QString encodedPropertyName = QUrl::toPercentEncoding( propertyName ); + + QString url = QString( "%1/cases/%2/ensembles/%3/grids/%4/realizations/%5/properties/%6/blob_id" ) + .arg( server() ) + .arg( caseId.get() ) + .arg( encodedEnsembleName ) + .arg( encodedGridName ) + .arg( realization ) + .arg( encodedPropertyName ); + + // The timestamp/interval is an optional query parameter; omit it for static properties. + if ( !isoDateOrInterval.isEmpty() ) + { + url += QString( "?property_iso_date_or_interval=%1" ).arg( QString( QUrl::toPercentEncoding( isoDateOrInterval ) ) ); + } + + QNetworkRequest networkRequest; + networkRequest.setUrl( QUrl( url ) ); + addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); + + return m_networkAccessManager->get( networkRequest ); +} + +//-------------------------------------------------------------------------------------------------- +/// Read the blob id off a finished blob id reply. The reply is consumed and scheduled for deletion. +//-------------------------------------------------------------------------------------------------- +QString RiaSumoConnector::blobIdFromReply( QNetworkReply* reply, const QString& propertyName ) +{ + if ( !reply ) return {}; + + if ( !reply->isFinished() || reply->error() != QNetworkReply::NoError ) + { + if ( reply->error() != QNetworkReply::NoError ) + { + RiaLogging::error( std::format( "Request grid property blob ID failed: '{}'", reply->errorString().toStdString() ) ); + } + reply->deleteLater(); + return {}; + } + + // The REST API returns the blob id as a plain string, quoted by FastAPI. + QString blobId = QString::fromUtf8( reply->readAll() ).trimmed(); + reply->deleteLater(); + + if ( blobId.startsWith( '"' ) && blobId.endsWith( '"' ) ) + { + blobId = blobId.mid( 1, blobId.length() - 2 ); + } + + RiaLogging::debug( std::format( "Received blob ID for vector '{}': {}", propertyName.toStdString(), blobId.toStdString() ) ); + + return blobId; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QByteArray RiaSumoConnector::requestGridPropertyDataBlocking( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) +{ + // Serve from cache when possible. Keyed by the full property identity (not the blob id), a repeat request is + // answered without even asking Sumo for the blob id. This avoids re-downloading every time step when a + // property's global legend range is computed, and again when it is displayed. + const QString cacheKey = gridPropertyCacheKey( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + if ( auto cachedContents = gridPropertyBlobFromCache( cacheKey ); !cachedContents.isEmpty() ) + { + return cachedContents; + } + + // Resolve the blob id for this exact property. The getter waits on its own reply, so it can only ever return + // this property's id (or empty on failure) - never a neighbouring request's id. + const QString blobId = requestGridPropertyBlobIdBlocking( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + if ( blobId.isEmpty() ) return {}; + + QEventLoop eventLoop; + QTimer timer; + timer.setSingleShot( true ); + QObject::connect( &timer, SIGNAL( timeout() ), &eventLoop, SLOT( quit() ) ); + QObject::connect( this, SIGNAL( parquetDownloadFinished( const QByteArray&, const QString& ) ), &eventLoop, SLOT( quit() ) ); + + requestBlobDownload( blobId ); + + timer.start( RiaSumoDefines::requestTimeoutMillis() ); + eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); + + // Move the downloaded blob out of the transient redirect list and into the cache. Erasing the consumed entry + // also keeps m_redirectInfo from growing without bound as more properties are downloaded. + for ( auto it = m_redirectInfo.begin(); it != m_redirectInfo.end(); ++it ) + { + if ( it->objectId == blobId ) + { + QByteArray contents = it->contents; + m_redirectInfo.erase( it ); + insertGridPropertyBlobInCache( cacheKey, contents ); + return contents; + } + } + + return {}; +} + +//-------------------------------------------------------------------------------------------------- +/// Fetch several time steps of one grid property at the same time. The blob id requests are issued together +/// and waited for as a group, and so are the blob downloads, turning 2N sequential round trips into 2 batched +/// ones. The results are placed in the blob cache, so the per time step requests that follow are cache hits. +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::prefetchGridPropertyDataBlocking( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const std::vector& isoDatesOrIntervals ) +{ + // Only fetch what is not already cached, and drop duplicates so a time step is never requested twice. + std::vector cacheKeys; + std::vector timestampsToFetch; + for ( const auto& isoDateOrInterval : isoDatesOrIntervals ) + { + const QString cacheKey = gridPropertyCacheKey( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + + if ( std::ranges::find( cacheKeys, cacheKey ) != cacheKeys.end() ) continue; + if ( m_gridPropertyBlobCache.contains( cacheKey ) ) continue; + + cacheKeys.push_back( cacheKey ); + timestampsToFetch.push_back( isoDateOrInterval ); + } + + if ( timestampsToFetch.size() < 2 ) return; // nothing to gain over the single time step path + + // Phase 1: resolve all blob ids concurrently. + std::vector blobIdReplies; + for ( const auto& isoDateOrInterval : timestampsToFetch ) + { + blobIdReplies.push_back( makeGridPropertyBlobIdRequest( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ) ); + } + + waitForRepliesToFinish( blobIdReplies ); + + std::vector blobIds; + for ( auto reply : blobIdReplies ) + { + blobIds.push_back( blobIdFromReply( reply, propertyName ) ); + } + + // Phase 2: download all resolved blobs concurrently. requestBlobDownload is fire and forget; each finished + // download appends to m_redirectInfo, so wait until every requested blob has arrived there. + std::vector pendingBlobIds; + for ( const auto& blobId : blobIds ) + { + if ( blobId.isEmpty() ) continue; + + pendingBlobIds.push_back( blobId ); + requestBlobDownload( blobId ); + } + + if ( !pendingBlobIds.empty() ) + { + auto haveAllBlobsArrived = [this, &pendingBlobIds]() + { + return std::ranges::all_of( pendingBlobIds, + [this]( const QString& blobId ) + { + return std::ranges::any_of( m_redirectInfo, + [&blobId]( const SumoRedirect& redirect ) + { return redirect.objectId == blobId; } ); + } ); + }; + + // requestBlobDownload can spin an event loop of its own while acquiring a token, so a download may + // already have completed here. Check before waiting, or the wait would run until it times out. + if ( !haveAllBlobsArrived() ) + { + QEventLoop eventLoop; + QTimer timer; + timer.setSingleShot( true ); + QObject::connect( &timer, &QTimer::timeout, &eventLoop, &QEventLoop::quit ); + + // Every completed download emits this signal; quit once all of them have landed in m_redirectInfo. + auto connection = QObject::connect( this, + &RiaSumoConnector::parquetDownloadFinished, + &eventLoop, + [&eventLoop, &haveAllBlobsArrived]() + { + if ( haveAllBlobsArrived() ) eventLoop.quit(); + } ); + + // The downloads run concurrently, but allow the single request timeout per blob so a batch is + // never given less time than the same blobs would get one by one. + timer.start( static_cast( pendingBlobIds.size() ) * RiaSumoDefines::requestTimeoutMillis() ); + eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); + + QObject::disconnect( connection ); + } + } + + // Move the downloaded blobs out of the transient redirect list and into the cache. Anything missing was not + // downloaded in time; the per time step path fetches it again later. + for ( size_t i = 0; i < blobIds.size(); i++ ) + { + if ( blobIds[i].isEmpty() ) continue; + + for ( auto it = m_redirectInfo.begin(); it != m_redirectInfo.end(); ++it ) + { + if ( it->objectId == blobIds[i] ) + { + insertGridPropertyBlobInCache( cacheKeys[i], it->contents ); + m_redirectInfo.erase( it ); + break; + } + } + } +} + +//-------------------------------------------------------------------------------------------------- +/// Wait until every reply has finished, or the timeout expires. +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::waitForRepliesToFinish( const std::vector& replies ) +{ + if ( replies.empty() ) return; + + QEventLoop eventLoop; + QTimer timer; + timer.setSingleShot( true ); + QObject::connect( &timer, &QTimer::timeout, &eventLoop, &QEventLoop::quit ); + + auto isAllFinished = [&replies]() + { return std::ranges::all_of( replies, []( QNetworkReply* reply ) { return reply && reply->isFinished(); } ); }; + + std::vector connections; + for ( auto reply : replies ) + { + if ( !reply ) continue; + + connections.push_back( QObject::connect( reply, + &QNetworkReply::finished, + &eventLoop, + [&eventLoop, &isAllFinished]() + { + if ( isAllFinished() ) eventLoop.quit(); + } ) ); + } + + if ( !isAllFinished() ) + { + timer.start( RiaSumoDefines::requestTimeoutMillis() ); + eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); + } + + for ( const auto& connection : connections ) + { + QObject::disconnect( connection ); + } +} + +//-------------------------------------------------------------------------------------------------- +/// The full identity of one grid property time step, used as blob cache key. +//-------------------------------------------------------------------------------------------------- +QString RiaSumoConnector::gridPropertyCacheKey( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) +{ + return QString( "%1|%2|%3|%4|%5|%6" ).arg( caseId.get(), ensembleName, gridName ).arg( realization ).arg( propertyName, isoDateOrInterval ); +} + +//-------------------------------------------------------------------------------------------------- +/// Look up a downloaded grid property blob. Returns an empty array when the blob is not cached, which the +/// callers treat as a miss. A hit is moved to the front of the recency order, so it is evicted last. +//-------------------------------------------------------------------------------------------------- +QByteArray RiaSumoConnector::gridPropertyBlobFromCache( const QString& cacheKey ) +{ + auto it = m_gridPropertyBlobCache.find( cacheKey ); + if ( it == m_gridPropertyBlobCache.end() ) return {}; + + m_gridPropertyBlobCacheOrder.splice( m_gridPropertyBlobCacheOrder.begin(), m_gridPropertyBlobCacheOrder, it->second.orderIterator ); + + return it->second.contents; +} + +//-------------------------------------------------------------------------------------------------- +/// Cache a downloaded grid property blob, evicting the least recently used blobs until the cache is back +/// within the size limit. +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::insertGridPropertyBlobInCache( const QString& cacheKey, const QByteArray& contents ) +{ + if ( contents.isEmpty() ) return; + + const size_t contentsSize = static_cast( contents.size() ); + + // A blob larger than the whole cache would evict everything else to make room for itself. Leave it uncached + // instead, so the smaller blobs already present stay available. + if ( contentsSize > RiaSumoDefines::gridPropertyCacheLimitBytes() ) return; + + // Re-inserting an existing key would leak its order list entry, so drop the previous version first. + if ( auto it = m_gridPropertyBlobCache.find( cacheKey ); it != m_gridPropertyBlobCache.end() ) + { + m_gridPropertyBlobCacheSizeBytes -= static_cast( it->second.contents.size() ); + m_gridPropertyBlobCacheOrder.erase( it->second.orderIterator ); + m_gridPropertyBlobCache.erase( it ); + } + + m_gridPropertyBlobCacheOrder.push_front( cacheKey ); + m_gridPropertyBlobCache[cacheKey] = { contents, m_gridPropertyBlobCacheOrder.begin() }; + m_gridPropertyBlobCacheSizeBytes += contentsSize; + + while ( m_gridPropertyBlobCacheSizeBytes > RiaSumoDefines::gridPropertyCacheLimitBytes() && !m_gridPropertyBlobCacheOrder.empty() ) + { + const QString& oldestKey = m_gridPropertyBlobCacheOrder.back(); + + if ( auto it = m_gridPropertyBlobCache.find( oldestKey ); it != m_gridPropertyBlobCache.end() ) + { + m_gridPropertyBlobCacheSizeBytes -= static_cast( it->second.contents.size() ); + m_gridPropertyBlobCache.erase( it ); + } + + m_gridPropertyBlobCacheOrder.pop_back(); + } +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -578,6 +1142,14 @@ QString RiaSumoConnector::constructSasUri( const QString& blobStoreBaseUri, cons //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- +/// Note that this event loop dispatches everything except user input, so while a request is in flight the +/// view update code can run and re-enter a cell result load that is already in progress. That makes the same +/// result load twice. Declining the second load in RigCaseCellResultsData is not a way out: the only "no" +/// that method can return is cvf::UNDEFINED_SIZE_T, which consumers read as "no such result" and act on - +/// RimEclipseResultDefinitionTools::updateCellResultLegend computes the legend range straight after +/// ensureKnownResultLoaded without checking it, and caches a range over no data. Preventing the re-entrancy +/// means not dispatching events here at all, which requires moving the transfers off the GUI thread. +//-------------------------------------------------------------------------------------------------- void RiaSumoConnector::wrapAndCallNetworkRequest( std::function requestCallable, const QMetaMethod& signalMethod ) { QEventLoop eventLoop; @@ -728,6 +1300,89 @@ void RiaSumoConnector::parseRealizationNumbers( QNetworkReply* reply, const Sumo emit realizationIdsFinished(); } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::parseGridInfo( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName ) +{ + QByteArray result = reply->readAll(); + reply->deleteLater(); + + m_gridInfos.clear(); + + if ( reply->error() == QNetworkReply::NoError ) + { + QJsonDocument doc = QJsonDocument::fromJson( result ); + QJsonArray jsonArray = doc.array(); + + for ( const QJsonValue& value : jsonArray ) + { + QJsonObject gridObj = value.toObject(); + + SumoGridInfo gridInfo; + gridInfo.name = gridObj["name"].toString(); + + for ( const QJsonValue& realizationValue : gridObj["realizations"].toArray() ) + { + gridInfo.realizations.push_back( realizationValue.toInt() ); + } + + m_gridInfos.push_back( gridInfo ); + } + + RiaLogging::debug( std::format( "Grid info count : {}", m_gridInfos.size() ) ); + } + else + { + RiaLogging::error( std::format( "Request grid info failed: '{}'", reply->errorString().toStdString() ) ); + } + + emit gridInfoFinished(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::parseGridPropertyInfo( QNetworkReply* reply, + const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization ) +{ + QByteArray result = reply->readAll(); + reply->deleteLater(); + + m_gridPropertyInfos.clear(); + + if ( reply->error() == QNetworkReply::NoError ) + { + QJsonDocument doc = QJsonDocument::fromJson( result ); + QJsonArray jsonArray = doc.array(); + + for ( const QJsonValue& value : jsonArray ) + { + QJsonObject propertyObj = value.toObject(); + + SumoGridPropertyInfo propertyInfo; + propertyInfo.name = propertyObj["propertyName"].toString(); + + // isoDateOrInterval is null for static properties. + const auto isoValue = propertyObj["isoDateOrInterval"]; + if ( !isoValue.isNull() ) propertyInfo.isoDateOrInterval = isoValue.toString(); + + m_gridPropertyInfos.push_back( propertyInfo ); + } + + RiaLogging::debug( std::format( "Grid property info count : {}", m_gridPropertyInfos.size() ) ); + } + else + { + RiaLogging::error( std::format( "Request grid property info failed: '{}'", reply->errorString().toStdString() ) ); + } + + emit gridPropertyInfoFinished(); +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -874,6 +1529,22 @@ std::vector RiaSumoConnector::realizationIds() const return m_realizationIds; } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoConnector::gridInfos() const +{ + return m_gridInfos; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoConnector::gridPropertyInfos() const +{ + return m_gridPropertyInfos; +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h index 909f6d04060..bb9ba17c93e 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h @@ -26,6 +26,7 @@ #include #include +#include #include class QEventLoop; @@ -64,6 +65,21 @@ struct SumoEnsemble QString name; }; +struct SumoGridInfo +{ + QString name; + std::vector realizations; +}; + +struct SumoGridPropertyInfo +{ + QString name; + + // Empty for a static property. For a dynamic property this is either a single timestamp ("2018-01-01") + // or an interval ("2018-01-01/2019-01-01"). ResInsight currently only supports the single-timestamp form. + QString isoDateOrInterval; +}; + //================================================================================================== /// //================================================================================================== @@ -111,13 +127,52 @@ class RiaSumoConnector : public RiaCloudConnector QByteArray requestParquetDataBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ); - std::vector assets() const; - std::vector cases() const; - std::vector ensembleNamesForCase( const SumoCaseId& caseId ) const; - std::vector vectorNames() const; - std::vector realizationIds() const; - std::vector blobIds() const; - std::vector blobContents() const; + void requestGridInfoForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ); + void requestGridInfoForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ); + + void requestGridBlobIdForEnsemble( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); + void requestGridBlobIdForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); + + QByteArray requestGridDataBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); + + void requestGridPropertyInfoForEnsemble( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); + void requestGridPropertyInfoForEnsembleBlocking( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization ); + + QString requestGridPropertyBlobIdBlocking( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ); + + QByteArray requestGridPropertyDataBlocking( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ); + + // Download several time steps of one grid property concurrently and put them in the blob cache, so the + // following per time step requests are served without going to Sumo. Entries already cached are skipped. + void prefetchGridPropertyDataBlocking( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const std::vector& isoDatesOrIntervals ); + + std::vector assets() const; + std::vector cases() const; + std::vector ensembleNamesForCase( const SumoCaseId& caseId ) const; + std::vector vectorNames() const; + std::vector realizationIds() const; + std::vector gridInfos() const; + std::vector gridPropertyInfos() const; + std::vector blobIds() const; + std::vector blobContents() const; public slots: void parseAssets( QNetworkReply* reply ); @@ -125,6 +180,8 @@ public slots: void parseCases( QNetworkReply* reply ); void parseVectorNames( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName ); void parseRealizationNumbers( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName ); + void parseGridInfo( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName ); + void parseGridPropertyInfo( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); void parseBlobId( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName, bool isParameters ); void requestFailed( const QAbstractOAuth::Error error ); @@ -142,6 +199,8 @@ public slots: void blobIdFinished(); void assetsFinished(); void realizationIdsFinished(); + void gridInfoFinished(); + void gridPropertyInfoFinished(); private: void addStandardHeader( QNetworkRequest& networkRequest, const QString& token, const QString& contentType ); @@ -153,6 +212,27 @@ public slots: void wrapAndCallNetworkRequest( std::function requestCallable, const QMetaMethod& signalMethod ); + QByteArray gridPropertyBlobFromCache( const QString& cacheKey ); + void insertGridPropertyBlobInCache( const QString& cacheKey, const QByteArray& contents ); + + static QString gridPropertyCacheKey( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ); + + QNetworkReply* makeGridPropertyBlobIdRequest( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ); + + static QString blobIdFromReply( QNetworkReply* reply, const QString& propertyName ); + + static void waitForRepliesToFinish( const std::vector& replies ); + private: std::function m_serverUrlProvider; @@ -161,8 +241,27 @@ public slots: std::vector m_vectorNames; std::vector m_realizationIds; std::vector m_ensembleNames; + std::vector m_gridInfos; + + std::vector m_gridPropertyInfos; std::vector m_blobId; std::vector m_redirectInfo; + + // Downloaded grid-property blobs, keyed by the full property identity (case, ensemble, grid, realization, + // property, timestamp). Displaying a property computes its global legend range across all time steps, so a + // property is requested repeatedly; the cache ensures each blob is fetched from Sumo at most once. + // + // Bounded by total byte size, not by entry count, as the blob size follows the grid size and varies by orders + // of magnitude. The least recently used entries are evicted when the limit is exceeded. + struct GridPropertyBlobCacheEntry + { + QByteArray contents; + std::list::iterator orderIterator; + }; + + std::map m_gridPropertyBlobCache; + std::list m_gridPropertyBlobCacheOrder; // most recently used at front + size_t m_gridPropertyBlobCacheSizeBytes = 0; }; diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.cpp index e8876409d33..f696d3bc710 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.cpp +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.cpp @@ -36,3 +36,19 @@ int RiaSumoDefines::requestTimeoutMillis() { return 10 * 1000; } + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +size_t RiaSumoDefines::gridPropertyCacheLimitBytes() +{ + return 256ull * 1024 * 1024; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +size_t RiaSumoDefines::gridPropertyPrefetchBatchSize() +{ + return 8; +} diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.h index 2bb97f0ea4f..fe0a8fa25af 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.h +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.h @@ -22,6 +22,8 @@ #include +#include + using SumoAssetId = nonstd::ordered; using SumoCaseId = nonstd::ordered; @@ -29,4 +31,11 @@ namespace RiaSumoDefines { QString tokenPath(); int requestTimeoutMillis(); + +// The maximum number of bytes of downloaded grid property blobs kept in memory. +size_t gridPropertyCacheLimitBytes(); + +// The number of grid property time steps fetched concurrently when prefetching. Bounds both the number of +// requests in flight and the amount of blob data pulled in for time steps that may not be needed. +size_t gridPropertyPrefetchBatchSize(); }; // namespace RiaSumoDefines diff --git a/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.cpp b/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.cpp new file mode 100644 index 00000000000..1c60708c119 --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.cpp @@ -0,0 +1,156 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024 Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "RifReaderSumoGridProperty.h" + +#include "RiaLogging.h" +#include "RiaSumoConnector.h" +#include "RiaSumoDefines.h" + +#include "RifRoffFileTools.h" + +#include +#include + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RifReaderSumoGridProperty::RifReaderSumoGridProperty( RiaSumoConnector* connector, + const QString& caseId, + const QString& ensembleName, + const QString& gridName, + int realization ) + : m_connector( connector ) + , m_caseId( caseId ) + , m_ensembleName( ensembleName ) + , m_gridName( gridName ) + , m_realization( realization ) + , m_caseData( nullptr ) +{ +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RifReaderSumoGridProperty::setStaticProperties( const std::vector& propertyNames ) +{ + m_staticProperties = propertyNames; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RifReaderSumoGridProperty::setDynamicProperties( const std::map>& propertyNameToTimestamps ) +{ + m_dynamicTimestamps = propertyNameToTimestamps; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RifReaderSumoGridProperty::open( const QString& /*fileName*/, RigEclipseCaseData* eclipseCase ) +{ + // The grid geometry is loaded elsewhere; only keep the case data for cell count and active cell masking. + m_caseData = eclipseCase; + return true; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RifReaderSumoGridProperty::staticResult( const QString& result, RiaDefines::PorosityModelType matrixOrFracture, std::vector* values ) +{ + if ( matrixOrFracture != RiaDefines::PorosityModelType::MATRIX_MODEL ) return false; + + // Only fetch properties this reader owns; other static results (e.g. computed DEPTH) are not on Sumo. + if ( std::find( m_staticProperties.begin(), m_staticProperties.end(), result ) == m_staticProperties.end() ) return false; + + return fetchAndDecode( result, "", values ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RifReaderSumoGridProperty::dynamicResult( const QString& result, + RiaDefines::PorosityModelType matrixOrFracture, + size_t stepIndex, + std::vector* values ) +{ + if ( matrixOrFracture != RiaDefines::PorosityModelType::MATRIX_MODEL ) return false; + + auto it = m_dynamicTimestamps.find( result ); + if ( it == m_dynamicTimestamps.end() || stepIndex >= it->second.size() ) return false; + + // The timestamp list is aligned with the case's common time steps. An empty entry means this property has + // no data at that time step, so report "no data" instead of fetching another step's values. + const QString& isoDateOrInterval = it->second[stepIndex]; + if ( isoDateOrInterval.isEmpty() ) return false; + + prefetchFromTimeStep( result, it->second, stepIndex ); + + return fetchAndDecode( result, isoDateOrInterval, values ); +} + +//-------------------------------------------------------------------------------------------------- +/// A displayed dynamic property is read one time step at a time, and each read is a blocking round trip to +/// Sumo. Fetch a window of the following time steps in one concurrent batch instead, so the reads that follow +/// are served from the connector's blob cache. The window bounds both the requests in flight and the data +/// pulled in for time steps that may never be displayed. +//-------------------------------------------------------------------------------------------------- +void RifReaderSumoGridProperty::prefetchFromTimeStep( const QString& propertyName, const std::vector& timestamps, size_t stepIndex ) +{ + if ( !m_connector ) return; + + const size_t batchSize = RiaSumoDefines::gridPropertyPrefetchBatchSize(); + + std::vector batch; + for ( size_t i = stepIndex; i < timestamps.size() && batch.size() < batchSize; i++ ) + { + // Skip the time steps this property has no data for, they are never downloaded. + if ( !timestamps[i].isEmpty() ) batch.push_back( timestamps[i] ); + } + + m_connector->prefetchGridPropertyDataBlocking( SumoCaseId( m_caseId ), m_ensembleName, m_gridName, m_realization, propertyName, batch ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RifReaderSumoGridProperty::fetchAndDecode( const QString& propertyName, const QString& isoDateOrInterval, std::vector* values ) +{ + if ( !m_connector || !m_caseData || !values ) return false; + + QByteArray contents = m_connector->requestGridPropertyDataBlocking( SumoCaseId( m_caseId ), + m_ensembleName, + m_gridName, + m_realization, + propertyName, + isoDateOrInterval ); + + RiaLogging::debug( std::format( "Sumo grid property '{}' (time '{}'): downloaded {} bytes.", + propertyName.toStdString(), + isoDateOrInterval.toStdString(), + contents.size() ) ); + + if ( contents.isEmpty() ) return false; + + std::string buffer = contents.toStdString(); + std::istringstream stream( buffer, std::ios::binary ); + + return RifRoffFileTools::propertyValuesFromStream( stream, m_caseData, propertyName, values ); +} diff --git a/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.h b/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.h new file mode 100644 index 00000000000..cd594578fa7 --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.h @@ -0,0 +1,71 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024 Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "RifReaderInterface.h" + +#include +#include +#include + +#include +#include + +class RiaSumoConnector; + +//================================================================================================== +// +// Lazily fetches grid cell properties for a single Sumo grid realization. ResInsight's result +// machinery calls staticResult()/dynamicResult() the first time a property is displayed; this reader +// then downloads the corresponding roff blob from Sumo and decodes the cell values. +// +//================================================================================================== +class RifReaderSumoGridProperty : public RifReaderInterface +{ +public: + RifReaderSumoGridProperty( RiaSumoConnector* connector, + const QString& caseId, + const QString& ensembleName, + const QString& gridName, + int realization ); + + void setStaticProperties( const std::vector& propertyNames ); + void setDynamicProperties( const std::map>& propertyNameToTimestamps ); + + bool open( const QString& fileName, RigEclipseCaseData* eclipseCase ) override; + + bool staticResult( const QString& result, RiaDefines::PorosityModelType matrixOrFracture, std::vector* values ) override; + bool dynamicResult( const QString& result, RiaDefines::PorosityModelType matrixOrFracture, size_t stepIndex, std::vector* values ) override; + +private: + bool fetchAndDecode( const QString& propertyName, const QString& isoDateOrInterval, std::vector* values ); + void prefetchFromTimeStep( const QString& propertyName, const std::vector& timestamps, size_t stepIndex ); + +private: + QPointer m_connector; + QString m_caseId; + QString m_ensembleName; + QString m_gridName; + int m_realization; + + RigEclipseCaseData* m_caseData; // set in open(); used for cell count and active cell masking + + std::vector m_staticProperties; + std::map> m_dynamicTimestamps; // property name -> sorted iso timestamps +}; diff --git a/ApplicationLibCode/Commands/Sumo/CMakeLists_files.cmake b/ApplicationLibCode/Commands/Sumo/CMakeLists_files.cmake index 41f8eb1ee83..a3e4bff0a06 100644 --- a/ApplicationLibCode/Commands/Sumo/CMakeLists_files.cmake +++ b/ApplicationLibCode/Commands/Sumo/CMakeLists_files.cmake @@ -1,5 +1,6 @@ set(SOURCE_GROUP_SOURCE_FILES ${CMAKE_CURRENT_LIST_DIR}/RicCreateSumoEnsembleFeature.cpp + ${CMAKE_CURRENT_LIST_DIR}/RicCreateSumoGridEnsembleFeature.cpp ${CMAKE_CURRENT_LIST_DIR}/RicDeleteSumoTokenFeature.cpp ) diff --git a/ApplicationLibCode/Commands/Sumo/RicCreateSumoGridEnsembleFeature.cpp b/ApplicationLibCode/Commands/Sumo/RicCreateSumoGridEnsembleFeature.cpp new file mode 100644 index 00000000000..296ea56215a --- /dev/null +++ b/ApplicationLibCode/Commands/Sumo/RicCreateSumoGridEnsembleFeature.cpp @@ -0,0 +1,126 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "RicCreateSumoGridEnsembleFeature.h" + +#include "RiaDefines.h" +#include "RiaLogging.h" + +#include "RicNewViewFeature.h" + +#include "Rim3dView.h" +#include "RimEclipseCaseCollection.h" +#include "RimEclipseCaseEnsemble.h" +#include "RimEclipseViewCollection.h" +#include "RimOilField.h" +#include "RimProject.h" +#include "RimRoffCaseSumo.h" +#include "RimViewNameConfig.h" +#include "Sumo/RimSumoDataSource.h" + +#include "cafSelectionManagerTools.h" + +#include + +CAF_CMD_SOURCE_INIT( RicCreateSumoGridEnsembleFeature, "RicCreateSumoGridEnsembleFeature" ); + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RicCreateSumoGridEnsembleFeature::onActionTriggered( bool isChecked ) +{ + auto dataSources = caf::selectedObjectsByType(); + + for ( auto dataSource : dataSources ) + { + createGridEnsemble( dataSource ); + } +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RicCreateSumoGridEnsembleFeature::createGridEnsemble( RimSumoDataSource* dataSource ) +{ + if ( !dataSource ) return; + + const QString gridName = dataSource->selectedGridName(); + const std::vector realizationIds = dataSource->selectedRealizationIds(); + + if ( gridName.isEmpty() ) + { + RiaLogging::warning( "No grid selected. Unable to create grid ensemble from Sumo." ); + return; + } + + if ( realizationIds.empty() ) + { + RiaLogging::warning( "No realizations selected. Unable to create grid ensemble from Sumo." ); + return; + } + + RimProject* project = RimProject::current(); + if ( !project ) return; + + RimOilField* oilfield = project->activeOilField(); + if ( !oilfield ) return; + + auto eclipseCaseEnsemble = new RimEclipseCaseEnsemble; + eclipseCaseEnsemble->setName( QString( "%1 - %2" ).arg( dataSource->ensembleName(), gridName ) ); + eclipseCaseEnsemble->setDoComputeMobileVolumeWeightedMean( dataSource->doComputeMobileVolumeWeightedMean() ); + + for ( const QString& realizationId : realizationIds ) + { + bool ok = false; + int realization = realizationId.toInt( &ok ); + if ( !ok ) continue; + + if ( auto* gridCase = RimRoffCaseSumo::createFromDataSource( dataSource, gridName, realization ) ) + { + eclipseCaseEnsemble->addCase( gridCase ); + } + } + + if ( eclipseCaseEnsemble->cases().empty() ) + { + RiaLogging::warning( "No valid realizations selected. No grid ensemble created." ); + delete eclipseCaseEnsemble; + return; + } + + oilfield->analysisModels()->caseEnsembles.push_back( eclipseCaseEnsemble ); + oilfield->analysisModels()->updateConnectedEditors(); + + auto firstCase = eclipseCaseEnsemble->cases().front(); + if ( !firstCase ) return; + + auto view = RicNewViewFeature::addReservoirView( firstCase, nullptr, eclipseCaseEnsemble->viewCollection() ); + if ( view ) + { + view->nameConfig()->setAddCaseName( true ); + } +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RicCreateSumoGridEnsembleFeature::setupActionLook( QAction* actionToSetup ) +{ + actionToSetup->setText( "Create Grid Ensemble" + RiaDefines::betaFeaturePostfix() ); + actionToSetup->setIcon( QIcon( ":/CreateGridCaseGroup16x16.png" ) ); +} diff --git a/ApplicationLibCode/Commands/Sumo/RicCreateSumoGridEnsembleFeature.h b/ApplicationLibCode/Commands/Sumo/RicCreateSumoGridEnsembleFeature.h new file mode 100644 index 00000000000..05c5f4ede12 --- /dev/null +++ b/ApplicationLibCode/Commands/Sumo/RicCreateSumoGridEnsembleFeature.h @@ -0,0 +1,38 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "cafCmdFeature.h" + +class RimSumoDataSource; + +//================================================================================================== +/// +//================================================================================================== +class RicCreateSumoGridEnsembleFeature : public caf::CmdFeature +{ + CAF_CMD_HEADER_INIT; + +protected: + void onActionTriggered( bool isChecked ) override; + void setupActionLook( QAction* actionToSetup ) override; + +private: + static void createGridEnsemble( RimSumoDataSource* dataSource ); +}; diff --git a/ApplicationLibCode/FileInterface/RifRoffFileTools.cpp b/ApplicationLibCode/FileInterface/RifRoffFileTools.cpp index 14c0dc9f094..150de3fee01 100644 --- a/ApplicationLibCode/FileInterface/RifRoffFileTools.cpp +++ b/ApplicationLibCode/FileInterface/RifRoffFileTools.cpp @@ -108,6 +108,14 @@ bool RifRoffFileTools::openGridFile( const QString& fileName, RigEclipseCaseData return false; } + return openGridFile( stream, eclipseCase, errorMessages ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RifRoffFileTools::openGridFile( std::istream& stream, RigEclipseCaseData* eclipseCase, QString* errorMessages ) +{ auto getInt = []( auto values, const std::string& name ) { auto v = std::find_if( values.begin(), values.end(), [&name]( const auto& arg ) { return arg.first == name; } ); @@ -274,6 +282,7 @@ bool RifRoffFileTools::openGridFile( const QString& fileName, RigEclipseCaseData catch ( std::runtime_error& err ) { RiaLogging::error( std::format( "Roff file import failed: {}", err.what() ) ); + if ( errorMessages ) *errorMessages = QString::fromStdString( err.what() ); return false; } @@ -492,17 +501,26 @@ std::pair> RifRoffFileTools::createInputPropert { RiaLogging::info( std::format( "Reading properties from roff file: {}", fileName ) ); - std::string filename = fileName.toStdString(); - - std::map keywordMapping; - - std::ifstream stream( filename, std::ios::binary ); + std::ifstream stream( fileName.toStdString(), std::ios::binary ); if ( !stream.good() ) { RiaLogging::error( "Unable to open roff file" ); - return std::make_pair( false, keywordMapping ); + return std::make_pair( false, std::map{} ); } + return createInputProperties( stream, eclipseCaseData, fileName ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::pair> RifRoffFileTools::createInputProperties( std::istream& stream, + RigEclipseCaseData* eclipseCaseData, + const QString& sourceName, + RiaDefines::ResultCatType resultCategory ) +{ + std::map keywordMapping; + auto codeNamesAndValuesForKeyword = []( const std::string& keyword, roff::Reader& reader ) -> std::map { const std::string codeNamesKeyword = keyword + roff::Parser::postFixCodeNames(); @@ -545,7 +563,7 @@ std::pair> RifRoffFileTools::createInputPropert { if ( !appendZoneIndexPropertyFromSubgrids( eclipseCaseData, reader, keywordMapping ) ) { - RiaLogging::warning( std::format( "Unable to import ROFF subgrids zonation from {}", fileName ) ); + RiaLogging::warning( std::format( "Unable to import ROFF subgrids zonation from {}", sourceName ) ); } } else if ( eclipseCaseData->mainGrid()->cellCount() == keywordLength ) @@ -558,9 +576,9 @@ std::pair> RifRoffFileTools::createInputPropert newResultName = "ACTNUM"; } - if ( !appendNewInputPropertyResult( eclipseCaseData, newResultName, keyword, kind, reader ) ) + if ( !appendNewInputPropertyResult( eclipseCaseData, newResultName, keyword, kind, reader, resultCategory ) ) { - RiaLogging::error( std::format( "Unable to import result '{}' from {}", keyword, fileName ) ); + RiaLogging::error( std::format( "Unable to import result '{}' from {}", keyword, sourceName ) ); return std::make_pair( false, keywordMapping ); } @@ -604,6 +622,15 @@ std::pair> RifRoffFileTools::createInputPropert const auto codeNames = codeNamesAndValuesForKeyword( keyword, reader ); RicFaciesPropertiesImportTools::createColorLegendMatchDefaultRockColors( codeNames ); } + else + { + // Skipped: typically grid metadata, but a property array whose length does not match the grid + // cell count is also skipped here. Log it so a size mismatch does not fail silently. + RiaLogging::debug( std::format( "Skipping roff array '{}' (length {}), grid cell count is {}.", + keyword, + keywordLength, + eclipseCaseData->mainGrid()->cellCount() ) ); + } } } catch ( std::runtime_error& err ) @@ -612,6 +639,13 @@ std::pair> RifRoffFileTools::createInputPropert return std::make_pair( false, keywordMapping ); } + if ( keywordMapping.empty() ) + { + RiaLogging::warning( std::format( "No grid properties matching the grid cell count ({}) were imported from {}.", + eclipseCaseData->mainGrid()->cellCount(), + sourceName.toStdString() ) ); + } + return std::make_pair( true, keywordMapping ); } @@ -680,11 +714,90 @@ std::vector //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -bool RifRoffFileTools::appendNewInputPropertyResult( RigEclipseCaseData* caseData, - const QString& resultName, - const std::string& keyword, - roff::Token::Kind kind, - roff::Reader& reader ) +bool RifRoffFileTools::propertyValuesFromStream( std::istream& stream, + RigEclipseCaseData* eclipseCaseData, + const QString& propertyName, + std::vector* values ) +{ + if ( !eclipseCaseData || !eclipseCaseData->mainGrid() || !values ) return false; + + auto mainGrid = eclipseCaseData->mainGrid(); + + const int nx = static_cast( mainGrid->cellCountI() ); + const int ny = static_cast( mainGrid->cellCountJ() ); + const int nz = static_cast( mainGrid->cellCountK() ); + const size_t cellCount = mainGrid->cellCount(); + + try + { + roff::Reader reader( stream ); + reader.parse(); + + // Collect the arrays whose length matches the grid cell count. A Sumo property blob may contain more + // than one such array, so pick the one whose keyword matches the requested property name rather than + // blindly taking the first match (which could decode a different property). + std::vector> candidates; + for ( const auto& [keyword, kind] : reader.getNamedArrayTypes() ) + { + if ( reader.getArrayLength( keyword ) == cellCount ) candidates.push_back( { keyword, kind } ); + } + + if ( candidates.empty() ) return false; + + std::string candidateNames; + for ( const auto& [keyword, kind] : candidates ) + { + if ( !candidateNames.empty() ) candidateNames += ", "; + candidateNames += keyword; + } + + auto selected = candidates.front(); + for ( const auto& candidate : candidates ) + { + if ( QString::fromStdString( candidate.first ).compare( propertyName, Qt::CaseInsensitive ) == 0 ) + { + selected = candidate; + break; + } + } + + RiaLogging::debug( std::format( "Roff property '{}': arrays matching cell count [{}], using '{}'.", + propertyName.toStdString(), + candidateNames, + selected.first ) ); + + std::vector roffValues = readAndConvertToDouble( nx, ny, nz, selected.first, selected.second, reader ); + if ( roffValues.size() != cellCount ) return false; + + // Set better invalid value for inactive cells: roff file has -999. + auto activeCellInfo = eclipseCaseData->activeCellInfo( RiaDefines::PorosityModelType::MATRIX_MODEL ); + for ( size_t i = 0; i < cellCount; i++ ) + { + if ( !activeCellInfo->isActive( ReservoirCellIndex( mainGrid->reservoirCellIndex( i ) ) ) ) + { + roffValues[i] = HUGE_VAL; + } + } + + *values = std::move( roffValues ); + return true; + } + catch ( std::runtime_error& err ) + { + RiaLogging::error( std::format( "Roff property parsing failed: {}", err.what() ) ); + return false; + } +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RifRoffFileTools::appendNewInputPropertyResult( RigEclipseCaseData* caseData, + const QString& resultName, + const std::string& keyword, + roff::Token::Kind kind, + roff::Reader& reader, + RiaDefines::ResultCatType resultCategory ) { CAF_ASSERT( caseData ); @@ -707,7 +820,7 @@ bool RifRoffFileTools::appendNewInputPropertyResult( RigEclipseCaseData* caseDat } } - RigEclipseResultAddress resAddr( RiaDefines::ResultCatType::INPUT_PROPERTY, RifRoffFileTools::mapFromType( kind ), resultName ); + RigEclipseResultAddress resAddr( resultCategory, RifRoffFileTools::mapFromType( kind ), resultName ); caseData->results( RiaDefines::PorosityModelType::MATRIX_MODEL )->createResultEntry( resAddr, false ); auto newPropertyData = caseData->results( RiaDefines::PorosityModelType::MATRIX_MODEL )->modifiableCellScalarResultTimesteps( resAddr ); diff --git a/ApplicationLibCode/FileInterface/RifRoffFileTools.h b/ApplicationLibCode/FileInterface/RifRoffFileTools.h index 823a6f978b7..48447ddac8a 100644 --- a/ApplicationLibCode/FileInterface/RifRoffFileTools.h +++ b/ApplicationLibCode/FileInterface/RifRoffFileTools.h @@ -25,6 +25,7 @@ #include +#include #include #include @@ -52,10 +53,31 @@ class RifRoffFileTools : public cvf::Object static bool openGridFile( const QString& fileName, RigEclipseCaseData* eclipseCase, QString* errorMessages ); + // Parse a roff grid directly from an already opened binary stream. Used when the roff data does not + // originate from a file on disk (e.g. a blob downloaded from Sumo). + static bool openGridFile( std::istream& stream, RigEclipseCaseData* eclipseCase, QString* errorMessages ); + static std::pair> createInputProperties( const QString& fileName, RigEclipseCaseData* eclipseCase ); + // Read roff property data from an already opened binary stream (e.g. a blob downloaded from Sumo). The + // sourceName is only used for log messages. resultCategory selects the result category the properties are + // imported into (e.g. STATIC_NATIVE for static and DYNAMIC_NATIVE for time dependent properties). + static std::pair> + createInputProperties( std::istream& stream, + RigEclipseCaseData* eclipseCase, + const QString& sourceName, + RiaDefines::ResultCatType resultCategory = RiaDefines::ResultCatType::INPUT_PROPERTY ); + static bool hasGridData( const QString& filename ); + // Read a single grid property from an in-memory roff blob and return its values (with inactive cells + // masked), without registering it. Among the arrays matching the grid cell count, the one whose keyword + // matches propertyName (case-insensitive) is preferred; otherwise the first matching array is used. + static bool propertyValuesFromStream( std::istream& stream, + RigEclipseCaseData* eclipseCase, + const QString& propertyName, + std::vector* values ); + static size_t computeActiveCellMatrixIndex( std::vector& activeCells ); static std::vector computeZoneValuesFromSubgrids( const std::vector& nLayers, size_t nx, size_t ny, size_t nz ); @@ -81,11 +103,12 @@ class RifRoffFileTools : public cvf::Object static std::vector readAndConvertToDouble( int nx, int ny, int nz, const std::string& keyword, roff::Token::Kind kind, roff::Reader& reader ); - static bool appendNewInputPropertyResult( RigEclipseCaseData* caseData, - const QString& resultName, - const std::string& keyword, - roff::Token::Kind token, - roff::Reader& reader ); + static bool appendNewInputPropertyResult( RigEclipseCaseData* caseData, + const QString& resultName, + const std::string& keyword, + roff::Token::Kind token, + roff::Reader& reader, + RiaDefines::ResultCatType resultCategory ); static bool appendZoneIndexPropertyFromSubgrids( RigEclipseCaseData* caseData, roff::Reader& reader, std::map& keywordMapping ); diff --git a/ApplicationLibCode/ProjectDataModel/CMakeLists_files.cmake b/ApplicationLibCode/ProjectDataModel/CMakeLists_files.cmake index 8e4ccd74936..84b9d281c12 100644 --- a/ApplicationLibCode/ProjectDataModel/CMakeLists_files.cmake +++ b/ApplicationLibCode/ProjectDataModel/CMakeLists_files.cmake @@ -11,6 +11,7 @@ set(SOURCE_GROUP_SOURCE_FILES ${CMAKE_CURRENT_LIST_DIR}/RimEclipseInputPropertyCollection.cpp ${CMAKE_CURRENT_LIST_DIR}/RimEclipseInputCase.cpp ${CMAKE_CURRENT_LIST_DIR}/RimEclipseResultCase.cpp + ${CMAKE_CURRENT_LIST_DIR}/RimRoffCaseSumo.cpp ${CMAKE_CURRENT_LIST_DIR}/RimEclipseView.cpp ${CMAKE_CURRENT_LIST_DIR}/RimEclipseResultDefinition.cpp ${CMAKE_CURRENT_LIST_DIR}/RimEclipseCellColors.cpp diff --git a/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.cpp b/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.cpp index ef9988186fa..1aa21ecc439 100644 --- a/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.cpp +++ b/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.cpp @@ -413,10 +413,17 @@ std::vector RimCloudDataSourceCollection::addDataSources() m_sumoConnector->requestRealizationIdsForEnsembleBlocking( sumoCaseId, ensembleName ); m_sumoConnector->requestVectorNamesForEnsembleBlocking( sumoCaseId, ensembleName ); + m_sumoConnector->requestGridInfoForEnsembleBlocking( sumoCaseId, ensembleName ); auto availableRealizationIds = m_sumoConnector->realizationIds(); auto vectorNames = m_sumoConnector->vectorNames(); + std::vector gridNames; + for ( const auto& gridInfo : m_sumoConnector->gridInfos() ) + { + gridNames.push_back( gridInfo.name ); + } + auto dataSource = new RimSumoDataSource(); dataSource->setCaseId( sumoCaseId ); dataSource->setAssetName( m_sumoFieldName ); @@ -424,6 +431,7 @@ std::vector RimCloudDataSourceCollection::addDataSources() dataSource->setEnsembleName( ensembleName ); dataSource->setAvailableRealizationIds( availableRealizationIds ); dataSource->setVectorNames( vectorNames ); + dataSource->setGridNames( gridNames ); dataSource->updateName(); objectToSelect = dataSource; diff --git a/ApplicationLibCode/ProjectDataModel/Rim3dOverlayInfoConfig.cpp b/ApplicationLibCode/ProjectDataModel/Rim3dOverlayInfoConfig.cpp index faeb0c33d20..ace7be0e80f 100644 --- a/ApplicationLibCode/ProjectDataModel/Rim3dOverlayInfoConfig.cpp +++ b/ApplicationLibCode/ProjectDataModel/Rim3dOverlayInfoConfig.cpp @@ -44,6 +44,7 @@ #include "RimCase.h" #include "RimCellEdgeColors.h" #include "RimEclipseCase.h" +#include "RimEclipseCaseEnsemble.h" #include "RimEclipseCellColors.h" #include "RimEclipseFaultColors.h" #include "RimEclipsePropertyFilterCollection.h" @@ -162,6 +163,10 @@ RigHistogramData Rim3dOverlayInfoConfig::histogramData() auto geoMechContourMap = dynamic_cast( geoMechView ); auto seismicView = dynamic_cast( m_viewDef.p() ); + // The mobile volume weighted mean requires MOBPORV (PORV, SWCR and MULTPV). Skip the calculation when the value + // is not displayed, as computing it can be expensive for cases backed by remote data. + m_histogramCalculator->setDoComputeMobileVolumeWeightedMean( m_showVolumeWeightedMean() ); + if ( eclipseContourMap ) return m_histogramCalculator->histogramData( eclipseContourMap ); else if ( geoMechContourMap ) @@ -754,6 +759,18 @@ void Rim3dOverlayInfoConfig::update3DInfo() { m_showVolumeWeightedMean = false; } + + if ( auto eclipseCase = reservoirView->eclipseCase() ) + { + // Ensembles can opt out of the mobile volume weighted mean, as the data it is derived from can be + // expensive to fetch. Used by Sumo grid ensembles, see RimSumoDataSource. + auto ensemble = eclipseCase->firstAncestorOrThisOfType(); + if ( ensemble && !ensemble->doComputeMobileVolumeWeightedMean() ) + { + m_showVolumeWeightedMean = false; + } + } + updateEclipse3DInfo( reservoirView ); } diff --git a/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.cpp b/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.cpp index f89690bc046..ceb64829436 100644 --- a/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.cpp +++ b/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.cpp @@ -51,10 +51,18 @@ RimEclipseCaseEnsemble::RimEclipseCaseEnsemble() CAF_PDM_InitFieldNoDefault( &m_viewCollection, "ViewCollection", "Views" ); m_viewCollection = new RimEclipseViewCollection; + // Limit the case dropdown of the ensemble views to the cases of this ensemble. + m_viewCollection->setEclipseCaseProvider( [this]() { return this->cases(); } ); + CAF_PDM_InitFieldNoDefault( &m_wellTargetMappings, "WellTargetMappings", "Well Target Mappings" ); CAF_PDM_InitFieldNoDefault( &m_statisticsContourMaps, "StatisticsContourMaps", "Statistics Contour maps" ); + // Set by the data source creating the ensemble. Defaults to true to keep the behavior of ensembles created + // before this field was introduced. + CAF_PDM_InitField( &m_doComputeMobileVolumeWeightedMean, "DoComputeMobileVolumeWeightedMean", true, "Compute Mobile Volume Weighted Mean" ); + m_doComputeMobileVolumeWeightedMean.uiCapability()->setUiHidden( true ); + setDeletable( true ); } @@ -70,6 +78,18 @@ RimEclipseCaseEnsemble::~RimEclipseCaseEnsemble() m_viewCollection = nullptr; } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimEclipseCaseEnsemble::initAfterRead() +{ + // The provider is a lambda and not serialized, so re-apply it after loading a project. + if ( m_viewCollection ) + { + m_viewCollection->setEclipseCaseProvider( [this]() { return this->cases(); } ); + } +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -274,6 +294,22 @@ std::vector RimEclipseCaseEnsemble::statisticsContourM return m_statisticsContourMaps.childrenByType(); } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RimEclipseCaseEnsemble::doComputeMobileVolumeWeightedMean() const +{ + return m_doComputeMobileVolumeWeightedMean(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimEclipseCaseEnsemble::setDoComputeMobileVolumeWeightedMean( bool enable ) +{ + m_doComputeMobileVolumeWeightedMean = enable; +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.h b/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.h index b4553f38a2d..032420bcf2b 100644 --- a/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.h +++ b/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.h @@ -75,7 +75,11 @@ class RimEclipseCaseEnsemble : public RimNamedObject, public RimReservoirGridEns void addStatisticsContourMap( RimStatisticsContourMap* statisticsContourMap ) override; std::vector statisticsContourMaps() const override; + bool doComputeMobileVolumeWeightedMean() const; + void setDoComputeMobileVolumeWeightedMean( bool enable ); + protected: + void initAfterRead() override; void appendMenuItems( caf::CmdFeatureMenuBuilder& menuBuilder ) const override; void defineUiOrdering( QString uiConfigName, caf::PdmUiOrdering& uiOrdering ) override; @@ -86,4 +90,5 @@ class RimEclipseCaseEnsemble : public RimNamedObject, public RimReservoirGridEns caf::PdmChildArrayField m_wellTargetMappings; caf::PdmChildArrayField m_statisticsContourMaps; caf::PdmPtrField m_selectedCase; + caf::PdmField m_doComputeMobileVolumeWeightedMean; }; diff --git a/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.cpp b/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.cpp index 8a245af5f2b..f9b3e0d4eff 100644 --- a/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.cpp +++ b/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.cpp @@ -75,6 +75,7 @@ void caf::AppEnum::setUp() RimHistogramCalculator::RimHistogramCalculator() : m_isVisCellStatUpToDate( false ) , m_numBins( RigStatisticsDataCache::defaultNumBins() ) + , m_doComputeMobileVolumeWeightedMean( true ) { } @@ -141,6 +142,14 @@ void RimHistogramCalculator::applyCustomBinning( RigStatisticsDataCache* statist } } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimHistogramCalculator::setDoComputeMobileVolumeWeightedMean( bool enable ) +{ + m_doComputeMobileVolumeWeightedMean = enable; +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -272,7 +281,8 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* fldResults->p10p90ScalarValues( resAddr, timeStep, &histData.p10, &histData.p90 ); fldResults->meanScalarValue( resAddr, timeStep, &histData.mean ); fldResults->sumScalarValue( resAddr, timeStep, &histData.sum ); - fldResults->mobileVolumeWeightedMean( resAddr, timeStep, &histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) + fldResults->mobileVolumeWeightedMean( resAddr, timeStep, &histData.weightedMean ); histData.histogram = fldResults->scalarValuesHistogram( resAddr, timeStep ); @@ -290,7 +300,8 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* m_visibleCellStatistics->minMaxCellScalarValues( timeStep, histData.min, histData.max ); m_visibleCellStatistics->p10p90CellScalarValues( timeStep, histData.p10, histData.p90 ); m_visibleCellStatistics->sumCellScalarValues( timeStep, histData.sum ); - m_visibleCellStatistics->mobileVolumeWeightedMean( timeStep, histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) + m_visibleCellStatistics->mobileVolumeWeightedMean( timeStep, histData.weightedMean ); histData.histogram = m_visibleCellStatistics->cellScalarValuesHistogram( timeStep ); @@ -310,7 +321,7 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* fldResults->p10p90ScalarValues( resAddr, &histData.p10, &histData.p90 ); fldResults->meanScalarValue( resAddr, &histData.mean ); fldResults->sumScalarValue( resAddr, &histData.sum ); - fldResults->mobileVolumeWeightedMean( resAddr, &histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) fldResults->mobileVolumeWeightedMean( resAddr, &histData.weightedMean ); histData.histogram = fldResults->scalarValuesHistogram( resAddr ); @@ -327,7 +338,7 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* m_visibleCellStatistics->minMaxCellScalarValues( histData.min, histData.max ); m_visibleCellStatistics->p10p90CellScalarValues( histData.p10, histData.p90 ); m_visibleCellStatistics->sumCellScalarValues( histData.sum ); - m_visibleCellStatistics->mobileVolumeWeightedMean( histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) m_visibleCellStatistics->mobileVolumeWeightedMean( histData.weightedMean ); histData.histogram = m_visibleCellStatistics->cellScalarValuesHistogram(); @@ -348,7 +359,7 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* cellResults->p10p90CellScalarValues( eclResAddr, histData.p10, histData.p90 ); cellResults->meanCellScalarValues( eclResAddr, histData.mean ); cellResults->sumCellScalarValues( eclResAddr, histData.sum ); - cellResults->mobileVolumeWeightedMean( eclResAddr, histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) cellResults->mobileVolumeWeightedMean( eclResAddr, histData.weightedMean ); histData.histogram = cellResults->cellScalarValuesHistogram( eclResAddr ); statisticsCache = cellResults->statistics( eclResAddr ); @@ -360,7 +371,8 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* cellResults->p10p90CellScalarValues( eclResAddr, timeStep, histData.p10, histData.p90 ); cellResults->meanCellScalarValues( eclResAddr, timeStep, histData.mean ); cellResults->sumCellScalarValues( eclResAddr, timeStep, histData.sum ); - cellResults->mobileVolumeWeightedMean( eclResAddr, timeStep, histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) + cellResults->mobileVolumeWeightedMean( eclResAddr, timeStep, histData.weightedMean ); histData.histogram = cellResults->cellScalarValuesHistogram( eclResAddr, timeStep ); statisticsCache = cellResults->statistics( eclResAddr ); @@ -380,7 +392,7 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* m_visibleCellStatistics->minMaxCellScalarValues( histData.min, histData.max ); m_visibleCellStatistics->p10p90CellScalarValues( histData.p10, histData.p90 ); m_visibleCellStatistics->sumCellScalarValues( histData.sum ); - m_visibleCellStatistics->mobileVolumeWeightedMean( histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) m_visibleCellStatistics->mobileVolumeWeightedMean( histData.weightedMean ); histData.histogram = m_visibleCellStatistics->cellScalarValuesHistogram(); @@ -393,7 +405,7 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* m_visibleCellStatistics->minMaxCellScalarValues( timeStep, histData.min, histData.max ); m_visibleCellStatistics->p10p90CellScalarValues( timeStep, histData.p10, histData.p90 ); m_visibleCellStatistics->sumCellScalarValues( timeStep, histData.sum ); - m_visibleCellStatistics->mobileVolumeWeightedMean( timeStep, histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) m_visibleCellStatistics->mobileVolumeWeightedMean( timeStep, histData.weightedMean ); histData.histogram = m_visibleCellStatistics->cellScalarValuesHistogram( timeStep ); diff --git a/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.h b/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.h index 175c8ef541b..e4d934ecf34 100644 --- a/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.h +++ b/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.h @@ -63,6 +63,10 @@ class RimHistogramCalculator RigHistogramCalculator::OutOfRangeHandling outOfRangeHandling, std::optional> customBinRange ); + // The mobile volume weighted mean is derived from MOBPROV, which in turn requires PORV, SWCR and MULTPV. For + // cases backed by remote data these are expensive to fetch, so the calculation can be turned off. + void setDoComputeMobileVolumeWeightedMean( bool enable ); + RigHistogramData histogramData( RimEclipseContourMapView* contourMap ); RigHistogramData histogramData( RimGeoMechContourMapView* contourMap ); RigHistogramData histogramData( RimEclipseView* eclipseView, StatisticsCellRangeType cellRange, StatisticsTimeRangeType timeRange ); @@ -93,4 +97,5 @@ class RimHistogramCalculator RigHistogramCalculator::BinningMode m_binningMode = RigHistogramCalculator::BinningMode::LINEAR; RigHistogramCalculator::OutOfRangeHandling m_outOfRangeHandling = RigHistogramCalculator::OutOfRangeHandling::EXCLUDE; std::optional> m_customBinRange; + bool m_doComputeMobileVolumeWeightedMean; }; diff --git a/ApplicationLibCode/ProjectDataModel/RimRoffCaseSumo.cpp b/ApplicationLibCode/ProjectDataModel/RimRoffCaseSumo.cpp new file mode 100644 index 00000000000..cbc215d360e --- /dev/null +++ b/ApplicationLibCode/ProjectDataModel/RimRoffCaseSumo.cpp @@ -0,0 +1,370 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024 Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "RimRoffCaseSumo.h" + +#include "RiaApplication.h" +#include "RiaLogging.h" +#include "RiaPreferencesGrid.h" +#include "RiaResultNames.h" + +#include "Cloud/RiaSumoConnector.h" +#include "Cloud/RifReaderSumoGridProperty.h" + +#include "RifRoffFileTools.h" + +#include "RigCaseCellResultsData.h" +#include "RigEclipseCaseData.h" +#include "RigEclipseResultAddress.h" +#include "RigEclipseResultInfo.h" +#include "RigMainGrid.h" + +#include "RimReservoirCellResultsStorage.h" +#include "Sumo/RimSumoDataSource.h" + +#include "cafPdmObjectScriptingCapability.h" + +#include +#include + +#include +#include +#include +#include +#include + +CAF_PDM_SOURCE_INIT( RimRoffCaseSumo, "RimRoffCaseSumo" ); + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RimRoffCaseSumo::RimRoffCaseSumo() +{ + CAF_PDM_InitScriptableObject( "Sumo Grid Case", ":/Case48x48.png" ); + + CAF_PDM_InitFieldNoDefault( &m_sumoDataSource, "SumoDataSource", "Sumo Data Source" ); + m_sumoDataSource.uiCapability()->setUiHidden( true ); + + CAF_PDM_InitFieldNoDefault( &m_sumoCaseId, "SumoCaseId", "Sumo Case Id" ); + m_sumoCaseId.uiCapability()->setUiReadOnly( true ); + + CAF_PDM_InitFieldNoDefault( &m_ensembleName, "EnsembleName", "Ensemble Name" ); + m_ensembleName.uiCapability()->setUiReadOnly( true ); + + CAF_PDM_InitFieldNoDefault( &m_gridName, "GridName", "Grid Name" ); + m_gridName.uiCapability()->setUiReadOnly( true ); + + CAF_PDM_InitField( &m_realization, "Realization", -1, "Realization" ); + m_realization.uiCapability()->setUiReadOnly( true ); + + m_sumoConnector = RiaApplication::instance()->makeSumoConnector(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RimRoffCaseSumo::~RimRoffCaseSumo() +{ +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RimRoffCaseSumo* RimRoffCaseSumo::createFromDataSource( RimSumoDataSource* dataSource, const QString& gridName, int realization ) +{ + if ( !dataSource ) return nullptr; + + auto* gridCase = new RimRoffCaseSumo(); + gridCase->setSumoDataSource( dataSource ); + gridCase->setSumoCaseId( dataSource->caseId().get() ); + gridCase->setEnsembleName( dataSource->ensembleName() ); + gridCase->setGridName( gridName ); + gridCase->setRealization( realization ); + + // The grid is stored on Sumo, not on disk, so there is no real grid file name. Still assign a unique + // synthetic grid file name: an empty one collapses all custom case names onto a single + // path-variable key on project save (RiaProjectFileTools), overwriting every name with the last. + gridCase->setGridFileName( + QString( "sumo/%1/%2/realization-%3/%4.roff" ).arg( dataSource->caseId().get() ).arg( gridName ).arg( realization ).arg( gridName ) ); + + // Name the case using grid name, asset, ensemble and realization, e.g. "Geogrid_Drogon_iter-0_Real_0". + QString caseDisplayName = + QString( "%1_%2_%3_Real_%4" ).arg( gridName, dataSource->assetName(), dataSource->ensembleName() ).arg( realization ); + gridCase->setCustomCaseName( caseDisplayName ); + + return gridCase; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimRoffCaseSumo::setSumoDataSource( RimSumoDataSource* dataSource ) +{ + m_sumoDataSource = dataSource; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimRoffCaseSumo::setSumoCaseId( const QString& sumoCaseId ) +{ + m_sumoCaseId = sumoCaseId; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimRoffCaseSumo::setEnsembleName( const QString& ensembleName ) +{ + m_ensembleName = ensembleName; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimRoffCaseSumo::setGridName( const QString& gridName ) +{ + m_gridName = gridName; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimRoffCaseSumo::setRealization( int realization ) +{ + m_realization = realization; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RimRoffCaseSumo::gridName() const +{ + return m_gridName(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +int RimRoffCaseSumo::realization() const +{ + return m_realization(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RimRoffCaseSumo::openEclipseGridFile() +{ + if ( eclipseCaseData() ) + { + // Early exit if reservoir data is created + return true; + } + + if ( !m_sumoConnector ) + { + RiaLogging::error( "No Sumo connector available, unable to load grid from Sumo." ); + return false; + } + + setReservoirData( new RigEclipseCaseData( this ) ); + + if ( eclipseCaseData()->mainGrid()->cellCount() == 0 ) + { + QByteArray contents = + m_sumoConnector->requestGridDataBlocking( SumoCaseId( m_sumoCaseId() ), m_ensembleName(), m_gridName(), m_realization() ); + if ( contents.isEmpty() ) + { + RiaLogging::error( + std::format( "Failed to download grid '{}' (realization {}) from Sumo.", m_gridName().toStdString(), m_realization() ) ); + return false; + } + + // The downloaded blob is a binary roff grid. Parse it directly from memory. + std::string buffer = contents.toStdString(); + std::istringstream stream( buffer, std::ios::binary ); + + QString errorMessages; + if ( RifRoffFileTools::openGridFile( stream, eclipseCaseData(), &errorMessages ) ) + { + eclipseCaseData()->mainGrid()->setFlipAxis( m_flipXAxis, m_flipYAxis ); + computeCachedData(); + } + else + { + RiaLogging::error( errorMessages.toStdString() ); + return false; + } + } + + results( RiaDefines::PorosityModelType::MATRIX_MODEL )->createPlaceholderResultEntries(); + + if ( RiaPreferencesGrid::current()->autoComputeDepthRelatedProperties() ) + { + eclipseCaseData()->computeDepthRelatedResults(); + } + + results( RiaDefines::PorosityModelType::MATRIX_MODEL )->computeCellVolumes(); + + // Make the Sumo grid properties available as cell results (fetched on demand when displayed). + registerSumoGridProperties(); + + // Rebuild the Data Sources result folders now that the result meta data is available. + updateResultAddressCollection(); + + return true; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RimRoffCaseSumo::locationOnDisc() const +{ + // The grid is stored on Sumo, not on disk. + return QString(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimRoffCaseSumo::defineUiOrdering( QString uiConfigName, caf::PdmUiOrdering& uiOrdering ) +{ + uiOrdering.add( &m_caseUserDescription ); + uiOrdering.add( &m_displayNameOption ); + uiOrdering.add( &m_caseId ); + + auto group = uiOrdering.addNewGroup( "Sumo" ); + group->add( &m_sumoCaseId ); + group->add( &m_ensembleName ); + group->add( &m_gridName ); + group->add( &m_realization ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimRoffCaseSumo::registerSumoGridProperties() +{ + if ( !m_sumoConnector || !eclipseCaseData() ) return; + + m_sumoConnector->requestGridPropertyInfoForEnsembleBlocking( SumoCaseId( m_sumoCaseId() ), m_ensembleName(), m_gridName(), m_realization() ); + + // Properties without a timestamp are static. Properties with a single timestamp are dynamic (one time + // step per timestamp). Time intervals (the iso string contains '/') are not supported and skipped. + std::vector staticPropertyNames; + std::map> dynamicPropertyTimestamps; // property name -> the timestamps it has + std::set allTimestamps; // union of timestamps across all properties + for ( const auto& info : m_sumoConnector->gridPropertyInfos() ) + { + if ( info.isoDateOrInterval.isEmpty() ) + { + staticPropertyNames.push_back( info.name ); + } + else if ( !info.isoDateOrInterval.contains( '/' ) ) + { + dynamicPropertyTimestamps[info.name].insert( info.isoDateOrInterval ); + allTimestamps.insert( info.isoDateOrInterval ); + } + } + + if ( staticPropertyNames.empty() && dynamicPropertyTimestamps.empty() ) return; + + // The Eclipse readers take the available phases from the INIT file. There is no INIT file here, so derive + // the phases from the saturation properties the case actually has. Without this the phase set stays empty, + // and RigCaseCellResultsData::defaultResult() skips its SOIL and SGAS branches - both are guarded by the + // phase set, while the SWAT branch is not - so a new view would always open on SWAT. + auto hasProperty = [&]( const QString& name ) + { return dynamicPropertyTimestamps.contains( name ) || std::ranges::find( staticPropertyNames, name ) != staticPropertyNames.end(); }; + + std::set availablePhases; + if ( hasProperty( RiaResultNames::soil() ) ) availablePhases.insert( RiaDefines::PhaseType::OIL_PHASE ); + if ( hasProperty( RiaResultNames::sgas() ) ) availablePhases.insert( RiaDefines::PhaseType::GAS_PHASE ); + if ( hasProperty( RiaResultNames::swat() ) ) availablePhases.insert( RiaDefines::PhaseType::WATER_PHASE ); + + if ( !availablePhases.empty() ) eclipseCaseData()->setAvailablePhases( availablePhases ); + + auto cellResults = results( RiaDefines::PorosityModelType::MATRIX_MODEL ); + if ( !cellResults ) return; + + auto reader = new RifReaderSumoGridProperty( m_sumoConnector, m_sumoCaseId(), m_ensembleName(), m_gridName(), m_realization() ); + reader->open( "", eclipseCaseData() ); + + // Register the property names as cell results so they are listed in the cell result editor. The values are + // not loaded here; the reader fetches them on demand the first time a property is displayed. + + for ( const auto& name : staticPropertyNames ) + { + RigEclipseResultAddress resultAddress( RiaDefines::ResultCatType::STATIC_NATIVE, RiaDefines::ResultDataType::FLOAT, name ); + cellResults->createResultEntry( resultAddress, false ); + } + reader->setStaticProperties( staticPropertyNames ); + + if ( !dynamicPropertyTimestamps.empty() ) + { + auto parseTimestamp = []( const QString& isoString ) -> QDateTime + { + QDateTime dateTime = QDateTime::fromString( isoString, Qt::ISODate ); + if ( !dateTime.isValid() ) + { + // Date-only string, e.g. "2018-01-01". + QDate date = QDate::fromString( isoString, Qt::ISODate ); + if ( date.isValid() ) dateTime = QDateTime( date, QTime( 0, 0, 0 ) ); + } + return dateTime; + }; + + // All dynamic results share one common, case-wide set of time steps so they use the same time step + // index space as the 3D view time slider. std::set is sorted, and ISO date strings sort chronologically. + std::vector commonTimestamps( allTimestamps.begin(), allTimestamps.end() ); + + std::vector dates; + std::vector reportNumbers; + std::vector daysSinceStart; + for ( int i = 0; i < static_cast( commonTimestamps.size() ); i++ ) + { + QDateTime date = parseTimestamp( commonTimestamps[i] ); + dates.push_back( date ); + reportNumbers.push_back( i ); + daysSinceStart.push_back( ( dates.front().isValid() && date.isValid() ) ? dates.front().daysTo( date ) : static_cast( i ) ); + } + auto commonTimeStepInfos = RigEclipseTimeStepInfo::createTimeStepInfos( dates, reportNumbers, daysSinceStart ); + + // For each property, build a list aligned with commonTimestamps. An empty entry marks a time step the + // property has no data for, so the reader reports "no data" there instead of another step's values. + std::map> readerDynamicTimestamps; + for ( const auto& [name, propertyTimestamps] : dynamicPropertyTimestamps ) + { + RigEclipseResultAddress resultAddress( RiaDefines::ResultCatType::DYNAMIC_NATIVE, RiaDefines::ResultDataType::FLOAT, name ); + cellResults->createResultEntry( resultAddress, false ); + cellResults->setTimeStepInfos( resultAddress, commonTimeStepInfos ); + + std::vector alignedTimestamps; + alignedTimestamps.reserve( commonTimestamps.size() ); + for ( const auto& timestamp : commonTimestamps ) + { + alignedTimestamps.push_back( propertyTimestamps.count( timestamp ) > 0 ? timestamp : QString() ); + } + readerDynamicTimestamps[name] = alignedTimestamps; + } + + reader->setDynamicProperties( readerDynamicTimestamps ); + } + + cellResults->setReaderInterface( reader ); +} diff --git a/ApplicationLibCode/ProjectDataModel/RimRoffCaseSumo.h b/ApplicationLibCode/ProjectDataModel/RimRoffCaseSumo.h new file mode 100644 index 00000000000..8d62e8cb3c3 --- /dev/null +++ b/ApplicationLibCode/ProjectDataModel/RimRoffCaseSumo.h @@ -0,0 +1,79 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024 Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "RimEclipseCase.h" + +#include "cafPdmField.h" +#include "cafPdmPtrField.h" + +#include +#include + +class RiaSumoConnector; +class RimSumoDataSource; + +//================================================================================================== +// +// Eclipse grid case backed by a roff grid stored on Sumo. The grid geometry is downloaded as a +// blob through RiaSumoConnector and parsed in memory, so there is no grid file on disk. +// +//================================================================================================== +class RimRoffCaseSumo : public RimEclipseCase +{ + CAF_PDM_HEADER_INIT; + +public: + RimRoffCaseSumo(); + ~RimRoffCaseSumo() override; + + // Create a grid case for a single realization of the given grid, linked back to the data source + // so the case can be updated when the data source realization filter changes. + static RimRoffCaseSumo* createFromDataSource( RimSumoDataSource* dataSource, const QString& gridName, int realization ); + + void setSumoDataSource( RimSumoDataSource* dataSource ); + void setSumoCaseId( const QString& sumoCaseId ); + void setEnsembleName( const QString& ensembleName ); + void setGridName( const QString& gridName ); + void setRealization( int realization ); + + QString gridName() const; + int realization() const; + + bool openEclipseGridFile() override; + + QString locationOnDisc() const override; + +protected: + void defineUiOrdering( QString uiConfigName, caf::PdmUiOrdering& uiOrdering ) override; + +private: + // Discover the available grid properties from Sumo, register them as cell results and attach a reader + // that fetches the property data on demand. Static properties only in this first version. + void registerSumoGridProperties(); + +private: + caf::PdmPtrField m_sumoDataSource; + caf::PdmField m_sumoCaseId; + caf::PdmField m_ensembleName; + caf::PdmField m_gridName; + caf::PdmField m_realization; + + QPointer m_sumoConnector; +}; diff --git a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.cpp b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.cpp index cb9be041e0c..b093def4bb5 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.cpp +++ b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.cpp @@ -20,6 +20,11 @@ #include "RiaStdStringTools.h" +#include "Rim3dOverlayInfoConfig.h" +#include "RimEclipseCaseEnsemble.h" +#include "RimEclipseView.h" +#include "RimEclipseViewCollection.h" +#include "RimRoffCaseSumo.h" #include "RimSummaryEnsembleSumo.h" #include "cafCmdFeatureMenuBuilder.h" @@ -29,7 +34,10 @@ #include -CAF_PDM_SOURCE_INIT( RimSumoDataSource, "RimSumoDataSource", "RimSummarySumoDataSource" ); +#include +#include + +CAF_PDM_SOURCE_INIT( RimSumoDataSource, "RimSumoDataSource", "RimSummarySumoDataSource", "RimSumoGridDataSource" ); //-------------------------------------------------------------------------------------------------- /// @@ -62,6 +70,19 @@ RimSumoDataSource::RimSumoDataSource() CAF_PDM_InitFieldNoDefault( &m_vectorNames, "VectorNames", "Vector Names" ); m_vectorNames.uiCapability()->setUiHidden( true ); + CAF_PDM_InitFieldNoDefault( &m_gridNames, "GridNames", "Grid Names" ); + m_gridNames.uiCapability()->setUiHidden( true ); + + CAF_PDM_InitFieldNoDefault( &m_selectedGridName, "GridName", "Grid Name" ); + + CAF_PDM_InitField( &m_doComputeMobileVolumeWeightedMean, + "DoComputeMobileVolumeWeightedMean", + false, + "Compute Mobile Volume Weighted Mean", + "", + "Show the mobile volume weighted mean in the 3D info box. This downloads the PORV and SWCR grid " + "properties for every realization." ); + setDeletable( true ); } @@ -198,6 +219,38 @@ void RimSumoDataSource::setVectorNames( const std::vector& vectorNames m_vectorNames = vectorNames; } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RimSumoDataSource::gridNames() const +{ + return m_gridNames(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimSumoDataSource::setGridNames( const std::vector& gridNames ) +{ + m_gridNames = gridNames; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RimSumoDataSource::selectedGridName() const +{ + return m_selectedGridName(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RimSumoDataSource::doComputeMobileVolumeWeightedMean() const +{ + return m_doComputeMobileVolumeWeightedMean(); +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -220,6 +273,7 @@ void RimSumoDataSource::updateName() void RimSumoDataSource::appendMenuItems( caf::CmdFeatureMenuBuilder& menuBuilder ) const { menuBuilder.addCmdFeature( "RicCreateSumoEnsembleFeature" ); + menuBuilder.addCmdFeature( "RicCreateSumoGridEnsembleFeature" ); } //-------------------------------------------------------------------------------------------------- @@ -270,6 +324,28 @@ void RimSumoDataSource::defineUiOrdering( QString uiConfigName, caf::PdmUiOrderi auto ensembleGroup = uiOrdering.addNewGroup( "Ensemble Selection" ); ensembleGroup->add( &m_realizationFilter ); ensembleGroup->add( &m_realizationFilterInfo ); + + auto gridGroup = uiOrdering.addNewGroup( "Grid Selection" ); + gridGroup->add( &m_selectedGridName ); + gridGroup->add( &m_doComputeMobileVolumeWeightedMean ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QList RimSumoDataSource::calculateValueOptions( const caf::PdmFieldHandle* fieldNeedingOptions ) +{ + QList options; + + if ( fieldNeedingOptions == &m_selectedGridName ) + { + for ( const auto& gridName : m_gridNames() ) + { + options.push_back( caf::PdmOptionItemInfo( gridName, gridName ) ); + } + } + + return options; } //-------------------------------------------------------------------------------------------------- @@ -285,6 +361,10 @@ void RimSumoDataSource::fieldChangedByUi( const caf::PdmFieldHandle* changedFiel { onRealizationFilterChanged(); } + else if ( changedField == &m_doComputeMobileVolumeWeightedMean ) + { + updateGridCaseEnsembles(); + } } //-------------------------------------------------------------------------------------------------- @@ -299,6 +379,113 @@ void RimSumoDataSource::onRealizationFilterChanged() { ensemble->onRealizationSelectionChanged(); } + + // Update any grid case ensembles created from this data source. + updateGridCaseEnsembles(); +} + +//-------------------------------------------------------------------------------------------------- +/// Synchronize the realization grid cases (RimRoffCaseSumo) of every grid ensemble created from this +/// data source with the current realization filter: remove cases for deselected realizations and add +/// cases for newly selected ones. The grid name of each ensemble is preserved. +//-------------------------------------------------------------------------------------------------- +void RimSumoDataSource::updateGridCaseEnsembles() +{ + std::set selectedRealizations; + for ( const auto& realizationId : selectedRealizationIds() ) + { + bool ok = false; + int value = realizationId.toInt( &ok ); + if ( ok ) selectedRealizations.insert( value ); + } + + // Group the grid cases created from this data source by their owning ensemble. + std::map> casesByEnsemble; + for ( auto gridCase : objectsWithReferringPtrFieldsOfType() ) + { + if ( auto ensemble = gridCase->firstAncestorOrThisOfType() ) + { + casesByEnsemble[ensemble].push_back( gridCase ); + } + } + + for ( auto& [ensemble, gridCases] : casesByEnsemble ) + { + if ( gridCases.empty() ) continue; + + if ( ensemble->doComputeMobileVolumeWeightedMean() != m_doComputeMobileVolumeWeightedMean() ) + { + ensemble->setDoComputeMobileVolumeWeightedMean( m_doComputeMobileVolumeWeightedMean() ); + + // The 3D info box reads the setting when it is updated, so refresh it to apply the change immediately. + if ( auto viewColl = ensemble->viewCollection() ) + { + for ( auto view : viewColl->views() ) + { + if ( view->overlayInfoConfig() ) view->overlayInfoConfig()->update3DInfo(); + } + } + } + + // All cases in an ensemble share the same grid; keep it when adding new realizations. + const QString gridName = gridCases.front()->gridName(); + + std::set currentRealizations; + for ( auto gridCase : gridCases ) + { + currentRealizations.insert( gridCase->realization() ); + } + + // Add cases for newly selected realizations first, so a view displaying a removed realization + // can be repointed to one of these instead of being deleted. + for ( int realization : selectedRealizations ) + { + if ( currentRealizations.find( realization ) != currentRealizations.end() ) continue; + + if ( auto* gridCase = RimRoffCaseSumo::createFromDataSource( this, gridName, realization ) ) + { + ensemble->addCase( gridCase ); + } + } + + // Remove cases for realizations no longer selected. Remember any views that displayed them, so + // they can be repointed to a surviving case below (a deleted case auto-nulls the view's case, + // which would otherwise make the view disappear, e.g. when changing 30-40 to 31-40). + std::vector orphanedViews; + for ( auto gridCase : gridCases ) + { + if ( selectedRealizations.find( gridCase->realization() ) != selectedRealizations.end() ) continue; + + if ( auto viewColl = ensemble->viewCollection() ) + { + for ( auto view : viewColl->views() ) + { + if ( view->eclipseCase() == gridCase ) orphanedViews.push_back( view ); + } + } + + ensemble->removeCase( gridCase ); + delete gridCase; + } + + // Repoint orphaned views to a surviving case, or delete them if no realizations remain. + const auto remainingCases = ensemble->cases(); + for ( auto view : orphanedViews ) + { + if ( !remainingCases.empty() ) + { + view->setEclipseCase( remainingCases.front() ); + view->loadDataAndUpdate(); + } + else if ( auto viewColl = ensemble->viewCollection() ) + { + viewColl->removeView( view ); + delete view; + } + } + + ensemble->updateConnectedEditors(); + } } //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.h b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.h index 19cc8e848cb..4cac8ee6119 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.h +++ b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.h @@ -25,12 +25,12 @@ //================================================================================================== // -// Common data source describing a single Sumo ensemble. Holds the information required by summary -// ensembles (asset, vector names). The available ensemble realizations - fetched from the -// realizations endpoint - are the source of truth, and the user selects a subset of them ("ensemble -// selection"). Consumers listen to the selected realization id subset. All values are populated by -// RimCloudDataSourceCollection, which owns the RiaSumoConnector; this object does not talk to the -// connector directly. +// Common data source describing a single Sumo ensemble. Holds the information required both for +// summary ensembles (vector names) and for grid case ensembles (asset, grid names). The available +// ensemble realizations - fetched from the realizations endpoint - are the source of truth, and the +// user selects a subset of them ("ensemble selection"). Both summary and grid consumers listen to +// the selected realization id subset. All values are populated by RimCloudDataSourceCollection, +// which owns the RiaSumoConnector; this object does not talk to the connector directly. // //================================================================================================== @@ -57,7 +57,7 @@ class RimSumoDataSource : public RimNamedObject std::vector availableRealizationIds() const; void setAvailableRealizationIds( const std::vector& realizationIds ); - // The subset of realizations matching the realization filter. Summary ensemble creation listens to this. + // The subset of realizations matching the realization filter. Both summary and grid creation listen to this. std::vector selectedRealizationIds() const; // Available summary vectors for the ensemble. Not shown in the UI, but used to populate the @@ -65,15 +65,24 @@ class RimSumoDataSource : public RimNamedObject std::vector vectorNames() const; void setVectorNames( const std::vector& vectorNames ); + std::vector gridNames() const; + void setGridNames( const std::vector& gridNames ); + + QString selectedGridName() const; + + bool doComputeMobileVolumeWeightedMean() const; + void updateName(); private: void appendMenuItems( caf::CmdFeatureMenuBuilder& menuBuilder ) const override; void defineEditorAttribute( const caf::PdmFieldHandle* field, QString uiConfigName, caf::PdmUiEditorAttribute* attribute ) override; void defineUiOrdering( QString uiConfigName, caf::PdmUiOrdering& uiOrdering ) override; + QList calculateValueOptions( const caf::PdmFieldHandle* fieldNeedingOptions ) override; void fieldChangedByUi( const caf::PdmFieldHandle* changedField, const QVariant& oldValue, const QVariant& newValue ) override; void onRealizationFilterChanged(); + void updateGridCaseEnsembles(); QString realizationFilterInfoText() const; QString availableRealizationsRangeText() const; @@ -90,4 +99,8 @@ class RimSumoDataSource : public RimNamedObject caf::PdmProxyValueField m_realizationFilterInfo; caf::PdmField> m_vectorNames; + + caf::PdmField> m_gridNames; + caf::PdmField m_selectedGridName; + caf::PdmField m_doComputeMobileVolumeWeightedMean; }; From a05600c25d1e74294c3aa728a7293b270260a227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Herje?= Date: Tue, 18 Aug 2026 10:45:10 +0200 Subject: [PATCH 2/6] Run Sumo transfers on a dedicated thread Blocking Sumo requests waited on a nested event loop on the GUI thread. That let the view update code re-enter a load that was already running, and the same grid property was downloaded twice, once by each pass. Transfers now run on a thread of their own, with its own network manager, and the calling thread waits on a semaphore without dispatching events, so nothing can re-enter the load while it runs. Authentication stays on the GUI thread, as the OAuth flow opens a browser and its objects live there. Log messages written from the transfer thread were being dropped, as the message panel may only be touched from the thread owning it. They are handed over to that thread instead, and delivered while the request they describe is still the most recent thing that happened, so the log does not read out of order. Co-Authored-By: Claude Opus 5 --- .../Tools/Cloud/RiaSumoConnector.cpp | 333 +++++++++++------- .../Tools/Cloud/RiaSumoConnector.h | 30 ++ .../Application/Tools/RiaLogging.cpp | 11 + .../Application/Tools/RiaLogging.h | 8 + .../UserInterface/RiuMessagePanel.cpp | 36 +- .../UserInterface/RiuMessagePanel.h | 2 + 6 files changed, 299 insertions(+), 121 deletions(-) diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp index 43c507ac627..1d08f17f919 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp @@ -31,6 +31,8 @@ #include #include #include +#include +#include #include #include @@ -47,6 +49,22 @@ RiaSumoConnector::RiaSumoConnector( QObject* parent, : RiaCloudConnector( parent, {}, authority, scopes, clientId, port ) , m_serverUrlProvider( std::move( serverUrlProvider ) ) { + // The transfer thread runs the network requests issued by the blocking wrappers, so the calling thread can + // wait for them without dispatching events. The context object gives us something with transfer thread + // affinity to post work to, and the network manager has to be constructed on the thread that uses it. + m_transferThread = new QThread( this ); + m_transferContext = new QObject; + m_transferContext->moveToThread( m_transferThread ); + + QObject::connect( m_transferThread, + &QThread::started, + m_transferContext, + [this]() { m_transferNetworkAccessManager = new QNetworkAccessManager( m_transferContext ); } ); + + // The context object lives on the transfer thread, so let that thread delete it when it stops. + QObject::connect( m_transferThread, &QThread::finished, m_transferContext, &QObject::deleteLater ); + + m_transferThread->start(); } //-------------------------------------------------------------------------------------------------- @@ -87,6 +105,66 @@ void RiaSumoConnector::parquetDownloadComplete( const QString& blobId, const QBy //-------------------------------------------------------------------------------------------------- RiaSumoConnector::~RiaSumoConnector() { + if ( m_transferThread ) + { + m_transferThread->quit(); + m_transferThread->wait(); + } +} + +//-------------------------------------------------------------------------------------------------- +/// The network manager of the calling thread. A QNetworkAccessManager can only be used from the thread it +/// was created on, and the connector is used from both the GUI thread (the token flow and the Sumo Data +/// dialog) and the transfer thread (everything issued by a blocking wrapper). +//-------------------------------------------------------------------------------------------------- +QNetworkAccessManager* RiaSumoConnector::networkAccessManager() +{ + if ( m_transferThread && QThread::currentThread() == m_transferThread && m_transferNetworkAccessManager ) + { + return m_transferNetworkAccessManager; + } + + return m_networkAccessManager; +} + +//-------------------------------------------------------------------------------------------------- +/// Run work on the transfer thread and wait for it to finish, without dispatching any events on the calling +/// thread. This is what keeps the view update code from re-entering a load that is already running. +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::runOnTransferThreadBlocking( const std::function& work ) +{ + // A blocking request can be issued from inside another one, for instance the blob id lookup done while + // downloading a grid property. Already on the transfer thread, run directly instead of deadlocking on a + // thread that is busy waiting for us. + if ( !m_transferThread || !m_transferContext || QThread::currentThread() == m_transferThread ) + { + work(); + return; + } + + QSemaphore semaphore; + + QMetaObject::invokeMethod( + m_transferContext, + [&work, &semaphore]() + { + // Release on every exit path. A request that fails or times out must not leave the caller waiting. + struct Releaser + { + QSemaphore& semaphore; + ~Releaser() { semaphore.release(); } + } releaser{ semaphore }; + + work(); + }, + Qt::QueuedConnection ); + + semaphore.acquire(); + + // The transfer thread hands its log messages to the thread owning the message panel. Deliver them here, + // while the request they describe is still the most recent thing that happened, or they would appear + // after whatever the caller logs next and the log would read out of order. + RiaLogging::flushPendingMessages(); } //-------------------------------------------------------------------------------------------------- @@ -104,7 +182,7 @@ void RiaSumoConnector::requestCasesForField( const QString& fieldName ) addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - auto reply = m_networkAccessManager->get( m_networkRequest ); + auto reply = networkAccessManager()->get( m_networkRequest ); connect( reply, &QNetworkReply::finished, @@ -138,7 +216,7 @@ void RiaSumoConnector::requestAssets() addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - auto reply = m_networkAccessManager->get( m_networkRequest ); + auto reply = networkAccessManager()->get( m_networkRequest ); connect( reply, &QNetworkReply::finished, @@ -173,7 +251,7 @@ void RiaSumoConnector::requestEnsembleByCasesId( const SumoCaseId& caseId ) addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - auto reply = m_networkAccessManager->get( m_networkRequest ); + auto reply = networkAccessManager()->get( m_networkRequest ); connect( reply, &QNetworkReply::finished, @@ -240,7 +318,7 @@ void RiaSumoConnector::requestVectorNamesForEnsemble( const SumoCaseId& caseId, addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - auto reply = m_networkAccessManager->get( m_networkRequest ); + auto reply = networkAccessManager()->get( m_networkRequest ); connect( reply, &QNetworkReply::finished, @@ -277,7 +355,7 @@ void RiaSumoConnector::requestRealizationIdsForEnsemble( const SumoCaseId& caseI addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - auto reply = m_networkAccessManager->get( m_networkRequest ); + auto reply = networkAccessManager()->get( m_networkRequest ); connect( reply, &QNetworkReply::finished, @@ -315,7 +393,7 @@ void RiaSumoConnector::requestGridInfoForEnsemble( const SumoCaseId& caseId, con addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - auto reply = m_networkAccessManager->get( m_networkRequest ); + auto reply = networkAccessManager()->get( m_networkRequest ); connect( reply, &QNetworkReply::finished, @@ -366,7 +444,7 @@ void RiaSumoConnector::requestGridBlobIdForEnsemble( const SumoCaseId& caseId, c addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - auto reply = m_networkAccessManager->get( m_networkRequest ); + auto reply = networkAccessManager()->get( m_networkRequest ); connect( reply, &QNetworkReply::finished, @@ -409,28 +487,7 @@ QByteArray if ( m_blobId.empty() ) return {}; // The REST API returns the blob Id - auto blobId = m_blobId.back(); - - QEventLoop eventLoop; - QTimer timer; - timer.setSingleShot( true ); - QObject::connect( &timer, SIGNAL( timeout() ), &eventLoop, SLOT( quit() ) ); - QObject::connect( this, SIGNAL( parquetDownloadFinished( const QByteArray&, const QString& ) ), &eventLoop, SLOT( quit() ) ); - - requestBlobDownload( blobId ); - - timer.start( RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); - - for ( const auto& blobData : m_redirectInfo ) - { - if ( blobData.objectId == blobId ) - { - return blobData.contents; - } - } - - return {}; + return downloadBlobBlocking( m_blobId.back() ); } //-------------------------------------------------------------------------------------------------- @@ -460,7 +517,7 @@ void RiaSumoConnector::requestGridPropertyInfoForEnsemble( const SumoCaseId& cas addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - auto reply = m_networkAccessManager->get( m_networkRequest ); + auto reply = networkAccessManager()->get( m_networkRequest ); connect( reply, &QNetworkReply::finished, @@ -502,22 +559,33 @@ QString RiaSumoConnector::requestGridPropertyBlobIdBlocking( const SumoCaseId& c const QString& propertyName, const QString& isoDateOrInterval ) { - auto reply = makeGridPropertyBlobIdRequest( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + requestTokenBlocking(); - // Wait for THIS reply only. Binding the event loop to the reply, rather than to the shared blobIdFinished - // signal, is what makes the mapping correct: a still-pending reply from an earlier property's request can no - // longer satisfy this wait and hand us its blob id (which previously caused e.g. SWAT to be served the SWCR - // blob). The blob id is read straight off this reply and never routed through shared state. - QEventLoop eventLoop; - QTimer timer; - timer.setSingleShot( true ); - QObject::connect( &timer, &QTimer::timeout, &eventLoop, &QEventLoop::quit ); - QObject::connect( reply, &QNetworkReply::finished, &eventLoop, &QEventLoop::quit ); + QString blobId; - timer.start( RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); + runOnTransferThreadBlocking( + [&]() + { + auto reply = makeGridPropertyBlobIdRequest( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); - return blobIdFromReply( reply, propertyName ); + // Wait for THIS reply only. Binding the event loop to the reply, rather than to the shared + // blobIdFinished signal, is what makes the mapping correct: a still-pending reply from an earlier + // property's request can no longer satisfy this wait and hand us its blob id (which previously caused + // e.g. SWAT to be served the SWCR blob). The blob id is read straight off this reply and never routed + // through shared state. + QEventLoop eventLoop; + QTimer timer; + timer.setSingleShot( true ); + QObject::connect( &timer, &QTimer::timeout, &eventLoop, &QEventLoop::quit ); + QObject::connect( reply, &QNetworkReply::finished, &eventLoop, &QEventLoop::quit ); + + timer.start( RiaSumoDefines::requestTimeoutMillis() ); + eventLoop.exec(); + + blobId = blobIdFromReply( reply, propertyName ); + } ); + + return blobId; } //-------------------------------------------------------------------------------------------------- @@ -556,7 +624,7 @@ QNetworkReply* RiaSumoConnector::makeGridPropertyBlobIdRequest( const SumoCaseId networkRequest.setUrl( QUrl( url ) ); addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - return m_networkAccessManager->get( networkRequest ); + return networkAccessManager()->get( networkRequest ); } //-------------------------------------------------------------------------------------------------- @@ -614,31 +682,37 @@ QByteArray RiaSumoConnector::requestGridPropertyDataBlocking( const SumoCaseId& const QString blobId = requestGridPropertyBlobIdBlocking( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); if ( blobId.isEmpty() ) return {}; - QEventLoop eventLoop; - QTimer timer; - timer.setSingleShot( true ); - QObject::connect( &timer, SIGNAL( timeout() ), &eventLoop, SLOT( quit() ) ); - QObject::connect( this, SIGNAL( parquetDownloadFinished( const QByteArray&, const QString& ) ), &eventLoop, SLOT( quit() ) ); + QByteArray contents; - requestBlobDownload( blobId ); + runOnTransferThreadBlocking( + [this, blobId, cacheKey, &contents]() + { + QEventLoop eventLoop; + QTimer timer; + timer.setSingleShot( true ); + QObject::connect( &timer, SIGNAL( timeout() ), &eventLoop, SLOT( quit() ) ); + QObject::connect( this, SIGNAL( parquetDownloadFinished( const QByteArray&, const QString& ) ), &eventLoop, SLOT( quit() ) ); - timer.start( RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); + requestBlobDownload( blobId ); - // Move the downloaded blob out of the transient redirect list and into the cache. Erasing the consumed entry - // also keeps m_redirectInfo from growing without bound as more properties are downloaded. - for ( auto it = m_redirectInfo.begin(); it != m_redirectInfo.end(); ++it ) - { - if ( it->objectId == blobId ) - { - QByteArray contents = it->contents; - m_redirectInfo.erase( it ); - insertGridPropertyBlobInCache( cacheKey, contents ); - return contents; - } - } + timer.start( RiaSumoDefines::requestTimeoutMillis() ); + eventLoop.exec(); - return {}; + // Move the downloaded blob out of the transient redirect list and into the cache. Erasing the consumed + // entry also keeps m_redirectInfo from growing without bound as more properties are downloaded. + for ( auto it = m_redirectInfo.begin(); it != m_redirectInfo.end(); ++it ) + { + if ( it->objectId == blobId ) + { + contents = it->contents; + m_redirectInfo.erase( it ); + insertGridPropertyBlobInCache( cacheKey, contents ); + return; + } + } + } ); + + return contents; } //-------------------------------------------------------------------------------------------------- @@ -669,6 +743,24 @@ void RiaSumoConnector::prefetchGridPropertyDataBlocking( const SumoCaseId& if ( timestampsToFetch.size() < 2 ) return; // nothing to gain over the single time step path + requestTokenBlocking(); + + runOnTransferThreadBlocking( + [&]() { fetchGridPropertyBatch( caseId, ensembleName, gridName, realization, propertyName, timestampsToFetch, cacheKeys ); } ); +} + +//-------------------------------------------------------------------------------------------------- +/// Resolve the blob ids of a batch of time steps, download the blobs, and put them in the cache. Always +/// called on the transfer thread, where the event loops it waits on dispatch no GUI events. +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::fetchGridPropertyBatch( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const std::vector& timestampsToFetch, + const std::vector& cacheKeys ) +{ // Phase 1: resolve all blob ids concurrently. std::vector blobIdReplies; for ( const auto& isoDateOrInterval : timestampsToFetch ) @@ -729,7 +821,7 @@ void RiaSumoConnector::prefetchGridPropertyDataBlocking( const SumoCaseId& // The downloads run concurrently, but allow the single request timeout per blob so a batch is // never given less time than the same blobs would get one by one. timer.start( static_cast( pendingBlobIds.size() ) * RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); + eventLoop.exec(); QObject::disconnect( connection ); } @@ -785,7 +877,7 @@ void RiaSumoConnector::waitForRepliesToFinish( const std::vector if ( !isAllFinished() ) { timer.start( RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); + eventLoop.exec(); } for ( const auto& connection : connections ) @@ -870,28 +962,45 @@ QByteArray RiaSumoConnector::requestParametersParquetDataBlocking( const SumoCas if ( m_blobId.empty() ) return {}; - auto blobId = m_blobId.back(); + return downloadBlobBlocking( m_blobId.back() ); +} - QEventLoop eventLoop; - QTimer timer; - timer.setSingleShot( true ); - QObject::connect( &timer, SIGNAL( timeout() ), &eventLoop, SLOT( quit() ) ); - QObject::connect( this, SIGNAL( parquetDownloadFinished( const QByteArray&, const QString& ) ), &eventLoop, SLOT( quit() ) ); +//-------------------------------------------------------------------------------------------------- +/// Download one blob and return its contents. The download and the wait for it run on the transfer thread. +//-------------------------------------------------------------------------------------------------- +QByteArray RiaSumoConnector::downloadBlobBlocking( const QString& blobId ) +{ + if ( blobId.isEmpty() ) return {}; - requestBlobDownload( blobId ); + requestTokenBlocking(); - timer.start( RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); + QByteArray contents; - for ( const auto& blobData : m_redirectInfo ) - { - if ( blobData.objectId == blobId ) + runOnTransferThreadBlocking( + [this, blobId, &contents]() { - return blobData.contents; - } - } + QEventLoop eventLoop; + QTimer timer; + timer.setSingleShot( true ); + QObject::connect( &timer, SIGNAL( timeout() ), &eventLoop, SLOT( quit() ) ); + QObject::connect( this, SIGNAL( parquetDownloadFinished( const QByteArray&, const QString& ) ), &eventLoop, SLOT( quit() ) ); + + requestBlobDownload( blobId ); + + timer.start( RiaSumoDefines::requestTimeoutMillis() ); + eventLoop.exec(); + + for ( const auto& blobData : m_redirectInfo ) + { + if ( blobData.objectId == blobId ) + { + contents = blobData.contents; + return; + } + } + } ); - return {}; + return contents; } //-------------------------------------------------------------------------------------------------- @@ -921,7 +1030,7 @@ void RiaSumoConnector::requestParametersBlobIdForEnsemble( const SumoCaseId& cas addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - auto reply = m_networkAccessManager->get( networkRequest ); + auto reply = networkAccessManager()->get( networkRequest ); connect( reply, &QNetworkReply::finished, @@ -952,7 +1061,7 @@ void RiaSumoConnector::requestBlobIdForEnsemble( const SumoCaseId& caseId, const addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - auto reply = m_networkAccessManager->get( networkRequest ); + auto reply = networkAccessManager()->get( networkRequest ); connect( reply, &QNetworkReply::finished, @@ -988,7 +1097,7 @@ void RiaSumoConnector::requestBlobDownload( const QString& blobId ) addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - auto reply = m_networkAccessManager->get( networkRequest ); + auto reply = networkAccessManager()->get( networkRequest ); connect( reply, &QNetworkReply::finished, @@ -1042,7 +1151,7 @@ void RiaSumoConnector::requestBlobBySasUri( const QString& blobId, const QString // storage host. Redirect policy is set explicitly so behaviour is not Qt-version dependent. networkRequest.setAttribute( QNetworkRequest::RedirectPolicyAttribute, QNetworkRequest::NoLessSafeRedirectPolicy ); - auto reply = m_networkAccessManager->get( networkRequest ); + auto reply = networkAccessManager()->get( networkRequest ); connect( reply, &QNetworkReply::finished, @@ -1100,28 +1209,7 @@ QByteArray RiaSumoConnector::requestParquetDataBlocking( const SumoCaseId& caseI if ( m_blobId.empty() ) return {}; // The REST API now returns the complete blob URL, not just an ID - auto blobId = m_blobId.back(); - - QEventLoop eventLoop; - QTimer timer; - timer.setSingleShot( true ); - QObject::connect( &timer, SIGNAL( timeout() ), &eventLoop, SLOT( quit() ) ); - QObject::connect( this, SIGNAL( parquetDownloadFinished( const QByteArray&, const QString& ) ), &eventLoop, SLOT( quit() ) ); - - requestBlobDownload( blobId ); - - timer.start( RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); - - for ( const auto& blobData : m_redirectInfo ) - { - if ( blobData.objectId == blobId ) - { - return blobData.contents; - } - } - - return {}; + return downloadBlobBlocking( m_blobId.back() ); } //-------------------------------------------------------------------------------------------------- @@ -1142,15 +1230,24 @@ QString RiaSumoConnector::constructSasUri( const QString& blobStoreBaseUri, cons //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -/// Note that this event loop dispatches everything except user input, so while a request is in flight the -/// view update code can run and re-enter a cell result load that is already in progress. That makes the same -/// result load twice. Declining the second load in RigCaseCellResultsData is not a way out: the only "no" -/// that method can return is cvf::UNDEFINED_SIZE_T, which consumers read as "no such result" and act on - -/// RimEclipseResultDefinitionTools::updateCellResultLegend computes the legend range straight after -/// ensureKnownResultLoaded without checking it, and caches a range over no data. Preventing the re-entrancy -/// means not dispatching events here at all, which requires moving the transfers off the GUI thread. +/// The request runs on the transfer thread, and the event loop that waits for it runs there too. Waiting on +/// the GUI thread instead would dispatch everything except user input, letting the view update code re-enter +/// a cell result load that was still running and download the same grid property a second time. An event +/// loop on the transfer thread dispatches no GUI events, so nothing can re-enter. //-------------------------------------------------------------------------------------------------- void RiaSumoConnector::wrapAndCallNetworkRequest( std::function requestCallable, const QMetaMethod& signalMethod ) +{ + // Acquire the token before handing over to the transfer thread. The OAuth flow can open a browser and + // its objects live on the GUI thread, so it must not run on the transfer thread. + requestTokenBlocking(); + + runOnTransferThreadBlocking( [this, requestCallable, signalMethod]() { waitForRequest( requestCallable, signalMethod ); } ); +} + +//-------------------------------------------------------------------------------------------------- +/// Issue the request and wait for its completion signal. Always called on the transfer thread. +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::waitForRequest( const std::function& requestCallable, const QMetaMethod& signalMethod ) { QEventLoop eventLoop; @@ -1169,7 +1266,7 @@ void RiaSumoConnector::wrapAndCallNetworkRequest( std::function requestC requestCallable(); timer.start( RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); + eventLoop.exec(); } //-------------------------------------------------------------------------------------------------- @@ -1449,7 +1546,7 @@ QNetworkReply* RiaSumoConnector::makeDownloadRequest( const QString& url, const addStandardHeader( m_networkRequest, token, contentType ); - auto reply = m_networkAccessManager->get( m_networkRequest ); + auto reply = networkAccessManager()->get( m_networkRequest ); return reply; } diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h index bb9ba17c93e..036932668ce 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h @@ -30,6 +30,7 @@ #include class QEventLoop; +class QThread; using SumoObjectId = QString; @@ -211,6 +212,7 @@ public slots: static QString constructSasUri( const QString& blobStoreBaseUri, const QString& blobId, const QString& sasToken ); void wrapAndCallNetworkRequest( std::function requestCallable, const QMetaMethod& signalMethod ); + void waitForRequest( const std::function& requestCallable, const QMetaMethod& signalMethod ); QByteArray gridPropertyBlobFromCache( const QString& cacheKey ); void insertGridPropertyBlobInCache( const QString& cacheKey, const QByteArray& contents ); @@ -231,8 +233,28 @@ public slots: static QString blobIdFromReply( QNetworkReply* reply, const QString& propertyName ); + void fetchGridPropertyBatch( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const std::vector& timestampsToFetch, + const std::vector& cacheKeys ); + static void waitForRepliesToFinish( const std::vector& replies ); + // Download one blob by id and return its contents, waiting on the transfer thread. + QByteArray downloadBlobBlocking( const QString& blobId ); + + // The network manager belonging to the calling thread: the transfer thread manager when called from + // there, otherwise the one owned by RiaCloudConnector on the GUI thread. + QNetworkAccessManager* networkAccessManager(); + + // Run work on the transfer thread and block the caller until it returns. The caller waits on a semaphore + // and dispatches no events, so nothing can re-enter the code that started the request. Called from the + // transfer thread itself, the work is run directly, which keeps nested blocking requests working. + void runOnTransferThreadBlocking( const std::function& work ); + private: std::function m_serverUrlProvider; @@ -264,4 +286,12 @@ public slots: std::map m_gridPropertyBlobCache; std::list m_gridPropertyBlobCacheOrder; // most recently used at front size_t m_gridPropertyBlobCacheSizeBytes = 0; + + // Transfers run on their own thread so the calling thread can wait without dispatching events. Waiting on + // a nested event loop on the GUI thread let the view update code re-enter a load that was still running, + // and the same grid property was downloaded twice. Authentication stays on the GUI thread: the OAuth flow + // opens a browser and its objects live there. + QThread* m_transferThread = nullptr; + QObject* m_transferContext = nullptr; // lives on the transfer thread + QNetworkAccessManager* m_transferNetworkAccessManager = nullptr; // created on the transfer thread }; diff --git a/ApplicationLibCode/Application/Tools/RiaLogging.cpp b/ApplicationLibCode/Application/Tools/RiaLogging.cpp index ee0b169a1f1..de942253ca6 100644 --- a/ApplicationLibCode/Application/Tools/RiaLogging.cpp +++ b/ApplicationLibCode/Application/Tools/RiaLogging.cpp @@ -240,6 +240,17 @@ void RiaLogging::appendLoggerInstance( std::unique_ptr loggerInstance sm_logger.push_back( std::move( loggerInstance ) ); } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaLogging::flushPendingMessages() +{ + for ( const auto& logger : sm_logger ) + { + logger->flushPendingMessages(); + } +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/Application/Tools/RiaLogging.h b/ApplicationLibCode/Application/Tools/RiaLogging.h index b20fee415c5..9d9ff48b86d 100644 --- a/ApplicationLibCode/Application/Tools/RiaLogging.h +++ b/ApplicationLibCode/Application/Tools/RiaLogging.h @@ -52,6 +52,10 @@ class RiaLogger virtual void warning( const char* message ) = 0; virtual void info( const char* message ) = 0; virtual void debug( const char* message ) = 0; + + // Deliver messages a logger has accepted but not yet written out. A logger that has to hand messages from + // a worker thread over to another thread can otherwise let them arrive after messages logged later. + virtual void flushPendingMessages() {} }; //================================================================================================== @@ -73,6 +77,10 @@ class RiaLogging static void info( std::string_view message, std::string_view logKeyword = "" ); static void debug( std::string_view message, std::string_view logKeyword = "" ); + // Write out anything the loggers are holding, so messages logged from a worker thread appear before the + // messages the waiting thread logs once the worker is done. + static void flushPendingMessages(); + static std::chrono::time_point currentTime(); static void logElapsedTime( std::string_view message, const std::chrono::time_point& startTime ); diff --git a/ApplicationLibCode/UserInterface/RiuMessagePanel.cpp b/ApplicationLibCode/UserInterface/RiuMessagePanel.cpp index e7e7537ddbf..395ad1c04f0 100644 --- a/ApplicationLibCode/UserInterface/RiuMessagePanel.cpp +++ b/ApplicationLibCode/UserInterface/RiuMessagePanel.cpp @@ -26,6 +26,7 @@ #include "cafStyleSheetTools.h" +#include #include #include #include @@ -205,6 +206,24 @@ void RiuMessagePanelLogger::debug( const char* message ) writeToMessagePanel( RILogLevel::RI_LL_DEBUG, message ); } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +/// Deliver the messages handed over from other threads. Only the queued addMessage calls above are posted to +/// a panel, so this writes out the pending log messages and nothing else. Must be called from the thread +/// owning the panels. +//-------------------------------------------------------------------------------------------------- +void RiuMessagePanelLogger::flushPendingMessages() +{ + for ( auto& panel : m_messagePanels ) + { + if ( panel && panel->thread() == QThread::currentThread() ) + { + QCoreApplication::sendPostedEvents( panel, QEvent::MetaCall ); + } + } +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -219,13 +238,24 @@ void RiuMessagePanelLogger::writeToMessagePanel( RILogLevel messageLevel, const { if ( panel ) { - // Make sure we only output messages for the GUI-thread. - // We can loose some messages, but we avoid updating UI from a different thread that will cause asserts and - // potential crashes if ( panel->thread() == QThread::currentThread() ) { panel->addMessage( messageLevel, message ); } + else + { + // The panel can only be touched from the thread owning it. Hand the message over instead of + // dropping it, so messages logged from a worker thread still reach the panel. The text is + // copied into the queued call, as the caller owns the buffer. + const QString messageText = QString::fromUtf8( message ); + QMetaObject::invokeMethod( + panel, + [panel, messageLevel, messageText]() + { + if ( panel ) panel->addMessage( messageLevel, messageText ); + }, + Qt::QueuedConnection ); + } } } } diff --git a/ApplicationLibCode/UserInterface/RiuMessagePanel.h b/ApplicationLibCode/UserInterface/RiuMessagePanel.h index 5756d02ebfe..5e472cff4dd 100644 --- a/ApplicationLibCode/UserInterface/RiuMessagePanel.h +++ b/ApplicationLibCode/UserInterface/RiuMessagePanel.h @@ -70,6 +70,8 @@ class RiuMessagePanelLogger : public RiaLogger void info( const char* message ) override; void debug( const char* message ) override; + void flushPendingMessages() override; + private: void writeToMessagePanel( RILogLevel messageLevel, const char* message ); From 50841ecb630910e78edce333b16271b607ff7257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Herje?= Date: Tue, 18 Aug 2026 10:45:29 +0200 Subject: [PATCH 3/6] Split RiaSumoConnector into transport and data delegates RiaSumoConnector had grown to know about every kind of Sumo data, holding the requests, the parsing and the results of all of them in one class of some 1600 lines. It now keeps only what the connection itself needs: authentication, the transfer thread and the blob transfers. What is asked for is moved into one delegate per kind of data, RiaSumoExplore for what Sumo holds, RiaSumoGrid for grid data and RiaSumoSummary for summary data, each reached through the connector that owns it. The delegates return their results directly rather than leaving them in shared state for the caller to pick up afterwards, which was the source of much of the bookkeeping. The blob cache is extracted into RiaSumoBlobCache and given unit tests, the only part of this area a test can reach without a live connection. Also fixes the Sumo dev dialog deleting the connector it was handed, which is shared with the rest of the application. Clear the dependent Sumo selections and options when the asset or case changes Co-Authored-By: Claude Opus 5 --- .../Tools/Cloud/CMakeLists_files.cmake | 4 + .../Tools/Cloud/RiaSumoBlobCache.cpp | 121 ++ .../Tools/Cloud/RiaSumoBlobCache.h | 68 + .../Tools/Cloud/RiaSumoConnector.cpp | 1598 +++-------------- .../Tools/Cloud/RiaSumoConnector.h | 244 +-- .../Tools/Cloud/RiaSumoExplore.cpp | 168 ++ .../Application/Tools/Cloud/RiaSumoExplore.h | 70 + .../Application/Tools/Cloud/RiaSumoGrid.cpp | 386 ++++ .../Application/Tools/Cloud/RiaSumoGrid.h | 133 ++ .../Tools/Cloud/RiaSumoSummary.cpp | 133 ++ .../Application/Tools/Cloud/RiaSumoSummary.h | 56 + .../Tools/Cloud/RifReaderSumoGridProperty.cpp | 10 +- .../RicSumoDataFeature.cpp | 48 +- .../ApplicationCommands/RicSumoDataFeature.h | 5 + .../Cloud/RimCloudDataSourceCollection.cpp | 107 +- .../Cloud/RimCloudDataSourceCollection.h | 18 + .../ProjectDataModel/RimRoffCaseSumo.cpp | 8 +- .../Summary/Sumo/RimSummaryEnsembleSumo.cpp | 4 +- ApplicationLibCode/UnitTests/CMakeLists.txt | 1 + .../UnitTests/RiaSumoBlobCache-Test.cpp | 201 +++ .../UserInterface/RiuMessagePanel.cpp | 2 - 21 files changed, 1724 insertions(+), 1661 deletions(-) create mode 100644 ApplicationLibCode/Application/Tools/Cloud/RiaSumoBlobCache.cpp create mode 100644 ApplicationLibCode/Application/Tools/Cloud/RiaSumoBlobCache.h create mode 100644 ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.cpp create mode 100644 ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.h create mode 100644 ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.cpp create mode 100644 ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.h create mode 100644 ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp create mode 100644 ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.h create mode 100644 ApplicationLibCode/UnitTests/RiaSumoBlobCache-Test.cpp diff --git a/ApplicationLibCode/Application/Tools/Cloud/CMakeLists_files.cmake b/ApplicationLibCode/Application/Tools/Cloud/CMakeLists_files.cmake index e914ae5cd37..4ddf728532c 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/CMakeLists_files.cmake +++ b/ApplicationLibCode/Application/Tools/Cloud/CMakeLists_files.cmake @@ -1,8 +1,12 @@ set(SOURCE_GROUP_SOURCE_FILES ${CMAKE_CURRENT_LIST_DIR}/RiaCloudApiService.cpp ${CMAKE_CURRENT_LIST_DIR}/RiaCloudConnector.cpp + ${CMAKE_CURRENT_LIST_DIR}/RiaSumoBlobCache.cpp ${CMAKE_CURRENT_LIST_DIR}/RiaSumoConnector.cpp ${CMAKE_CURRENT_LIST_DIR}/RiaSumoDefines.cpp + ${CMAKE_CURRENT_LIST_DIR}/RiaSumoExplore.cpp + ${CMAKE_CURRENT_LIST_DIR}/RiaSumoGrid.cpp + ${CMAKE_CURRENT_LIST_DIR}/RiaSumoSummary.cpp ${CMAKE_CURRENT_LIST_DIR}/RiaConnectorTools.cpp ${CMAKE_CURRENT_LIST_DIR}/RiaOsduConnector.cpp ${CMAKE_CURRENT_LIST_DIR}/RiaOAuthHttpServerReplyHandler.cpp diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoBlobCache.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoBlobCache.cpp new file mode 100644 index 00000000000..61ef93409ec --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoBlobCache.cpp @@ -0,0 +1,121 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "RiaSumoBlobCache.h" + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RiaSumoBlobCache::RiaSumoBlobCache( size_t limitBytes ) + : m_limitBytes( limitBytes ) +{ +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RiaSumoBlobCache::contains( const QString& key ) const +{ + return m_entries.contains( key ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QByteArray RiaSumoBlobCache::lookup( const QString& key ) +{ + auto it = m_entries.find( key ); + if ( it == m_entries.end() ) return {}; + + m_order.splice( m_order.begin(), m_order, it->second.orderIterator ); + + return it->second.contents; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoBlobCache::insert( const QString& key, const QByteArray& contents ) +{ + if ( contents.isEmpty() ) return; + + const size_t contentsSize = static_cast( contents.size() ); + + // A blob larger than the whole cache would evict everything else to make room for itself. Leave it + // uncached instead, so the smaller blobs already present stay available. + if ( contentsSize > m_limitBytes ) return; + + // Re-inserting an existing key would leak its order list entry, so drop the previous version first. + if ( auto it = m_entries.find( key ); it != m_entries.end() ) + { + m_sizeBytes -= static_cast( it->second.contents.size() ); + m_order.erase( it->second.orderIterator ); + m_entries.erase( it ); + } + + m_order.push_front( key ); + m_entries[key] = { contents, m_order.begin() }; + m_sizeBytes += contentsSize; + + while ( m_sizeBytes > m_limitBytes && !m_order.empty() ) + { + const QString& oldestKey = m_order.back(); + + if ( auto it = m_entries.find( oldestKey ); it != m_entries.end() ) + { + m_sizeBytes -= static_cast( it->second.contents.size() ); + m_entries.erase( it ); + } + + m_order.pop_back(); + } +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoBlobCache::clear() +{ + m_entries.clear(); + m_order.clear(); + m_sizeBytes = 0; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +size_t RiaSumoBlobCache::sizeBytes() const +{ + return m_sizeBytes; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +size_t RiaSumoBlobCache::entryCount() const +{ + return m_entries.size(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +size_t RiaSumoBlobCache::limitBytes() const +{ + return m_limitBytes; +} diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoBlobCache.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoBlobCache.h new file mode 100644 index 00000000000..4ebf3d46408 --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoBlobCache.h @@ -0,0 +1,68 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include +#include + +#include +#include +#include + +//================================================================================================== +/// Blobs downloaded from Sumo, kept in memory so a repeated request is answered without going to the +/// server. Displaying a grid property computes its legend range across all time steps and then reads the +/// values again, so the same blob is asked for several times. +/// +/// Bounded by total byte size rather than entry count, as blob sizes follow the grid size and vary by +/// orders of magnitude. The least recently used entries are evicted when the limit is exceeded. +//================================================================================================== +class RiaSumoBlobCache +{ +public: + explicit RiaSumoBlobCache( size_t limitBytes ); + + bool contains( const QString& key ) const; + + // The cached contents, or an empty array when the key is not cached. A hit is moved to the front of + // the recency order, so it is evicted last. + QByteArray lookup( const QString& key ); + + // Empty contents are not cached, as an empty array is how a miss is reported. Contents larger than the + // whole limit are not cached either, as making room for them would evict everything else. + void insert( const QString& key, const QByteArray& contents ); + + void clear(); + + size_t sizeBytes() const; + size_t entryCount() const; + size_t limitBytes() const; + +private: + struct Entry + { + QByteArray contents; + std::list::iterator orderIterator; + }; + + std::map m_entries; + std::list m_order; // most recently used at front + size_t m_sizeBytes = 0; + const size_t m_limitBytes; +}; diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp index 1d08f17f919..265cc9faa07 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp @@ -48,6 +48,9 @@ RiaSumoConnector::RiaSumoConnector( QObject* parent, unsigned int port ) : RiaCloudConnector( parent, {}, authority, scopes, clientId, port ) , m_serverUrlProvider( std::move( serverUrlProvider ) ) + , m_explore( *this ) + , m_grid( *this ) + , m_summary( *this ) { // The transfer thread runs the network requests issued by the blocking wrappers, so the calling thread can // wait for them without dispatching events. The context object gives us something with transfer thread @@ -70,34 +73,45 @@ RiaSumoConnector::RiaSumoConnector( QObject* parent, //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -QString RiaSumoConnector::server() const +RiaSumoExplore& RiaSumoConnector::explore() { - // Ask for the address on every call. The server is bound to the first available port, and gets a new - // port if it is restarted, so a cached address goes stale. - if ( !m_serverUrlProvider ) return {}; + return m_explore; +} - return m_serverUrlProvider(); +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RiaSumoGrid& RiaSumoConnector::grid() +{ + return m_grid; } //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestFailed( const QAbstractOAuth::Error error ) +RiaSumoSummary& RiaSumoConnector::summary() { - RiaLogging::error( "Request failed: " ); + return m_summary; } //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parquetDownloadComplete( const QString& blobId, const QByteArray& contents, const QString& url ) +QString RiaSumoConnector::server() const { - SumoRedirect obj; - obj.objectId = blobId; - obj.contents = contents; - obj.url = url; + // Ask for the address on every call. The server is bound to the first available port, and gets a new + // port if it is restarted, so a cached address goes stale. + if ( !m_serverUrlProvider ) return {}; - m_redirectInfo.push_back( obj ); + return m_serverUrlProvider(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::requestFailed( const QAbstractOAuth::Error error ) +{ + RiaLogging::error( "Request failed: " ); } //-------------------------------------------------------------------------------------------------- @@ -168,1492 +182,316 @@ void RiaSumoConnector::runOnTransferThreadBlocking( const std::function& } //-------------------------------------------------------------------------------------------------- -/// +/// Wait until every reply has finished, or the timeout expires. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestCasesForField( const QString& fieldName ) +void RiaSumoConnector::waitForRepliesToFinish( const std::vector& replies ) { - m_cases.clear(); + if ( replies.empty() ) return; - requestTokenBlocking(); + QEventLoop eventLoop; + QTimer timer; + timer.setSingleShot( true ); + QObject::connect( &timer, &QTimer::timeout, &eventLoop, &QEventLoop::quit ); - QNetworkRequest m_networkRequest; - QString url = QString( "%1/cases?asset_name=%2" ).arg( server() ).arg( fieldName ); - m_networkRequest.setUrl( QUrl( url ) ); + auto isAllFinished = [&replies]() + { return std::ranges::all_of( replies, []( QNetworkReply* reply ) { return reply && reply->isFinished(); } ); }; - addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); + std::vector connections; + for ( auto reply : replies ) + { + if ( !reply ) continue; - auto reply = networkAccessManager()->get( m_networkRequest ); + connections.push_back( QObject::connect( reply, + &QNetworkReply::finished, + &eventLoop, + [&eventLoop, &isAllFinished]() + { + if ( isAllFinished() ) eventLoop.quit(); + } ) ); + } - connect( reply, - &QNetworkReply::finished, - [this, reply]() - { - // parseCases handles the error case and always emits casesFinished, so the blocking caller - // returns immediately instead of waiting for the request to time out. - parseCases( reply ); - } ); -} + if ( !isAllFinished() ) + { + timer.start( RiaSumoDefines::requestTimeoutMillis() ); + eventLoop.exec(); + } -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestCasesForFieldBlocking( const QString& fieldName ) -{ - auto requestCallable = [this, fieldName] { requestCasesForField( fieldName ); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::casesFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); + for ( const auto& connection : connections ) + { + QObject::disconnect( connection ); + } } //-------------------------------------------------------------------------------------------------- -/// +/// Download one blob and return its contents. The download and the wait for it run on the transfer thread. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestAssets() +QByteArray RiaSumoConnector::downloadBlobBlocking( const QString& blobId ) { - requestTokenBlocking(); - - QNetworkRequest m_networkRequest; - m_networkRequest.setUrl( QUrl( QString( "%1/assets" ).arg( server() ) ) ); - - addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); + const auto contentsByBlobId = downloadBlobsBlocking( { blobId } ); - auto reply = networkAccessManager()->get( m_networkRequest ); + if ( auto it = contentsByBlobId.find( blobId ); it != contentsByBlobId.end() ) return it->second; - connect( reply, - &QNetworkReply::finished, - [this, reply]() - { - // parseAssets handles the error case and always emits assetsFinished, so the blocking caller - // returns immediately instead of waiting for the request to time out. - parseAssets( reply ); - } ); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestAssetsBlocking() -{ - auto requestCallable = [this] { requestAssets(); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::assetsFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); + return {}; } //-------------------------------------------------------------------------------------------------- -/// +/// Issue a GET and return the response body, waiting on the transfer thread. Returns an empty array when +/// the request fails. This is the primitive the data specific requests are built from. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestEnsembleByCasesId( const SumoCaseId& caseId ) +QByteArray RiaSumoConnector::getBlocking( const QString& url ) { requestTokenBlocking(); - QNetworkRequest m_networkRequest; - QString url = QString( "%1/cases/%2/ensembles" ).arg( server() ).arg( caseId.get() ); - m_networkRequest.setUrl( QUrl( url ) ); - - addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - - auto reply = networkAccessManager()->get( m_networkRequest ); - - connect( reply, - &QNetworkReply::finished, - [this, reply, caseId]() - { - // parseEnsembleNames handles the error case and always emits ensembleNamesFinished, so the - // blocking caller returns immediately instead of waiting for the request to time out. - parseEnsembleNames( reply, caseId ); - } ); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parseEnsembleNames( QNetworkReply* reply, const SumoCaseId& caseId ) -{ - QByteArray result = reply->readAll(); - reply->deleteLater(); - - if ( reply->error() == QNetworkReply::NoError ) - { - m_ensembleNames.clear(); - - QJsonDocument doc = QJsonDocument::fromJson( result ); - QJsonArray jsonArray = doc.array(); + QByteArray body; - for ( const QJsonValue& value : jsonArray ) + runOnTransferThreadBlocking( + [&]() { - QJsonObject ensembleObj = value.toObject(); - QString ensembleName = ensembleObj["name"].toString(); - m_ensembleNames.push_back( { caseId, ensembleName } ); - } - - RiaLogging::debug( std::format( "Ensemble count : {}", m_ensembleNames.size() ) ); - } - else - { - RiaLogging::error( std::format( "Request ensemble names failed: '{}'", reply->errorString() ) ); - } - - emit ensembleNamesFinished(); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestEnsembleByCasesIdBlocking( const SumoCaseId& caseId ) -{ - auto requestCallable = [this, caseId] { requestEnsembleByCasesId( caseId ); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::ensembleNamesFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestVectorNamesForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ) -{ - requestTokenBlocking(); + QNetworkRequest networkRequest; + networkRequest.setUrl( QUrl( url ) ); + addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - QNetworkRequest m_networkRequest; - QString url = QString( "%1/cases/%2/ensembles/%3/vector_list" ).arg( server() ).arg( caseId.get() ).arg( ensembleName ); - m_networkRequest.setUrl( QUrl( url ) ); + auto reply = networkAccessManager()->get( networkRequest ); - addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); + QEventLoop eventLoop; + QTimer timer; + timer.setSingleShot( true ); + QObject::connect( &timer, &QTimer::timeout, &eventLoop, &QEventLoop::quit ); + QObject::connect( reply, &QNetworkReply::finished, &eventLoop, &QEventLoop::quit ); - auto reply = networkAccessManager()->get( m_networkRequest ); + timer.start( RiaSumoDefines::requestTimeoutMillis() ); + eventLoop.exec(); - connect( reply, - &QNetworkReply::finished, - [this, reply, ensembleName, caseId]() - { - // parseVectorNames handles the error case and always emits vectorNamesFinished, so the - // blocking caller returns immediately instead of waiting for the request to time out. - parseVectorNames( reply, caseId, ensembleName ); - } ); -} + body = replyBody( reply, url ); + } ); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestVectorNamesForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ) -{ - auto requestCallable = [this, caseId, ensembleName] { requestVectorNamesForEnsemble( caseId, ensembleName ); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::vectorNamesFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); + return body; } //-------------------------------------------------------------------------------------------------- -/// +/// The REST API returns a blob id as a plain string, quoted by FastAPI. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestRealizationIdsForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ) +QString RiaSumoConnector::blobIdFromBody( const QByteArray& body ) { - m_realizationIds.clear(); - - requestTokenBlocking(); - - QNetworkRequest m_networkRequest; - QString url = QString( "%1/cases/%2/ensembles/%3/realizations" ).arg( server() ).arg( caseId.get() ).arg( ensembleName ); - m_networkRequest.setUrl( QUrl( url ) ); + if ( body.isEmpty() ) return {}; - addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); + QString blobId = QString::fromUtf8( body ).trimmed(); - auto reply = networkAccessManager()->get( m_networkRequest ); + if ( blobId.startsWith( '"' ) && blobId.endsWith( '"' ) ) + { + blobId = blobId.mid( 1, blobId.length() - 2 ); + } - connect( reply, - &QNetworkReply::finished, - [this, reply, ensembleName, caseId]() - { - // parseRealizationNumbers handles the error case and always emits realizationIdsFinished, so - // the blocking caller returns immediately instead of waiting for the request to time out. - parseRealizationNumbers( reply, caseId, ensembleName ); - } ); + return blobId; } //-------------------------------------------------------------------------------------------------- -/// +/// Read the body off a finished reply. The reply is consumed and scheduled for deletion. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestRealizationIdsForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ) +QByteArray RiaSumoConnector::replyBody( QNetworkReply* reply, const QString& url ) { - auto requestCallable = [this, caseId, ensembleName] { requestRealizationIdsForEnsemble( caseId, ensembleName ); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::realizationIdsFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); -} + if ( !reply ) return {}; -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestGridInfoForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ) -{ - m_gridInfos.clear(); + const bool failed = !reply->isFinished() || reply->error() != QNetworkReply::NoError; + QByteArray body = failed ? QByteArray() : reply->readAll(); - requestTokenBlocking(); + if ( failed ) + { + RiaLogging::error( std::format( "Request failed: '{}': {}", url.toStdString(), reply->errorString().toStdString() ) ); + } - QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); - QNetworkRequest m_networkRequest; - QString url = QString( "%1/cases/%2/ensembles/%3/grid_info_list" ).arg( server() ).arg( caseId.get() ).arg( encodedEnsembleName ); - m_networkRequest.setUrl( QUrl( url ) ); - - addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - - auto reply = networkAccessManager()->get( m_networkRequest ); - - connect( reply, - &QNetworkReply::finished, - [this, reply, ensembleName, caseId]() - { - if ( reply->error() == QNetworkReply::NoError ) - { - parseGridInfo( reply, caseId, ensembleName ); - } - else - { - RiaLogging::error( std::format( "Request grid info failed: '{}'", reply->errorString().toStdString() ) ); - emit gridInfoFinished(); - } - } ); -} + reply->deleteLater(); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestGridInfoForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ) -{ - auto requestCallable = [this, caseId, ensembleName] { requestGridInfoForEnsemble( caseId, ensembleName ); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::gridInfoFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); + return body; } //-------------------------------------------------------------------------------------------------- -/// +/// Entry point for callers on any thread: hand the work to the transfer thread and wait for it there, so +/// the calling thread dispatches no events while the transfers are in flight. See downloadBlobs for what +/// the transfers actually are. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestGridBlobIdForEnsemble( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ) +std::map RiaSumoConnector::downloadBlobsBlocking( const std::vector& blobIds ) { - requestTokenBlocking(); - - QNetworkRequest m_networkRequest; - - // Properly URL-encode the path components - QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); - QString encodedGridName = QUrl::toPercentEncoding( gridName ); - - QString url = QString( "%1/cases/%2/ensembles/%3/grids/%4/realizations/%5/blob_id" ) - .arg( server() ) - .arg( caseId.get() ) - .arg( encodedEnsembleName ) - .arg( encodedGridName ) - .arg( realization ); - m_networkRequest.setUrl( QUrl( url ) ); - - addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - - auto reply = networkAccessManager()->get( m_networkRequest ); - - connect( reply, - &QNetworkReply::finished, - [this, reply, ensembleName, caseId, gridName]() - { - if ( reply->error() == QNetworkReply::NoError ) - { - parseBlobId( reply, caseId, ensembleName, gridName, false ); - } - else - { - RiaLogging::error( std::format( "Request grid blob ID failed: '{}'", reply->errorString().toStdString() ) ); - emit blobIdFinished(); - } - } ); -} + if ( blobIds.empty() ) return {}; -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestGridBlobIdForEnsembleBlocking( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization ) -{ - auto requestCallable = [this, caseId, ensembleName, gridName, realization] - { requestGridBlobIdForEnsemble( caseId, ensembleName, gridName, realization ); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::blobIdFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); -} + requestTokenBlocking(); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -QByteArray - RiaSumoConnector::requestGridDataBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ) -{ - requestGridBlobIdForEnsembleBlocking( caseId, ensembleName, gridName, realization ); + std::map contentsByBlobId; - if ( m_blobId.empty() ) return {}; + runOnTransferThreadBlocking( [&]() { contentsByBlobId = downloadBlobs( blobIds ); } ); - // The REST API returns the blob Id - return downloadBlobBlocking( m_blobId.back() ); + return contentsByBlobId; } //-------------------------------------------------------------------------------------------------- +/// Download several blobs and return their contents by blob id. Getting a blob takes two round trips, one +/// for the pre-signed URI and one for the data itself, so both are done as a group: all the access info +/// requests are issued and waited for together, then all the transfers. A blob that fails is left out of +/// the returned map. /// +/// Always called on the transfer thread, where the event loops it waits on dispatch no GUI events. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestGridPropertyInfoForEnsemble( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization ) +std::map RiaSumoConnector::downloadBlobs( const std::vector& blobIds ) { - m_gridPropertyInfos.clear(); + std::map contentsByBlobId; + if ( blobIds.empty() ) return contentsByBlobId; - requestTokenBlocking(); + // Phase 1: ask for the pre-signed URI of every blob. + std::vector accessInfoReplies; + for ( const auto& blobId : blobIds ) + { + QString url = QString( "%1/blobs/%2/sas_token_and_blob_base_uri" ).arg( server() ).arg( blobId ); - QNetworkRequest m_networkRequest; - - QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); - QString encodedGridName = QUrl::toPercentEncoding( gridName ); - - QString url = QString( "%1/cases/%2/ensembles/%3/grids/%4/realizations/%5/property_info_list" ) - .arg( server() ) - .arg( caseId.get() ) - .arg( encodedEnsembleName ) - .arg( encodedGridName ) - .arg( realization ); - m_networkRequest.setUrl( QUrl( url ) ); - - addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - - auto reply = networkAccessManager()->get( m_networkRequest ); - - connect( reply, - &QNetworkReply::finished, - [this, reply, caseId, ensembleName, gridName, realization]() - { - if ( reply->error() == QNetworkReply::NoError ) - { - parseGridPropertyInfo( reply, caseId, ensembleName, gridName, realization ); - } - else - { - RiaLogging::error( std::format( "Request grid property info failed: '{}'", reply->errorString().toStdString() ) ); - emit gridPropertyInfoFinished(); - } - } ); -} + QNetworkRequest networkRequest; + networkRequest.setUrl( QUrl( url ) ); + addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestGridPropertyInfoForEnsembleBlocking( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization ) -{ - auto requestCallable = [this, caseId, ensembleName, gridName, realization] - { requestGridPropertyInfoForEnsemble( caseId, ensembleName, gridName, realization ); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::gridPropertyInfoFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); -} + accessInfoReplies.push_back( networkAccessManager()->get( networkRequest ) ); + } -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -QString RiaSumoConnector::requestGridPropertyBlobIdBlocking( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization, - const QString& propertyName, - const QString& isoDateOrInterval ) -{ - requestTokenBlocking(); + waitForRepliesToFinish( accessInfoReplies ); - QString blobId; + std::vector sasUris; + for ( size_t i = 0; i < accessInfoReplies.size(); i++ ) + { + sasUris.push_back( sasUriFromReply( accessInfoReplies[i], blobIds[i] ) ); + } - runOnTransferThreadBlocking( - [&]() - { - auto reply = makeGridPropertyBlobIdRequest( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + // Phase 2: transfer the blobs themselves. + std::vector blobReplies; + std::vector blobIndices; // index into blobIds for each reply + for ( size_t i = 0; i < sasUris.size(); i++ ) + { + if ( sasUris[i].isEmpty() ) continue; - // Wait for THIS reply only. Binding the event loop to the reply, rather than to the shared - // blobIdFinished signal, is what makes the mapping correct: a still-pending reply from an earlier - // property's request can no longer satisfy this wait and hand us its blob id (which previously caused - // e.g. SWAT to be served the SWCR blob). The blob id is read straight off this reply and never routed - // through shared state. - QEventLoop eventLoop; - QTimer timer; - timer.setSingleShot( true ); - QObject::connect( &timer, &QTimer::timeout, &eventLoop, &QEventLoop::quit ); - QObject::connect( reply, &QNetworkReply::finished, &eventLoop, &QEventLoop::quit ); + RiaLogging::debug( std::format( "Requesting blob. Id: {} SAS URI: {}", blobIds[i], sasUris[i] ) ); - timer.start( RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec(); + QNetworkRequest networkRequest; + networkRequest.setUrl( sasUris[i] ); - blobId = blobIdFromReply( reply, propertyName ); - } ); + // The pre-signed SAS URI carries its own credential (signature in the query string), so no + // Authorization header is added here. Do NOT forward the bearer token to the storage host. Redirect + // policy is set explicitly so behaviour is not Qt-version dependent. + networkRequest.setAttribute( QNetworkRequest::RedirectPolicyAttribute, QNetworkRequest::NoLessSafeRedirectPolicy ); - return blobId; -} + blobReplies.push_back( networkAccessManager()->get( networkRequest ) ); + blobIndices.push_back( i ); + } -//-------------------------------------------------------------------------------------------------- -/// Issue the blob id request for one grid property time step. The reply is returned unfinished, so the caller -/// decides how to wait for it: one at a time, or several at once when prefetching. -//-------------------------------------------------------------------------------------------------- -QNetworkReply* RiaSumoConnector::makeGridPropertyBlobIdRequest( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization, - const QString& propertyName, - const QString& isoDateOrInterval ) -{ - requestTokenBlocking(); + waitForRepliesToFinish( blobReplies ); - // Properly URL-encode the path components - QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); - QString encodedGridName = QUrl::toPercentEncoding( gridName ); - QString encodedPropertyName = QUrl::toPercentEncoding( propertyName ); - - QString url = QString( "%1/cases/%2/ensembles/%3/grids/%4/realizations/%5/properties/%6/blob_id" ) - .arg( server() ) - .arg( caseId.get() ) - .arg( encodedEnsembleName ) - .arg( encodedGridName ) - .arg( realization ) - .arg( encodedPropertyName ); - - // The timestamp/interval is an optional query parameter; omit it for static properties. - if ( !isoDateOrInterval.isEmpty() ) + for ( size_t i = 0; i < blobReplies.size(); i++ ) { - url += QString( "?property_iso_date_or_interval=%1" ).arg( QString( QUrl::toPercentEncoding( isoDateOrInterval ) ) ); - } + const size_t blobIndex = blobIndices[i]; - QNetworkRequest networkRequest; - networkRequest.setUrl( QUrl( url ) ); - addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); + QByteArray contents = blobContentsFromReply( blobReplies[i], sasUris[blobIndex] ); + if ( !contents.isEmpty() ) + { + contentsByBlobId[blobIds[blobIndex]] = contents; + } + } - return networkAccessManager()->get( networkRequest ); + return contentsByBlobId; } //-------------------------------------------------------------------------------------------------- -/// Read the blob id off a finished blob id reply. The reply is consumed and scheduled for deletion. +/// Read the pre-signed download URI off a finished blob access info reply. The reply is consumed and +/// scheduled for deletion. Returns an empty string on failure. //-------------------------------------------------------------------------------------------------- -QString RiaSumoConnector::blobIdFromReply( QNetworkReply* reply, const QString& propertyName ) +QString RiaSumoConnector::sasUriFromReply( QNetworkReply* reply, const QString& blobId ) { if ( !reply ) return {}; + reply->deleteLater(); + if ( !reply->isFinished() || reply->error() != QNetworkReply::NoError ) { - if ( reply->error() != QNetworkReply::NoError ) - { - RiaLogging::error( std::format( "Request grid property blob ID failed: '{}'", reply->errorString().toStdString() ) ); - } - reply->deleteLater(); + RiaLogging::error( + std::format( "Requesting access info for blob '{}' failed: {}", blobId.toStdString(), reply->errorString().toStdString() ) ); return {}; } - // The REST API returns the blob id as a plain string, quoted by FastAPI. - QString blobId = QString::fromUtf8( reply->readAll() ).trimmed(); - reply->deleteLater(); - - if ( blobId.startsWith( '"' ) && blobId.endsWith( '"' ) ) + // The backend returns BlobAccessInfo as JSON: { "sasToken": "...", "blobStoreBaseUri": "..." } + const QByteArray contents = reply->readAll(); + QJsonParseError parseError; + const QJsonDocument doc = QJsonDocument::fromJson( contents, &parseError ); + if ( parseError.error != QJsonParseError::NoError || !doc.isObject() ) { - blobId = blobId.mid( 1, blobId.length() - 2 ); + RiaLogging::error( std::format( "Could not parse blob access info response as JSON: {}", parseError.errorString().toStdString() ) ); + return {}; } - RiaLogging::debug( std::format( "Received blob ID for vector '{}': {}", propertyName.toStdString(), blobId.toStdString() ) ); + const QJsonObject obj = doc.object(); + const QString sasToken = obj.value( "sasToken" ).toString(); + const QString blobBaseUri = obj.value( "blobStoreBaseUri" ).toString(); + if ( blobBaseUri.isEmpty() ) + { + RiaLogging::error( "Blob access info response did not contain a blobStoreBaseUri." ); + return {}; + } - return blobId; + return constructSasUri( blobBaseUri, blobId, sasToken ); } //-------------------------------------------------------------------------------------------------- -/// +/// Read the blob contents off a finished transfer reply. The reply is consumed and scheduled for deletion. +/// Returns an empty array on failure. //-------------------------------------------------------------------------------------------------- -QByteArray RiaSumoConnector::requestGridPropertyDataBlocking( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization, - const QString& propertyName, - const QString& isoDateOrInterval ) +QByteArray RiaSumoConnector::blobContentsFromReply( QNetworkReply* reply, const QString& sasUri ) { - // Serve from cache when possible. Keyed by the full property identity (not the blob id), a repeat request is - // answered without even asking Sumo for the blob id. This avoids re-downloading every time step when a - // property's global legend range is computed, and again when it is displayed. - const QString cacheKey = gridPropertyCacheKey( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); - if ( auto cachedContents = gridPropertyBlobFromCache( cacheKey ); !cachedContents.isEmpty() ) + if ( !reply ) return {}; + + reply->deleteLater(); + + if ( !reply->isFinished() || reply->error() != QNetworkReply::NoError ) { - return cachedContents; + RiaLogging::error( ( "Download failed: " + sasUri + " failed." + reply->errorString() ).toStdString() ); + return {}; } - // Resolve the blob id for this exact property. The getter waits on its own reply, so it can only ever return - // this property's id (or empty on failure) - never a neighbouring request's id. - const QString blobId = requestGridPropertyBlobIdBlocking( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); - if ( blobId.isEmpty() ) return {}; + auto statusCode = reply->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt(); + auto contentLength = reply->header( QNetworkRequest::ContentLengthHeader ).toLongLong(); + auto bytesAvailable = reply->bytesAvailable(); - QByteArray contents; + RiaLogging::debug( std::format( "Response: status={}, content-length={}, bytes-available={}", statusCode, contentLength, bytesAvailable ) ); - runOnTransferThreadBlocking( - [this, blobId, cacheKey, &contents]() - { - QEventLoop eventLoop; - QTimer timer; - timer.setSingleShot( true ); - QObject::connect( &timer, SIGNAL( timeout() ), &eventLoop, SLOT( quit() ) ); - QObject::connect( this, SIGNAL( parquetDownloadFinished( const QByteArray&, const QString& ) ), &eventLoop, SLOT( quit() ) ); + auto contents = reply->readAll(); - requestBlobDownload( blobId ); + RiaLogging::debug( std::format( "Read {} bytes from reply", contents.size() ) ); - timer.start( RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec(); + // Guard against a silently truncated transfer: a dropped connection on a chunked response can still + // report NoError. If the server told us how many bytes to expect and we got fewer, treat it as a failure. + if ( contentLength > 0 && contents.size() != contentLength ) + { + RiaLogging::error( std::format( "Download truncated: expected {} bytes, got {}.", contentLength, contents.size() ) ); + return {}; + } - // Move the downloaded blob out of the transient redirect list and into the cache. Erasing the consumed - // entry also keeps m_redirectInfo from growing without bound as more properties are downloaded. - for ( auto it = m_redirectInfo.begin(); it != m_redirectInfo.end(); ++it ) - { - if ( it->objectId == blobId ) - { - contents = it->contents; - m_redirectInfo.erase( it ); - insertGridPropertyBlobInCache( cacheKey, contents ); - return; - } - } - } ); + RiaLogging::debug( ( "Received data from : " + sasUri ).toStdString() ); return contents; } //-------------------------------------------------------------------------------------------------- -/// Fetch several time steps of one grid property at the same time. The blob id requests are issued together -/// and waited for as a group, and so are the blob downloads, turning 2N sequential round trips into 2 batched -/// ones. The results are placed in the blob cache, so the per time step requests that follow are cache hits. +/// Assemble the pre-signed download URI: {blobStoreBaseUri}/{blobId}?{sasToken} //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::prefetchGridPropertyDataBlocking( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization, - const QString& propertyName, - const std::vector& isoDatesOrIntervals ) +QString RiaSumoConnector::constructSasUri( const QString& blobStoreBaseUri, const QString& blobId, const QString& sasToken ) { - // Only fetch what is not already cached, and drop duplicates so a time step is never requested twice. - std::vector cacheKeys; - std::vector timestampsToFetch; - for ( const auto& isoDateOrInterval : isoDatesOrIntervals ) + QString sasUri = blobStoreBaseUri; + if ( !sasUri.endsWith( '/' ) ) sasUri += '/'; + sasUri += blobId; + if ( !sasToken.isEmpty() ) { - const QString cacheKey = gridPropertyCacheKey( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); - - if ( std::ranges::find( cacheKeys, cacheKey ) != cacheKeys.end() ) continue; - if ( m_gridPropertyBlobCache.contains( cacheKey ) ) continue; - - cacheKeys.push_back( cacheKey ); - timestampsToFetch.push_back( isoDateOrInterval ); + sasUri += ( sasToken.startsWith( '?' ) ? sasToken : ( "?" + sasToken ) ); } - - if ( timestampsToFetch.size() < 2 ) return; // nothing to gain over the single time step path - - requestTokenBlocking(); - - runOnTransferThreadBlocking( - [&]() { fetchGridPropertyBatch( caseId, ensembleName, gridName, realization, propertyName, timestampsToFetch, cacheKeys ); } ); + return sasUri; } //-------------------------------------------------------------------------------------------------- -/// Resolve the blob ids of a batch of time steps, download the blobs, and put them in the cache. Always -/// called on the transfer thread, where the event loops it waits on dispatch no GUI events. +/// //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::fetchGridPropertyBatch( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization, - const QString& propertyName, - const std::vector& timestampsToFetch, - const std::vector& cacheKeys ) -{ - // Phase 1: resolve all blob ids concurrently. - std::vector blobIdReplies; - for ( const auto& isoDateOrInterval : timestampsToFetch ) - { - blobIdReplies.push_back( makeGridPropertyBlobIdRequest( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ) ); - } - - waitForRepliesToFinish( blobIdReplies ); - - std::vector blobIds; - for ( auto reply : blobIdReplies ) - { - blobIds.push_back( blobIdFromReply( reply, propertyName ) ); - } - - // Phase 2: download all resolved blobs concurrently. requestBlobDownload is fire and forget; each finished - // download appends to m_redirectInfo, so wait until every requested blob has arrived there. - std::vector pendingBlobIds; - for ( const auto& blobId : blobIds ) - { - if ( blobId.isEmpty() ) continue; - - pendingBlobIds.push_back( blobId ); - requestBlobDownload( blobId ); - } - - if ( !pendingBlobIds.empty() ) - { - auto haveAllBlobsArrived = [this, &pendingBlobIds]() - { - return std::ranges::all_of( pendingBlobIds, - [this]( const QString& blobId ) - { - return std::ranges::any_of( m_redirectInfo, - [&blobId]( const SumoRedirect& redirect ) - { return redirect.objectId == blobId; } ); - } ); - }; - - // requestBlobDownload can spin an event loop of its own while acquiring a token, so a download may - // already have completed here. Check before waiting, or the wait would run until it times out. - if ( !haveAllBlobsArrived() ) - { - QEventLoop eventLoop; - QTimer timer; - timer.setSingleShot( true ); - QObject::connect( &timer, &QTimer::timeout, &eventLoop, &QEventLoop::quit ); - - // Every completed download emits this signal; quit once all of them have landed in m_redirectInfo. - auto connection = QObject::connect( this, - &RiaSumoConnector::parquetDownloadFinished, - &eventLoop, - [&eventLoop, &haveAllBlobsArrived]() - { - if ( haveAllBlobsArrived() ) eventLoop.quit(); - } ); - - // The downloads run concurrently, but allow the single request timeout per blob so a batch is - // never given less time than the same blobs would get one by one. - timer.start( static_cast( pendingBlobIds.size() ) * RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec(); - - QObject::disconnect( connection ); - } - } - - // Move the downloaded blobs out of the transient redirect list and into the cache. Anything missing was not - // downloaded in time; the per time step path fetches it again later. - for ( size_t i = 0; i < blobIds.size(); i++ ) - { - if ( blobIds[i].isEmpty() ) continue; - - for ( auto it = m_redirectInfo.begin(); it != m_redirectInfo.end(); ++it ) - { - if ( it->objectId == blobIds[i] ) - { - insertGridPropertyBlobInCache( cacheKeys[i], it->contents ); - m_redirectInfo.erase( it ); - break; - } - } - } -} - -//-------------------------------------------------------------------------------------------------- -/// Wait until every reply has finished, or the timeout expires. -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::waitForRepliesToFinish( const std::vector& replies ) -{ - if ( replies.empty() ) return; - - QEventLoop eventLoop; - QTimer timer; - timer.setSingleShot( true ); - QObject::connect( &timer, &QTimer::timeout, &eventLoop, &QEventLoop::quit ); - - auto isAllFinished = [&replies]() - { return std::ranges::all_of( replies, []( QNetworkReply* reply ) { return reply && reply->isFinished(); } ); }; - - std::vector connections; - for ( auto reply : replies ) - { - if ( !reply ) continue; - - connections.push_back( QObject::connect( reply, - &QNetworkReply::finished, - &eventLoop, - [&eventLoop, &isAllFinished]() - { - if ( isAllFinished() ) eventLoop.quit(); - } ) ); - } - - if ( !isAllFinished() ) - { - timer.start( RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec(); - } - - for ( const auto& connection : connections ) - { - QObject::disconnect( connection ); - } -} - -//-------------------------------------------------------------------------------------------------- -/// The full identity of one grid property time step, used as blob cache key. -//-------------------------------------------------------------------------------------------------- -QString RiaSumoConnector::gridPropertyCacheKey( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization, - const QString& propertyName, - const QString& isoDateOrInterval ) -{ - return QString( "%1|%2|%3|%4|%5|%6" ).arg( caseId.get(), ensembleName, gridName ).arg( realization ).arg( propertyName, isoDateOrInterval ); -} - -//-------------------------------------------------------------------------------------------------- -/// Look up a downloaded grid property blob. Returns an empty array when the blob is not cached, which the -/// callers treat as a miss. A hit is moved to the front of the recency order, so it is evicted last. -//-------------------------------------------------------------------------------------------------- -QByteArray RiaSumoConnector::gridPropertyBlobFromCache( const QString& cacheKey ) -{ - auto it = m_gridPropertyBlobCache.find( cacheKey ); - if ( it == m_gridPropertyBlobCache.end() ) return {}; - - m_gridPropertyBlobCacheOrder.splice( m_gridPropertyBlobCacheOrder.begin(), m_gridPropertyBlobCacheOrder, it->second.orderIterator ); - - return it->second.contents; -} - -//-------------------------------------------------------------------------------------------------- -/// Cache a downloaded grid property blob, evicting the least recently used blobs until the cache is back -/// within the size limit. -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::insertGridPropertyBlobInCache( const QString& cacheKey, const QByteArray& contents ) -{ - if ( contents.isEmpty() ) return; - - const size_t contentsSize = static_cast( contents.size() ); - - // A blob larger than the whole cache would evict everything else to make room for itself. Leave it uncached - // instead, so the smaller blobs already present stay available. - if ( contentsSize > RiaSumoDefines::gridPropertyCacheLimitBytes() ) return; - - // Re-inserting an existing key would leak its order list entry, so drop the previous version first. - if ( auto it = m_gridPropertyBlobCache.find( cacheKey ); it != m_gridPropertyBlobCache.end() ) - { - m_gridPropertyBlobCacheSizeBytes -= static_cast( it->second.contents.size() ); - m_gridPropertyBlobCacheOrder.erase( it->second.orderIterator ); - m_gridPropertyBlobCache.erase( it ); - } - - m_gridPropertyBlobCacheOrder.push_front( cacheKey ); - m_gridPropertyBlobCache[cacheKey] = { contents, m_gridPropertyBlobCacheOrder.begin() }; - m_gridPropertyBlobCacheSizeBytes += contentsSize; - - while ( m_gridPropertyBlobCacheSizeBytes > RiaSumoDefines::gridPropertyCacheLimitBytes() && !m_gridPropertyBlobCacheOrder.empty() ) - { - const QString& oldestKey = m_gridPropertyBlobCacheOrder.back(); - - if ( auto it = m_gridPropertyBlobCache.find( oldestKey ); it != m_gridPropertyBlobCache.end() ) - { - m_gridPropertyBlobCacheSizeBytes -= static_cast( it->second.contents.size() ); - m_gridPropertyBlobCache.erase( it ); - } - - m_gridPropertyBlobCacheOrder.pop_back(); - } -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -QByteArray RiaSumoConnector::requestParametersParquetDataBlocking( const SumoCaseId& caseId, const QString& ensembleName ) -{ - requestParametersBlobIdForEnsembleBlocking( caseId, ensembleName ); - - if ( m_blobId.empty() ) return {}; - - return downloadBlobBlocking( m_blobId.back() ); -} - -//-------------------------------------------------------------------------------------------------- -/// Download one blob and return its contents. The download and the wait for it run on the transfer thread. -//-------------------------------------------------------------------------------------------------- -QByteArray RiaSumoConnector::downloadBlobBlocking( const QString& blobId ) -{ - if ( blobId.isEmpty() ) return {}; - - requestTokenBlocking(); - - QByteArray contents; - - runOnTransferThreadBlocking( - [this, blobId, &contents]() - { - QEventLoop eventLoop; - QTimer timer; - timer.setSingleShot( true ); - QObject::connect( &timer, SIGNAL( timeout() ), &eventLoop, SLOT( quit() ) ); - QObject::connect( this, SIGNAL( parquetDownloadFinished( const QByteArray&, const QString& ) ), &eventLoop, SLOT( quit() ) ); - - requestBlobDownload( blobId ); - - timer.start( RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec(); - - for ( const auto& blobData : m_redirectInfo ) - { - if ( blobData.objectId == blobId ) - { - contents = blobData.contents; - return; - } - } - } ); - - return contents; -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestParametersBlobIdForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ) -{ - auto requestCallable = [this, caseId, ensembleName] { requestParametersBlobIdForEnsemble( caseId, ensembleName ); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::blobIdFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestParametersBlobIdForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ) -{ - requestTokenBlocking(); - - QNetworkRequest networkRequest; - - // Properly URL-encode the path components - QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); - - QString url = QString( "%1/cases/%2/ensembles/%3/parameters/blob_id" ).arg( server() ).arg( caseId.get() ).arg( encodedEnsembleName ); - networkRequest.setUrl( QUrl( url ) ); - - addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - - auto reply = networkAccessManager()->get( networkRequest ); - - connect( reply, - &QNetworkReply::finished, - [this, reply, ensembleName, caseId]() - { - // parseBlobId handles the error case and always emits blobIdFinished, so the blocking - // caller returns immediately instead of waiting for the request to time out. - parseBlobId( reply, caseId, ensembleName, "", true ); - } ); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestBlobIdForEnsemble( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) -{ - requestTokenBlocking(); - - QNetworkRequest networkRequest; - - // Properly URL-encode the path components - QString encodedVectorName = QUrl::toPercentEncoding( vectorName ); - QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); - - QString url = - QString( "%1/cases/%2/ensembles/%3/vectors/%4/blob_id" ).arg( server() ).arg( caseId.get() ).arg( encodedEnsembleName ).arg( encodedVectorName ); - networkRequest.setUrl( QUrl( url ) ); - - addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - - auto reply = networkAccessManager()->get( networkRequest ); - - connect( reply, - &QNetworkReply::finished, - [this, reply, ensembleName, caseId, vectorName]() - { - // parseBlobId handles the error case and always emits blobIdFinished, so the blocking - // caller returns immediately instead of waiting for the request to time out. - parseBlobId( reply, caseId, ensembleName, vectorName, false ); // false = vector data - } ); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestBlobIdForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) -{ - auto requestCallable = [this, caseId, ensembleName, vectorName] { requestBlobIdForEnsemble( caseId, ensembleName, vectorName ); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::blobIdFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestBlobDownload( const QString& blobId ) -{ - requestTokenBlocking(); - - QString url = QString( "%1/blobs/%2/sas_token_and_blob_base_uri" ).arg( server() ).arg( blobId ); - - QNetworkRequest networkRequest; - networkRequest.setUrl( QUrl( url ) ); - - addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - - auto reply = networkAccessManager()->get( networkRequest ); - - connect( reply, - &QNetworkReply::finished, - [this, reply, blobId, url]() - { - reply->deleteLater(); // don't leak the reply - - if ( reply->error() != QNetworkReply::NoError ) - { - RiaLogging::error( ( "Download failed: " + url + " failed. " + reply->errorString() ).toStdString() ); - return; - } - - // The backend returns BlobAccessInfo as JSON: { "sasToken": "...", "blobStoreBaseUri": "..." } - const QByteArray contents = reply->readAll(); - QJsonParseError parseError; - const QJsonDocument doc = QJsonDocument::fromJson( contents, &parseError ); - if ( parseError.error != QJsonParseError::NoError || !doc.isObject() ) - { - RiaLogging::error( - std::format( "Could not parse blob access info response as JSON: {}", parseError.errorString().toStdString() ) ); - return; - } - - const QJsonObject obj = doc.object(); - const QString sasToken = obj.value( "sasToken" ).toString(); - const QString blobBaseUri = obj.value( "blobStoreBaseUri" ).toString(); - if ( blobBaseUri.isEmpty() ) - { - RiaLogging::error( "Blob access info response did not contain a blobStoreBaseUri." ); - return; - } - - const QString sasUri = constructSasUri( blobBaseUri, blobId, sasToken ); - requestBlobBySasUri( blobId, sasUri ); - } ); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestBlobBySasUri( const QString& blobId, const QString& sasUri ) -{ - RiaLogging::debug( std::format( "Requesting blob. Id: {} SAS URI: {}", blobId, sasUri ) ); - - QNetworkRequest networkRequest; - networkRequest.setUrl( sasUri ); - - // The pre-signed SAS URI carries its own credential (signature in the query string), - // so no Authorization header is added here. Do NOT forward the bearer token to the - // storage host. Redirect policy is set explicitly so behaviour is not Qt-version dependent. - networkRequest.setAttribute( QNetworkRequest::RedirectPolicyAttribute, QNetworkRequest::NoLessSafeRedirectPolicy ); - - auto reply = networkAccessManager()->get( networkRequest ); - - connect( reply, - &QNetworkReply::finished, - [this, reply, blobId, sasUri]() - { - reply->deleteLater(); - - if ( reply->error() != QNetworkReply::NoError ) - { - QString errorMessage = "Download failed: " + sasUri + " failed." + reply->errorString(); - RiaLogging::error( errorMessage.toStdString() ); - - emit parquetDownloadFinished( {}, sasUri ); - return; - } - - auto statusCode = reply->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt(); - auto contentLength = reply->header( QNetworkRequest::ContentLengthHeader ).toLongLong(); - auto bytesAvailable = reply->bytesAvailable(); - - RiaLogging::debug( - std::format( "Response: status={}, content-length={}, bytes-available={}", statusCode, contentLength, bytesAvailable ) ); - - auto contents = reply->readAll(); - - RiaLogging::debug( std::format( "Read {} bytes from reply", contents.size() ) ); - - // Guard against a silently truncated transfer: a dropped connection on a - // chunked response can still report NoError. If the server told us how many - // bytes to expect and we got fewer, treat it as a failure. - if ( contentLength > 0 && contents.size() != contentLength ) - { - RiaLogging::error( std::format( "Download truncated: expected {} bytes, got {}.", contentLength, contents.size() ) ); - - emit parquetDownloadFinished( {}, sasUri ); - return; - } - - QString msg = "Received data from : " + sasUri; - RiaLogging::debug( msg.toStdString() ); - - parquetDownloadComplete( blobId, contents, sasUri ); - - emit parquetDownloadFinished( contents, sasUri ); - } ); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -QByteArray RiaSumoConnector::requestParquetDataBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) -{ - requestBlobIdForEnsembleBlocking( caseId, ensembleName, vectorName ); - - if ( m_blobId.empty() ) return {}; - - // The REST API now returns the complete blob URL, not just an ID - return downloadBlobBlocking( m_blobId.back() ); -} - -//-------------------------------------------------------------------------------------------------- -/// Assemble the pre-signed download URI: {blobStoreBaseUri}/{blobId}?{sasToken} -//-------------------------------------------------------------------------------------------------- -QString RiaSumoConnector::constructSasUri( const QString& blobStoreBaseUri, const QString& blobId, const QString& sasToken ) -{ - QString sasUri = blobStoreBaseUri; - if ( !sasUri.endsWith( '/' ) ) sasUri += '/'; - sasUri += blobId; - if ( !sasToken.isEmpty() ) - { - sasUri += ( sasToken.startsWith( '?' ) ? sasToken : ( "?" + sasToken ) ); - } - return sasUri; -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -/// The request runs on the transfer thread, and the event loop that waits for it runs there too. Waiting on -/// the GUI thread instead would dispatch everything except user input, letting the view update code re-enter -/// a cell result load that was still running and download the same grid property a second time. An event -/// loop on the transfer thread dispatches no GUI events, so nothing can re-enter. -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::wrapAndCallNetworkRequest( std::function requestCallable, const QMetaMethod& signalMethod ) -{ - // Acquire the token before handing over to the transfer thread. The OAuth flow can open a browser and - // its objects live on the GUI thread, so it must not run on the transfer thread. - requestTokenBlocking(); - - runOnTransferThreadBlocking( [this, requestCallable, signalMethod]() { waitForRequest( requestCallable, signalMethod ); } ); -} - -//-------------------------------------------------------------------------------------------------- -/// Issue the request and wait for its completion signal. Always called on the transfer thread. -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::waitForRequest( const std::function& requestCallable, const QMetaMethod& signalMethod ) -{ - QEventLoop eventLoop; - - QTimer timer; - timer.setSingleShot( true ); - - QObject::connect( &timer, &QTimer::timeout, [] { RiaLogging::error( "Sumo request timed out." ); } ); - QObject::connect( &timer, &QTimer::timeout, &eventLoop, &QEventLoop::quit ); - - // Not able to use the modern connect syntax here, as the signal is communicated as a QMetaMethod - int methodIndex = eventLoop.metaObject()->indexOfMethod( "quit()" ); - QMetaMethod quitMethod = eventLoop.metaObject()->method( methodIndex ); - QObject::connect( this, signalMethod, &eventLoop, quitMethod ); - - // Call the function that will execute the request - requestCallable(); - - timer.start( RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec(); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parseAssets( QNetworkReply* reply ) -{ - QByteArray result = reply->readAll(); - reply->deleteLater(); - - if ( reply->error() == QNetworkReply::NoError ) - { - QJsonDocument doc = QJsonDocument::fromJson( result ); - QJsonArray jsonArray = doc.array(); - - m_assets.clear(); - - // This json is an array of AssetInfo - for ( const QJsonValue& assetInfo : jsonArray ) - { - QString assetName = assetInfo["name"].toString(); - m_assets.push_back( SumoAsset{ SumoAssetId( "" ), "", assetName } ); - } - - for ( auto a : m_assets ) - { - RiaLogging::debug( std::format( "Asset: {}", a.name ) ); - } - } - else - { - RiaLogging::error( std::format( "Request assets failed: '{}'", reply->errorString() ) ); - } - - emit assetsFinished(); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parseCases( QNetworkReply* reply ) -{ - QByteArray result = reply->readAll(); - reply->deleteLater(); - - if ( reply->error() == QNetworkReply::NoError ) - { - QJsonDocument doc = QJsonDocument::fromJson( result ); - QJsonArray jsonArray = doc.array(); - - m_cases.clear(); - - for ( const QJsonValue& value : jsonArray ) - { - QJsonObject caseObj = value.toObject(); - - QString id = caseObj["id"].toString(); - QString kind = ""; - QString name = caseObj["name"].toString(); - m_cases.push_back( SumoCase{ SumoCaseId( id ), kind, name } ); - } - - RiaLogging::debug( std::format( "Case count : {}", m_cases.size() ) ); - } - else - { - RiaLogging::error( std::format( "Request cases failed: '{}'", reply->errorString() ) ); - } - - emit casesFinished(); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parseVectorNames( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName ) -{ - QByteArray result = reply->readAll(); - reply->deleteLater(); - - m_vectorNames.clear(); - - if ( reply->error() == QNetworkReply::NoError ) - { - QJsonDocument doc = QJsonDocument::fromJson( result ); - QJsonArray jsonArray = doc.array(); - - for ( const QJsonValue& value : jsonArray ) - { - QJsonObject vectorObj = value.toObject(); - QString vectorName = vectorObj["name"].toString(); - m_vectorNames.push_back( vectorName ); - } - } - else - { - RiaLogging::error( std::format( "Request vector names failed: '{}'", reply->errorString() ) ); - } - - emit vectorNamesFinished(); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parseRealizationNumbers( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName ) -{ - QByteArray result = reply->readAll(); - reply->deleteLater(); - - if ( reply->error() == QNetworkReply::NoError ) - { - QJsonDocument doc = QJsonDocument::fromJson( result ); - QJsonArray jsonArray = doc.array(); - - for ( const QJsonValue& value : jsonArray ) - { - int intValue = value.toInt(); - auto realizationId = QString::number( intValue ); - m_realizationIds.push_back( realizationId ); - } - } - else - { - RiaLogging::error( std::format( "Request realization IDs failed: '{}'", reply->errorString() ) ); - } - - emit realizationIdsFinished(); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parseGridInfo( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName ) -{ - QByteArray result = reply->readAll(); - reply->deleteLater(); - - m_gridInfos.clear(); - - if ( reply->error() == QNetworkReply::NoError ) - { - QJsonDocument doc = QJsonDocument::fromJson( result ); - QJsonArray jsonArray = doc.array(); - - for ( const QJsonValue& value : jsonArray ) - { - QJsonObject gridObj = value.toObject(); - - SumoGridInfo gridInfo; - gridInfo.name = gridObj["name"].toString(); - - for ( const QJsonValue& realizationValue : gridObj["realizations"].toArray() ) - { - gridInfo.realizations.push_back( realizationValue.toInt() ); - } - - m_gridInfos.push_back( gridInfo ); - } - - RiaLogging::debug( std::format( "Grid info count : {}", m_gridInfos.size() ) ); - } - else - { - RiaLogging::error( std::format( "Request grid info failed: '{}'", reply->errorString().toStdString() ) ); - } - - emit gridInfoFinished(); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parseGridPropertyInfo( QNetworkReply* reply, - const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization ) -{ - QByteArray result = reply->readAll(); - reply->deleteLater(); - - m_gridPropertyInfos.clear(); - - if ( reply->error() == QNetworkReply::NoError ) - { - QJsonDocument doc = QJsonDocument::fromJson( result ); - QJsonArray jsonArray = doc.array(); - - for ( const QJsonValue& value : jsonArray ) - { - QJsonObject propertyObj = value.toObject(); - - SumoGridPropertyInfo propertyInfo; - propertyInfo.name = propertyObj["propertyName"].toString(); - - // isoDateOrInterval is null for static properties. - const auto isoValue = propertyObj["isoDateOrInterval"]; - if ( !isoValue.isNull() ) propertyInfo.isoDateOrInterval = isoValue.toString(); - - m_gridPropertyInfos.push_back( propertyInfo ); - } - - RiaLogging::debug( std::format( "Grid property info count : {}", m_gridPropertyInfos.size() ) ); - } - else - { - RiaLogging::error( std::format( "Request grid property info failed: '{}'", reply->errorString().toStdString() ) ); - } - - emit gridPropertyInfoFinished(); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parseBlobId( QNetworkReply* reply, - const SumoCaseId& caseId, - const QString& ensembleName, - const QString& vectorName, - bool isParameters ) -{ - QByteArray result = reply->readAll(); - reply->deleteLater(); - - m_blobId.clear(); - - if ( reply->error() == QNetworkReply::NoError ) - { - // The REST API returns a plain string (the blob id) - QString blobId = QString::fromUtf8( result ).trimmed(); - - // Remove quotes if present (FastAPI returns strings with quotes) - if ( blobId.startsWith( '"' ) && blobId.endsWith( '"' ) ) - { - blobId = blobId.mid( 1, blobId.length() - 2 ); - } - - m_blobId.push_back( blobId ); - - // Context-aware logging - if ( isParameters ) - { - RiaLogging::debug( std::format( "Received blob ID for parameters: {}", blobId.toStdString() ) ); - } - else - { - RiaLogging::debug( std::format( "Received blob ID for vector '{}': {}", vectorName.toStdString(), blobId.toStdString() ) ); - } - } - else - { - // Context-aware error logging - QString errorContext = isParameters ? "parameters" : QString( "vector '%1'" ).arg( vectorName ); - RiaLogging::error( std::format( "Request blob ID failed for {}: {}", errorContext.toStdString(), reply->errorString().toStdString() ) ); - } - - emit blobIdFinished(); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::addStandardHeader( QNetworkRequest& networkRequest, const QString& token, const QString& contentType ) +void RiaSumoConnector::addStandardHeader( QNetworkRequest& networkRequest, const QString& token, const QString& contentType ) { networkRequest.setHeader( QNetworkRequest::ContentTypeHeader, contentType ); networkRequest.setRawHeader( "Authorization", "Bearer " + token.toUtf8() ); } - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -QNetworkReply* RiaSumoConnector::makeDownloadRequest( const QString& url, const QString& token, const QString& contentType ) -{ - QNetworkRequest m_networkRequest; - m_networkRequest.setUrl( QUrl( url ) ); - - addStandardHeader( m_networkRequest, token, contentType ); - - auto reply = networkAccessManager()->get( m_networkRequest ); - return reply; -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestParquetData( const QString& url, const QString& token ) -{ - RiaLogging::debug( "Requesting download of parquet from: " + url.toStdString() ); - - auto reply = makeDownloadRequest( url, token, RiaCloudDefines::contentTypeJson() ); - connect( reply, - &QNetworkReply::finished, - [this, reply, url]() - { - if ( reply->error() == QNetworkReply::NoError ) - { - QByteArray contents = reply->readAll(); - RiaLogging::debug( std::format( "Download succeeded: {} bytes.", contents.length() ) ); - RiaLogging::debug( std::format( "Download succeeded for url: {}", url.toStdString() ) ); - emit parquetDownloadFinished( contents, "" ); - } - else - { - QString errorMessage = "Download failed: " + url + " failed." + reply->errorString(); - RiaLogging::error( errorMessage.toStdString() ); - emit parquetDownloadFinished( QByteArray(), errorMessage ); - } - } ); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::assets() const -{ - return m_assets; -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::cases() const -{ - return m_cases; -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::ensembleNamesForCase( const SumoCaseId& caseId ) const -{ - std::vector ensembleNames; - for ( const auto& ensemble : m_ensembleNames ) - { - if ( ensemble.caseId == caseId ) - { - ensembleNames.push_back( ensemble.name ); - } - } - return ensembleNames; -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::vectorNames() const -{ - return m_vectorNames; -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::realizationIds() const -{ - return m_realizationIds; -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::gridInfos() const -{ - return m_gridInfos; -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::gridPropertyInfos() const -{ - return m_gridPropertyInfos; -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::blobIds() const -{ - return m_blobId; -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::blobContents() const -{ - return m_redirectInfo; -} diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h index 036932668ce..4d6ef11607b 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h @@ -20,6 +20,9 @@ #include "RiaCloudConnector.h" #include "RiaSumoDefines.h" +#include "RiaSumoExplore.h" +#include "RiaSumoGrid.h" +#include "RiaSumoSummary.h" #include #include @@ -34,53 +37,6 @@ class QThread; using SumoObjectId = QString; -struct SumoAsset -{ - SumoAssetId assetId; - - QString kind; - QString name; -}; - -struct SumoCase -{ - SumoCaseId caseId; - - QString kind; - QString name; -}; - -struct SumoRedirect -{ - SumoObjectId objectId; - QString blobName; - QString url; - QString redirectBaseUri; - QString redirectAuth; - QByteArray contents; -}; - -struct SumoEnsemble -{ - SumoCaseId caseId; - QString name; -}; - -struct SumoGridInfo -{ - QString name; - std::vector realizations; -}; - -struct SumoGridPropertyInfo -{ - QString name; - - // Empty for a static property. For a dynamic property this is either a single timestamp ("2018-01-01") - // or an interval ("2018-01-01/2019-01-01"). ResInsight currently only supports the single-timestamp form. - QString isoDateOrInterval; -}; - //================================================================================================== /// //================================================================================================== @@ -101,191 +57,55 @@ class RiaSumoConnector : public RiaCloudConnector QString server() const override; - void requestAssets(); - void requestAssetsBlocking(); - - void requestCasesForField( const QString& fieldName ); - void requestCasesForFieldBlocking( const QString& fieldName ); + // Download blobs by id and return their contents. Getting a blob takes two round trips, one for the + // pre-signed URI and one for the data, and a batch does each of those as one concurrent group. + QByteArray downloadBlobBlocking( const QString& blobId ); + std::map downloadBlobsBlocking( const std::vector& blobIds ); - void requestEnsembleByCasesId( const SumoCaseId& caseId ); - void requestEnsembleByCasesIdBlocking( const SumoCaseId& caseId ); + // What Sumo holds: assets, cases, ensembles and realizations. + RiaSumoExplore& explore(); - void requestVectorNamesForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ); - void requestVectorNamesForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ); + // The grid data of a case. Owned here so its blob cache lives as long as the connection. + RiaSumoGrid& grid(); - void requestRealizationIdsForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ); - void requestRealizationIdsForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ); + // The summary data of a case. + RiaSumoSummary& summary(); - void requestParametersBlobIdForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ); - void requestParametersBlobIdForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ); - QByteArray requestParametersParquetDataBlocking( const SumoCaseId& caseId, const QString& ensembleName ); + // Transport used by the data specific delegates. Every request goes through the transfer thread, so + // the calling thread waits without dispatching events. + QByteArray getBlocking( const QString& url ); - void requestBlobIdForEnsemble( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ); - void requestBlobIdForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ); + // The REST API returns a blob id as a plain string, quoted by FastAPI. + static QString blobIdFromBody( const QByteArray& body ); - void requestBlobDownload( const QString& blobId ); - void requestBlobBySasUri( const QString& blobId, const QString& sasUri ); - - QByteArray requestParquetDataBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ); - - void requestGridInfoForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ); - void requestGridInfoForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ); - - void requestGridBlobIdForEnsemble( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); - void requestGridBlobIdForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); - - QByteArray requestGridDataBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); - - void requestGridPropertyInfoForEnsemble( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); - void requestGridPropertyInfoForEnsembleBlocking( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization ); - - QString requestGridPropertyBlobIdBlocking( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization, - const QString& propertyName, - const QString& isoDateOrInterval ); + void addStandardHeader( QNetworkRequest& networkRequest, const QString& token, const QString& contentType ); + void runOnTransferThreadBlocking( const std::function& work ); - QByteArray requestGridPropertyDataBlocking( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization, - const QString& propertyName, - const QString& isoDateOrInterval ); + // The network manager belonging to the calling thread: the transfer thread manager when called from + // there, otherwise the one owned by RiaCloudConnector on the GUI thread. + QNetworkAccessManager* networkAccessManager(); - // Download several time steps of one grid property concurrently and put them in the blob cache, so the - // following per time step requests are served without going to Sumo. Entries already cached are skipped. - void prefetchGridPropertyDataBlocking( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization, - const QString& propertyName, - const std::vector& isoDatesOrIntervals ); + static void waitForRepliesToFinish( const std::vector& replies ); - std::vector assets() const; - std::vector cases() const; - std::vector ensembleNamesForCase( const SumoCaseId& caseId ) const; - std::vector vectorNames() const; - std::vector realizationIds() const; - std::vector gridInfos() const; - std::vector gridPropertyInfos() const; - std::vector blobIds() const; - std::vector blobContents() const; + // Issue and collect the two round trips a blob transfer needs. Called on the transfer thread. + std::map downloadBlobs( const std::vector& blobIds ); public slots: - void parseAssets( QNetworkReply* reply ); - void parseEnsembleNames( QNetworkReply* reply, const SumoCaseId& caseId ); - void parseCases( QNetworkReply* reply ); - void parseVectorNames( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName ); - void parseRealizationNumbers( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName ); - void parseGridInfo( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName ); - void parseGridPropertyInfo( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); - void parseBlobId( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName, bool isParameters ); - void requestFailed( const QAbstractOAuth::Error error ); - void parquetDownloadComplete( const QString& blobId, const QByteArray&, const QString& url ); - -signals: - void fileDownloadFinished( const QString& fileId, const QString& filePath ); - void casesFinished(); - void wellsFinished(); - void wellboresFinished( const QString& wellId ); - void wellboreTrajectoryFinished( const QString& wellboreId ); - void parquetDownloadFinished( const QByteArray& contents, const QString& url ); - void ensembleNamesFinished(); - void vectorNamesFinished(); - void blobIdFinished(); - void assetsFinished(); - void realizationIdsFinished(); - void gridInfoFinished(); - void gridPropertyInfoFinished(); private: - void addStandardHeader( QNetworkRequest& networkRequest, const QString& token, const QString& contentType ); - - QNetworkReply* makeDownloadRequest( const QString& url, const QString& token, const QString& contentType ); - void requestParquetData( const QString& url, const QString& token ); - static QString constructSasUri( const QString& blobStoreBaseUri, const QString& blobId, const QString& sasToken ); - void wrapAndCallNetworkRequest( std::function requestCallable, const QMetaMethod& signalMethod ); - void waitForRequest( const std::function& requestCallable, const QMetaMethod& signalMethod ); - - QByteArray gridPropertyBlobFromCache( const QString& cacheKey ); - void insertGridPropertyBlobInCache( const QString& cacheKey, const QByteArray& contents ); - - static QString gridPropertyCacheKey( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization, - const QString& propertyName, - const QString& isoDateOrInterval ); - - QNetworkReply* makeGridPropertyBlobIdRequest( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization, - const QString& propertyName, - const QString& isoDateOrInterval ); - - static QString blobIdFromReply( QNetworkReply* reply, const QString& propertyName ); - - void fetchGridPropertyBatch( const SumoCaseId& caseId, - const QString& ensembleName, - const QString& gridName, - int realization, - const QString& propertyName, - const std::vector& timestampsToFetch, - const std::vector& cacheKeys ); - - static void waitForRepliesToFinish( const std::vector& replies ); - - // Download one blob by id and return its contents, waiting on the transfer thread. - QByteArray downloadBlobBlocking( const QString& blobId ); - - // The network manager belonging to the calling thread: the transfer thread manager when called from - // there, otherwise the one owned by RiaCloudConnector on the GUI thread. - QNetworkAccessManager* networkAccessManager(); - - // Run work on the transfer thread and block the caller until it returns. The caller waits on a semaphore - // and dispatches no events, so nothing can re-enter the code that started the request. Called from the - // transfer thread itself, the work is run directly, which keeps nested blocking requests working. - void runOnTransferThreadBlocking( const std::function& work ); + QString sasUriFromReply( QNetworkReply* reply, const QString& blobId ); + static QByteArray blobContentsFromReply( QNetworkReply* reply, const QString& sasUri ); + static QByteArray replyBody( QNetworkReply* reply, const QString& url ); private: std::function m_serverUrlProvider; - std::vector m_assets; - std::vector m_cases; - std::vector m_vectorNames; - std::vector m_realizationIds; - std::vector m_ensembleNames; - std::vector m_gridInfos; - - std::vector m_gridPropertyInfos; - - std::vector m_blobId; - - std::vector m_redirectInfo; - - // Downloaded grid-property blobs, keyed by the full property identity (case, ensemble, grid, realization, - // property, timestamp). Displaying a property computes its global legend range across all time steps, so a - // property is requested repeatedly; the cache ensures each blob is fetched from Sumo at most once. - // - // Bounded by total byte size, not by entry count, as the blob size follows the grid size and varies by orders - // of magnitude. The least recently used entries are evicted when the limit is exceeded. - struct GridPropertyBlobCacheEntry - { - QByteArray contents; - std::list::iterator orderIterator; - }; - - std::map m_gridPropertyBlobCache; - std::list m_gridPropertyBlobCacheOrder; // most recently used at front - size_t m_gridPropertyBlobCacheSizeBytes = 0; + RiaSumoExplore m_explore; + RiaSumoGrid m_grid; + RiaSumoSummary m_summary; // Transfers run on their own thread so the calling thread can wait without dispatching events. Waiting on // a nested event loop on the GUI thread let the view update code re-enter a load that was still running, diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.cpp new file mode 100644 index 00000000000..3daae970353 --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.cpp @@ -0,0 +1,168 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "RiaSumoExplore.h" + +#include "RiaLogging.h" +#include "RiaQStringFormatter.h" +#include "RiaSumoConnector.h" + +#include +#include +#include +#include + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RiaSumoExplore::RiaSumoExplore( RiaSumoConnector& connector ) + : m_connector( connector ) +{ +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::assets() +{ + const QString url = QString( "%1/assets" ).arg( m_connector.server() ); + + return parseAssets( m_connector.getBlocking( url ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::cases( const QString& assetName ) +{ + const QString url = QString( "%1/cases?asset_name=%2" ).arg( m_connector.server() ).arg( QString( QUrl::toPercentEncoding( assetName ) ) ); + + return parseCases( m_connector.getBlocking( url ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::ensembleNames( const SumoCaseId& caseId ) +{ + const QString url = QString( "%1/cases/%2/ensembles" ).arg( m_connector.server() ).arg( caseId.get() ); + + return parseEnsembleNames( m_connector.getBlocking( url ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::realizationIds( const SumoCaseId& caseId, const QString& ensembleName ) +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + + const QString url = + QString( "%1/cases/%2/ensembles/%3/realizations" ).arg( m_connector.server() ).arg( caseId.get() ).arg( encodedEnsembleName ); + + return parseRealizationIds( m_connector.getBlocking( url ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::parseAssets( const QByteArray& body ) +{ + std::vector assets; + + QJsonDocument doc = QJsonDocument::fromJson( body ); + QJsonArray jsonArray = doc.array(); + + // This json is an array of AssetInfo + for ( const QJsonValue& assetInfo : jsonArray ) + { + QString assetName = assetInfo["name"].toString(); + assets.push_back( SumoAsset{ SumoAssetId( "" ), "", assetName } ); + } + + for ( const auto& asset : assets ) + { + RiaLogging::debug( std::format( "Asset: {}", asset.name ) ); + } + + return assets; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::parseCases( const QByteArray& body ) +{ + std::vector cases; + + QJsonDocument doc = QJsonDocument::fromJson( body ); + QJsonArray jsonArray = doc.array(); + + for ( const QJsonValue& value : jsonArray ) + { + QJsonObject caseObj = value.toObject(); + + QString id = caseObj["id"].toString(); + QString kind = ""; + QString name = caseObj["name"].toString(); + cases.push_back( SumoCase{ SumoCaseId( id ), kind, name } ); + } + + RiaLogging::debug( std::format( "Case count : {}", cases.size() ) ); + + return cases; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::parseEnsembleNames( const QByteArray& body ) +{ + std::vector ensembleNames; + + QJsonDocument doc = QJsonDocument::fromJson( body ); + QJsonArray jsonArray = doc.array(); + + for ( const QJsonValue& value : jsonArray ) + { + QJsonObject ensembleObj = value.toObject(); + ensembleNames.push_back( ensembleObj["name"].toString() ); + } + + RiaLogging::debug( std::format( "Ensemble count : {}", ensembleNames.size() ) ); + + return ensembleNames; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::parseRealizationIds( const QByteArray& body ) +{ + std::vector realizationIds; + + QJsonDocument doc = QJsonDocument::fromJson( body ); + QJsonArray jsonArray = doc.array(); + + for ( const QJsonValue& value : jsonArray ) + { + realizationIds.push_back( QString::number( value.toInt() ) ); + } + + return realizationIds; +} diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.h new file mode 100644 index 00000000000..60c66aa7ad2 --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.h @@ -0,0 +1,70 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "RiaSumoDefines.h" + +#include +#include + +#include + +class RiaSumoConnector; + +struct SumoAsset +{ + SumoAssetId assetId; + + QString kind; + QString name; +}; + +struct SumoCase +{ + SumoCaseId caseId; + + QString kind; + QString name; +}; + +//================================================================================================== +/// Finding your way around what Sumo holds: the assets available, the cases of an asset, the ensembles +/// of a case and the realizations of an ensemble. Requests are made through RiaSumoConnector, which owns +/// the connection and does the transfers, and every call returns its result rather than leaving it in +/// shared state. +//================================================================================================== +class RiaSumoExplore +{ +public: + explicit RiaSumoExplore( RiaSumoConnector& connector ); + + std::vector assets(); + std::vector cases( const QString& assetName ); + std::vector ensembleNames( const SumoCaseId& caseId ); + std::vector realizationIds( const SumoCaseId& caseId, const QString& ensembleName ); + +private: + static std::vector parseAssets( const QByteArray& body ); + static std::vector parseCases( const QByteArray& body ); + static std::vector parseEnsembleNames( const QByteArray& body ); + static std::vector parseRealizationIds( const QByteArray& body ); + +private: + RiaSumoConnector& m_connector; +}; diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.cpp new file mode 100644 index 00000000000..d2250994de7 --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.cpp @@ -0,0 +1,386 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "RiaSumoGrid.h" + +#include "RiaCloudDefines.h" +#include "RiaLogging.h" +#include "RiaSumoConnector.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RiaSumoGrid::RiaSumoGrid( RiaSumoConnector& connector ) + : m_connector( connector ) + , m_blobCache( RiaSumoDefines::gridPropertyCacheLimitBytes() ) +{ +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoGrid::gridInfo( const SumoCaseId& caseId, const QString& ensembleName ) +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + + const QString url = + QString( "%1/cases/%2/ensembles/%3/grid_info_list" ).arg( m_connector.server() ).arg( caseId.get() ).arg( encodedEnsembleName ); + + return parseGridInfo( m_connector.getBlocking( url ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QByteArray RiaSumoGrid::gridData( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ) +{ + const QString blobId = gridBlobId( caseId, ensembleName, gridName, realization ); + if ( blobId.isEmpty() ) return {}; + + return m_connector.downloadBlobBlocking( blobId ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoGrid::gridBlobId( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ) +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + const QString encodedGridName = QUrl::toPercentEncoding( gridName ); + + const QString url = QString( "%1/cases/%2/ensembles/%3/grids/%4/realizations/%5/blob_id" ) + .arg( m_connector.server() ) + .arg( caseId.get() ) + .arg( encodedEnsembleName ) + .arg( encodedGridName ) + .arg( realization ); + + return blobIdFromBody( m_connector.getBlocking( url ), gridName ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector + RiaSumoGrid::propertyInfo( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ) +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + const QString encodedGridName = QUrl::toPercentEncoding( gridName ); + + const QString url = QString( "%1/cases/%2/ensembles/%3/grids/%4/realizations/%5/property_info_list" ) + .arg( m_connector.server() ) + .arg( caseId.get() ) + .arg( encodedEnsembleName ) + .arg( encodedGridName ) + .arg( realization ); + + return parsePropertyInfo( m_connector.getBlocking( url ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QByteArray RiaSumoGrid::propertyData( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) +{ + // Serve from cache when possible. Keyed by the full property identity (not the blob id), a repeat request + // is answered without even asking Sumo for the blob id. This avoids re-downloading every time step when a + // property's global legend range is computed, and again when it is displayed. + const QString key = cacheKey( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + if ( auto cachedContents = m_blobCache.lookup( key ); !cachedContents.isEmpty() ) + { + return cachedContents; + } + + const QString blobId = propertyBlobId( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + if ( blobId.isEmpty() ) return {}; + + QByteArray contents = m_connector.downloadBlobBlocking( blobId ); + + m_blobCache.insert( key, contents ); + + return contents; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoGrid::prefetchPropertyData( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const std::vector& isoDatesOrIntervals ) +{ + // Only fetch what is not already cached, and drop duplicates so a time step is never requested twice. + std::vector cacheKeys; + std::vector timestampsToFetch; + for ( const auto& isoDateOrInterval : isoDatesOrIntervals ) + { + const QString key = cacheKey( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + + if ( std::ranges::find( cacheKeys, key ) != cacheKeys.end() ) continue; + if ( m_blobCache.contains( key ) ) continue; + + cacheKeys.push_back( key ); + timestampsToFetch.push_back( isoDateOrInterval ); + } + + if ( timestampsToFetch.size() < 2 ) return; // nothing to gain over the single time step path + + m_connector.runOnTransferThreadBlocking( + [&]() { fetchPropertyBatch( caseId, ensembleName, gridName, realization, propertyName, timestampsToFetch, cacheKeys ); } ); +} + +//-------------------------------------------------------------------------------------------------- +/// Resolve the blob ids of a batch of time steps, download the blobs, and put them in the cache. Always +/// called on the transfer thread, where the event loops it waits on dispatch no GUI events. +//-------------------------------------------------------------------------------------------------- +void RiaSumoGrid::fetchPropertyBatch( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const std::vector& timestampsToFetch, + const std::vector& cacheKeys ) +{ + // Phase 1: resolve all blob ids concurrently. + std::vector blobIdReplies; + for ( const auto& isoDateOrInterval : timestampsToFetch ) + { + blobIdReplies.push_back( makePropertyBlobIdRequest( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ) ); + } + + RiaSumoConnector::waitForRepliesToFinish( blobIdReplies ); + + std::vector blobIds; + for ( auto reply : blobIdReplies ) + { + blobIds.push_back( blobIdFromReply( reply, propertyName ) ); + } + + // Phase 2: download all resolved blobs as one group. + std::vector blobIdsToDownload; + for ( const auto& blobId : blobIds ) + { + if ( !blobId.isEmpty() ) blobIdsToDownload.push_back( blobId ); + } + + const auto contentsByBlobId = m_connector.downloadBlobs( blobIdsToDownload ); + + // Cache what arrived. Anything missing failed to download; the per time step path fetches it again later. + for ( size_t i = 0; i < blobIds.size(); i++ ) + { + if ( blobIds[i].isEmpty() ) continue; + + if ( auto it = contentsByBlobId.find( blobIds[i] ); it != contentsByBlobId.end() ) + { + m_blobCache.insert( cacheKeys[i], it->second ); + } + } +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoGrid::propertyBlobIdUrl( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) const +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + const QString encodedGridName = QUrl::toPercentEncoding( gridName ); + const QString encodedPropertyName = QUrl::toPercentEncoding( propertyName ); + + QString url = QString( "%1/cases/%2/ensembles/%3/grids/%4/realizations/%5/properties/%6/blob_id" ) + .arg( m_connector.server() ) + .arg( caseId.get() ) + .arg( encodedEnsembleName ) + .arg( encodedGridName ) + .arg( realization ) + .arg( encodedPropertyName ); + + // The timestamp/interval is an optional query parameter; omit it for static properties. + if ( !isoDateOrInterval.isEmpty() ) + { + url += QString( "?property_iso_date_or_interval=%1" ).arg( QString( QUrl::toPercentEncoding( isoDateOrInterval ) ) ); + } + + return url; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoGrid::propertyBlobId( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) +{ + const QString url = propertyBlobIdUrl( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + + return blobIdFromBody( m_connector.getBlocking( url ), propertyName ); +} + +//-------------------------------------------------------------------------------------------------- +/// Issue the blob id request for one grid property time step. The reply is returned unfinished, so the +/// caller decides how to wait for it: one at a time, or several at once when prefetching. +//-------------------------------------------------------------------------------------------------- +QNetworkReply* RiaSumoGrid::makePropertyBlobIdRequest( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) +{ + const QString url = propertyBlobIdUrl( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + + QNetworkRequest networkRequest; + networkRequest.setUrl( QUrl( url ) ); + m_connector.addStandardHeader( networkRequest, m_connector.token(), RiaCloudDefines::contentTypeJson() ); + + return m_connector.networkAccessManager()->get( networkRequest ); +} + +//-------------------------------------------------------------------------------------------------- +/// Read the blob id off a finished blob id reply. The reply is consumed and scheduled for deletion. +/// +/// Waiting for one specific reply, rather than for a signal shared by all blob id requests, is what makes +/// the mapping correct: a still-pending reply from an earlier property's request can no longer satisfy +/// this wait and hand us its blob id, which previously caused e.g. SWAT to be served the SWCR blob. +//-------------------------------------------------------------------------------------------------- +QString RiaSumoGrid::blobIdFromReply( QNetworkReply* reply, const QString& propertyName ) +{ + if ( !reply ) return {}; + + const bool failed = !reply->isFinished() || reply->error() != QNetworkReply::NoError; + QByteArray body = failed ? QByteArray() : reply->readAll(); + + if ( failed ) + { + RiaLogging::error( std::format( "Request grid property blob ID failed: '{}'", reply->errorString().toStdString() ) ); + } + + reply->deleteLater(); + + return blobIdFromBody( body, propertyName ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoGrid::blobIdFromBody( const QByteArray& body, const QString& name ) +{ + const QString blobId = RiaSumoConnector::blobIdFromBody( body ); + if ( blobId.isEmpty() ) return {}; + + RiaLogging::debug( std::format( "Received blob ID for vector '{}': {}", name.toStdString(), blobId.toStdString() ) ); + + return blobId; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoGrid::cacheKey( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) +{ + return QString( "%1|%2|%3|%4|%5|%6" ).arg( caseId.get(), ensembleName, gridName ).arg( realization ).arg( propertyName, isoDateOrInterval ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoGrid::parseGridInfo( const QByteArray& body ) +{ + std::vector gridInfos; + + QJsonDocument doc = QJsonDocument::fromJson( body ); + QJsonArray jsonArray = doc.array(); + + for ( const QJsonValue& value : jsonArray ) + { + QJsonObject gridObj = value.toObject(); + + SumoGridInfo gridInfo; + gridInfo.name = gridObj["name"].toString(); + + for ( const QJsonValue& realizationValue : gridObj["realizations"].toArray() ) + { + gridInfo.realizations.push_back( realizationValue.toInt() ); + } + + gridInfos.push_back( gridInfo ); + } + + RiaLogging::debug( std::format( "Grid info count : {}", gridInfos.size() ) ); + + return gridInfos; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoGrid::parsePropertyInfo( const QByteArray& body ) +{ + std::vector propertyInfos; + + QJsonDocument doc = QJsonDocument::fromJson( body ); + QJsonArray jsonArray = doc.array(); + + for ( const QJsonValue& value : jsonArray ) + { + QJsonObject propertyObj = value.toObject(); + + SumoGridPropertyInfo propertyInfo; + propertyInfo.name = propertyObj["propertyName"].toString(); + + // isoDateOrInterval is null for static properties. + const auto isoValue = propertyObj["isoDateOrInterval"]; + if ( !isoValue.isNull() ) propertyInfo.isoDateOrInterval = isoValue.toString(); + + propertyInfos.push_back( propertyInfo ); + } + + RiaLogging::debug( std::format( "Grid property info count : {}", propertyInfos.size() ) ); + + return propertyInfos; +} diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.h new file mode 100644 index 00000000000..5eaffe6e791 --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.h @@ -0,0 +1,133 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "RiaSumoBlobCache.h" +#include "RiaSumoDefines.h" + +#include +#include + +#include + +class RiaSumoConnector; +class QNetworkReply; + +struct SumoGridInfo +{ + QString name; + std::vector realizations; +}; + +struct SumoGridPropertyInfo +{ + QString name; + + // Empty for a static property. For a dynamic property this is either a single timestamp ("2018-01-01") + // or an interval ("2018-01-01/2019-01-01"). ResInsight currently only supports the single-timestamp form. + QString isoDateOrInterval; +}; + +//================================================================================================== +/// The grid data of a Sumo case: the grids of an ensemble, the grid geometry itself, and the grid +/// properties. Requests are made through RiaSumoConnector, which owns the connection and does the +/// transfers, and every call returns its result rather than leaving it in shared state. +//================================================================================================== +class RiaSumoGrid +{ +public: + explicit RiaSumoGrid( RiaSumoConnector& connector ); + + std::vector gridInfo( const SumoCaseId& caseId, const QString& ensembleName ); + + // The grid geometry as a binary roff blob. + QByteArray gridData( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); + + std::vector + propertyInfo( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); + + // One time step of one grid property, as a binary roff blob. Served from the blob cache when possible. + QByteArray propertyData( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ); + + // Download several time steps of one property concurrently and put them in the blob cache, so the + // per time step requests that follow are served without going to Sumo. Cached entries are skipped. + void prefetchPropertyData( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const std::vector& isoDatesOrIntervals ); + +private: + QString gridBlobId( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); + + QString propertyBlobId( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ); + + QNetworkReply* makePropertyBlobIdRequest( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ); + + void fetchPropertyBatch( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const std::vector& timestampsToFetch, + const std::vector& cacheKeys ); + + QString propertyBlobIdUrl( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) const; + + // The full identity of one grid property time step, used as blob cache key. + static QString cacheKey( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ); + + static QString blobIdFromBody( const QByteArray& body, const QString& name ); + static QString blobIdFromReply( QNetworkReply* reply, const QString& propertyName ); + + static std::vector parseGridInfo( const QByteArray& body ); + static std::vector parsePropertyInfo( const QByteArray& body ); + +private: + RiaSumoConnector& m_connector; + + // Downloaded grid property blobs, keyed by the full property identity, see cacheKey. + RiaSumoBlobCache m_blobCache; +}; diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp new file mode 100644 index 00000000000..87a11664eac --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp @@ -0,0 +1,133 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "RiaSumoSummary.h" + +#include "RiaLogging.h" +#include "RiaSumoConnector.h" + +#include +#include +#include +#include + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RiaSumoSummary::RiaSumoSummary( RiaSumoConnector& connector ) + : m_connector( connector ) +{ +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoSummary::vectorNames( const SumoCaseId& caseId, const QString& ensembleName ) +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + + const QString url = + QString( "%1/cases/%2/ensembles/%3/vector_list" ).arg( m_connector.server() ).arg( caseId.get() ).arg( encodedEnsembleName ); + + return parseVectorNames( m_connector.getBlocking( url ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QByteArray RiaSumoSummary::vectorData( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) +{ + const QString blobId = vectorBlobId( caseId, ensembleName, vectorName ); + if ( blobId.isEmpty() ) return {}; + + return m_connector.downloadBlobBlocking( blobId ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QByteArray RiaSumoSummary::parameterData( const SumoCaseId& caseId, const QString& ensembleName ) +{ + const QString blobId = parameterBlobId( caseId, ensembleName ); + if ( blobId.isEmpty() ) return {}; + + return m_connector.downloadBlobBlocking( blobId ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoSummary::vectorBlobId( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + const QString encodedVectorName = QUrl::toPercentEncoding( vectorName ); + + const QString url = QString( "%1/cases/%2/ensembles/%3/vectors/%4/blob_id" ) + .arg( m_connector.server() ) + .arg( caseId.get() ) + .arg( encodedEnsembleName ) + .arg( encodedVectorName ); + + const QString blobId = RiaSumoConnector::blobIdFromBody( m_connector.getBlocking( url ) ); + + if ( !blobId.isEmpty() ) + { + RiaLogging::debug( std::format( "Received blob ID for vector '{}': {}", vectorName.toStdString(), blobId.toStdString() ) ); + } + + return blobId; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoSummary::parameterBlobId( const SumoCaseId& caseId, const QString& ensembleName ) +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + + const QString url = + QString( "%1/cases/%2/ensembles/%3/parameters/blob_id" ).arg( m_connector.server() ).arg( caseId.get() ).arg( encodedEnsembleName ); + + const QString blobId = RiaSumoConnector::blobIdFromBody( m_connector.getBlocking( url ) ); + + if ( !blobId.isEmpty() ) + { + RiaLogging::debug( std::format( "Received blob ID for parameters: {}", blobId.toStdString() ) ); + } + + return blobId; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoSummary::parseVectorNames( const QByteArray& body ) +{ + std::vector vectorNames; + + QJsonDocument doc = QJsonDocument::fromJson( body ); + QJsonArray jsonArray = doc.array(); + + for ( const QJsonValue& value : jsonArray ) + { + QJsonObject vectorObj = value.toObject(); + vectorNames.push_back( vectorObj["name"].toString() ); + } + + return vectorNames; +} diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.h new file mode 100644 index 00000000000..6fbb1a24b29 --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.h @@ -0,0 +1,56 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "RiaSumoDefines.h" + +#include +#include + +#include + +class RiaSumoConnector; + +//================================================================================================== +/// The summary data of a Sumo case: the vectors an ensemble has, their values, and the ensemble +/// parameters. Requests are made through RiaSumoConnector, which owns the connection and does the +/// transfers, and every call returns its result rather than leaving it in shared state. +//================================================================================================== +class RiaSumoSummary +{ +public: + explicit RiaSumoSummary( RiaSumoConnector& connector ); + + std::vector vectorNames( const SumoCaseId& caseId, const QString& ensembleName ); + + // The values of one summary vector, for all realizations, as a parquet blob. + QByteArray vectorData( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ); + + // The ensemble parameters, as a parquet blob. + QByteArray parameterData( const SumoCaseId& caseId, const QString& ensembleName ); + + QString vectorBlobId( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ); + QString parameterBlobId( const SumoCaseId& caseId, const QString& ensembleName ); + +private: + static std::vector parseVectorNames( const QByteArray& body ); + +private: + RiaSumoConnector& m_connector; +}; diff --git a/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.cpp b/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.cpp index 1c60708c119..641aee7e489 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.cpp +++ b/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.cpp @@ -125,7 +125,7 @@ void RifReaderSumoGridProperty::prefetchFromTimeStep( const QString& propertyNam if ( !timestamps[i].isEmpty() ) batch.push_back( timestamps[i] ); } - m_connector->prefetchGridPropertyDataBlocking( SumoCaseId( m_caseId ), m_ensembleName, m_gridName, m_realization, propertyName, batch ); + m_connector->grid().prefetchPropertyData( SumoCaseId( m_caseId ), m_ensembleName, m_gridName, m_realization, propertyName, batch ); } //-------------------------------------------------------------------------------------------------- @@ -135,12 +135,8 @@ bool RifReaderSumoGridProperty::fetchAndDecode( const QString& propertyName, con { if ( !m_connector || !m_caseData || !values ) return false; - QByteArray contents = m_connector->requestGridPropertyDataBlocking( SumoCaseId( m_caseId ), - m_ensembleName, - m_gridName, - m_realization, - propertyName, - isoDateOrInterval ); + QByteArray contents = + m_connector->grid().propertyData( SumoCaseId( m_caseId ), m_ensembleName, m_gridName, m_realization, propertyName, isoDateOrInterval ); RiaLogging::debug( std::format( "Sumo grid property '{}' (time '{}'): downloaded {} bytes.", propertyName.toStdString(), diff --git a/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.cpp b/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.cpp index e7777e37b64..06330224fa9 100644 --- a/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.cpp +++ b/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.cpp @@ -93,9 +93,11 @@ SimpleDialog::SimpleDialog( QWidget* parent ) //-------------------------------------------------------------------------------------------------- SimpleDialog::~SimpleDialog() { + // The connector belongs to RiaApplication and is shared with everything else reading from Sumo, so it + // must outlive this dialog. Only the connection made in createConnection is ours to undo. if ( m_sumoConnector ) { - m_sumoConnector->deleteLater(); + disconnect( m_sumoConnector, &RiaSumoConnector::tokenReady, this, &SimpleDialog::onTokenReady ); } } @@ -105,7 +107,10 @@ SimpleDialog::~SimpleDialog() void SimpleDialog::createConnection() { m_sumoConnector = RiaApplication::instance()->makeSumoConnector(); - connect( m_sumoConnector, &RiaSumoConnector::tokenReady, this, &SimpleDialog::onTokenReady ); + + // Authenticating more than once would otherwise leave one connection per attempt, and onTokenReady + // would be called once for each of them. + connect( m_sumoConnector, &RiaSumoConnector::tokenReady, this, &SimpleDialog::onTokenReady, Qt::UniqueConnection ); } //-------------------------------------------------------------------------------------------------- @@ -124,10 +129,9 @@ void SimpleDialog::onAssetsClicked() { if ( !isTokenValid() ) return; - m_sumoConnector->requestAssetsBlocking(); - m_sumoConnector->assets(); + const auto assets = m_sumoConnector->explore().assets(); - label->setText( "Requesting fields (see log for response" ); + label->setText( QString( "Received %1 assets" ).arg( assets.size() ) ); } //-------------------------------------------------------------------------------------------------- @@ -137,10 +141,10 @@ void SimpleDialog::onCasesClicked() { if ( !isTokenValid() ) return; - QString fieldName = "Drogon"; - m_sumoConnector->requestCasesForField( fieldName ); + QString fieldName = "Drogon"; + const auto cases = m_sumoConnector->explore().cases( fieldName ); - label->setText( "Requesting cases (see log for response" ); + label->setText( QString( "Received %1 cases" ).arg( cases.size() ) ); } //-------------------------------------------------------------------------------------------------- @@ -153,9 +157,9 @@ void SimpleDialog::onVectorNamesClicked() SumoCaseId caseId( "5b783aab-ce10-4b78-b129-baf8d8ce4baa" ); QString iteration = "iter-0"; - m_sumoConnector->requestVectorNamesForEnsemble( caseId, iteration ); + const auto vectorNames = m_sumoConnector->summary().vectorNames( caseId, iteration ); - label->setText( "Requesting vector names (see log for response" ); + label->setText( QString( "Received %1 vector names" ).arg( vectorNames.size() ) ); } //-------------------------------------------------------------------------------------------------- @@ -169,9 +173,9 @@ void SimpleDialog::onFindBlobIdClicked() QString iteration = "iter-0"; QString vectorName = "FOPT"; - m_sumoConnector->requestBlobIdForEnsemble( caseId, iteration, vectorName ); + m_blobId = m_sumoConnector->summary().vectorBlobId( caseId, iteration, vectorName ); - label->setText( "Requesting blob ID for vector name (see log for response" ); + label->setText( m_blobId.isEmpty() ? QString( "No blob ID received" ) : QString( "Blob ID: %1" ).arg( m_blobId ) ); } //-------------------------------------------------------------------------------------------------- @@ -181,16 +185,16 @@ void SimpleDialog::onParquetClicked() { if ( !isTokenValid() ) return; - if ( m_sumoConnector->blobIds().empty() ) + if ( m_blobId.isEmpty() ) { onFindBlobIdClicked(); } - if ( !m_sumoConnector->blobIds().empty() ) + if ( !m_blobId.isEmpty() ) { - m_sumoConnector->requestBlobDownload( m_sumoConnector->blobIds().back() ); + m_blobContents = m_sumoConnector->downloadBlobBlocking( m_blobId ); - label->setText( "Requesting blob ID for vector name (see log for response" ); + label->setText( QString( "Downloaded blob, %1 bytes" ).arg( m_blobContents.size() ) ); } } @@ -199,14 +203,10 @@ void SimpleDialog::onParquetClicked() //-------------------------------------------------------------------------------------------------- void SimpleDialog::onShowContentParquetClicked() { - if ( m_sumoConnector->blobContents().empty() ) return; - - auto blob = m_sumoConnector->blobContents().back(); - - auto content = blob.contents; + if ( m_blobContents.isEmpty() ) return; // TODO: show content using parquet reader - auto tableText = RifArrowTools::readFirstRowsOfTable( content ); + auto tableText = RifArrowTools::readFirstRowsOfTable( m_blobContents ); RiaLogging::info( tableText.toStdString() ); } @@ -220,9 +220,7 @@ void SimpleDialog::onRealizationsClicked() SumoCaseId caseId( "485041ce-ad72-48a3-ac8c-484c0ed95cf8" ); QString iteration = "iter-0"; - m_sumoConnector->requestRealizationIdsForEnsembleBlocking( caseId, iteration ); - - auto ids = m_sumoConnector->realizationIds(); + auto ids = m_sumoConnector->explore().realizationIds( caseId, iteration ); for ( const auto& id : ids ) { RiaLogging::info( id.toStdString() ); diff --git a/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.h b/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.h index 3b75d55f793..84dce40c4ec 100644 --- a/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.h +++ b/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.h @@ -67,6 +67,11 @@ class SimpleDialog : public QDialog QPushButton* realizationIdsButton; QPointer m_sumoConnector; + + // The most recently resolved blob id, and the blob downloaded from it, kept here so the buttons that + // follow have something to work on. + QString m_blobId; + QByteArray m_blobContents; }; //================================================================================================== diff --git a/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.cpp b/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.cpp index 1aa21ecc439..bf18ccab7cc 100644 --- a/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.cpp +++ b/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.cpp @@ -167,14 +167,21 @@ void RimCloudDataSourceCollection::fieldChangedByUi( const caf::PdmFieldHandle* if ( changedField == &m_sumoFieldName ) { + // What was picked below belonged to the asset that was just left, both the selection and the options + // it was chosen from. Forget the cached answers as well, or the case list would keep offering the + // cases of the previous asset. The editors are refreshed by the caller, which updates them as soon + // as this returns, so asking for that here would only fetch everything a second time. m_sumoCaseId = ""; - m_sumoEnsembleNames.v().clear(); + m_sumoEnsembleNames.setValue( {} ); - m_sumoConnector->requestCasesForFieldBlocking( m_sumoFieldName ); + clearCachedCases(); + clearCachedEnsembleNames(); } else if ( changedField == &m_sumoCaseId ) { - m_sumoEnsembleNames.v().clear(); + m_sumoEnsembleNames.setValue( {} ); + + clearCachedEnsembleNames(); } if ( changedField == &m_addEnsembles ) { @@ -214,12 +221,7 @@ QList RimCloudDataSourceCollection::calculateValueOption QList options; if ( fieldNeedingOptions == &m_sumoFieldName ) { - if ( m_sumoConnector->assets().empty() ) - { - m_sumoConnector->requestAssetsBlocking(); - } - - for ( const auto& asset : m_sumoConnector->assets() ) + for ( const auto& asset : cachedAssets() ) { if ( m_sumoFieldName().isEmpty() ) { @@ -231,24 +233,14 @@ QList RimCloudDataSourceCollection::calculateValueOption } else if ( fieldNeedingOptions == &m_sumoCaseId && !m_sumoFieldName().isEmpty() ) { - if ( m_sumoConnector->cases().empty() ) - { - m_sumoConnector->requestCasesForFieldBlocking( m_sumoFieldName ); - } - - for ( const auto& sumoCase : m_sumoConnector->cases() ) + for ( const auto& sumoCase : cachedCases( m_sumoFieldName ) ) { options.push_back( { sumoCase.name, sumoCase.caseId.get() } ); } } else if ( fieldNeedingOptions == &m_sumoEnsembleNames && !m_sumoCaseId().isEmpty() ) { - if ( m_sumoConnector->ensembleNamesForCase( SumoCaseId( m_sumoCaseId ) ).empty() ) - { - m_sumoConnector->requestEnsembleByCasesIdBlocking( SumoCaseId( m_sumoCaseId ) ); - } - - for ( const auto& name : m_sumoConnector->ensembleNamesForCase( SumoCaseId( m_sumoCaseId ) ) ) + for ( const auto& name : cachedEnsembleNames( SumoCaseId( m_sumoCaseId ) ) ) { options.push_back( { name, name } ); } @@ -257,6 +249,66 @@ QList RimCloudDataSourceCollection::calculateValueOption return options; } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +const std::vector& RimCloudDataSourceCollection::cachedAssets() +{ + if ( m_assets.empty() && m_sumoConnector ) + { + m_assets = m_sumoConnector->explore().assets(); + } + + return m_assets; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +const std::vector& RimCloudDataSourceCollection::cachedCases( const QString& assetName ) +{ + if ( m_casesAssetName != assetName && m_sumoConnector ) + { + m_cases = m_sumoConnector->explore().cases( assetName ); + m_casesAssetName = assetName; + } + + return m_cases; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +const std::vector& RimCloudDataSourceCollection::cachedEnsembleNames( const SumoCaseId& caseId ) +{ + if ( m_ensembleNamesCaseId != caseId.get() && m_sumoConnector ) + { + m_ensembleNames = m_sumoConnector->explore().ensembleNames( caseId ); + m_ensembleNamesCaseId = caseId.get(); + } + + return m_ensembleNames; +} + +//-------------------------------------------------------------------------------------------------- +/// Forget the cases Sumo answered with, so the next request for them asks again. Called when the asset they +/// belong to is left behind. +//-------------------------------------------------------------------------------------------------- +void RimCloudDataSourceCollection::clearCachedCases() +{ + m_casesAssetName.clear(); + m_cases.clear(); +} + +//-------------------------------------------------------------------------------------------------- +/// The same for the ensemble names, which belong to a case. +//-------------------------------------------------------------------------------------------------- +void RimCloudDataSourceCollection::clearCachedEnsembleNames() +{ + m_ensembleNamesCaseId.clear(); + m_ensembleNames.clear(); +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -402,7 +454,7 @@ std::vector RimCloudDataSourceCollection::addDataSources() } QString caseName; - for ( const auto& sumoCase : m_sumoConnector->cases() ) + for ( const auto& sumoCase : cachedCases( m_sumoFieldName ) ) { if ( sumoCase.caseId == sumoCaseId ) { @@ -411,15 +463,12 @@ std::vector RimCloudDataSourceCollection::addDataSources() } } - m_sumoConnector->requestRealizationIdsForEnsembleBlocking( sumoCaseId, ensembleName ); - m_sumoConnector->requestVectorNamesForEnsembleBlocking( sumoCaseId, ensembleName ); - m_sumoConnector->requestGridInfoForEnsembleBlocking( sumoCaseId, ensembleName ); - - auto availableRealizationIds = m_sumoConnector->realizationIds(); - auto vectorNames = m_sumoConnector->vectorNames(); + const auto availableRealizationIds = m_sumoConnector->explore().realizationIds( sumoCaseId, ensembleName ); + const auto gridInfos = m_sumoConnector->grid().gridInfo( sumoCaseId, ensembleName ); + const auto vectorNames = m_sumoConnector->summary().vectorNames( sumoCaseId, ensembleName ); std::vector gridNames; - for ( const auto& gridInfo : m_sumoConnector->gridInfos() ) + for ( const auto& gridInfo : gridInfos ) { gridNames.push_back( gridInfo.name ); } diff --git a/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.h b/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.h index 917848236b5..2b48cca47fb 100644 --- a/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.h +++ b/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.h @@ -58,6 +58,16 @@ class RimCloudDataSourceCollection : public caf::PdmObject static bool isCloudApiServerAvailable(); + // The option lists are rebuilt every time the property editor refreshes, so what Sumo answered is kept + // here and only asked for again when the selection it belongs to changes. + const std::vector& cachedAssets(); + const std::vector& cachedCases( const QString& assetName ); + const std::vector& cachedEnsembleNames( const SumoCaseId& caseId ); + + void clearCachedCases(); + + void clearCachedEnsembleNames(); + private: caf::PdmField m_authenticate; caf::PdmField m_sumoFieldName; @@ -73,4 +83,12 @@ class RimCloudDataSourceCollection : public caf::PdmObject caf::PdmField m_restartServer; QPointer m_sumoConnector; + + std::vector m_assets; + + QString m_casesAssetName; + std::vector m_cases; + + QString m_ensembleNamesCaseId; + std::vector m_ensembleNames; }; diff --git a/ApplicationLibCode/ProjectDataModel/RimRoffCaseSumo.cpp b/ApplicationLibCode/ProjectDataModel/RimRoffCaseSumo.cpp index cbc215d360e..c45b78a921e 100644 --- a/ApplicationLibCode/ProjectDataModel/RimRoffCaseSumo.cpp +++ b/ApplicationLibCode/ProjectDataModel/RimRoffCaseSumo.cpp @@ -187,8 +187,7 @@ bool RimRoffCaseSumo::openEclipseGridFile() if ( eclipseCaseData()->mainGrid()->cellCount() == 0 ) { - QByteArray contents = - m_sumoConnector->requestGridDataBlocking( SumoCaseId( m_sumoCaseId() ), m_ensembleName(), m_gridName(), m_realization() ); + QByteArray contents = m_sumoConnector->grid().gridData( SumoCaseId( m_sumoCaseId() ), m_ensembleName(), m_gridName(), m_realization() ); if ( contents.isEmpty() ) { RiaLogging::error( @@ -263,14 +262,15 @@ void RimRoffCaseSumo::registerSumoGridProperties() { if ( !m_sumoConnector || !eclipseCaseData() ) return; - m_sumoConnector->requestGridPropertyInfoForEnsembleBlocking( SumoCaseId( m_sumoCaseId() ), m_ensembleName(), m_gridName(), m_realization() ); + const auto gridPropertyInfos = + m_sumoConnector->grid().propertyInfo( SumoCaseId( m_sumoCaseId() ), m_ensembleName(), m_gridName(), m_realization() ); // Properties without a timestamp are static. Properties with a single timestamp are dynamic (one time // step per timestamp). Time intervals (the iso string contains '/') are not supported and skipped. std::vector staticPropertyNames; std::map> dynamicPropertyTimestamps; // property name -> the timestamps it has std::set allTimestamps; // union of timestamps across all properties - for ( const auto& info : m_sumoConnector->gridPropertyInfos() ) + for ( const auto& info : gridPropertyInfos ) { if ( info.isoDateOrInterval.isEmpty() ) { diff --git a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp index 0fb675c8fee..62fc7c5fc45 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp +++ b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp @@ -209,7 +209,7 @@ void RimSummaryEnsembleSumo::loadSummaryData( const RifEclipseSummaryAddress& re auto parametersKey = ParquetKey{ sumoCaseId, sumoEnsembleName, "", true }; if ( m_parquetTable.find( parametersKey ) == m_parquetTable.end() ) { - auto contents = m_sumoConnector->requestParametersParquetDataBlocking( sumoCaseId, sumoEnsembleName ); + auto contents = m_sumoConnector->summary().parameterData( sumoCaseId, sumoEnsembleName ); RiaLogging::debug( std::format( "Load ensemble parameter sensitivities. Contents size: {}", contents.size() ) ); std::shared_ptr table = readParquetTable( contents, QString( "%1 parameter sensitivities" ).arg( sumoEnsembleName ) ); @@ -226,7 +226,7 @@ QByteArray RimSummaryEnsembleSumo::loadParquetData( const ParquetKey& parquetKey { if ( !m_sumoConnector ) return {}; - return m_sumoConnector->requestParquetDataBlocking( SumoCaseId( parquetKey.caseId ), parquetKey.ensembleId, parquetKey.vectorName ); + return m_sumoConnector->summary().vectorData( SumoCaseId( parquetKey.caseId ), parquetKey.ensembleId, parquetKey.vectorName ); } //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/UnitTests/CMakeLists.txt b/ApplicationLibCode/UnitTests/CMakeLists.txt index 922bab36c9e..d1732e946b3 100644 --- a/ApplicationLibCode/UnitTests/CMakeLists.txt +++ b/ApplicationLibCode/UnitTests/CMakeLists.txt @@ -18,6 +18,7 @@ set(SOURCE_UNITTEST_FILES ${CMAKE_CURRENT_LIST_DIR}/RifOpmFlowDeckFile-Test.cpp ${CMAKE_CURRENT_LIST_DIR}/RifReaderEclipseOutput-Test.cpp ${CMAKE_CURRENT_LIST_DIR}/RifReaderEclipseSummary-Test.cpp + ${CMAKE_CURRENT_LIST_DIR}/RiaSumoBlobCache-Test.cpp ${CMAKE_CURRENT_LIST_DIR}/RigActiveCellInfo-Test.cpp ${CMAKE_CURRENT_LIST_DIR}/RigEclipseCaseDataTools-Test.cpp ${CMAKE_CURRENT_LIST_DIR}/RigEclipseCrossPlotDataExtractor-Test.cpp diff --git a/ApplicationLibCode/UnitTests/RiaSumoBlobCache-Test.cpp b/ApplicationLibCode/UnitTests/RiaSumoBlobCache-Test.cpp new file mode 100644 index 00000000000..158a2d6208a --- /dev/null +++ b/ApplicationLibCode/UnitTests/RiaSumoBlobCache-Test.cpp @@ -0,0 +1,201 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "gtest/gtest.h" + +#include "Cloud/RiaSumoBlobCache.h" + +namespace +{ +QByteArray blobOfSize( int size, char fill = 'x' ) +{ + return QByteArray( size, fill ); +} +} // namespace + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, LookupOfMissingKeyReturnsEmpty ) +{ + RiaSumoBlobCache cache( 100 ); + + EXPECT_FALSE( cache.contains( "missing" ) ); + EXPECT_TRUE( cache.lookup( "missing" ).isEmpty() ); + EXPECT_EQ( size_t( 0 ), cache.entryCount() ); + EXPECT_EQ( size_t( 0 ), cache.sizeBytes() ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, InsertedBlobIsReturnedUnchanged ) +{ + RiaSumoBlobCache cache( 100 ); + + const QByteArray contents = blobOfSize( 10, 'a' ); + cache.insert( "key", contents ); + + EXPECT_TRUE( cache.contains( "key" ) ); + EXPECT_EQ( contents, cache.lookup( "key" ) ); + EXPECT_EQ( size_t( 1 ), cache.entryCount() ); + EXPECT_EQ( size_t( 10 ), cache.sizeBytes() ); +} + +//-------------------------------------------------------------------------------------------------- +/// An empty array is how a cache miss is reported, so it must never be stored as a value. +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, EmptyContentsAreNotCached ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "key", QByteArray() ); + + EXPECT_FALSE( cache.contains( "key" ) ); + EXPECT_EQ( size_t( 0 ), cache.entryCount() ); +} + +//-------------------------------------------------------------------------------------------------- +/// Caching a blob bigger than the whole limit would evict everything else to make room for it. +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, BlobLargerThanLimitIsNotCachedAndKeepsExistingEntries ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "small", blobOfSize( 10 ) ); + cache.insert( "huge", blobOfSize( 101 ) ); + + EXPECT_FALSE( cache.contains( "huge" ) ); + EXPECT_TRUE( cache.contains( "small" ) ); + EXPECT_EQ( size_t( 1 ), cache.entryCount() ); + EXPECT_EQ( size_t( 10 ), cache.sizeBytes() ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, BlobExactlyAtLimitIsCached ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "exact", blobOfSize( 100 ) ); + + EXPECT_TRUE( cache.contains( "exact" ) ); + EXPECT_EQ( size_t( 100 ), cache.sizeBytes() ); +} + +//-------------------------------------------------------------------------------------------------- +/// Exceeding the limit evicts from the back of the recency order, oldest first. +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, LeastRecentlyUsedIsEvictedFirst ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "a", blobOfSize( 40 ) ); + cache.insert( "b", blobOfSize( 40 ) ); + cache.insert( "c", blobOfSize( 40 ) ); // pushes the total to 120, so "a" has to go + + EXPECT_FALSE( cache.contains( "a" ) ); + EXPECT_TRUE( cache.contains( "b" ) ); + EXPECT_TRUE( cache.contains( "c" ) ); + EXPECT_EQ( size_t( 80 ), cache.sizeBytes() ); +} + +//-------------------------------------------------------------------------------------------------- +/// A lookup counts as use, so the blob it returns must survive the next eviction. +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, LookupRefreshesRecency ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "a", blobOfSize( 40 ) ); + cache.insert( "b", blobOfSize( 40 ) ); + + cache.lookup( "a" ); // "b" is now the least recently used + + cache.insert( "c", blobOfSize( 40 ) ); + + EXPECT_TRUE( cache.contains( "a" ) ); + EXPECT_FALSE( cache.contains( "b" ) ); + EXPECT_TRUE( cache.contains( "c" ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// Re-inserting a key must replace the entry rather than leave a stale recency entry behind, and the +/// accounted size must follow the new contents. +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, ReinsertingKeyReplacesEntryAndSize ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "key", blobOfSize( 10, 'a' ) ); + cache.insert( "key", blobOfSize( 30, 'b' ) ); + + EXPECT_EQ( size_t( 1 ), cache.entryCount() ); + EXPECT_EQ( size_t( 30 ), cache.sizeBytes() ); + EXPECT_EQ( blobOfSize( 30, 'b' ), cache.lookup( "key" ) ); + + // A stale recency entry would make this eviction drop the wrong key, or drop nothing at all. + cache.insert( "other", blobOfSize( 80 ) ); + + EXPECT_FALSE( cache.contains( "key" ) ); + EXPECT_TRUE( cache.contains( "other" ) ); + EXPECT_EQ( size_t( 1 ), cache.entryCount() ); + EXPECT_EQ( size_t( 80 ), cache.sizeBytes() ); +} + +//-------------------------------------------------------------------------------------------------- +/// One insert may have to evict several entries to get back within the limit. +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, InsertEvictsAsManyEntriesAsNeeded ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "a", blobOfSize( 20 ) ); + cache.insert( "b", blobOfSize( 20 ) ); + cache.insert( "c", blobOfSize( 20 ) ); + cache.insert( "big", blobOfSize( 90 ) ); + + EXPECT_TRUE( cache.contains( "big" ) ); + EXPECT_FALSE( cache.contains( "a" ) ); + EXPECT_FALSE( cache.contains( "b" ) ); + EXPECT_EQ( size_t( 1 ), cache.entryCount() ); + EXPECT_EQ( size_t( 90 ), cache.sizeBytes() ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, ClearEmptiesTheCache ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "a", blobOfSize( 20 ) ); + cache.insert( "b", blobOfSize( 20 ) ); + + cache.clear(); + + EXPECT_EQ( size_t( 0 ), cache.entryCount() ); + EXPECT_EQ( size_t( 0 ), cache.sizeBytes() ); + EXPECT_FALSE( cache.contains( "a" ) ); + + // The recency order must be cleared too, otherwise a later insert evicts against stale keys. + cache.insert( "c", blobOfSize( 20 ) ); + EXPECT_TRUE( cache.contains( "c" ) ); + EXPECT_EQ( size_t( 20 ), cache.sizeBytes() ); +} diff --git a/ApplicationLibCode/UserInterface/RiuMessagePanel.cpp b/ApplicationLibCode/UserInterface/RiuMessagePanel.cpp index 395ad1c04f0..a2eddc53acb 100644 --- a/ApplicationLibCode/UserInterface/RiuMessagePanel.cpp +++ b/ApplicationLibCode/UserInterface/RiuMessagePanel.cpp @@ -206,8 +206,6 @@ void RiuMessagePanelLogger::debug( const char* message ) writeToMessagePanel( RILogLevel::RI_LL_DEBUG, message ); } -//-------------------------------------------------------------------------------------------------- -/// //-------------------------------------------------------------------------------------------------- /// Deliver the messages handed over from other threads. Only the queued addMessage calls above are posted to /// a panel, so this writes out the pending log messages and nothing else. Must be called from the thread From c4e297a6a269bbf50d963332b83e506a1ba3b401 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Herje?= Date: Tue, 18 Aug 2026 10:45:29 +0200 Subject: [PATCH 4/6] Load Sumo summary vectors concurrently and without waiting A summary plot reads its curve values one curve at a time, so a plot with several curves made one blocking request after another. The service aggregates a vector and stores it back in Sumo, so the first request for a vector that has not been aggregated yet is slow, and fetching serially cost the sum of those aggregations rather than the slowest of them. A plot window now collects the addresses its plots are about to read and hands them to each ensemble in one go, through a new RimSummaryEnsemble::prefetchSummaryData that does nothing by default. The Sumo ensemble requests them together, so the wait is the slowest vector rather than the sum, and the plot code needs no knowledge of Sumo. Dropping a vector into a plot stopped the application until the data had been fetched, even though the vectors themselves were already being loaded without waiting. Those requests are made without waiting for them. The curves are drawn with whatever has arrived, and each vector is taken in as it lands and the plots using it redrawn, so the vectors that are ready appear while the rest are still being aggregated. A vector on its way is never requested a second time and reports having no data yet, which is what lets the curves that are ready draw. Co-Authored-By: Claude Opus 5 --- .../Tools/Cloud/RiaSumoConnector.cpp | 102 ++++++- .../Tools/Cloud/RiaSumoConnector.h | 16 ++ .../Tools/Cloud/RiaSumoDefines.cpp | 8 + .../Application/Tools/Cloud/RiaSumoDefines.h | 5 + .../Tools/Cloud/RiaSumoSummary.cpp | 262 +++++++++++++++++- .../Application/Tools/Cloud/RiaSumoSummary.h | 29 ++ .../ProjectDataModel/RimMultiPlot.cpp | 4 + .../ProjectDataModel/RimMultiPlot.h | 5 + .../Summary/RimSummaryEnsemble.h | 5 + .../Summary/RimSummaryMultiPlot.cpp | 26 ++ .../Summary/RimSummaryMultiPlot.h | 2 + .../Summary/RimSummaryPlot.cpp | 62 +++++ .../ProjectDataModel/Summary/RimSummaryPlot.h | 4 + .../Summary/Sumo/RimSummaryEnsembleSumo.cpp | 199 +++++++++++-- .../Summary/Sumo/RimSummaryEnsembleSumo.h | 20 +- 15 files changed, 706 insertions(+), 43 deletions(-) diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp index 265cc9faa07..4a8e250c580 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp @@ -141,6 +141,102 @@ QNetworkAccessManager* RiaSumoConnector::networkAccessManager() return m_networkAccessManager; } +//-------------------------------------------------------------------------------------------------- +/// Run work on the transfer thread and return at once. Nothing is waited for here, so the result has to be +/// delivered by a callback. The token is requested first, while still on the calling thread, because +/// refreshing it may need the authentication objects that live there. +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::runOnTransferThread( const std::function& work ) +{ + requestTokenBlocking(); + + // Without a transfer thread there is nothing to hand the work to, and running it here is the only option. + // Already on the transfer thread, run directly rather than queue behind work that may be waiting for us. + if ( !m_transferThread || !m_transferContext || QThread::currentThread() == m_transferThread ) + { + work(); + return; + } + + QMetaObject::invokeMethod( m_transferContext, work, Qt::QueuedConnection ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::invokeOnConnectorThread( const std::function& work ) +{ + if ( QThread::currentThread() == thread() ) + { + work(); + return; + } + + QMetaObject::invokeMethod( this, work, Qt::QueuedConnection ); +} + +//-------------------------------------------------------------------------------------------------- +/// A blob is fetched in two steps, first the pre-signed URI and then the data itself. Both are started here +/// and neither is waited for: each step continues from the reply of the one before. +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::downloadBlobAsync( const QString& blobId, const std::function& onFinished ) +{ + const QString url = QString( "%1/blobs/%2/sas_token_and_blob_base_uri" ).arg( server() ).arg( blobId ); + + QNetworkRequest networkRequest; + networkRequest.setUrl( QUrl( url ) ); + addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); + + auto accessInfoReply = networkAccessManager()->get( networkRequest ); + abortIfNotFinishedWithin( accessInfoReply, RiaSumoDefines::requestTimeoutMillis() ); + + QObject::connect( accessInfoReply, + &QNetworkReply::finished, + m_transferContext, + [this, accessInfoReply, blobId, onFinished]() + { + const QString sasUri = sasUriFromReply( accessInfoReply, blobId ); + if ( sasUri.isEmpty() ) + { + onFinished( {} ); + return; + } + + RiaLogging::debug( std::format( "Requesting blob. Id: {} SAS URI: {}", blobId, sasUri ) ); + + QNetworkRequest blobRequest; + blobRequest.setUrl( sasUri ); + + // The pre-signed SAS URI carries its own credential, so no Authorization header is + // added. Do NOT forward the bearer token to the storage host. + blobRequest.setAttribute( QNetworkRequest::RedirectPolicyAttribute, QNetworkRequest::NoLessSafeRedirectPolicy ); + + auto blobReply = networkAccessManager()->get( blobRequest ); + abortIfNotFinishedWithin( blobReply, RiaSumoDefines::requestTimeoutMillis() ); + + QObject::connect( blobReply, + &QNetworkReply::finished, + m_transferContext, + [blobReply, sasUri, onFinished]() { onFinished( blobContentsFromReply( blobReply, sasUri ) ); } ); + } ); +} + +//-------------------------------------------------------------------------------------------------- +/// Nothing waits on an async reply, so a request that never answers would otherwise keep its data pending +/// for good. Aborting makes the reply finish with an error, which the chain reports as a failed transfer. +//-------------------------------------------------------------------------------------------------- +void RiaSumoConnector::abortIfNotFinishedWithin( QNetworkReply* reply, int timeoutMillis ) +{ + if ( !reply ) return; + + QTimer::singleShot( timeoutMillis, + reply, + [reply]() + { + if ( !reply->isFinished() ) reply->abort(); + } ); +} + //-------------------------------------------------------------------------------------------------- /// Run work on the transfer thread and wait for it to finish, without dispatching any events on the calling /// thread. This is what keeps the view update code from re-entering a load that is already running. @@ -212,7 +308,11 @@ void RiaSumoConnector::waitForRepliesToFinish( const std::vector if ( !isAllFinished() ) { - timer.start( RiaSumoDefines::requestTimeoutMillis() ); + // The requests run concurrently, but give the group the timeout each of them would have been + // given on its own. A batch must not be more likely to time out than the same requests made one + // by one, and the server can take a while to answer: a summary vector that has not been + // aggregated yet is produced on demand by the first request that asks for it. + timer.start( static_cast( replies.size() ) * RiaSumoDefines::requestTimeoutMillis() ); eventLoop.exec(); } diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h index 4d6ef11607b..1e33008de7a 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h @@ -81,6 +81,22 @@ class RiaSumoConnector : public RiaCloudConnector void addStandardHeader( QNetworkRequest& networkRequest, const QString& token, const QString& contentType ); void runOnTransferThreadBlocking( const std::function& work ); + // Run work on the transfer thread without waiting for it. The async data paths use this: the result is + // delivered by a callback rather than by returning, so the calling thread carries on immediately. + void runOnTransferThread( const std::function& work ); + + // Hand a call back to the thread the connector lives on, the one owning the user interface. Results of + // async work are delivered through this, so a caller never has its data handed to it on another thread. + void invokeOnConnectorThread( const std::function& work ); + + // Download one blob, calling onFinished with its contents. Call on the transfer thread, where onFinished + // is called as well. Empty contents mean the transfer failed. + void downloadBlobAsync( const QString& blobId, const std::function& onFinished ); + + // Abort a reply that has not finished in time, so an async chain reports a failure instead of hanging and + // leaving whoever waits for the data waiting forever. + static void abortIfNotFinishedWithin( QNetworkReply* reply, int timeoutMillis ); + // The network manager belonging to the calling thread: the transfer thread manager when called from // there, otherwise the one owned by RiaCloudConnector on the GUI thread. QNetworkAccessManager* networkAccessManager(); diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.cpp index f696d3bc710..496f1dd4382 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.cpp +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.cpp @@ -37,6 +37,14 @@ int RiaSumoDefines::requestTimeoutMillis() return 10 * 1000; } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +int RiaSumoDefines::asyncRequestTimeoutMillis() +{ + return 5 * 60 * 1000; +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.h index fe0a8fa25af..4141994665d 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.h +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.h @@ -32,6 +32,11 @@ namespace RiaSumoDefines QString tokenPath(); int requestTimeoutMillis(); +// The timeout of a request nothing is waiting for. Only there so a request that never answers is eventually +// given up on, and generous because a summary vector that has not been aggregated yet is produced on demand +// by the request asking for it. Nothing is blocked while it runs, so waiting longer costs nothing. +int asyncRequestTimeoutMillis(); + // The maximum number of bytes of downloaded grid property blobs kept in memory. size_t gridPropertyCacheLimitBytes(); diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp index 87a11664eac..0323c8b5467 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp @@ -18,14 +18,19 @@ #include "RiaSumoSummary.h" +#include "RiaCloudDefines.h" #include "RiaLogging.h" #include "RiaSumoConnector.h" #include #include #include +#include +#include #include +#include + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -52,10 +57,133 @@ std::vector RiaSumoSummary::vectorNames( const SumoCaseId& caseId, cons //-------------------------------------------------------------------------------------------------- QByteArray RiaSumoSummary::vectorData( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) { - const QString blobId = vectorBlobId( caseId, ensembleName, vectorName ); - if ( blobId.isEmpty() ) return {}; + const auto contentsByVectorName = vectorData( caseId, ensembleName, std::vector{ vectorName } ); - return m_connector.downloadBlobBlocking( blobId ); + if ( auto it = contentsByVectorName.find( vectorName ); it != contentsByVectorName.end() ) return it->second; + + return {}; +} + +//-------------------------------------------------------------------------------------------------- +/// Fetch several summary vectors at the same time. The blob id requests are issued together and waited +/// for as a group, and so are the transfers, turning 2N sequential round trips into 2 batched ones. +/// +/// This matters more than the round trip count alone: a vector that has not been aggregated yet is +/// produced on demand by the request that asks for it, so fetching serially costs the sum of those +/// aggregations while fetching together costs roughly the slowest one. +//-------------------------------------------------------------------------------------------------- +std::map + RiaSumoSummary::vectorData( const SumoCaseId& caseId, const QString& ensembleName, const std::vector& vectorNames ) +{ + std::map contentsByVectorName; + if ( vectorNames.empty() ) return contentsByVectorName; + + // Drop duplicates, so a vector is never requested twice in one batch. + std::vector namesToFetch; + for ( const auto& vectorName : vectorNames ) + { + if ( vectorName.isEmpty() ) continue; + if ( std::ranges::find( namesToFetch, vectorName ) != namesToFetch.end() ) continue; + + namesToFetch.push_back( vectorName ); + } + + if ( namesToFetch.empty() ) return contentsByVectorName; + + m_connector.runOnTransferThreadBlocking( + [&]() + { + // Phase 1: resolve all blob ids concurrently. + std::vector blobIdReplies; + for ( const auto& vectorName : namesToFetch ) + { + blobIdReplies.push_back( makeVectorBlobIdRequest( caseId, ensembleName, vectorName ) ); + } + + RiaSumoConnector::waitForRepliesToFinish( blobIdReplies ); + + std::vector blobIds; + for ( size_t i = 0; i < blobIdReplies.size(); i++ ) + { + blobIds.push_back( blobIdFromReply( blobIdReplies[i], namesToFetch[i] ) ); + } + + // Phase 2: download all resolved blobs as one group. + std::vector blobIdsToDownload; + for ( const auto& blobId : blobIds ) + { + if ( !blobId.isEmpty() ) blobIdsToDownload.push_back( blobId ); + } + + const auto contentsByBlobId = m_connector.downloadBlobs( blobIdsToDownload ); + + // Anything missing failed; the caller falls back to fetching it on its own later. + for ( size_t i = 0; i < blobIds.size(); i++ ) + { + if ( blobIds[i].isEmpty() ) continue; + + if ( auto it = contentsByBlobId.find( blobIds[i] ); it != contentsByBlobId.end() ) + { + contentsByVectorName[namesToFetch[i]] = it->second; + } + } + } ); + + return contentsByVectorName; +} + +//-------------------------------------------------------------------------------------------------- +/// Fetch several summary vectors without waiting for any of them. Every vector is requested at once and +/// onVectorReady is called for each one as it arrives, on the thread the connector lives on, so a caller can +/// show each vector the moment it is there instead of when the slowest one is. +/// +/// Empty contents mean that vector failed. The callback is called exactly once per requested vector, so a +/// caller tracking what is still on its way can rely on all of them being accounted for. +//-------------------------------------------------------------------------------------------------- +void RiaSumoSummary::vectorDataAsync( const SumoCaseId& caseId, + const QString& ensembleName, + const std::vector& vectorNames, + const std::function& onVectorReady ) +{ + if ( vectorNames.empty() || !onVectorReady ) return; + + m_connector.runOnTransferThread( + [this, caseId, ensembleName, vectorNames, onVectorReady]() + { + for ( const auto& vectorName : vectorNames ) + { + auto deliver = [this, onVectorReady, vectorName]( const QByteArray& contents ) + { + m_connector.invokeOnConnectorThread( [onVectorReady, vectorName, contents]() { onVectorReady( vectorName, contents ); } ); + }; + + auto blobIdReply = makeVectorBlobIdRequest( caseId, ensembleName, vectorName ); + if ( !blobIdReply ) + { + deliver( {} ); + continue; + } + + // A vector that has not been aggregated yet is produced on demand by this request, which can + // take a good while. Nothing is waiting on it, so it is given room to finish. + RiaSumoConnector::abortIfNotFinishedWithin( blobIdReply, RiaSumoDefines::asyncRequestTimeoutMillis() ); + + QObject::connect( blobIdReply, + &QNetworkReply::finished, + blobIdReply, + [this, blobIdReply, vectorName, deliver]() + { + const QString blobId = blobIdFromReply( blobIdReply, vectorName ); + if ( blobId.isEmpty() ) + { + deliver( {} ); + return; + } + + m_connector.downloadBlobAsync( blobId, deliver ); + } ); + } + } ); } //-------------------------------------------------------------------------------------------------- @@ -73,18 +201,66 @@ QByteArray RiaSumoSummary::parameterData( const SumoCaseId& caseId, const QStrin /// //-------------------------------------------------------------------------------------------------- QString RiaSumoSummary::vectorBlobId( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) +{ + const QString url = vectorBlobIdUrl( caseId, ensembleName, vectorName ); + + return logBlobId( RiaSumoConnector::blobIdFromBody( m_connector.getBlocking( url ) ), vectorName ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoSummary::vectorBlobIdUrl( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) const { const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); const QString encodedVectorName = QUrl::toPercentEncoding( vectorName ); - const QString url = QString( "%1/cases/%2/ensembles/%3/vectors/%4/blob_id" ) - .arg( m_connector.server() ) - .arg( caseId.get() ) - .arg( encodedEnsembleName ) - .arg( encodedVectorName ); + return QString( "%1/cases/%2/ensembles/%3/vectors/%4/blob_id" ) + .arg( m_connector.server() ) + .arg( caseId.get() ) + .arg( encodedEnsembleName ) + .arg( encodedVectorName ); +} + +//-------------------------------------------------------------------------------------------------- +/// Issue the blob id request for one vector. The reply is returned unfinished, so the caller decides how +/// to wait for it: one at a time, or several at once when batching. +//-------------------------------------------------------------------------------------------------- +QNetworkReply* RiaSumoSummary::makeVectorBlobIdRequest( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) +{ + QNetworkRequest networkRequest; + networkRequest.setUrl( QUrl( vectorBlobIdUrl( caseId, ensembleName, vectorName ) ) ); + m_connector.addStandardHeader( networkRequest, m_connector.token(), RiaCloudDefines::contentTypeJson() ); + + return m_connector.networkAccessManager()->get( networkRequest ); +} + +//-------------------------------------------------------------------------------------------------- +/// Read the blob id off a finished blob id reply. The reply is consumed and scheduled for deletion. +//-------------------------------------------------------------------------------------------------- +QString RiaSumoSummary::blobIdFromReply( QNetworkReply* reply, const QString& vectorName ) +{ + if ( !reply ) return {}; + + const bool failed = !reply->isFinished() || reply->error() != QNetworkReply::NoError; + QByteArray body = failed ? QByteArray() : reply->readAll(); + + if ( failed ) + { + RiaLogging::error( + std::format( "Request blob ID failed for vector '{}': {}", vectorName.toStdString(), reply->errorString().toStdString() ) ); + } + + reply->deleteLater(); - const QString blobId = RiaSumoConnector::blobIdFromBody( m_connector.getBlocking( url ) ); + return logBlobId( RiaSumoConnector::blobIdFromBody( body ), vectorName ); +} +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoSummary::logBlobId( const QString& blobId, const QString& vectorName ) +{ if ( !blobId.isEmpty() ) { RiaLogging::debug( std::format( "Received blob ID for vector '{}': {}", vectorName.toStdString(), blobId.toStdString() ) ); @@ -96,14 +272,74 @@ QString RiaSumoSummary::vectorBlobId( const SumoCaseId& caseId, const QString& e //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -QString RiaSumoSummary::parameterBlobId( const SumoCaseId& caseId, const QString& ensembleName ) +QString RiaSumoSummary::parameterBlobIdUrl( const SumoCaseId& caseId, const QString& ensembleName ) const { const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); - const QString url = - QString( "%1/cases/%2/ensembles/%3/parameters/blob_id" ).arg( m_connector.server() ).arg( caseId.get() ).arg( encodedEnsembleName ); + return QString( "%1/cases/%2/ensembles/%3/parameters/blob_id" ).arg( m_connector.server() ).arg( caseId.get() ).arg( encodedEnsembleName ); +} - const QString blobId = RiaSumoConnector::blobIdFromBody( m_connector.getBlocking( url ) ); +//-------------------------------------------------------------------------------------------------- +/// Issue the blob id request for the ensemble parameters. The reply is returned unfinished, so the caller +/// decides how to wait for it. +//-------------------------------------------------------------------------------------------------- +QNetworkReply* RiaSumoSummary::makeParameterBlobIdRequest( const SumoCaseId& caseId, const QString& ensembleName ) +{ + QNetworkRequest networkRequest; + networkRequest.setUrl( QUrl( parameterBlobIdUrl( caseId, ensembleName ) ) ); + m_connector.addStandardHeader( networkRequest, m_connector.token(), RiaCloudDefines::contentTypeJson() ); + + return m_connector.networkAccessManager()->get( networkRequest ); +} + +//-------------------------------------------------------------------------------------------------- +/// Fetch the ensemble parameters without waiting, calling onParametersReady on the thread the connector +/// lives on. Empty contents mean the request failed, and the callback is called exactly once. +//-------------------------------------------------------------------------------------------------- +void RiaSumoSummary::parameterDataAsync( const SumoCaseId& caseId, + const QString& ensembleName, + const std::function& onParametersReady ) +{ + if ( !onParametersReady ) return; + + m_connector.runOnTransferThread( + [this, caseId, ensembleName, onParametersReady]() + { + auto deliver = [this, onParametersReady]( const QByteArray& contents ) + { m_connector.invokeOnConnectorThread( [onParametersReady, contents]() { onParametersReady( contents ); } ); }; + + auto blobIdReply = makeParameterBlobIdRequest( caseId, ensembleName ); + if ( !blobIdReply ) + { + deliver( {} ); + return; + } + + RiaSumoConnector::abortIfNotFinishedWithin( blobIdReply, RiaSumoDefines::asyncRequestTimeoutMillis() ); + + QObject::connect( blobIdReply, + &QNetworkReply::finished, + blobIdReply, + [this, blobIdReply, deliver]() + { + const QString blobId = blobIdFromReply( blobIdReply, "parameters" ); + if ( blobId.isEmpty() ) + { + deliver( {} ); + return; + } + + m_connector.downloadBlobAsync( blobId, deliver ); + } ); + } ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoSummary::parameterBlobId( const SumoCaseId& caseId, const QString& ensembleName ) +{ + const QString blobId = RiaSumoConnector::blobIdFromBody( m_connector.getBlocking( parameterBlobIdUrl( caseId, ensembleName ) ) ); if ( !blobId.isEmpty() ) { diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.h index 6fbb1a24b29..680bd94c093 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.h +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.h @@ -23,9 +23,12 @@ #include #include +#include +#include #include class RiaSumoConnector; +class QNetworkReply; //================================================================================================== /// The summary data of a Sumo case: the vectors an ensemble has, their values, and the ensemble @@ -42,13 +45,39 @@ class RiaSumoSummary // The values of one summary vector, for all realizations, as a parquet blob. QByteArray vectorData( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ); + // The same for several vectors at once, returned by vector name. The blob id requests are issued as one + // concurrent group and so are the transfers, which matters because a vector that has not been aggregated + // yet is produced on demand by the request asking for it. + std::map vectorData( const SumoCaseId& caseId, const QString& ensembleName, const std::vector& vectorNames ); + + // The same again, but without waiting: all vectors are requested at once and onVectorReady is called for + // each as it arrives, on the thread the connector lives on. Empty contents mean that vector failed, and + // the callback is called exactly once per requested vector. + void vectorDataAsync( const SumoCaseId& caseId, + const QString& ensembleName, + const std::vector& vectorNames, + const std::function& onVectorReady ); + // The ensemble parameters, as a parquet blob. QByteArray parameterData( const SumoCaseId& caseId, const QString& ensembleName ); + // The same without waiting. Like the vectors, the parameters are aggregated on demand by the service, so + // the first request for them can take a while and is not something to hold the user interface for. + void parameterDataAsync( const SumoCaseId& caseId, + const QString& ensembleName, + const std::function& onParametersReady ); + QString vectorBlobId( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ); QString parameterBlobId( const SumoCaseId& caseId, const QString& ensembleName ); private: + QString vectorBlobIdUrl( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) const; + QString parameterBlobIdUrl( const SumoCaseId& caseId, const QString& ensembleName ) const; + QNetworkReply* makeParameterBlobIdRequest( const SumoCaseId& caseId, const QString& ensembleName ); + QNetworkReply* makeVectorBlobIdRequest( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ); + static QString blobIdFromReply( QNetworkReply* reply, const QString& vectorName ); + static QString logBlobId( const QString& blobId, const QString& vectorName ); + static std::vector parseVectorNames( const QByteArray& body ); private: diff --git a/ApplicationLibCode/ProjectDataModel/RimMultiPlot.cpp b/ApplicationLibCode/ProjectDataModel/RimMultiPlot.cpp index c8f5864d0ba..028cc03e8bf 100644 --- a/ApplicationLibCode/ProjectDataModel/RimMultiPlot.cpp +++ b/ApplicationLibCode/ProjectDataModel/RimMultiPlot.cpp @@ -857,6 +857,10 @@ void RimMultiPlot::updatePlots() { if ( m_showWindow ) { + // The plots are loaded one after the other, so a source loading data remotely would make its requests + // one plot at a time. Give it the chance to load everything this window needs in one go first. + prefetchPlotData(); + for ( RimPlot* plot : plots() ) { plot->loadDataAndUpdate(); diff --git a/ApplicationLibCode/ProjectDataModel/RimMultiPlot.h b/ApplicationLibCode/ProjectDataModel/RimMultiPlot.h index d795e7f689b..8c10e015f0a 100644 --- a/ApplicationLibCode/ProjectDataModel/RimMultiPlot.h +++ b/ApplicationLibCode/ProjectDataModel/RimMultiPlot.h @@ -142,6 +142,11 @@ class RimMultiPlot : public RimPlotWindow, public RimTypedPlotCollection ensembleSummaryAddresses() const; virtual std::set ensembleTimeSteps() const; + // Hint that these addresses are about to be read, given before the curves pull their values. Sources that + // fetch data remotely can use it to load everything in one go instead of one blocking request per address. + // Data is still loaded on demand, so this is an optimization only: not calling it changes nothing but speed. + virtual void prefetchSummaryData( const std::vector& resultAddresses ) {} + void setEnsembleId( int ensembleId ); int ensembleId() const; bool hasEnsembleParameters() const; diff --git a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.cpp b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.cpp index 51ad0f97b8a..3b12e83fe2e 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.cpp +++ b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.cpp @@ -1588,6 +1588,32 @@ void RimSummaryMultiPlot::onPlotAdditionOrRemoval() RimMultiPlot::onPlotAdditionOrRemoval(); } +//-------------------------------------------------------------------------------------------------- +/// Gather what every plot in the window is about to read and hand it to each ensemble as one set. A plot +/// prefetches for itself as well, but the plots are loaded one at a time, so doing it here is what lets a +/// remote source load the whole window in one request group rather than one per plot. +//-------------------------------------------------------------------------------------------------- +void RimSummaryMultiPlot::prefetchPlotData() +{ + std::map> addressesByEnsemble; + + for ( RimSummaryPlot* plot : summaryPlots() ) + { + if ( !plot ) continue; + + for ( const auto& [ensemble, addresses] : plot->summaryAddressesByEnsemble() ) + { + auto& allAddresses = addressesByEnsemble[ensemble]; + allAddresses.insert( allAddresses.end(), addresses.begin(), addresses.end() ); + } + } + + for ( const auto& [ensemble, addresses] : addressesByEnsemble ) + { + ensemble->prefetchSummaryData( addresses ); + } +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.h b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.h index 9ac468526aa..7fdb44f2cad 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.h +++ b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.h @@ -123,6 +123,8 @@ class RimSummaryMultiPlot : public RimMultiPlot, public RimSummaryDataSourceStep void onPlotAdditionOrRemoval() override; + void prefetchPlotData() override; + private: void defineUiOrdering( QString uiConfigName, caf::PdmUiOrdering& uiOrdering ) override; void defineEditorAttribute( const caf::PdmFieldHandle* field, QString uiConfigName, caf::PdmUiEditorAttribute* attribute ) override; diff --git a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.cpp b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.cpp index 817d8af98b7..4feb4c1fd11 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.cpp +++ b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.cpp @@ -1778,6 +1778,66 @@ void RimSummaryPlot::defineUiTreeOrdering( caf::PdmUiTreeOrdering& uiTreeOrderin uiTreeOrdering.skipRemainingChildren( true ); } +//-------------------------------------------------------------------------------------------------- +/// The addresses this plot's curves are about to read, grouped by the ensemble owning them. Used to tell a +/// source what is coming before any of it is read, see prefetchSummaryData(). +//-------------------------------------------------------------------------------------------------- +std::map> RimSummaryPlot::summaryAddressesByEnsemble() const +{ + std::map> addressesByEnsemble; + + auto addAddress = []( auto& addressesByEnsemble, RimSummaryEnsemble* ensemble, const RifEclipseSummaryAddress& address ) + { + if ( !ensemble || !address.isValid() ) return; + + addressesByEnsemble[ensemble].push_back( address ); + }; + + if ( m_summaryCurveCollection ) + { + for ( RimSummaryCurve* curve : m_summaryCurveCollection->curves() ) + { + if ( !curve ) continue; + + if ( auto summaryCase = curve->summaryCaseY() ) + { + addAddress( addressesByEnsemble, summaryCase->firstAncestorOrThisOfType(), curve->summaryAddressY() ); + } + + if ( auto summaryCase = curve->summaryCaseX() ) + { + addAddress( addressesByEnsemble, summaryCase->firstAncestorOrThisOfType(), curve->summaryAddressX() ); + } + } + } + + for ( RimEnsembleCurveSet* curveSet : m_ensembleCurveSetCollection->curveSets() ) + { + if ( !curveSet ) continue; + + addAddress( addressesByEnsemble, curveSet->summaryEnsemble(), curveSet->summaryAddressY() ); + } + + return addressesByEnsemble; +} + +//-------------------------------------------------------------------------------------------------- +/// Tell each ensemble in the plot which addresses its curves are about to read, before any of them read +/// one. Curves pull their values one at a time, so a source loading data remotely would otherwise make one +/// blocking request per curve; given the whole set up front it can load them together. +/// +/// This is a hint only. Every curve still loads its own data, and an ensemble that does not need the hint +/// ignores it. A plot in a plot window is usually covered by the window prefetching for all its plots at +/// once, in which case there is nothing left for this to do. +//-------------------------------------------------------------------------------------------------- +void RimSummaryPlot::prefetchSummaryData() +{ + for ( const auto& [ensemble, addresses] : summaryAddressesByEnsemble() ) + { + ensemble->prefetchSummaryData( addresses ); + } +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -1788,6 +1848,8 @@ void RimSummaryPlot::onLoadDataAndUpdate() auto plotWindow = firstAncestorOrThisOfType(); if ( plotWindow == nullptr ) updateDockWindowVisibility(); + prefetchSummaryData(); + if ( m_summaryCurveCollection ) { m_summaryCurveCollection->loadDataAndUpdate( false ); diff --git a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.h b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.h index 04e39531c11..c5b3805e9e4 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.h +++ b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.h @@ -150,6 +150,8 @@ class RimSummaryPlot : public RimPlot, public RimSummaryDataSourceStepping, publ void deleteAllSummaryCurves(); RimSummaryCurveCollection* summaryCurveCollection() const; + std::map> summaryAddressesByEnsemble() const; + void updatePlotTitle(); const RimSummaryNameHelper* activePlotTitleHelperAllCurves() const; @@ -250,6 +252,8 @@ class RimSummaryPlot : public RimPlot, public RimSummaryDataSourceStepping, publ void defineUiOrdering( QString uiConfigName, caf::PdmUiOrdering& uiOrdering ) override; void onLoadDataAndUpdate() override; + void prefetchSummaryData(); + bool handleGlobalKeyEvent( QKeyEvent* keyEvent ) override; private slots: diff --git a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp index 62fc7c5fc45..076649b8972 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp +++ b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp @@ -30,13 +30,17 @@ #include "RifEclipseSummaryAddress.h" #include "Cloud/RimCloudDataSourceCollection.h" +#include "RimProject.h" #include "RimSummaryCaseMainCollection.h" #include "RimSummaryCaseSumo.h" +#include "RimSummaryPlot.h" #include "RimSumoDataSource.h" #include +#include #include +#include #include CAF_PDM_SOURCE_INIT( RimSummaryEnsembleSumo, "RimSummaryEnsembleSumo" ); @@ -93,6 +97,8 @@ RimSummaryEnsembleSumo::RimSummaryEnsembleSumo() setAsEnsemble( true ); m_sumoConnector = RiaApplication::instance()->makeSumoConnector(); + + m_lifetimeToken = std::make_shared( true ); } //-------------------------------------------------------------------------------------------------- @@ -180,53 +186,188 @@ void RimSummaryEnsembleSumo::updateName( const std::set& existingEnsemb //-------------------------------------------------------------------------------------------------- void RimSummaryEnsembleSumo::loadSummaryData( const RifEclipseSummaryAddress& resultAddress ) { - if ( resultAddress.isStatistics() ) return; + loadSummaryData( std::vector{ resultAddress } ); +} - // An address without a vector name has no blob to fetch. The special time address used as curve - // x-axis is one such address. Requesting it would produce a URL with an empty vector segment, which - // the service answers with 404, surfacing as a misleading "parquet file size is 0 bytes" error. - if ( resultAddress.vectorName().empty() ) return; +//-------------------------------------------------------------------------------------------------- +/// Load several vectors at once. Each vector is one parquet blob covering every realization, so the +/// addresses that are not cached yet are fetched as one concurrent batch rather than one after another. +/// That matters because a vector the service has not aggregated yet is produced on demand by the request +/// asking for it, so fetching serially costs the sum of those aggregations. +//-------------------------------------------------------------------------------------------------- +void RimSummaryEnsembleSumo::loadSummaryData( const std::vector& resultAddresses ) +{ + // Nothing is fetched while the caller waits. A curve asking for values it does not have yet gets none, + // the request is put on its way, and the curve is drawn again once it arrives. Waiting here instead + // stopped the application for as long as the service took, which for a vector it has not aggregated yet + // is a good while. + prefetchSummaryData( resultAddresses ); +} - if ( !m_sumoDataSource() ) return; +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimSummaryEnsembleSumo::loadEnsembleParameters() +{ + if ( !m_sumoDataSource() || !m_sumoConnector ) return; + + auto sumoCaseId = m_sumoDataSource->caseId(); + auto sumoEnsembleName = m_sumoDataSource->ensembleName(); + + auto parametersKey = ParquetKey{ sumoCaseId, sumoEnsembleName, "", true }; + if ( m_parquetTable.find( parametersKey ) != m_parquetTable.end() ) return; + if ( m_pendingVectors.find( parametersKey ) != m_pendingVectors.end() ) return; + + // Asked for without waiting, like the vectors. The service aggregates the parameters on demand too, so + // the first request for them can take a while, and it used to be made from inside the read of a curve + // value: dropping a vector into a plot stopped the application until the parameters had been fetched. + m_pendingVectors[parametersKey] = RifEclipseSummaryAddress(); - auto resultText = QString::fromStdString( resultAddress.toEclipseTextAddress() ); + std::weak_ptr isAlive = m_lifetimeToken; + + m_sumoConnector->summary().parameterDataAsync( sumoCaseId, + sumoEnsembleName, + [this, isAlive, parametersKey]( const QByteArray& contents ) + { + if ( isAlive.expired() ) return; + + onParameterDataReceived( parametersKey, contents ); + } ); +} + +//-------------------------------------------------------------------------------------------------- +/// The ensemble parameters have arrived. Called on the thread owning the user interface. +//-------------------------------------------------------------------------------------------------- +void RimSummaryEnsembleSumo::onParameterDataReceived( const ParquetKey& parquetKey, const QByteArray& contents ) +{ + auto it = m_pendingVectors.find( parquetKey ); + + // No longer wanted: the data source changed, or the cache was cleared, while this was on its way. + if ( it == m_pendingVectors.end() ) return; + + m_pendingVectors.erase( it ); + + RiaLogging::debug( std::format( "Load ensemble parameter sensitivities. Contents size: {}", contents.size() ) ); + + std::shared_ptr table = readParquetTable( contents, QString( "%1 parameter sensitivities" ).arg( parquetKey.ensembleId ) ); + + m_parquetTable[parquetKey] = table; + + distributeParametersDataToRealizations( table ); + + updatePlotsUsingThisEnsemble(); +} + +//-------------------------------------------------------------------------------------------------- +/// Ask for everything the plots are about to read, and return without waiting for any of it. Each vector is +/// taken in as it arrives, so a plot shows the vectors that are ready while the rest are still on their way +/// rather than staying blank until the slowest one is done. +/// +/// A vector still on its way is reported as having no data, and the curves using it are drawn empty. They are +/// redrawn when it arrives, see onVectorDataReceived. +//-------------------------------------------------------------------------------------------------- +void RimSummaryEnsembleSumo::prefetchSummaryData( const std::vector& resultAddresses ) +{ + if ( !m_sumoDataSource() || !m_sumoConnector ) return; auto sumoCaseId = m_sumoDataSource->caseId(); auto sumoEnsembleName = m_sumoDataSource->ensembleName(); - auto key = ParquetKey{ sumoCaseId, sumoEnsembleName, resultText, false }; - if ( m_parquetTable.find( key ) == m_parquetTable.end() ) + std::vector vectorNamesToFetch; + for ( const auto& resultAddress : resultAddresses ) { - auto contents = loadParquetData( key ); - RiaLogging::debug( std::format( "Load Summary Data. Contents size: {}", contents.size() ) ); + if ( resultAddress.isStatistics() ) continue; + if ( resultAddress.vectorName().empty() ) continue; - std::shared_ptr table = readParquetTable( contents, QString::fromStdString( resultAddress.uiText() ) ); - m_parquetTable[key] = table; + auto resultText = QString::fromStdString( resultAddress.toEclipseTextAddress() ); + auto key = ParquetKey{ sumoCaseId, sumoEnsembleName, resultText, false }; - distributeDataToRealizations( resultAddress, table ); + if ( m_parquetTable.find( key ) != m_parquetTable.end() ) continue; + if ( m_pendingVectors.find( key ) != m_pendingVectors.end() ) continue; + + m_pendingVectors[key] = resultAddress; + vectorNamesToFetch.push_back( resultText ); } - auto parametersKey = ParquetKey{ sumoCaseId, sumoEnsembleName, "", true }; - if ( m_parquetTable.find( parametersKey ) == m_parquetTable.end() ) - { - auto contents = m_sumoConnector->summary().parameterData( sumoCaseId, sumoEnsembleName ); - RiaLogging::debug( std::format( "Load ensemble parameter sensitivities. Contents size: {}", contents.size() ) ); + // The parameters belong to the ensemble rather than to any one vector, and are wanted as soon as + // anything of it is read. Asked for here so they travel alongside the vectors. + loadEnsembleParameters(); - std::shared_ptr table = readParquetTable( contents, QString( "%1 parameter sensitivities" ).arg( sumoEnsembleName ) ); - m_parquetTable[parametersKey] = table; + if ( vectorNamesToFetch.empty() ) return; - distributeParametersDataToRealizations( table ); - } + std::weak_ptr isAlive = m_lifetimeToken; + + m_sumoConnector->summary().vectorDataAsync( sumoCaseId, + sumoEnsembleName, + vectorNamesToFetch, + [this, isAlive, sumoCaseId, sumoEnsembleName]( const QString& vectorName, + const QByteArray& contents ) + { + // The request outlived the ensemble that asked for it. + if ( isAlive.expired() ) return; + + onVectorDataReceived( ParquetKey{ sumoCaseId, sumoEnsembleName, vectorName, false }, + contents ); + } ); } //-------------------------------------------------------------------------------------------------- -/// +/// One requested vector has arrived. Called on the thread owning the user interface, once per vector. //-------------------------------------------------------------------------------------------------- -QByteArray RimSummaryEnsembleSumo::loadParquetData( const ParquetKey& parquetKey ) +void RimSummaryEnsembleSumo::onVectorDataReceived( const ParquetKey& parquetKey, const QByteArray& contents ) { - if ( !m_sumoConnector ) return {}; + auto it = m_pendingVectors.find( parquetKey ); + + // No longer wanted: the data source changed, or the cache was cleared, while this was on its way. + if ( it == m_pendingVectors.end() ) return; + + const auto resultAddress = it->second; + m_pendingVectors.erase( it ); + + RiaLogging::debug( std::format( "Load Summary Data. Contents size: {}", contents.size() ) ); - return m_sumoConnector->summary().vectorData( SumoCaseId( parquetKey.caseId ), parquetKey.ensembleId, parquetKey.vectorName ); + // Empty contents mean the request failed. The empty result is stored like any other, so a failure is not + // retried on every redraw, matching what a failed blocking load does. + std::shared_ptr table = readParquetTable( contents, QString::fromStdString( resultAddress.uiText() ) ); + + m_parquetTable[parquetKey] = table; + distributeDataToRealizations( resultAddress, table ); + + loadEnsembleParameters(); + + updatePlotsUsingThisEnsemble(); +} + +//-------------------------------------------------------------------------------------------------- +/// Redraw with what has arrived so far. The curves read their values again, those still waiting for data come +/// back empty, and the replot itself is coalesced by the redraw scheduler. +//-------------------------------------------------------------------------------------------------- +void RimSummaryEnsembleSumo::updatePlotsUsingThisEnsemble() +{ + // Loading a plot can bring in the next vector, which asks for this update again. Finish the pass that is + // running and repeat it afterwards, rather than reloading plots from inside their own load. + if ( m_isUpdatingPlots ) + { + m_hasMissedPlotUpdate = true; + return; + } + + m_isUpdatingPlots = true; + + do + { + m_hasMissedPlotUpdate = false; + + for ( RimSummaryPlot* summaryPlot : RimProject::current()->descendantsOfType() ) + { + if ( !summaryPlot->summaryAddressesByEnsemble().contains( this ) ) continue; + + summaryPlot->loadDataAndUpdate(); + summaryPlot->scheduleReplotIfVisible(); + } + } while ( m_hasMissedPlotUpdate ); + + m_isUpdatingPlots = false; } //-------------------------------------------------------------------------------------------------- @@ -654,6 +795,10 @@ void RimSummaryEnsembleSumo::clearCachedData() { m_resultAddresses.clear(); m_parquetTable.clear(); + + // Anything still on its way belongs to the data just thrown away. Forgetting it here makes those replies + // drop their contents on arrival, and lets the vectors be requested again if they are still wanted. + m_pendingVectors.clear(); } //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.h b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.h index c3b07b09dcc..85e62256a44 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.h +++ b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.h @@ -65,12 +65,14 @@ class RimSummaryEnsembleSumo : public RimSummaryEnsemble void onRealizationSelectionChanged(); void loadSummaryData( const RifEclipseSummaryAddress& resultAddress ); + void loadSummaryData( const std::vector& resultAddresses ); std::string unitName( const RifEclipseSummaryAddress& resultAddress ); RiaDefines::EclipseUnitSystem unitSystem() const; std::set allResultAddresses() const; std::pair nameKeys() const override; void updateName( const std::set& existingEnsembleNames ) override; + void prefetchSummaryData( const std::vector& resultAddresses ) override; protected: void onLoadDataAndUpdate() override; @@ -83,13 +85,16 @@ class RimSummaryEnsembleSumo : public RimSummaryEnsemble void updateResultAddresses(); void clearCachedData(); - QByteArray loadParquetData( const ParquetKey& parquetKey ); - void distributeDataToRealizations( const RifEclipseSummaryAddress& resultAddress, std::shared_ptr table ); void buildMetaData(); void distributeParametersDataToRealizations( std::shared_ptr table ); void redistributeCachedDataToRealizations(); + void loadEnsembleParameters(); + + void onVectorDataReceived( const ParquetKey& parquetKey, const QByteArray& contents ); + void onParameterDataReceived( const ParquetKey& parquetKey, const QByteArray& contents ); + void updatePlotsUsingThisEnsemble(); static std::shared_ptr readParquetTable( const QByteArray& contents, const QString& messageTag ); @@ -100,4 +105,15 @@ class RimSummaryEnsembleSumo : public RimSummaryEnsemble std::set m_resultAddresses; std::map> m_parquetTable; + + // The vectors requested but not yet arrived, and the address each belongs to. A vector in here is not + // requested again, and is reported as having no data yet rather than being waited for. + std::map m_pendingVectors; + + // Held by the callbacks of requests still on their way. They check it before touching this object, so a + // reply arriving after the ensemble is gone is dropped instead of writing into freed memory. + std::shared_ptr m_lifetimeToken; + + bool m_isUpdatingPlots = false; + bool m_hasMissedPlotUpdate = false; }; From 86cdaecdba996636c80c4a4f9e0872d40edd7923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Herje?= Date: Tue, 18 Aug 2026 10:45:30 +0200 Subject: [PATCH 5/6] Show that Sumo data is being loaded Show progress while browsing Sumo Loading grid properties, ensemble parameters or summary vectors could leave the application sitting still with nothing saying why. The blocking transfers show the standard caf::ProgressInfo dialog. It is created where the waiting happens, so a single place covers every blocking Sumo transfer, and it is shown without the usual delay: a delayed dialog is put up by a timer, and the waiting thread dispatches no events, so it would never appear for exactly the requests slow enough to want it. Summary vectors are loaded without waiting and so have no scope to put a dialog in. A plot waiting for data shows a spinner in its corner instead, asked for through a new RimSummaryEnsemble::isSummaryDataPending, and each subplot drops its spinner as its own vectors arrive. Co-Authored-By: Claude Opus 5 --- .../Tools/Cloud/RiaSumoConnector.cpp | 28 +++- .../Tools/Cloud/RiaSumoConnector.h | 8 +- .../Tools/Cloud/RiaSumoExplore.cpp | 8 +- .../Application/Tools/Cloud/RiaSumoGrid.cpp | 3 +- .../Tools/Cloud/RiaSumoSummary.cpp | 3 +- .../Summary/RimSummaryEnsemble.h | 4 + .../Summary/RimSummaryPlot.cpp | 47 ++++++ .../ProjectDataModel/Summary/RimSummaryPlot.h | 7 +- .../Summary/Sumo/RimSummaryEnsembleSumo.cpp | 20 +++ .../Summary/Sumo/RimSummaryEnsembleSumo.h | 1 + .../RiuAbstractOverlayContentFrame.cpp | 143 ++++++++++++++++++ .../RiuAbstractOverlayContentFrame.h | 32 ++++ 12 files changed, 290 insertions(+), 14 deletions(-) diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp index 4a8e250c580..00a925ea3a5 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp @@ -24,6 +24,8 @@ #include "RiaOsduDefines.h" #include "RiaQStringFormatter.h" +#include "cafProgressInfo.h" + #include #include #include @@ -36,6 +38,7 @@ #include #include +#include //-------------------------------------------------------------------------------------------------- /// @@ -241,7 +244,7 @@ void RiaSumoConnector::abortIfNotFinishedWithin( QNetworkReply* reply, int timeo /// Run work on the transfer thread and wait for it to finish, without dispatching any events on the calling /// thread. This is what keeps the view update code from re-entering a load that is already running. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::runOnTransferThreadBlocking( const std::function& work ) +void RiaSumoConnector::runOnTransferThreadBlocking( const std::function& work, const QString& progressText ) { // A blocking request can be issued from inside another one, for instance the blob id lookup done while // downloading a grid property. Already on the transfer thread, run directly instead of deadlocking on a @@ -252,6 +255,21 @@ void RiaSumoConnector::runOnTransferThreadBlocking( const std::function& return; } + // Tell the user something is being loaded while this thread waits. Created only here, after the branch + // above has returned for calls made from the transfer thread: caf::ProgressInfo hands construction to the + // thread owning the user interface and waits for it, which would deadlock against a thread already + // waiting for this work. + // + // The dialog must not be delayed. A delayed dialog is put up by a timer, and no events are dispatched on + // this thread while the work runs, so it would never appear for exactly the requests slow enough to want + // it. There is one step: the work is a single wait, with nothing to count along the way. + std::optional progressInfo; + if ( !progressText.isEmpty() ) + { + const bool delayShowingProgress = false; + progressInfo.emplace( 1, progressText, delayShowingProgress ); + } + QSemaphore semaphore; QMetaObject::invokeMethod( @@ -338,7 +356,7 @@ QByteArray RiaSumoConnector::downloadBlobBlocking( const QString& blobId ) /// Issue a GET and return the response body, waiting on the transfer thread. Returns an empty array when /// the request fails. This is the primitive the data specific requests are built from. //-------------------------------------------------------------------------------------------------- -QByteArray RiaSumoConnector::getBlocking( const QString& url ) +QByteArray RiaSumoConnector::getBlocking( const QString& url, const QString& progressText ) { requestTokenBlocking(); @@ -363,7 +381,8 @@ QByteArray RiaSumoConnector::getBlocking( const QString& url ) eventLoop.exec(); body = replyBody( reply, url ); - } ); + }, + progressText ); return body; } @@ -418,7 +437,8 @@ std::map RiaSumoConnector::downloadBlobsBlocking( const std std::map contentsByBlobId; - runOnTransferThreadBlocking( [&]() { contentsByBlobId = downloadBlobs( blobIds ); } ); + runOnTransferThreadBlocking( [&]() { contentsByBlobId = downloadBlobs( blobIds ); }, + QString( "Downloading %1 file(s) from Sumo" ).arg( blobIds.size() ) ); return contentsByBlobId; } diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h index 1e33008de7a..f4a147dba24 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h @@ -73,13 +73,17 @@ class RiaSumoConnector : public RiaCloudConnector // Transport used by the data specific delegates. Every request goes through the transfer thread, so // the calling thread waits without dispatching events. - QByteArray getBlocking( const QString& url ); + QByteArray getBlocking( const QString& url, const QString& progressText = {} ); // The REST API returns a blob id as a plain string, quoted by FastAPI. static QString blobIdFromBody( const QByteArray& body ); void addStandardHeader( QNetworkRequest& networkRequest, const QString& token, const QString& contentType ); - void runOnTransferThreadBlocking( const std::function& work ); + + // Run work on the transfer thread and wait for it. Pass progressText to show the standard progress dialog + // while waiting, worth doing for the transfers slow enough to be noticed and not for the small requests + // that would only make it flash. + void runOnTransferThreadBlocking( const std::function& work, const QString& progressText = {} ); // Run work on the transfer thread without waiting for it. The async data paths use this: the result is // delivered by a callback rather than by returning, so the calling thread carries on immediately. diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.cpp index 3daae970353..b76104408e6 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.cpp +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.cpp @@ -42,7 +42,7 @@ std::vector RiaSumoExplore::assets() { const QString url = QString( "%1/assets" ).arg( m_connector.server() ); - return parseAssets( m_connector.getBlocking( url ) ); + return parseAssets( m_connector.getBlocking( url, "Loading assets from Sumo" ) ); } //-------------------------------------------------------------------------------------------------- @@ -52,7 +52,7 @@ std::vector RiaSumoExplore::cases( const QString& assetName ) { const QString url = QString( "%1/cases?asset_name=%2" ).arg( m_connector.server() ).arg( QString( QUrl::toPercentEncoding( assetName ) ) ); - return parseCases( m_connector.getBlocking( url ) ); + return parseCases( m_connector.getBlocking( url, QString( "Loading the cases of %1 from Sumo" ).arg( assetName ) ) ); } //-------------------------------------------------------------------------------------------------- @@ -62,7 +62,7 @@ std::vector RiaSumoExplore::ensembleNames( const SumoCaseId& caseId ) { const QString url = QString( "%1/cases/%2/ensembles" ).arg( m_connector.server() ).arg( caseId.get() ); - return parseEnsembleNames( m_connector.getBlocking( url ) ); + return parseEnsembleNames( m_connector.getBlocking( url, "Loading ensembles from Sumo" ) ); } //-------------------------------------------------------------------------------------------------- @@ -75,7 +75,7 @@ std::vector RiaSumoExplore::realizationIds( const SumoCaseId& caseId, c const QString url = QString( "%1/cases/%2/ensembles/%3/realizations" ).arg( m_connector.server() ).arg( caseId.get() ).arg( encodedEnsembleName ); - return parseRealizationIds( m_connector.getBlocking( url ) ); + return parseRealizationIds( m_connector.getBlocking( url, "Loading realizations from Sumo" ) ); } //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.cpp index d2250994de7..bbd67a02356 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.cpp +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.cpp @@ -159,7 +159,8 @@ void RiaSumoGrid::prefetchPropertyData( const SumoCaseId& caseId, if ( timestampsToFetch.size() < 2 ) return; // nothing to gain over the single time step path m_connector.runOnTransferThreadBlocking( - [&]() { fetchPropertyBatch( caseId, ensembleName, gridName, realization, propertyName, timestampsToFetch, cacheKeys ); } ); + [&]() { fetchPropertyBatch( caseId, ensembleName, gridName, realization, propertyName, timestampsToFetch, cacheKeys ); }, + QString( "Loading %1 time step(s) of %2 from Sumo" ).arg( timestampsToFetch.size() ).arg( propertyName ) ); } //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp index 0323c8b5467..4ebf4b3efad 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp @@ -127,7 +127,8 @@ std::map contentsByVectorName[namesToFetch[i]] = it->second; } } - } ); + }, + QString( "Loading %1 summary vector(s) from Sumo" ).arg( namesToFetch.size() ) ); return contentsByVectorName; } diff --git a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryEnsemble.h b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryEnsemble.h index dbf8625d635..db10750384e 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryEnsemble.h +++ b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryEnsemble.h @@ -88,6 +88,10 @@ class RimSummaryEnsemble : public caf::PdmObject // Data is still loaded on demand, so this is an optimization only: not calling it changes nothing but speed. virtual void prefetchSummaryData( const std::vector& resultAddresses ) {} + // Whether any of these addresses is being loaded right now, for sources that load without waiting. A plot + // asks about the addresses of its own curves, so it can say it is still waiting for data. + virtual bool isSummaryDataPending( const std::vector& resultAddresses ) const { return false; } + void setEnsembleId( int ensembleId ); int ensembleId() const; bool hasEnsembleParameters() const; diff --git a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.cpp b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.cpp index 4feb4c1fd11..16daec581a2 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.cpp +++ b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.cpp @@ -62,6 +62,8 @@ #include "SummaryPlotCommands/RicSummaryPlotEditorUi.h" #include "Tools/RimPlotAxisTools.h" +#include "RiuAbstractOverlayContentFrame.h" +#include "RiuDraggableOverlayFrame.h" #include "RiuPlotAxis.h" #include "RiuPlotMainWindow.h" #include "RiuPlotMainWindowTools.h" @@ -1838,6 +1840,49 @@ void RimSummaryPlot::prefetchSummaryData() } } +//-------------------------------------------------------------------------------------------------- +/// Say that this plot is waiting for data. A source can load without waiting, and then the curves are drawn +/// with what has arrived so far, leaving a plot looking finished when it is not. Called after every load, so +/// the frame appears when the data is asked for and goes away when the last of it has arrived. +//-------------------------------------------------------------------------------------------------- +void RimSummaryPlot::updateLoadingOverlayFrame() +{ + if ( !plotWidget() ) return; + + bool isWaitingForData = false; + for ( const auto& [ensemble, addresses] : summaryAddressesByEnsemble() ) + { + if ( ensemble->isSummaryDataPending( addresses ) ) + { + isWaitingForData = true; + break; + } + } + + if ( !isWaitingForData ) + { + if ( m_loadingOverlayFrame ) + { + plotWidget()->removeOverlayFrame( m_loadingOverlayFrame ); + delete m_loadingOverlayFrame; + m_loadingOverlayFrame = nullptr; + } + return; + } + + if ( !m_loadingOverlayFrame ) + { + m_loadingOverlayFrame = new RiuDraggableOverlayFrame( plotWidget()->getParentForOverlay(), plotWidget()->overlayMargins() ); + m_loadingOverlayFrame->setAnchorCorner( RiuDraggableOverlayFrame::AnchorCorner::TopLeft ); + + auto* spinnerFrame = new RiuSpinnerOverlayContentFrame( m_loadingOverlayFrame ); + m_loadingOverlayFrame->setContentFrame( spinnerFrame ); + spinnerFrame->setText( "Loading" ); + } + + plotWidget()->addOverlayFrame( m_loadingOverlayFrame ); +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -1896,6 +1941,8 @@ void RimSummaryPlot::onLoadDataAndUpdate() updateAxes(); updateStackedCurveData(); + + updateLoadingOverlayFrame(); } //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.h b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.h index c5b3805e9e4..31a34833d9d 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.h +++ b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.h @@ -56,6 +56,7 @@ class RimSummaryTimeAxisProperties; class RimPlotAxisPropertiesInterface; class RimPlotAxisProperties; class RiuSummaryQwtPlot; +class RiuDraggableOverlayFrame; class RimSummaryNameHelper; class RimSummaryPlotNameHelper; class RimPlotTemplateFileItem; @@ -253,6 +254,7 @@ class RimSummaryPlot : public RimPlot, public RimSummaryDataSourceStepping, publ void onLoadDataAndUpdate() override; void prefetchSummaryData(); + void updateLoadingOverlayFrame(); bool handleGlobalKeyEvent( QKeyEvent* keyEvent ) override; @@ -345,8 +347,9 @@ private slots: caf::PdmChildArrayField m_axisPropertiesArray; - QPointer m_summaryPlot; - std::unique_ptr m_plotInfoLabel; + QPointer m_summaryPlot; + QPointer m_loadingOverlayFrame; + std::unique_ptr m_plotInfoLabel; std::unique_ptr m_nameHelperAllCurves; caf::PdmChildField m_sourceStepping; diff --git a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp index 076649b8972..9c19519c489 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp +++ b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp @@ -311,6 +311,26 @@ void RimSummaryEnsembleSumo::prefetchSummaryData( const std::vector& resultAddresses ) const +{ + if ( m_pendingVectors.empty() || !m_sumoDataSource() ) return false; + + auto sumoCaseId = m_sumoDataSource()->caseId(); + auto sumoEnsembleName = m_sumoDataSource()->ensembleName(); + + for ( const auto& resultAddress : resultAddresses ) + { + auto resultText = QString::fromStdString( resultAddress.toEclipseTextAddress() ); + + if ( m_pendingVectors.contains( ParquetKey{ sumoCaseId, sumoEnsembleName, resultText, false } ) ) return true; + } + + return false; +} + //-------------------------------------------------------------------------------------------------- /// One requested vector has arrived. Called on the thread owning the user interface, once per vector. //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.h b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.h index 85e62256a44..657f950626f 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.h +++ b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.h @@ -73,6 +73,7 @@ class RimSummaryEnsembleSumo : public RimSummaryEnsemble std::pair nameKeys() const override; void updateName( const std::set& existingEnsembleNames ) override; void prefetchSummaryData( const std::vector& resultAddresses ) override; + bool isSummaryDataPending( const std::vector& resultAddresses ) const override; protected: void onLoadDataAndUpdate() override; diff --git a/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.cpp b/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.cpp index eef813b43f6..7e83d230e62 100644 --- a/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.cpp +++ b/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.cpp @@ -21,9 +21,11 @@ #include "RiaFontCache.h" #include "RiaPreferences.h" +#include #include #include #include +#include #include //-------------------------------------------------------------------------------------------------- @@ -101,3 +103,144 @@ void RiuTextOverlayContentFrame::updateLabelFont() font.setPointSize( caf::FontTools::pointSizeFromEnum( RiaPreferences::current()->defaultPlotFontSize() ) ); m_textLabel->setFont( font ); } + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RiuSpinnerOverlayContentFrame::RiuSpinnerOverlayContentFrame( QWidget* parent /*= nullptr */ ) + : RiuAbstractOverlayContentFrame( parent ) +{ + QHBoxLayout* layout = new QHBoxLayout( this ); + + // Room for the spinner, which is painted rather than laid out: it has no content of its own to size it, + // and reserving the space keeps it from ending up under the text. + layout->setContentsMargins( 4 + spinnerSize() + spinnerMargin(), 4, 4, 4 ); + + m_textLabel = new QLabel; + layout->addWidget( m_textLabel ); + + m_animationTimer = new QTimer( this ); + m_animationTimer->setInterval( 50 ); + + QObject::connect( m_animationTimer, + &QTimer::timeout, + this, + [this]() + { + m_angleDegrees = ( m_angleDegrees + 30 ) % 360; + update(); + } ); + + updateLabelFont(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiuSpinnerOverlayContentFrame::setText( const QString& text ) +{ + m_textLabel->setText( text ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +int RiuSpinnerOverlayContentFrame::spinnerSize() +{ + return 14; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +int RiuSpinnerOverlayContentFrame::spinnerMargin() +{ + return 6; +} + +//-------------------------------------------------------------------------------------------------- +/// Animate only while on screen. A frame taken off a plot keeps its timer, and a timer left running would +/// wake the application up several times a second for something nobody is looking at. +//-------------------------------------------------------------------------------------------------- +void RiuSpinnerOverlayContentFrame::showEvent( QShowEvent* event ) +{ + RiuAbstractOverlayContentFrame::showEvent( event ); + + m_animationTimer->start(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiuSpinnerOverlayContentFrame::hideEvent( QHideEvent* event ) +{ + RiuAbstractOverlayContentFrame::hideEvent( event ); + + m_animationTimer->stop(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiuSpinnerOverlayContentFrame::paintEvent( QPaintEvent* event ) +{ + RiuAbstractOverlayContentFrame::paintEvent( event ); + + QPainter painter( this ); + drawSpinner( &painter, QPoint( 4, ( height() - spinnerSize() ) / 2 ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// An arc left open at one end, turned a step further on every tick. The gap is what makes the turning +/// visible: a full circle would look the same at every angle. +//-------------------------------------------------------------------------------------------------- +void RiuSpinnerOverlayContentFrame::drawSpinner( QPainter* painter, const QPoint& topLeft ) const +{ + painter->save(); + painter->setRenderHint( QPainter::Antialiasing ); + + QPen pen( palette().color( QPalette::WindowText ), 2.0, Qt::SolidLine, Qt::RoundCap ); + painter->setPen( pen ); + painter->setBrush( Qt::NoBrush ); + + // Qt angles are in sixteenths of a degree and turn counterclockwise, so the sign makes the arc turn the + // way a clock does. Inset by the pen width, otherwise the stroke is drawn half outside the rectangle. + const QRect arcRect( topLeft.x() + 1, topLeft.y() + 1, spinnerSize() - 2, spinnerSize() - 2 ); + painter->drawArc( arcRect, -m_angleDegrees * 16, 300 * 16 ); + + painter->restore(); +} + +//-------------------------------------------------------------------------------------------------- +/// Drawn into snapshots as it looks at this moment. There is no animation in a still image, but leaving the +/// spinner out would make a plot that was still loading look finished. +//-------------------------------------------------------------------------------------------------- +void RiuSpinnerOverlayContentFrame::renderTo( QPainter* painter, const QRect& targetRect ) +{ + updateLabelFont(); + + painter->save(); + painter->translate( targetRect.topLeft() ); + + drawSpinner( painter, QPoint( 4, ( targetRect.height() - spinnerSize() ) / 2 ) ); + + painter->translate( contentsMargins().left(), contentsMargins().top() ); + painter->setFont( m_textLabel->font() ); + + QTextDocument td; + td.setDefaultFont( m_textLabel->font() ); + td.setHtml( m_textLabel->text() ); + td.drawContents( painter ); + + painter->restore(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiuSpinnerOverlayContentFrame::updateLabelFont() +{ + QFont font = m_textLabel->font(); + font.setPointSize( caf::FontTools::pointSizeFromEnum( RiaPreferences::current()->defaultPlotFontSize() ) ); + m_textLabel->setFont( font ); +} diff --git a/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.h b/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.h index b11b0899b5d..55da1c344bd 100644 --- a/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.h +++ b/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.h @@ -22,6 +22,7 @@ #include class QLabel; +class QTimer; class RiuAbstractOverlayContentFrame : public QFrame { @@ -51,3 +52,34 @@ class RiuTextOverlayContentFrame : public RiuAbstractOverlayContentFrame private: QPointer m_textLabel; }; + +//================================================================================================== +/// Says that something is going on, for work that finishes on its own and reports no progress along the +/// way. The animation is driven by a timer that only runs while the frame is visible, so a frame that has +/// been taken off a plot costs nothing. +//================================================================================================== +class RiuSpinnerOverlayContentFrame : public RiuAbstractOverlayContentFrame +{ + Q_OBJECT +public: + RiuSpinnerOverlayContentFrame( QWidget* parent = nullptr ); + + void setText( const QString& text ); + void renderTo( QPainter* painter, const QRect& targetRect ) override; + +protected: + void paintEvent( QPaintEvent* event ) override; + void showEvent( QShowEvent* event ) override; + void hideEvent( QHideEvent* event ) override; + +private: + void drawSpinner( QPainter* painter, const QPoint& topLeft ) const; + static int spinnerSize(); + static int spinnerMargin(); + void updateLabelFont(); + +private: + QPointer m_textLabel; + QTimer* m_animationTimer = nullptr; + int m_angleDegrees = 0; +}; From a1fb793ebde211d0c56a8496283ba3a30fd2c394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Herje?= Date: Tue, 18 Aug 2026 14:16:59 +0200 Subject: [PATCH 6/6] formatting --- ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.h b/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.h index e4d934ecf34..15aebb8c316 100644 --- a/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.h +++ b/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.h @@ -97,5 +97,5 @@ class RimHistogramCalculator RigHistogramCalculator::BinningMode m_binningMode = RigHistogramCalculator::BinningMode::LINEAR; RigHistogramCalculator::OutOfRangeHandling m_outOfRangeHandling = RigHistogramCalculator::OutOfRangeHandling::EXCLUDE; std::optional> m_customBinRange; - bool m_doComputeMobileVolumeWeightedMean; + bool m_doComputeMobileVolumeWeightedMean; };