diff --git a/ApplicationLibCode/Application/Tools/Cloud/CMakeLists_files.cmake b/ApplicationLibCode/Application/Tools/Cloud/CMakeLists_files.cmake index 4c05f2d8e6..4ddf728532 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/CMakeLists_files.cmake +++ b/ApplicationLibCode/Application/Tools/Cloud/CMakeLists_files.cmake @@ -1,11 +1,16 @@ set(SOURCE_GROUP_SOURCE_FILES ${CMAKE_CURRENT_LIST_DIR}/RiaCloudApiService.cpp ${CMAKE_CURRENT_LIST_DIR}/RiaCloudConnector.cpp + ${CMAKE_CURRENT_LIST_DIR}/RiaSumoBlobCache.cpp ${CMAKE_CURRENT_LIST_DIR}/RiaSumoConnector.cpp ${CMAKE_CURRENT_LIST_DIR}/RiaSumoDefines.cpp + ${CMAKE_CURRENT_LIST_DIR}/RiaSumoExplore.cpp + ${CMAKE_CURRENT_LIST_DIR}/RiaSumoGrid.cpp + ${CMAKE_CURRENT_LIST_DIR}/RiaSumoSummary.cpp ${CMAKE_CURRENT_LIST_DIR}/RiaConnectorTools.cpp ${CMAKE_CURRENT_LIST_DIR}/RiaOsduConnector.cpp ${CMAKE_CURRENT_LIST_DIR}/RiaOAuthHttpServerReplyHandler.cpp + ${CMAKE_CURRENT_LIST_DIR}/RifReaderSumoGridProperty.cpp ) list(APPEND CODE_SOURCE_FILES ${SOURCE_GROUP_SOURCE_FILES}) diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoBlobCache.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoBlobCache.cpp new file mode 100644 index 0000000000..61ef93409e --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoBlobCache.cpp @@ -0,0 +1,121 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "RiaSumoBlobCache.h" + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RiaSumoBlobCache::RiaSumoBlobCache( size_t limitBytes ) + : m_limitBytes( limitBytes ) +{ +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RiaSumoBlobCache::contains( const QString& key ) const +{ + return m_entries.contains( key ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QByteArray RiaSumoBlobCache::lookup( const QString& key ) +{ + auto it = m_entries.find( key ); + if ( it == m_entries.end() ) return {}; + + m_order.splice( m_order.begin(), m_order, it->second.orderIterator ); + + return it->second.contents; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoBlobCache::insert( const QString& key, const QByteArray& contents ) +{ + if ( contents.isEmpty() ) return; + + const size_t contentsSize = static_cast( contents.size() ); + + // A blob larger than the whole cache would evict everything else to make room for itself. Leave it + // uncached instead, so the smaller blobs already present stay available. + if ( contentsSize > m_limitBytes ) return; + + // Re-inserting an existing key would leak its order list entry, so drop the previous version first. + if ( auto it = m_entries.find( key ); it != m_entries.end() ) + { + m_sizeBytes -= static_cast( it->second.contents.size() ); + m_order.erase( it->second.orderIterator ); + m_entries.erase( it ); + } + + m_order.push_front( key ); + m_entries[key] = { contents, m_order.begin() }; + m_sizeBytes += contentsSize; + + while ( m_sizeBytes > m_limitBytes && !m_order.empty() ) + { + const QString& oldestKey = m_order.back(); + + if ( auto it = m_entries.find( oldestKey ); it != m_entries.end() ) + { + m_sizeBytes -= static_cast( it->second.contents.size() ); + m_entries.erase( it ); + } + + m_order.pop_back(); + } +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoBlobCache::clear() +{ + m_entries.clear(); + m_order.clear(); + m_sizeBytes = 0; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +size_t RiaSumoBlobCache::sizeBytes() const +{ + return m_sizeBytes; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +size_t RiaSumoBlobCache::entryCount() const +{ + return m_entries.size(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +size_t RiaSumoBlobCache::limitBytes() const +{ + return m_limitBytes; +} diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoBlobCache.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoBlobCache.h new file mode 100644 index 0000000000..4ebf3d4640 --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoBlobCache.h @@ -0,0 +1,68 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include +#include + +#include +#include +#include + +//================================================================================================== +/// Blobs downloaded from Sumo, kept in memory so a repeated request is answered without going to the +/// server. Displaying a grid property computes its legend range across all time steps and then reads the +/// values again, so the same blob is asked for several times. +/// +/// Bounded by total byte size rather than entry count, as blob sizes follow the grid size and vary by +/// orders of magnitude. The least recently used entries are evicted when the limit is exceeded. +//================================================================================================== +class RiaSumoBlobCache +{ +public: + explicit RiaSumoBlobCache( size_t limitBytes ); + + bool contains( const QString& key ) const; + + // The cached contents, or an empty array when the key is not cached. A hit is moved to the front of + // the recency order, so it is evicted last. + QByteArray lookup( const QString& key ); + + // Empty contents are not cached, as an empty array is how a miss is reported. Contents larger than the + // whole limit are not cached either, as making room for them would evict everything else. + void insert( const QString& key, const QByteArray& contents ); + + void clear(); + + size_t sizeBytes() const; + size_t entryCount() const; + size_t limitBytes() const; + +private: + struct Entry + { + QByteArray contents; + std::list::iterator orderIterator; + }; + + std::map m_entries; + std::list m_order; // most recently used at front + size_t m_sizeBytes = 0; + const size_t m_limitBytes; +}; diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp index 31117524cf..00a925ea3a 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.cpp @@ -24,6 +24,8 @@ #include "RiaOsduDefines.h" #include "RiaQStringFormatter.h" +#include "cafProgressInfo.h" + #include #include #include @@ -31,8 +33,13 @@ #include #include #include +#include +#include #include +#include +#include + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -44,848 +51,567 @@ RiaSumoConnector::RiaSumoConnector( QObject* parent, unsigned int port ) : RiaCloudConnector( parent, {}, authority, scopes, clientId, port ) , m_serverUrlProvider( std::move( serverUrlProvider ) ) + , m_explore( *this ) + , m_grid( *this ) + , m_summary( *this ) { -} + // The transfer thread runs the network requests issued by the blocking wrappers, so the calling thread can + // wait for them without dispatching events. The context object gives us something with transfer thread + // affinity to post work to, and the network manager has to be constructed on the thread that uses it. + m_transferThread = new QThread( this ); + m_transferContext = new QObject; + m_transferContext->moveToThread( m_transferThread ); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -QString RiaSumoConnector::server() const -{ - // Ask for the address on every call. The server is bound to the first available port, and gets a new - // port if it is restarted, so a cached address goes stale. - if ( !m_serverUrlProvider ) return {}; + QObject::connect( m_transferThread, + &QThread::started, + m_transferContext, + [this]() { m_transferNetworkAccessManager = new QNetworkAccessManager( m_transferContext ); } ); - return m_serverUrlProvider(); + // The context object lives on the transfer thread, so let that thread delete it when it stops. + QObject::connect( m_transferThread, &QThread::finished, m_transferContext, &QObject::deleteLater ); + + m_transferThread->start(); } //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestFailed( const QAbstractOAuth::Error error ) +RiaSumoExplore& RiaSumoConnector::explore() { - RiaLogging::error( "Request failed: " ); + return m_explore; } //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parquetDownloadComplete( const QString& blobId, const QByteArray& contents, const QString& url ) +RiaSumoGrid& RiaSumoConnector::grid() { - SumoRedirect obj; - obj.objectId = blobId; - obj.contents = contents; - obj.url = url; - - m_redirectInfo.push_back( obj ); + return m_grid; } //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -RiaSumoConnector::~RiaSumoConnector() +RiaSumoSummary& RiaSumoConnector::summary() { + return m_summary; } //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestCasesForField( const QString& fieldName ) +QString RiaSumoConnector::server() const { - m_cases.clear(); - - requestTokenBlocking(); - - QNetworkRequest m_networkRequest; - QString url = QString( "%1/cases?asset_name=%2" ).arg( server() ).arg( fieldName ); - m_networkRequest.setUrl( QUrl( url ) ); - - addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - - auto reply = m_networkAccessManager->get( m_networkRequest ); + // Ask for the address on every call. The server is bound to the first available port, and gets a new + // port if it is restarted, so a cached address goes stale. + if ( !m_serverUrlProvider ) return {}; - connect( reply, - &QNetworkReply::finished, - [this, reply]() - { - // parseCases handles the error case and always emits casesFinished, so the blocking caller - // returns immediately instead of waiting for the request to time out. - parseCases( reply ); - } ); + return m_serverUrlProvider(); } //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestCasesForFieldBlocking( const QString& fieldName ) +void RiaSumoConnector::requestFailed( const QAbstractOAuth::Error error ) { - auto requestCallable = [this, fieldName] { requestCasesForField( fieldName ); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::casesFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); + RiaLogging::error( "Request failed: " ); } //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestAssets() +RiaSumoConnector::~RiaSumoConnector() { - requestTokenBlocking(); - - QNetworkRequest m_networkRequest; - m_networkRequest.setUrl( QUrl( QString( "%1/assets" ).arg( server() ) ) ); - - addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - - auto reply = m_networkAccessManager->get( m_networkRequest ); - - connect( reply, - &QNetworkReply::finished, - [this, reply]() - { - // parseAssets handles the error case and always emits assetsFinished, so the blocking caller - // returns immediately instead of waiting for the request to time out. - parseAssets( reply ); - } ); + if ( m_transferThread ) + { + m_transferThread->quit(); + m_transferThread->wait(); + } } //-------------------------------------------------------------------------------------------------- -/// +/// The network manager of the calling thread. A QNetworkAccessManager can only be used from the thread it +/// was created on, and the connector is used from both the GUI thread (the token flow and the Sumo Data +/// dialog) and the transfer thread (everything issued by a blocking wrapper). //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestAssetsBlocking() +QNetworkAccessManager* RiaSumoConnector::networkAccessManager() { - auto requestCallable = [this] { requestAssets(); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::assetsFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); + if ( m_transferThread && QThread::currentThread() == m_transferThread && m_transferNetworkAccessManager ) + { + return m_transferNetworkAccessManager; + } + + return m_networkAccessManager; } //-------------------------------------------------------------------------------------------------- -/// +/// Run work on the transfer thread and return at once. Nothing is waited for here, so the result has to be +/// delivered by a callback. The token is requested first, while still on the calling thread, because +/// refreshing it may need the authentication objects that live there. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestEnsembleByCasesId( const SumoCaseId& caseId ) +void RiaSumoConnector::runOnTransferThread( const std::function& work ) { requestTokenBlocking(); - QNetworkRequest m_networkRequest; - QString url = QString( "%1/cases/%2/ensembles" ).arg( server() ).arg( caseId.get() ); - m_networkRequest.setUrl( QUrl( url ) ); - - addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - - auto reply = m_networkAccessManager->get( m_networkRequest ); + // Without a transfer thread there is nothing to hand the work to, and running it here is the only option. + // Already on the transfer thread, run directly rather than queue behind work that may be waiting for us. + if ( !m_transferThread || !m_transferContext || QThread::currentThread() == m_transferThread ) + { + work(); + return; + } - connect( reply, - &QNetworkReply::finished, - [this, reply, caseId]() - { - // parseEnsembleNames handles the error case and always emits ensembleNamesFinished, so the - // blocking caller returns immediately instead of waiting for the request to time out. - parseEnsembleNames( reply, caseId ); - } ); + QMetaObject::invokeMethod( m_transferContext, work, Qt::QueuedConnection ); } //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parseEnsembleNames( QNetworkReply* reply, const SumoCaseId& caseId ) +void RiaSumoConnector::invokeOnConnectorThread( const std::function& work ) { - QByteArray result = reply->readAll(); - reply->deleteLater(); - - if ( reply->error() == QNetworkReply::NoError ) + if ( QThread::currentThread() == thread() ) { - m_ensembleNames.clear(); - - QJsonDocument doc = QJsonDocument::fromJson( result ); - QJsonArray jsonArray = doc.array(); - - for ( const QJsonValue& value : jsonArray ) - { - QJsonObject ensembleObj = value.toObject(); - QString ensembleName = ensembleObj["name"].toString(); - m_ensembleNames.push_back( { caseId, ensembleName } ); - } - - RiaLogging::debug( std::format( "Ensemble count : {}", m_ensembleNames.size() ) ); - } - else - { - RiaLogging::error( std::format( "Request ensemble names failed: '{}'", reply->errorString() ) ); + work(); + return; } - emit ensembleNamesFinished(); + QMetaObject::invokeMethod( this, work, Qt::QueuedConnection ); } //-------------------------------------------------------------------------------------------------- -/// +/// A blob is fetched in two steps, first the pre-signed URI and then the data itself. Both are started here +/// and neither is waited for: each step continues from the reply of the one before. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestEnsembleByCasesIdBlocking( const SumoCaseId& caseId ) +void RiaSumoConnector::downloadBlobAsync( const QString& blobId, const std::function& onFinished ) { - auto requestCallable = [this, caseId] { requestEnsembleByCasesId( caseId ); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::ensembleNamesFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); -} + const QString url = QString( "%1/blobs/%2/sas_token_and_blob_base_uri" ).arg( server() ).arg( blobId ); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestVectorNamesForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ) -{ - requestTokenBlocking(); + QNetworkRequest networkRequest; + networkRequest.setUrl( QUrl( url ) ); + addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - QNetworkRequest m_networkRequest; - QString url = QString( "%1/cases/%2/ensembles/%3/vector_list" ).arg( server() ).arg( caseId.get() ).arg( ensembleName ); - m_networkRequest.setUrl( QUrl( url ) ); + auto accessInfoReply = networkAccessManager()->get( networkRequest ); + abortIfNotFinishedWithin( accessInfoReply, RiaSumoDefines::requestTimeoutMillis() ); - addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); + QObject::connect( accessInfoReply, + &QNetworkReply::finished, + m_transferContext, + [this, accessInfoReply, blobId, onFinished]() + { + const QString sasUri = sasUriFromReply( accessInfoReply, blobId ); + if ( sasUri.isEmpty() ) + { + onFinished( {} ); + return; + } - auto reply = m_networkAccessManager->get( m_networkRequest ); + RiaLogging::debug( std::format( "Requesting blob. Id: {} SAS URI: {}", blobId, sasUri ) ); - connect( reply, - &QNetworkReply::finished, - [this, reply, ensembleName, caseId]() - { - // parseVectorNames handles the error case and always emits vectorNamesFinished, so the - // blocking caller returns immediately instead of waiting for the request to time out. - parseVectorNames( reply, caseId, ensembleName ); - } ); + QNetworkRequest blobRequest; + blobRequest.setUrl( sasUri ); + + // The pre-signed SAS URI carries its own credential, so no Authorization header is + // added. Do NOT forward the bearer token to the storage host. + blobRequest.setAttribute( QNetworkRequest::RedirectPolicyAttribute, QNetworkRequest::NoLessSafeRedirectPolicy ); + + auto blobReply = networkAccessManager()->get( blobRequest ); + abortIfNotFinishedWithin( blobReply, RiaSumoDefines::requestTimeoutMillis() ); + + QObject::connect( blobReply, + &QNetworkReply::finished, + m_transferContext, + [blobReply, sasUri, onFinished]() { onFinished( blobContentsFromReply( blobReply, sasUri ) ); } ); + } ); } //-------------------------------------------------------------------------------------------------- -/// +/// Nothing waits on an async reply, so a request that never answers would otherwise keep its data pending +/// for good. Aborting makes the reply finish with an error, which the chain reports as a failed transfer. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestVectorNamesForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ) +void RiaSumoConnector::abortIfNotFinishedWithin( QNetworkReply* reply, int timeoutMillis ) { - auto requestCallable = [this, caseId, ensembleName] { requestVectorNamesForEnsemble( caseId, ensembleName ); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::vectorNamesFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); + if ( !reply ) return; + + QTimer::singleShot( timeoutMillis, + reply, + [reply]() + { + if ( !reply->isFinished() ) reply->abort(); + } ); } //-------------------------------------------------------------------------------------------------- -/// +/// Run work on the transfer thread and wait for it to finish, without dispatching any events on the calling +/// thread. This is what keeps the view update code from re-entering a load that is already running. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestRealizationIdsForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ) +void RiaSumoConnector::runOnTransferThreadBlocking( const std::function& work, const QString& progressText ) { - m_realizationIds.clear(); + // A blocking request can be issued from inside another one, for instance the blob id lookup done while + // downloading a grid property. Already on the transfer thread, run directly instead of deadlocking on a + // thread that is busy waiting for us. + if ( !m_transferThread || !m_transferContext || QThread::currentThread() == m_transferThread ) + { + work(); + return; + } - requestTokenBlocking(); + // Tell the user something is being loaded while this thread waits. Created only here, after the branch + // above has returned for calls made from the transfer thread: caf::ProgressInfo hands construction to the + // thread owning the user interface and waits for it, which would deadlock against a thread already + // waiting for this work. + // + // The dialog must not be delayed. A delayed dialog is put up by a timer, and no events are dispatched on + // this thread while the work runs, so it would never appear for exactly the requests slow enough to want + // it. There is one step: the work is a single wait, with nothing to count along the way. + std::optional progressInfo; + if ( !progressText.isEmpty() ) + { + const bool delayShowingProgress = false; + progressInfo.emplace( 1, progressText, delayShowingProgress ); + } - QNetworkRequest m_networkRequest; - QString url = QString( "%1/cases/%2/ensembles/%3/realizations" ).arg( server() ).arg( caseId.get() ).arg( ensembleName ); - m_networkRequest.setUrl( QUrl( url ) ); + QSemaphore semaphore; - addStandardHeader( m_networkRequest, token(), RiaCloudDefines::contentTypeJson() ); + QMetaObject::invokeMethod( + m_transferContext, + [&work, &semaphore]() + { + // Release on every exit path. A request that fails or times out must not leave the caller waiting. + struct Releaser + { + QSemaphore& semaphore; + ~Releaser() { semaphore.release(); } + } releaser{ semaphore }; - auto reply = m_networkAccessManager->get( m_networkRequest ); + work(); + }, + Qt::QueuedConnection ); - connect( reply, - &QNetworkReply::finished, - [this, reply, ensembleName, caseId]() - { - // parseRealizationNumbers handles the error case and always emits realizationIdsFinished, so - // the blocking caller returns immediately instead of waiting for the request to time out. - parseRealizationNumbers( reply, caseId, ensembleName ); - } ); -} + semaphore.acquire(); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestRealizationIdsForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ) -{ - auto requestCallable = [this, caseId, ensembleName] { requestRealizationIdsForEnsemble( caseId, ensembleName ); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::realizationIdsFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); + // The transfer thread hands its log messages to the thread owning the message panel. Deliver them here, + // while the request they describe is still the most recent thing that happened, or they would appear + // after whatever the caller logs next and the log would read out of order. + RiaLogging::flushPendingMessages(); } //-------------------------------------------------------------------------------------------------- -/// +/// Wait until every reply has finished, or the timeout expires. //-------------------------------------------------------------------------------------------------- -QByteArray RiaSumoConnector::requestParametersParquetDataBlocking( const SumoCaseId& caseId, const QString& ensembleName ) +void RiaSumoConnector::waitForRepliesToFinish( const std::vector& replies ) { - requestParametersBlobIdForEnsembleBlocking( caseId, ensembleName ); - - if ( m_blobId.empty() ) return {}; - - auto blobId = m_blobId.back(); + if ( replies.empty() ) return; QEventLoop eventLoop; QTimer timer; timer.setSingleShot( true ); - QObject::connect( &timer, SIGNAL( timeout() ), &eventLoop, SLOT( quit() ) ); - QObject::connect( this, SIGNAL( parquetDownloadFinished( const QByteArray&, const QString& ) ), &eventLoop, SLOT( quit() ) ); - - requestBlobDownload( blobId ); + QObject::connect( &timer, &QTimer::timeout, &eventLoop, &QEventLoop::quit ); - timer.start( RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); + auto isAllFinished = [&replies]() + { return std::ranges::all_of( replies, []( QNetworkReply* reply ) { return reply && reply->isFinished(); } ); }; - for ( const auto& blobData : m_redirectInfo ) + std::vector connections; + for ( auto reply : replies ) { - if ( blobData.objectId == blobId ) - { - return blobData.contents; - } + if ( !reply ) continue; + + connections.push_back( QObject::connect( reply, + &QNetworkReply::finished, + &eventLoop, + [&eventLoop, &isAllFinished]() + { + if ( isAllFinished() ) eventLoop.quit(); + } ) ); } - return {}; -} + if ( !isAllFinished() ) + { + // The requests run concurrently, but give the group the timeout each of them would have been + // given on its own. A batch must not be more likely to time out than the same requests made one + // by one, and the server can take a while to answer: a summary vector that has not been + // aggregated yet is produced on demand by the first request that asks for it. + timer.start( static_cast( replies.size() ) * RiaSumoDefines::requestTimeoutMillis() ); + eventLoop.exec(); + } -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestParametersBlobIdForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ) -{ - auto requestCallable = [this, caseId, ensembleName] { requestParametersBlobIdForEnsemble( caseId, ensembleName ); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::blobIdFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); + for ( const auto& connection : connections ) + { + QObject::disconnect( connection ); + } } //-------------------------------------------------------------------------------------------------- -/// +/// Download one blob and return its contents. The download and the wait for it run on the transfer thread. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestParametersBlobIdForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ) +QByteArray RiaSumoConnector::downloadBlobBlocking( const QString& blobId ) { - requestTokenBlocking(); - - QNetworkRequest networkRequest; - - // Properly URL-encode the path components - QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + const auto contentsByBlobId = downloadBlobsBlocking( { blobId } ); - QString url = QString( "%1/cases/%2/ensembles/%3/parameters/blob_id" ).arg( server() ).arg( caseId.get() ).arg( encodedEnsembleName ); - networkRequest.setUrl( QUrl( url ) ); - - addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); + if ( auto it = contentsByBlobId.find( blobId ); it != contentsByBlobId.end() ) return it->second; - auto reply = m_networkAccessManager->get( networkRequest ); - - connect( reply, - &QNetworkReply::finished, - [this, reply, ensembleName, caseId]() - { - // parseBlobId handles the error case and always emits blobIdFinished, so the blocking - // caller returns immediately instead of waiting for the request to time out. - parseBlobId( reply, caseId, ensembleName, "", true ); - } ); + return {}; } //-------------------------------------------------------------------------------------------------- -/// +/// Issue a GET and return the response body, waiting on the transfer thread. Returns an empty array when +/// the request fails. This is the primitive the data specific requests are built from. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestBlobIdForEnsemble( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) +QByteArray RiaSumoConnector::getBlocking( const QString& url, const QString& progressText ) { requestTokenBlocking(); - QNetworkRequest networkRequest; - - // Properly URL-encode the path components - QString encodedVectorName = QUrl::toPercentEncoding( vectorName ); - QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + QByteArray body; - QString url = - QString( "%1/cases/%2/ensembles/%3/vectors/%4/blob_id" ).arg( server() ).arg( caseId.get() ).arg( encodedEnsembleName ).arg( encodedVectorName ); - networkRequest.setUrl( QUrl( url ) ); + runOnTransferThreadBlocking( + [&]() + { + QNetworkRequest networkRequest; + networkRequest.setUrl( QUrl( url ) ); + addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); + auto reply = networkAccessManager()->get( networkRequest ); - auto reply = m_networkAccessManager->get( networkRequest ); + QEventLoop eventLoop; + QTimer timer; + timer.setSingleShot( true ); + QObject::connect( &timer, &QTimer::timeout, &eventLoop, &QEventLoop::quit ); + QObject::connect( reply, &QNetworkReply::finished, &eventLoop, &QEventLoop::quit ); - connect( reply, - &QNetworkReply::finished, - [this, reply, ensembleName, caseId, vectorName]() - { - // parseBlobId handles the error case and always emits blobIdFinished, so the blocking - // caller returns immediately instead of waiting for the request to time out. - parseBlobId( reply, caseId, ensembleName, vectorName, false ); // false = vector data - } ); -} + timer.start( RiaSumoDefines::requestTimeoutMillis() ); + eventLoop.exec(); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestBlobIdForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) -{ - auto requestCallable = [this, caseId, ensembleName, vectorName] { requestBlobIdForEnsemble( caseId, ensembleName, vectorName ); }; - QMetaMethod signalMethod = QMetaMethod::fromSignal( &RiaSumoConnector::blobIdFinished ); - wrapAndCallNetworkRequest( requestCallable, signalMethod ); -} + body = replyBody( reply, url ); + }, + progressText ); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestBlobDownload( const QString& blobId ) -{ - requestTokenBlocking(); - - QString url = QString( "%1/blobs/%2/sas_token_and_blob_base_uri" ).arg( server() ).arg( blobId ); - - QNetworkRequest networkRequest; - networkRequest.setUrl( QUrl( url ) ); - - addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - - auto reply = m_networkAccessManager->get( networkRequest ); - - connect( reply, - &QNetworkReply::finished, - [this, reply, blobId, url]() - { - reply->deleteLater(); // don't leak the reply - - if ( reply->error() != QNetworkReply::NoError ) - { - RiaLogging::error( ( "Download failed: " + url + " failed. " + reply->errorString() ).toStdString() ); - return; - } - - // The backend returns BlobAccessInfo as JSON: { "sasToken": "...", "blobStoreBaseUri": "..." } - const QByteArray contents = reply->readAll(); - QJsonParseError parseError; - const QJsonDocument doc = QJsonDocument::fromJson( contents, &parseError ); - if ( parseError.error != QJsonParseError::NoError || !doc.isObject() ) - { - RiaLogging::error( - std::format( "Could not parse blob access info response as JSON: {}", parseError.errorString().toStdString() ) ); - return; - } - - const QJsonObject obj = doc.object(); - const QString sasToken = obj.value( "sasToken" ).toString(); - const QString blobBaseUri = obj.value( "blobStoreBaseUri" ).toString(); - if ( blobBaseUri.isEmpty() ) - { - RiaLogging::error( "Blob access info response did not contain a blobStoreBaseUri." ); - return; - } - - const QString sasUri = constructSasUri( blobBaseUri, blobId, sasToken ); - requestBlobBySasUri( blobId, sasUri ); - } ); + return body; } //-------------------------------------------------------------------------------------------------- -/// +/// The REST API returns a blob id as a plain string, quoted by FastAPI. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestBlobBySasUri( const QString& blobId, const QString& sasUri ) +QString RiaSumoConnector::blobIdFromBody( const QByteArray& body ) { - RiaLogging::debug( std::format( "Requesting blob. Id: {} SAS URI: {}", blobId, sasUri ) ); - - QNetworkRequest networkRequest; - networkRequest.setUrl( sasUri ); - - // The pre-signed SAS URI carries its own credential (signature in the query string), - // so no Authorization header is added here. Do NOT forward the bearer token to the - // storage host. Redirect policy is set explicitly so behaviour is not Qt-version dependent. - networkRequest.setAttribute( QNetworkRequest::RedirectPolicyAttribute, QNetworkRequest::NoLessSafeRedirectPolicy ); - - auto reply = m_networkAccessManager->get( networkRequest ); - - connect( reply, - &QNetworkReply::finished, - [this, reply, blobId, sasUri]() - { - reply->deleteLater(); + if ( body.isEmpty() ) return {}; - if ( reply->error() != QNetworkReply::NoError ) - { - QString errorMessage = "Download failed: " + sasUri + " failed." + reply->errorString(); - RiaLogging::error( errorMessage.toStdString() ); + QString blobId = QString::fromUtf8( body ).trimmed(); - emit parquetDownloadFinished( {}, sasUri ); - return; - } - - auto statusCode = reply->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt(); - auto contentLength = reply->header( QNetworkRequest::ContentLengthHeader ).toLongLong(); - auto bytesAvailable = reply->bytesAvailable(); - - RiaLogging::debug( - std::format( "Response: status={}, content-length={}, bytes-available={}", statusCode, contentLength, bytesAvailable ) ); - - auto contents = reply->readAll(); - - RiaLogging::debug( std::format( "Read {} bytes from reply", contents.size() ) ); - - // Guard against a silently truncated transfer: a dropped connection on a - // chunked response can still report NoError. If the server told us how many - // bytes to expect and we got fewer, treat it as a failure. - if ( contentLength > 0 && contents.size() != contentLength ) - { - RiaLogging::error( std::format( "Download truncated: expected {} bytes, got {}.", contentLength, contents.size() ) ); - - emit parquetDownloadFinished( {}, sasUri ); - return; - } - - QString msg = "Received data from : " + sasUri; - RiaLogging::debug( msg.toStdString() ); - - parquetDownloadComplete( blobId, contents, sasUri ); + if ( blobId.startsWith( '"' ) && blobId.endsWith( '"' ) ) + { + blobId = blobId.mid( 1, blobId.length() - 2 ); + } - emit parquetDownloadFinished( contents, sasUri ); - } ); + return blobId; } //-------------------------------------------------------------------------------------------------- -/// +/// Read the body off a finished reply. The reply is consumed and scheduled for deletion. //-------------------------------------------------------------------------------------------------- -QByteArray RiaSumoConnector::requestParquetDataBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) +QByteArray RiaSumoConnector::replyBody( QNetworkReply* reply, const QString& url ) { - requestBlobIdForEnsembleBlocking( caseId, ensembleName, vectorName ); - - if ( m_blobId.empty() ) return {}; - - // The REST API now returns the complete blob URL, not just an ID - auto blobId = m_blobId.back(); - - QEventLoop eventLoop; - QTimer timer; - timer.setSingleShot( true ); - QObject::connect( &timer, SIGNAL( timeout() ), &eventLoop, SLOT( quit() ) ); - QObject::connect( this, SIGNAL( parquetDownloadFinished( const QByteArray&, const QString& ) ), &eventLoop, SLOT( quit() ) ); + if ( !reply ) return {}; - requestBlobDownload( blobId ); + const bool failed = !reply->isFinished() || reply->error() != QNetworkReply::NoError; + QByteArray body = failed ? QByteArray() : reply->readAll(); - timer.start( RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); - - for ( const auto& blobData : m_redirectInfo ) + if ( failed ) { - if ( blobData.objectId == blobId ) - { - return blobData.contents; - } + RiaLogging::error( std::format( "Request failed: '{}': {}", url.toStdString(), reply->errorString().toStdString() ) ); } - return {}; -} + reply->deleteLater(); -//-------------------------------------------------------------------------------------------------- -/// Assemble the pre-signed download URI: {blobStoreBaseUri}/{blobId}?{sasToken} -//-------------------------------------------------------------------------------------------------- -QString RiaSumoConnector::constructSasUri( const QString& blobStoreBaseUri, const QString& blobId, const QString& sasToken ) -{ - QString sasUri = blobStoreBaseUri; - if ( !sasUri.endsWith( '/' ) ) sasUri += '/'; - sasUri += blobId; - if ( !sasToken.isEmpty() ) - { - sasUri += ( sasToken.startsWith( '?' ) ? sasToken : ( "?" + sasToken ) ); - } - return sasUri; + return body; } //-------------------------------------------------------------------------------------------------- -/// +/// Entry point for callers on any thread: hand the work to the transfer thread and wait for it there, so +/// the calling thread dispatches no events while the transfers are in flight. See downloadBlobs for what +/// the transfers actually are. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::wrapAndCallNetworkRequest( std::function requestCallable, const QMetaMethod& signalMethod ) +std::map RiaSumoConnector::downloadBlobsBlocking( const std::vector& blobIds ) { - QEventLoop eventLoop; - - QTimer timer; - timer.setSingleShot( true ); + if ( blobIds.empty() ) return {}; - QObject::connect( &timer, &QTimer::timeout, [] { RiaLogging::error( "Sumo request timed out." ); } ); - QObject::connect( &timer, &QTimer::timeout, &eventLoop, &QEventLoop::quit ); + requestTokenBlocking(); - // Not able to use the modern connect syntax here, as the signal is communicated as a QMetaMethod - int methodIndex = eventLoop.metaObject()->indexOfMethod( "quit()" ); - QMetaMethod quitMethod = eventLoop.metaObject()->method( methodIndex ); - QObject::connect( this, signalMethod, &eventLoop, quitMethod ); + std::map contentsByBlobId; - // Call the function that will execute the request - requestCallable(); + runOnTransferThreadBlocking( [&]() { contentsByBlobId = downloadBlobs( blobIds ); }, + QString( "Downloading %1 file(s) from Sumo" ).arg( blobIds.size() ) ); - timer.start( RiaSumoDefines::requestTimeoutMillis() ); - eventLoop.exec( QEventLoop::ProcessEventsFlag::ExcludeUserInputEvents ); + return contentsByBlobId; } //-------------------------------------------------------------------------------------------------- +/// Download several blobs and return their contents by blob id. Getting a blob takes two round trips, one +/// for the pre-signed URI and one for the data itself, so both are done as a group: all the access info +/// requests are issued and waited for together, then all the transfers. A blob that fails is left out of +/// the returned map. /// +/// Always called on the transfer thread, where the event loops it waits on dispatch no GUI events. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parseAssets( QNetworkReply* reply ) +std::map RiaSumoConnector::downloadBlobs( const std::vector& blobIds ) { - QByteArray result = reply->readAll(); - reply->deleteLater(); + std::map contentsByBlobId; + if ( blobIds.empty() ) return contentsByBlobId; - if ( reply->error() == QNetworkReply::NoError ) + // Phase 1: ask for the pre-signed URI of every blob. + std::vector accessInfoReplies; + for ( const auto& blobId : blobIds ) { - QJsonDocument doc = QJsonDocument::fromJson( result ); - QJsonArray jsonArray = doc.array(); - - m_assets.clear(); + QString url = QString( "%1/blobs/%2/sas_token_and_blob_base_uri" ).arg( server() ).arg( blobId ); - // This json is an array of AssetInfo - for ( const QJsonValue& assetInfo : jsonArray ) - { - QString assetName = assetInfo["name"].toString(); - m_assets.push_back( SumoAsset{ SumoAssetId( "" ), "", assetName } ); - } + QNetworkRequest networkRequest; + networkRequest.setUrl( QUrl( url ) ); + addStandardHeader( networkRequest, token(), RiaCloudDefines::contentTypeJson() ); - for ( auto a : m_assets ) - { - RiaLogging::debug( std::format( "Asset: {}", a.name ) ); - } - } - else - { - RiaLogging::error( std::format( "Request assets failed: '{}'", reply->errorString() ) ); + accessInfoReplies.push_back( networkAccessManager()->get( networkRequest ) ); } - emit assetsFinished(); -} + waitForRepliesToFinish( accessInfoReplies ); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parseCases( QNetworkReply* reply ) -{ - QByteArray result = reply->readAll(); - reply->deleteLater(); + std::vector sasUris; + for ( size_t i = 0; i < accessInfoReplies.size(); i++ ) + { + sasUris.push_back( sasUriFromReply( accessInfoReplies[i], blobIds[i] ) ); + } - if ( reply->error() == QNetworkReply::NoError ) + // Phase 2: transfer the blobs themselves. + std::vector blobReplies; + std::vector blobIndices; // index into blobIds for each reply + for ( size_t i = 0; i < sasUris.size(); i++ ) { - QJsonDocument doc = QJsonDocument::fromJson( result ); - QJsonArray jsonArray = doc.array(); + if ( sasUris[i].isEmpty() ) continue; - m_cases.clear(); + RiaLogging::debug( std::format( "Requesting blob. Id: {} SAS URI: {}", blobIds[i], sasUris[i] ) ); - for ( const QJsonValue& value : jsonArray ) - { - QJsonObject caseObj = value.toObject(); + QNetworkRequest networkRequest; + networkRequest.setUrl( sasUris[i] ); - QString id = caseObj["id"].toString(); - QString kind = ""; - QString name = caseObj["name"].toString(); - m_cases.push_back( SumoCase{ SumoCaseId( id ), kind, name } ); - } + // The pre-signed SAS URI carries its own credential (signature in the query string), so no + // Authorization header is added here. Do NOT forward the bearer token to the storage host. Redirect + // policy is set explicitly so behaviour is not Qt-version dependent. + networkRequest.setAttribute( QNetworkRequest::RedirectPolicyAttribute, QNetworkRequest::NoLessSafeRedirectPolicy ); - RiaLogging::debug( std::format( "Case count : {}", m_cases.size() ) ); - } - else - { - RiaLogging::error( std::format( "Request cases failed: '{}'", reply->errorString() ) ); + blobReplies.push_back( networkAccessManager()->get( networkRequest ) ); + blobIndices.push_back( i ); } - emit casesFinished(); -} + waitForRepliesToFinish( blobReplies ); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parseVectorNames( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName ) -{ - QByteArray result = reply->readAll(); - reply->deleteLater(); - - m_vectorNames.clear(); - - if ( reply->error() == QNetworkReply::NoError ) + for ( size_t i = 0; i < blobReplies.size(); i++ ) { - QJsonDocument doc = QJsonDocument::fromJson( result ); - QJsonArray jsonArray = doc.array(); + const size_t blobIndex = blobIndices[i]; - for ( const QJsonValue& value : jsonArray ) + QByteArray contents = blobContentsFromReply( blobReplies[i], sasUris[blobIndex] ); + if ( !contents.isEmpty() ) { - QJsonObject vectorObj = value.toObject(); - QString vectorName = vectorObj["name"].toString(); - m_vectorNames.push_back( vectorName ); + contentsByBlobId[blobIds[blobIndex]] = contents; } } - else - { - RiaLogging::error( std::format( "Request vector names failed: '{}'", reply->errorString() ) ); - } - emit vectorNamesFinished(); + return contentsByBlobId; } //-------------------------------------------------------------------------------------------------- -/// +/// Read the pre-signed download URI off a finished blob access info reply. The reply is consumed and +/// scheduled for deletion. Returns an empty string on failure. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parseRealizationNumbers( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName ) +QString RiaSumoConnector::sasUriFromReply( QNetworkReply* reply, const QString& blobId ) { - QByteArray result = reply->readAll(); - reply->deleteLater(); + if ( !reply ) return {}; - if ( reply->error() == QNetworkReply::NoError ) - { - QJsonDocument doc = QJsonDocument::fromJson( result ); - QJsonArray jsonArray = doc.array(); + reply->deleteLater(); - for ( const QJsonValue& value : jsonArray ) - { - int intValue = value.toInt(); - auto realizationId = QString::number( intValue ); - m_realizationIds.push_back( realizationId ); - } - } - else + if ( !reply->isFinished() || reply->error() != QNetworkReply::NoError ) { - RiaLogging::error( std::format( "Request realization IDs failed: '{}'", reply->errorString() ) ); + RiaLogging::error( + std::format( "Requesting access info for blob '{}' failed: {}", blobId.toStdString(), reply->errorString().toStdString() ) ); + return {}; } - emit realizationIdsFinished(); -} - -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::parseBlobId( QNetworkReply* reply, - const SumoCaseId& caseId, - const QString& ensembleName, - const QString& vectorName, - bool isParameters ) -{ - QByteArray result = reply->readAll(); - reply->deleteLater(); - - m_blobId.clear(); - - if ( reply->error() == QNetworkReply::NoError ) + // The backend returns BlobAccessInfo as JSON: { "sasToken": "...", "blobStoreBaseUri": "..." } + const QByteArray contents = reply->readAll(); + QJsonParseError parseError; + const QJsonDocument doc = QJsonDocument::fromJson( contents, &parseError ); + if ( parseError.error != QJsonParseError::NoError || !doc.isObject() ) { - // The REST API returns a plain string (the blob id) - QString blobId = QString::fromUtf8( result ).trimmed(); - - // Remove quotes if present (FastAPI returns strings with quotes) - if ( blobId.startsWith( '"' ) && blobId.endsWith( '"' ) ) - { - blobId = blobId.mid( 1, blobId.length() - 2 ); - } - - m_blobId.push_back( blobId ); - - // Context-aware logging - if ( isParameters ) - { - RiaLogging::debug( std::format( "Received blob ID for parameters: {}", blobId.toStdString() ) ); - } - else - { - RiaLogging::debug( std::format( "Received blob ID for vector '{}': {}", vectorName.toStdString(), blobId.toStdString() ) ); - } + RiaLogging::error( std::format( "Could not parse blob access info response as JSON: {}", parseError.errorString().toStdString() ) ); + return {}; } - else + + const QJsonObject obj = doc.object(); + const QString sasToken = obj.value( "sasToken" ).toString(); + const QString blobBaseUri = obj.value( "blobStoreBaseUri" ).toString(); + if ( blobBaseUri.isEmpty() ) { - // Context-aware error logging - QString errorContext = isParameters ? "parameters" : QString( "vector '%1'" ).arg( vectorName ); - RiaLogging::error( std::format( "Request blob ID failed for {}: {}", errorContext.toStdString(), reply->errorString().toStdString() ) ); + RiaLogging::error( "Blob access info response did not contain a blobStoreBaseUri." ); + return {}; } - emit blobIdFinished(); + return constructSasUri( blobBaseUri, blobId, sasToken ); } //-------------------------------------------------------------------------------------------------- -/// +/// Read the blob contents off a finished transfer reply. The reply is consumed and scheduled for deletion. +/// Returns an empty array on failure. //-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::addStandardHeader( QNetworkRequest& networkRequest, const QString& token, const QString& contentType ) +QByteArray RiaSumoConnector::blobContentsFromReply( QNetworkReply* reply, const QString& sasUri ) { - networkRequest.setHeader( QNetworkRequest::ContentTypeHeader, contentType ); - networkRequest.setRawHeader( "Authorization", "Bearer " + token.toUtf8() ); -} + if ( !reply ) return {}; -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -QNetworkReply* RiaSumoConnector::makeDownloadRequest( const QString& url, const QString& token, const QString& contentType ) -{ - QNetworkRequest m_networkRequest; - m_networkRequest.setUrl( QUrl( url ) ); + reply->deleteLater(); - addStandardHeader( m_networkRequest, token, contentType ); + if ( !reply->isFinished() || reply->error() != QNetworkReply::NoError ) + { + RiaLogging::error( ( "Download failed: " + sasUri + " failed." + reply->errorString() ).toStdString() ); + return {}; + } - auto reply = m_networkAccessManager->get( m_networkRequest ); - return reply; -} + auto statusCode = reply->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt(); + auto contentLength = reply->header( QNetworkRequest::ContentLengthHeader ).toLongLong(); + auto bytesAvailable = reply->bytesAvailable(); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -void RiaSumoConnector::requestParquetData( const QString& url, const QString& token ) -{ - RiaLogging::debug( "Requesting download of parquet from: " + url.toStdString() ); - - auto reply = makeDownloadRequest( url, token, RiaCloudDefines::contentTypeJson() ); - connect( reply, - &QNetworkReply::finished, - [this, reply, url]() - { - if ( reply->error() == QNetworkReply::NoError ) - { - QByteArray contents = reply->readAll(); - RiaLogging::debug( std::format( "Download succeeded: {} bytes.", contents.length() ) ); - RiaLogging::debug( std::format( "Download succeeded for url: {}", url.toStdString() ) ); - emit parquetDownloadFinished( contents, "" ); - } - else - { - QString errorMessage = "Download failed: " + url + " failed." + reply->errorString(); - RiaLogging::error( errorMessage.toStdString() ); - emit parquetDownloadFinished( QByteArray(), errorMessage ); - } - } ); -} + RiaLogging::debug( std::format( "Response: status={}, content-length={}, bytes-available={}", statusCode, contentLength, bytesAvailable ) ); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::assets() const -{ - return m_assets; -} + auto contents = reply->readAll(); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::cases() const -{ - return m_cases; -} + RiaLogging::debug( std::format( "Read {} bytes from reply", contents.size() ) ); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::ensembleNamesForCase( const SumoCaseId& caseId ) const -{ - std::vector ensembleNames; - for ( const auto& ensemble : m_ensembleNames ) + // Guard against a silently truncated transfer: a dropped connection on a chunked response can still + // report NoError. If the server told us how many bytes to expect and we got fewer, treat it as a failure. + if ( contentLength > 0 && contents.size() != contentLength ) { - if ( ensemble.caseId == caseId ) - { - ensembleNames.push_back( ensemble.name ); - } + RiaLogging::error( std::format( "Download truncated: expected {} bytes, got {}.", contentLength, contents.size() ) ); + return {}; } - return ensembleNames; -} -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::vectorNames() const -{ - return m_vectorNames; -} + RiaLogging::debug( ( "Received data from : " + sasUri ).toStdString() ); -//-------------------------------------------------------------------------------------------------- -/// -//-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::realizationIds() const -{ - return m_realizationIds; + return contents; } //-------------------------------------------------------------------------------------------------- -/// +/// Assemble the pre-signed download URI: {blobStoreBaseUri}/{blobId}?{sasToken} //-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::blobIds() const +QString RiaSumoConnector::constructSasUri( const QString& blobStoreBaseUri, const QString& blobId, const QString& sasToken ) { - return m_blobId; + QString sasUri = blobStoreBaseUri; + if ( !sasUri.endsWith( '/' ) ) sasUri += '/'; + sasUri += blobId; + if ( !sasToken.isEmpty() ) + { + sasUri += ( sasToken.startsWith( '?' ) ? sasToken : ( "?" + sasToken ) ); + } + return sasUri; } //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -std::vector RiaSumoConnector::blobContents() const +void RiaSumoConnector::addStandardHeader( QNetworkRequest& networkRequest, const QString& token, const QString& contentType ) { - return m_redirectInfo; + networkRequest.setHeader( QNetworkRequest::ContentTypeHeader, contentType ); + networkRequest.setRawHeader( "Authorization", "Bearer " + token.toUtf8() ); } diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h index 909f6d0406..f4a147dba2 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoConnector.h @@ -20,50 +20,23 @@ #include "RiaCloudConnector.h" #include "RiaSumoDefines.h" +#include "RiaSumoExplore.h" +#include "RiaSumoGrid.h" +#include "RiaSumoSummary.h" #include #include #include #include +#include #include class QEventLoop; +class QThread; using SumoObjectId = QString; -struct SumoAsset -{ - SumoAssetId assetId; - - QString kind; - QString name; -}; - -struct SumoCase -{ - SumoCaseId caseId; - - QString kind; - QString name; -}; - -struct SumoRedirect -{ - SumoObjectId objectId; - QString blobName; - QString url; - QString redirectBaseUri; - QString redirectAuth; - QByteArray contents; -}; - -struct SumoEnsemble -{ - SumoCaseId caseId; - QString name; -}; - //================================================================================================== /// //================================================================================================== @@ -84,85 +57,81 @@ class RiaSumoConnector : public RiaCloudConnector QString server() const override; - void requestAssets(); - void requestAssetsBlocking(); + // Download blobs by id and return their contents. Getting a blob takes two round trips, one for the + // pre-signed URI and one for the data, and a batch does each of those as one concurrent group. + QByteArray downloadBlobBlocking( const QString& blobId ); + std::map downloadBlobsBlocking( const std::vector& blobIds ); - void requestCasesForField( const QString& fieldName ); - void requestCasesForFieldBlocking( const QString& fieldName ); + // What Sumo holds: assets, cases, ensembles and realizations. + RiaSumoExplore& explore(); - void requestEnsembleByCasesId( const SumoCaseId& caseId ); - void requestEnsembleByCasesIdBlocking( const SumoCaseId& caseId ); + // The grid data of a case. Owned here so its blob cache lives as long as the connection. + RiaSumoGrid& grid(); - void requestVectorNamesForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ); - void requestVectorNamesForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ); + // The summary data of a case. + RiaSumoSummary& summary(); - void requestRealizationIdsForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ); - void requestRealizationIdsForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ); + // Transport used by the data specific delegates. Every request goes through the transfer thread, so + // the calling thread waits without dispatching events. + QByteArray getBlocking( const QString& url, const QString& progressText = {} ); - void requestParametersBlobIdForEnsemble( const SumoCaseId& caseId, const QString& ensembleName ); - void requestParametersBlobIdForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName ); - QByteArray requestParametersParquetDataBlocking( const SumoCaseId& caseId, const QString& ensembleName ); + // The REST API returns a blob id as a plain string, quoted by FastAPI. + static QString blobIdFromBody( const QByteArray& body ); - void requestBlobIdForEnsemble( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ); - void requestBlobIdForEnsembleBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ); + void addStandardHeader( QNetworkRequest& networkRequest, const QString& token, const QString& contentType ); - void requestBlobDownload( const QString& blobId ); - void requestBlobBySasUri( const QString& blobId, const QString& sasUri ); + // Run work on the transfer thread and wait for it. Pass progressText to show the standard progress dialog + // while waiting, worth doing for the transfers slow enough to be noticed and not for the small requests + // that would only make it flash. + void runOnTransferThreadBlocking( const std::function& work, const QString& progressText = {} ); - QByteArray requestParquetDataBlocking( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ); + // Run work on the transfer thread without waiting for it. The async data paths use this: the result is + // delivered by a callback rather than by returning, so the calling thread carries on immediately. + void runOnTransferThread( const std::function& work ); - std::vector assets() const; - std::vector cases() const; - std::vector ensembleNamesForCase( const SumoCaseId& caseId ) const; - std::vector vectorNames() const; - std::vector realizationIds() const; - std::vector blobIds() const; - std::vector blobContents() const; + // Hand a call back to the thread the connector lives on, the one owning the user interface. Results of + // async work are delivered through this, so a caller never has its data handed to it on another thread. + void invokeOnConnectorThread( const std::function& work ); -public slots: - void parseAssets( QNetworkReply* reply ); - void parseEnsembleNames( QNetworkReply* reply, const SumoCaseId& caseId ); - void parseCases( QNetworkReply* reply ); - void parseVectorNames( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName ); - void parseRealizationNumbers( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName ); - void parseBlobId( QNetworkReply* reply, const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName, bool isParameters ); + // Download one blob, calling onFinished with its contents. Call on the transfer thread, where onFinished + // is called as well. Empty contents mean the transfer failed. + void downloadBlobAsync( const QString& blobId, const std::function& onFinished ); - void requestFailed( const QAbstractOAuth::Error error ); - void parquetDownloadComplete( const QString& blobId, const QByteArray&, const QString& url ); - -signals: - void fileDownloadFinished( const QString& fileId, const QString& filePath ); - void casesFinished(); - void wellsFinished(); - void wellboresFinished( const QString& wellId ); - void wellboreTrajectoryFinished( const QString& wellboreId ); - void parquetDownloadFinished( const QByteArray& contents, const QString& url ); - void ensembleNamesFinished(); - void vectorNamesFinished(); - void blobIdFinished(); - void assetsFinished(); - void realizationIdsFinished(); + // Abort a reply that has not finished in time, so an async chain reports a failure instead of hanging and + // leaving whoever waits for the data waiting forever. + static void abortIfNotFinishedWithin( QNetworkReply* reply, int timeoutMillis ); -private: - void addStandardHeader( QNetworkRequest& networkRequest, const QString& token, const QString& contentType ); + // The network manager belonging to the calling thread: the transfer thread manager when called from + // there, otherwise the one owned by RiaCloudConnector on the GUI thread. + QNetworkAccessManager* networkAccessManager(); - QNetworkReply* makeDownloadRequest( const QString& url, const QString& token, const QString& contentType ); - void requestParquetData( const QString& url, const QString& token ); + static void waitForRepliesToFinish( const std::vector& replies ); - static QString constructSasUri( const QString& blobStoreBaseUri, const QString& blobId, const QString& sasToken ); + // Issue and collect the two round trips a blob transfer needs. Called on the transfer thread. + std::map downloadBlobs( const std::vector& blobIds ); - void wrapAndCallNetworkRequest( std::function requestCallable, const QMetaMethod& signalMethod ); +public slots: + void requestFailed( const QAbstractOAuth::Error error ); private: - std::function m_serverUrlProvider; + static QString constructSasUri( const QString& blobStoreBaseUri, const QString& blobId, const QString& sasToken ); - std::vector m_assets; - std::vector m_cases; - std::vector m_vectorNames; - std::vector m_realizationIds; - std::vector m_ensembleNames; + QString sasUriFromReply( QNetworkReply* reply, const QString& blobId ); + static QByteArray blobContentsFromReply( QNetworkReply* reply, const QString& sasUri ); + static QByteArray replyBody( QNetworkReply* reply, const QString& url ); - std::vector m_blobId; +private: + std::function m_serverUrlProvider; - std::vector m_redirectInfo; + RiaSumoExplore m_explore; + RiaSumoGrid m_grid; + RiaSumoSummary m_summary; + + // Transfers run on their own thread so the calling thread can wait without dispatching events. Waiting on + // a nested event loop on the GUI thread let the view update code re-enter a load that was still running, + // and the same grid property was downloaded twice. Authentication stays on the GUI thread: the OAuth flow + // opens a browser and its objects live there. + QThread* m_transferThread = nullptr; + QObject* m_transferContext = nullptr; // lives on the transfer thread + QNetworkAccessManager* m_transferNetworkAccessManager = nullptr; // created on the transfer thread }; diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.cpp index e8876409d3..496f1dd438 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.cpp +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.cpp @@ -36,3 +36,27 @@ int RiaSumoDefines::requestTimeoutMillis() { return 10 * 1000; } + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +int RiaSumoDefines::asyncRequestTimeoutMillis() +{ + return 5 * 60 * 1000; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +size_t RiaSumoDefines::gridPropertyCacheLimitBytes() +{ + return 256ull * 1024 * 1024; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +size_t RiaSumoDefines::gridPropertyPrefetchBatchSize() +{ + return 8; +} diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.h index 2bb97f0ea4..4141994665 100644 --- a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.h +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoDefines.h @@ -22,6 +22,8 @@ #include +#include + using SumoAssetId = nonstd::ordered; using SumoCaseId = nonstd::ordered; @@ -29,4 +31,16 @@ namespace RiaSumoDefines { QString tokenPath(); int requestTimeoutMillis(); + +// The timeout of a request nothing is waiting for. Only there so a request that never answers is eventually +// given up on, and generous because a summary vector that has not been aggregated yet is produced on demand +// by the request asking for it. Nothing is blocked while it runs, so waiting longer costs nothing. +int asyncRequestTimeoutMillis(); + +// The maximum number of bytes of downloaded grid property blobs kept in memory. +size_t gridPropertyCacheLimitBytes(); + +// The number of grid property time steps fetched concurrently when prefetching. Bounds both the number of +// requests in flight and the amount of blob data pulled in for time steps that may not be needed. +size_t gridPropertyPrefetchBatchSize(); }; // namespace RiaSumoDefines diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.cpp new file mode 100644 index 0000000000..b76104408e --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.cpp @@ -0,0 +1,168 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "RiaSumoExplore.h" + +#include "RiaLogging.h" +#include "RiaQStringFormatter.h" +#include "RiaSumoConnector.h" + +#include +#include +#include +#include + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RiaSumoExplore::RiaSumoExplore( RiaSumoConnector& connector ) + : m_connector( connector ) +{ +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::assets() +{ + const QString url = QString( "%1/assets" ).arg( m_connector.server() ); + + return parseAssets( m_connector.getBlocking( url, "Loading assets from Sumo" ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::cases( const QString& assetName ) +{ + const QString url = QString( "%1/cases?asset_name=%2" ).arg( m_connector.server() ).arg( QString( QUrl::toPercentEncoding( assetName ) ) ); + + return parseCases( m_connector.getBlocking( url, QString( "Loading the cases of %1 from Sumo" ).arg( assetName ) ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::ensembleNames( const SumoCaseId& caseId ) +{ + const QString url = QString( "%1/cases/%2/ensembles" ).arg( m_connector.server() ).arg( caseId.get() ); + + return parseEnsembleNames( m_connector.getBlocking( url, "Loading ensembles from Sumo" ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::realizationIds( const SumoCaseId& caseId, const QString& ensembleName ) +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + + const QString url = + QString( "%1/cases/%2/ensembles/%3/realizations" ).arg( m_connector.server() ).arg( caseId.get() ).arg( encodedEnsembleName ); + + return parseRealizationIds( m_connector.getBlocking( url, "Loading realizations from Sumo" ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::parseAssets( const QByteArray& body ) +{ + std::vector assets; + + QJsonDocument doc = QJsonDocument::fromJson( body ); + QJsonArray jsonArray = doc.array(); + + // This json is an array of AssetInfo + for ( const QJsonValue& assetInfo : jsonArray ) + { + QString assetName = assetInfo["name"].toString(); + assets.push_back( SumoAsset{ SumoAssetId( "" ), "", assetName } ); + } + + for ( const auto& asset : assets ) + { + RiaLogging::debug( std::format( "Asset: {}", asset.name ) ); + } + + return assets; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::parseCases( const QByteArray& body ) +{ + std::vector cases; + + QJsonDocument doc = QJsonDocument::fromJson( body ); + QJsonArray jsonArray = doc.array(); + + for ( const QJsonValue& value : jsonArray ) + { + QJsonObject caseObj = value.toObject(); + + QString id = caseObj["id"].toString(); + QString kind = ""; + QString name = caseObj["name"].toString(); + cases.push_back( SumoCase{ SumoCaseId( id ), kind, name } ); + } + + RiaLogging::debug( std::format( "Case count : {}", cases.size() ) ); + + return cases; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::parseEnsembleNames( const QByteArray& body ) +{ + std::vector ensembleNames; + + QJsonDocument doc = QJsonDocument::fromJson( body ); + QJsonArray jsonArray = doc.array(); + + for ( const QJsonValue& value : jsonArray ) + { + QJsonObject ensembleObj = value.toObject(); + ensembleNames.push_back( ensembleObj["name"].toString() ); + } + + RiaLogging::debug( std::format( "Ensemble count : {}", ensembleNames.size() ) ); + + return ensembleNames; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoExplore::parseRealizationIds( const QByteArray& body ) +{ + std::vector realizationIds; + + QJsonDocument doc = QJsonDocument::fromJson( body ); + QJsonArray jsonArray = doc.array(); + + for ( const QJsonValue& value : jsonArray ) + { + realizationIds.push_back( QString::number( value.toInt() ) ); + } + + return realizationIds; +} diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.h new file mode 100644 index 0000000000..60c66aa7ad --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoExplore.h @@ -0,0 +1,70 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "RiaSumoDefines.h" + +#include +#include + +#include + +class RiaSumoConnector; + +struct SumoAsset +{ + SumoAssetId assetId; + + QString kind; + QString name; +}; + +struct SumoCase +{ + SumoCaseId caseId; + + QString kind; + QString name; +}; + +//================================================================================================== +/// Finding your way around what Sumo holds: the assets available, the cases of an asset, the ensembles +/// of a case and the realizations of an ensemble. Requests are made through RiaSumoConnector, which owns +/// the connection and does the transfers, and every call returns its result rather than leaving it in +/// shared state. +//================================================================================================== +class RiaSumoExplore +{ +public: + explicit RiaSumoExplore( RiaSumoConnector& connector ); + + std::vector assets(); + std::vector cases( const QString& assetName ); + std::vector ensembleNames( const SumoCaseId& caseId ); + std::vector realizationIds( const SumoCaseId& caseId, const QString& ensembleName ); + +private: + static std::vector parseAssets( const QByteArray& body ); + static std::vector parseCases( const QByteArray& body ); + static std::vector parseEnsembleNames( const QByteArray& body ); + static std::vector parseRealizationIds( const QByteArray& body ); + +private: + RiaSumoConnector& m_connector; +}; diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.cpp new file mode 100644 index 0000000000..bbd67a0235 --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.cpp @@ -0,0 +1,387 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "RiaSumoGrid.h" + +#include "RiaCloudDefines.h" +#include "RiaLogging.h" +#include "RiaSumoConnector.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RiaSumoGrid::RiaSumoGrid( RiaSumoConnector& connector ) + : m_connector( connector ) + , m_blobCache( RiaSumoDefines::gridPropertyCacheLimitBytes() ) +{ +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoGrid::gridInfo( const SumoCaseId& caseId, const QString& ensembleName ) +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + + const QString url = + QString( "%1/cases/%2/ensembles/%3/grid_info_list" ).arg( m_connector.server() ).arg( caseId.get() ).arg( encodedEnsembleName ); + + return parseGridInfo( m_connector.getBlocking( url ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QByteArray RiaSumoGrid::gridData( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ) +{ + const QString blobId = gridBlobId( caseId, ensembleName, gridName, realization ); + if ( blobId.isEmpty() ) return {}; + + return m_connector.downloadBlobBlocking( blobId ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoGrid::gridBlobId( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ) +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + const QString encodedGridName = QUrl::toPercentEncoding( gridName ); + + const QString url = QString( "%1/cases/%2/ensembles/%3/grids/%4/realizations/%5/blob_id" ) + .arg( m_connector.server() ) + .arg( caseId.get() ) + .arg( encodedEnsembleName ) + .arg( encodedGridName ) + .arg( realization ); + + return blobIdFromBody( m_connector.getBlocking( url ), gridName ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector + RiaSumoGrid::propertyInfo( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ) +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + const QString encodedGridName = QUrl::toPercentEncoding( gridName ); + + const QString url = QString( "%1/cases/%2/ensembles/%3/grids/%4/realizations/%5/property_info_list" ) + .arg( m_connector.server() ) + .arg( caseId.get() ) + .arg( encodedEnsembleName ) + .arg( encodedGridName ) + .arg( realization ); + + return parsePropertyInfo( m_connector.getBlocking( url ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QByteArray RiaSumoGrid::propertyData( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) +{ + // Serve from cache when possible. Keyed by the full property identity (not the blob id), a repeat request + // is answered without even asking Sumo for the blob id. This avoids re-downloading every time step when a + // property's global legend range is computed, and again when it is displayed. + const QString key = cacheKey( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + if ( auto cachedContents = m_blobCache.lookup( key ); !cachedContents.isEmpty() ) + { + return cachedContents; + } + + const QString blobId = propertyBlobId( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + if ( blobId.isEmpty() ) return {}; + + QByteArray contents = m_connector.downloadBlobBlocking( blobId ); + + m_blobCache.insert( key, contents ); + + return contents; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaSumoGrid::prefetchPropertyData( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const std::vector& isoDatesOrIntervals ) +{ + // Only fetch what is not already cached, and drop duplicates so a time step is never requested twice. + std::vector cacheKeys; + std::vector timestampsToFetch; + for ( const auto& isoDateOrInterval : isoDatesOrIntervals ) + { + const QString key = cacheKey( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + + if ( std::ranges::find( cacheKeys, key ) != cacheKeys.end() ) continue; + if ( m_blobCache.contains( key ) ) continue; + + cacheKeys.push_back( key ); + timestampsToFetch.push_back( isoDateOrInterval ); + } + + if ( timestampsToFetch.size() < 2 ) return; // nothing to gain over the single time step path + + m_connector.runOnTransferThreadBlocking( + [&]() { fetchPropertyBatch( caseId, ensembleName, gridName, realization, propertyName, timestampsToFetch, cacheKeys ); }, + QString( "Loading %1 time step(s) of %2 from Sumo" ).arg( timestampsToFetch.size() ).arg( propertyName ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// Resolve the blob ids of a batch of time steps, download the blobs, and put them in the cache. Always +/// called on the transfer thread, where the event loops it waits on dispatch no GUI events. +//-------------------------------------------------------------------------------------------------- +void RiaSumoGrid::fetchPropertyBatch( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const std::vector& timestampsToFetch, + const std::vector& cacheKeys ) +{ + // Phase 1: resolve all blob ids concurrently. + std::vector blobIdReplies; + for ( const auto& isoDateOrInterval : timestampsToFetch ) + { + blobIdReplies.push_back( makePropertyBlobIdRequest( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ) ); + } + + RiaSumoConnector::waitForRepliesToFinish( blobIdReplies ); + + std::vector blobIds; + for ( auto reply : blobIdReplies ) + { + blobIds.push_back( blobIdFromReply( reply, propertyName ) ); + } + + // Phase 2: download all resolved blobs as one group. + std::vector blobIdsToDownload; + for ( const auto& blobId : blobIds ) + { + if ( !blobId.isEmpty() ) blobIdsToDownload.push_back( blobId ); + } + + const auto contentsByBlobId = m_connector.downloadBlobs( blobIdsToDownload ); + + // Cache what arrived. Anything missing failed to download; the per time step path fetches it again later. + for ( size_t i = 0; i < blobIds.size(); i++ ) + { + if ( blobIds[i].isEmpty() ) continue; + + if ( auto it = contentsByBlobId.find( blobIds[i] ); it != contentsByBlobId.end() ) + { + m_blobCache.insert( cacheKeys[i], it->second ); + } + } +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoGrid::propertyBlobIdUrl( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) const +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + const QString encodedGridName = QUrl::toPercentEncoding( gridName ); + const QString encodedPropertyName = QUrl::toPercentEncoding( propertyName ); + + QString url = QString( "%1/cases/%2/ensembles/%3/grids/%4/realizations/%5/properties/%6/blob_id" ) + .arg( m_connector.server() ) + .arg( caseId.get() ) + .arg( encodedEnsembleName ) + .arg( encodedGridName ) + .arg( realization ) + .arg( encodedPropertyName ); + + // The timestamp/interval is an optional query parameter; omit it for static properties. + if ( !isoDateOrInterval.isEmpty() ) + { + url += QString( "?property_iso_date_or_interval=%1" ).arg( QString( QUrl::toPercentEncoding( isoDateOrInterval ) ) ); + } + + return url; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoGrid::propertyBlobId( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) +{ + const QString url = propertyBlobIdUrl( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + + return blobIdFromBody( m_connector.getBlocking( url ), propertyName ); +} + +//-------------------------------------------------------------------------------------------------- +/// Issue the blob id request for one grid property time step. The reply is returned unfinished, so the +/// caller decides how to wait for it: one at a time, or several at once when prefetching. +//-------------------------------------------------------------------------------------------------- +QNetworkReply* RiaSumoGrid::makePropertyBlobIdRequest( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) +{ + const QString url = propertyBlobIdUrl( caseId, ensembleName, gridName, realization, propertyName, isoDateOrInterval ); + + QNetworkRequest networkRequest; + networkRequest.setUrl( QUrl( url ) ); + m_connector.addStandardHeader( networkRequest, m_connector.token(), RiaCloudDefines::contentTypeJson() ); + + return m_connector.networkAccessManager()->get( networkRequest ); +} + +//-------------------------------------------------------------------------------------------------- +/// Read the blob id off a finished blob id reply. The reply is consumed and scheduled for deletion. +/// +/// Waiting for one specific reply, rather than for a signal shared by all blob id requests, is what makes +/// the mapping correct: a still-pending reply from an earlier property's request can no longer satisfy +/// this wait and hand us its blob id, which previously caused e.g. SWAT to be served the SWCR blob. +//-------------------------------------------------------------------------------------------------- +QString RiaSumoGrid::blobIdFromReply( QNetworkReply* reply, const QString& propertyName ) +{ + if ( !reply ) return {}; + + const bool failed = !reply->isFinished() || reply->error() != QNetworkReply::NoError; + QByteArray body = failed ? QByteArray() : reply->readAll(); + + if ( failed ) + { + RiaLogging::error( std::format( "Request grid property blob ID failed: '{}'", reply->errorString().toStdString() ) ); + } + + reply->deleteLater(); + + return blobIdFromBody( body, propertyName ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoGrid::blobIdFromBody( const QByteArray& body, const QString& name ) +{ + const QString blobId = RiaSumoConnector::blobIdFromBody( body ); + if ( blobId.isEmpty() ) return {}; + + RiaLogging::debug( std::format( "Received blob ID for vector '{}': {}", name.toStdString(), blobId.toStdString() ) ); + + return blobId; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoGrid::cacheKey( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) +{ + return QString( "%1|%2|%3|%4|%5|%6" ).arg( caseId.get(), ensembleName, gridName ).arg( realization ).arg( propertyName, isoDateOrInterval ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoGrid::parseGridInfo( const QByteArray& body ) +{ + std::vector gridInfos; + + QJsonDocument doc = QJsonDocument::fromJson( body ); + QJsonArray jsonArray = doc.array(); + + for ( const QJsonValue& value : jsonArray ) + { + QJsonObject gridObj = value.toObject(); + + SumoGridInfo gridInfo; + gridInfo.name = gridObj["name"].toString(); + + for ( const QJsonValue& realizationValue : gridObj["realizations"].toArray() ) + { + gridInfo.realizations.push_back( realizationValue.toInt() ); + } + + gridInfos.push_back( gridInfo ); + } + + RiaLogging::debug( std::format( "Grid info count : {}", gridInfos.size() ) ); + + return gridInfos; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoGrid::parsePropertyInfo( const QByteArray& body ) +{ + std::vector propertyInfos; + + QJsonDocument doc = QJsonDocument::fromJson( body ); + QJsonArray jsonArray = doc.array(); + + for ( const QJsonValue& value : jsonArray ) + { + QJsonObject propertyObj = value.toObject(); + + SumoGridPropertyInfo propertyInfo; + propertyInfo.name = propertyObj["propertyName"].toString(); + + // isoDateOrInterval is null for static properties. + const auto isoValue = propertyObj["isoDateOrInterval"]; + if ( !isoValue.isNull() ) propertyInfo.isoDateOrInterval = isoValue.toString(); + + propertyInfos.push_back( propertyInfo ); + } + + RiaLogging::debug( std::format( "Grid property info count : {}", propertyInfos.size() ) ); + + return propertyInfos; +} diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.h new file mode 100644 index 0000000000..5eaffe6e79 --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoGrid.h @@ -0,0 +1,133 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "RiaSumoBlobCache.h" +#include "RiaSumoDefines.h" + +#include +#include + +#include + +class RiaSumoConnector; +class QNetworkReply; + +struct SumoGridInfo +{ + QString name; + std::vector realizations; +}; + +struct SumoGridPropertyInfo +{ + QString name; + + // Empty for a static property. For a dynamic property this is either a single timestamp ("2018-01-01") + // or an interval ("2018-01-01/2019-01-01"). ResInsight currently only supports the single-timestamp form. + QString isoDateOrInterval; +}; + +//================================================================================================== +/// The grid data of a Sumo case: the grids of an ensemble, the grid geometry itself, and the grid +/// properties. Requests are made through RiaSumoConnector, which owns the connection and does the +/// transfers, and every call returns its result rather than leaving it in shared state. +//================================================================================================== +class RiaSumoGrid +{ +public: + explicit RiaSumoGrid( RiaSumoConnector& connector ); + + std::vector gridInfo( const SumoCaseId& caseId, const QString& ensembleName ); + + // The grid geometry as a binary roff blob. + QByteArray gridData( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); + + std::vector + propertyInfo( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); + + // One time step of one grid property, as a binary roff blob. Served from the blob cache when possible. + QByteArray propertyData( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ); + + // Download several time steps of one property concurrently and put them in the blob cache, so the + // per time step requests that follow are served without going to Sumo. Cached entries are skipped. + void prefetchPropertyData( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const std::vector& isoDatesOrIntervals ); + +private: + QString gridBlobId( const SumoCaseId& caseId, const QString& ensembleName, const QString& gridName, int realization ); + + QString propertyBlobId( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ); + + QNetworkReply* makePropertyBlobIdRequest( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ); + + void fetchPropertyBatch( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const std::vector& timestampsToFetch, + const std::vector& cacheKeys ); + + QString propertyBlobIdUrl( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ) const; + + // The full identity of one grid property time step, used as blob cache key. + static QString cacheKey( const SumoCaseId& caseId, + const QString& ensembleName, + const QString& gridName, + int realization, + const QString& propertyName, + const QString& isoDateOrInterval ); + + static QString blobIdFromBody( const QByteArray& body, const QString& name ); + static QString blobIdFromReply( QNetworkReply* reply, const QString& propertyName ); + + static std::vector parseGridInfo( const QByteArray& body ); + static std::vector parsePropertyInfo( const QByteArray& body ); + +private: + RiaSumoConnector& m_connector; + + // Downloaded grid property blobs, keyed by the full property identity, see cacheKey. + RiaSumoBlobCache m_blobCache; +}; diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp new file mode 100644 index 0000000000..4ebf4b3efa --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.cpp @@ -0,0 +1,370 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "RiaSumoSummary.h" + +#include "RiaCloudDefines.h" +#include "RiaLogging.h" +#include "RiaSumoConnector.h" + +#include +#include +#include +#include +#include +#include + +#include + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RiaSumoSummary::RiaSumoSummary( RiaSumoConnector& connector ) + : m_connector( connector ) +{ +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoSummary::vectorNames( const SumoCaseId& caseId, const QString& ensembleName ) +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + + const QString url = + QString( "%1/cases/%2/ensembles/%3/vector_list" ).arg( m_connector.server() ).arg( caseId.get() ).arg( encodedEnsembleName ); + + return parseVectorNames( m_connector.getBlocking( url ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QByteArray RiaSumoSummary::vectorData( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) +{ + const auto contentsByVectorName = vectorData( caseId, ensembleName, std::vector{ vectorName } ); + + if ( auto it = contentsByVectorName.find( vectorName ); it != contentsByVectorName.end() ) return it->second; + + return {}; +} + +//-------------------------------------------------------------------------------------------------- +/// Fetch several summary vectors at the same time. The blob id requests are issued together and waited +/// for as a group, and so are the transfers, turning 2N sequential round trips into 2 batched ones. +/// +/// This matters more than the round trip count alone: a vector that has not been aggregated yet is +/// produced on demand by the request that asks for it, so fetching serially costs the sum of those +/// aggregations while fetching together costs roughly the slowest one. +//-------------------------------------------------------------------------------------------------- +std::map + RiaSumoSummary::vectorData( const SumoCaseId& caseId, const QString& ensembleName, const std::vector& vectorNames ) +{ + std::map contentsByVectorName; + if ( vectorNames.empty() ) return contentsByVectorName; + + // Drop duplicates, so a vector is never requested twice in one batch. + std::vector namesToFetch; + for ( const auto& vectorName : vectorNames ) + { + if ( vectorName.isEmpty() ) continue; + if ( std::ranges::find( namesToFetch, vectorName ) != namesToFetch.end() ) continue; + + namesToFetch.push_back( vectorName ); + } + + if ( namesToFetch.empty() ) return contentsByVectorName; + + m_connector.runOnTransferThreadBlocking( + [&]() + { + // Phase 1: resolve all blob ids concurrently. + std::vector blobIdReplies; + for ( const auto& vectorName : namesToFetch ) + { + blobIdReplies.push_back( makeVectorBlobIdRequest( caseId, ensembleName, vectorName ) ); + } + + RiaSumoConnector::waitForRepliesToFinish( blobIdReplies ); + + std::vector blobIds; + for ( size_t i = 0; i < blobIdReplies.size(); i++ ) + { + blobIds.push_back( blobIdFromReply( blobIdReplies[i], namesToFetch[i] ) ); + } + + // Phase 2: download all resolved blobs as one group. + std::vector blobIdsToDownload; + for ( const auto& blobId : blobIds ) + { + if ( !blobId.isEmpty() ) blobIdsToDownload.push_back( blobId ); + } + + const auto contentsByBlobId = m_connector.downloadBlobs( blobIdsToDownload ); + + // Anything missing failed; the caller falls back to fetching it on its own later. + for ( size_t i = 0; i < blobIds.size(); i++ ) + { + if ( blobIds[i].isEmpty() ) continue; + + if ( auto it = contentsByBlobId.find( blobIds[i] ); it != contentsByBlobId.end() ) + { + contentsByVectorName[namesToFetch[i]] = it->second; + } + } + }, + QString( "Loading %1 summary vector(s) from Sumo" ).arg( namesToFetch.size() ) ); + + return contentsByVectorName; +} + +//-------------------------------------------------------------------------------------------------- +/// Fetch several summary vectors without waiting for any of them. Every vector is requested at once and +/// onVectorReady is called for each one as it arrives, on the thread the connector lives on, so a caller can +/// show each vector the moment it is there instead of when the slowest one is. +/// +/// Empty contents mean that vector failed. The callback is called exactly once per requested vector, so a +/// caller tracking what is still on its way can rely on all of them being accounted for. +//-------------------------------------------------------------------------------------------------- +void RiaSumoSummary::vectorDataAsync( const SumoCaseId& caseId, + const QString& ensembleName, + const std::vector& vectorNames, + const std::function& onVectorReady ) +{ + if ( vectorNames.empty() || !onVectorReady ) return; + + m_connector.runOnTransferThread( + [this, caseId, ensembleName, vectorNames, onVectorReady]() + { + for ( const auto& vectorName : vectorNames ) + { + auto deliver = [this, onVectorReady, vectorName]( const QByteArray& contents ) + { + m_connector.invokeOnConnectorThread( [onVectorReady, vectorName, contents]() { onVectorReady( vectorName, contents ); } ); + }; + + auto blobIdReply = makeVectorBlobIdRequest( caseId, ensembleName, vectorName ); + if ( !blobIdReply ) + { + deliver( {} ); + continue; + } + + // A vector that has not been aggregated yet is produced on demand by this request, which can + // take a good while. Nothing is waiting on it, so it is given room to finish. + RiaSumoConnector::abortIfNotFinishedWithin( blobIdReply, RiaSumoDefines::asyncRequestTimeoutMillis() ); + + QObject::connect( blobIdReply, + &QNetworkReply::finished, + blobIdReply, + [this, blobIdReply, vectorName, deliver]() + { + const QString blobId = blobIdFromReply( blobIdReply, vectorName ); + if ( blobId.isEmpty() ) + { + deliver( {} ); + return; + } + + m_connector.downloadBlobAsync( blobId, deliver ); + } ); + } + } ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QByteArray RiaSumoSummary::parameterData( const SumoCaseId& caseId, const QString& ensembleName ) +{ + const QString blobId = parameterBlobId( caseId, ensembleName ); + if ( blobId.isEmpty() ) return {}; + + return m_connector.downloadBlobBlocking( blobId ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoSummary::vectorBlobId( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) +{ + const QString url = vectorBlobIdUrl( caseId, ensembleName, vectorName ); + + return logBlobId( RiaSumoConnector::blobIdFromBody( m_connector.getBlocking( url ) ), vectorName ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoSummary::vectorBlobIdUrl( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) const +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + const QString encodedVectorName = QUrl::toPercentEncoding( vectorName ); + + return QString( "%1/cases/%2/ensembles/%3/vectors/%4/blob_id" ) + .arg( m_connector.server() ) + .arg( caseId.get() ) + .arg( encodedEnsembleName ) + .arg( encodedVectorName ); +} + +//-------------------------------------------------------------------------------------------------- +/// Issue the blob id request for one vector. The reply is returned unfinished, so the caller decides how +/// to wait for it: one at a time, or several at once when batching. +//-------------------------------------------------------------------------------------------------- +QNetworkReply* RiaSumoSummary::makeVectorBlobIdRequest( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) +{ + QNetworkRequest networkRequest; + networkRequest.setUrl( QUrl( vectorBlobIdUrl( caseId, ensembleName, vectorName ) ) ); + m_connector.addStandardHeader( networkRequest, m_connector.token(), RiaCloudDefines::contentTypeJson() ); + + return m_connector.networkAccessManager()->get( networkRequest ); +} + +//-------------------------------------------------------------------------------------------------- +/// Read the blob id off a finished blob id reply. The reply is consumed and scheduled for deletion. +//-------------------------------------------------------------------------------------------------- +QString RiaSumoSummary::blobIdFromReply( QNetworkReply* reply, const QString& vectorName ) +{ + if ( !reply ) return {}; + + const bool failed = !reply->isFinished() || reply->error() != QNetworkReply::NoError; + QByteArray body = failed ? QByteArray() : reply->readAll(); + + if ( failed ) + { + RiaLogging::error( + std::format( "Request blob ID failed for vector '{}': {}", vectorName.toStdString(), reply->errorString().toStdString() ) ); + } + + reply->deleteLater(); + + return logBlobId( RiaSumoConnector::blobIdFromBody( body ), vectorName ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoSummary::logBlobId( const QString& blobId, const QString& vectorName ) +{ + if ( !blobId.isEmpty() ) + { + RiaLogging::debug( std::format( "Received blob ID for vector '{}': {}", vectorName.toStdString(), blobId.toStdString() ) ); + } + + return blobId; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoSummary::parameterBlobIdUrl( const SumoCaseId& caseId, const QString& ensembleName ) const +{ + const QString encodedEnsembleName = QUrl::toPercentEncoding( ensembleName ); + + return QString( "%1/cases/%2/ensembles/%3/parameters/blob_id" ).arg( m_connector.server() ).arg( caseId.get() ).arg( encodedEnsembleName ); +} + +//-------------------------------------------------------------------------------------------------- +/// Issue the blob id request for the ensemble parameters. The reply is returned unfinished, so the caller +/// decides how to wait for it. +//-------------------------------------------------------------------------------------------------- +QNetworkReply* RiaSumoSummary::makeParameterBlobIdRequest( const SumoCaseId& caseId, const QString& ensembleName ) +{ + QNetworkRequest networkRequest; + networkRequest.setUrl( QUrl( parameterBlobIdUrl( caseId, ensembleName ) ) ); + m_connector.addStandardHeader( networkRequest, m_connector.token(), RiaCloudDefines::contentTypeJson() ); + + return m_connector.networkAccessManager()->get( networkRequest ); +} + +//-------------------------------------------------------------------------------------------------- +/// Fetch the ensemble parameters without waiting, calling onParametersReady on the thread the connector +/// lives on. Empty contents mean the request failed, and the callback is called exactly once. +//-------------------------------------------------------------------------------------------------- +void RiaSumoSummary::parameterDataAsync( const SumoCaseId& caseId, + const QString& ensembleName, + const std::function& onParametersReady ) +{ + if ( !onParametersReady ) return; + + m_connector.runOnTransferThread( + [this, caseId, ensembleName, onParametersReady]() + { + auto deliver = [this, onParametersReady]( const QByteArray& contents ) + { m_connector.invokeOnConnectorThread( [onParametersReady, contents]() { onParametersReady( contents ); } ); }; + + auto blobIdReply = makeParameterBlobIdRequest( caseId, ensembleName ); + if ( !blobIdReply ) + { + deliver( {} ); + return; + } + + RiaSumoConnector::abortIfNotFinishedWithin( blobIdReply, RiaSumoDefines::asyncRequestTimeoutMillis() ); + + QObject::connect( blobIdReply, + &QNetworkReply::finished, + blobIdReply, + [this, blobIdReply, deliver]() + { + const QString blobId = blobIdFromReply( blobIdReply, "parameters" ); + if ( blobId.isEmpty() ) + { + deliver( {} ); + return; + } + + m_connector.downloadBlobAsync( blobId, deliver ); + } ); + } ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RiaSumoSummary::parameterBlobId( const SumoCaseId& caseId, const QString& ensembleName ) +{ + const QString blobId = RiaSumoConnector::blobIdFromBody( m_connector.getBlocking( parameterBlobIdUrl( caseId, ensembleName ) ) ); + + if ( !blobId.isEmpty() ) + { + RiaLogging::debug( std::format( "Received blob ID for parameters: {}", blobId.toStdString() ) ); + } + + return blobId; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RiaSumoSummary::parseVectorNames( const QByteArray& body ) +{ + std::vector vectorNames; + + QJsonDocument doc = QJsonDocument::fromJson( body ); + QJsonArray jsonArray = doc.array(); + + for ( const QJsonValue& value : jsonArray ) + { + QJsonObject vectorObj = value.toObject(); + vectorNames.push_back( vectorObj["name"].toString() ); + } + + return vectorNames; +} diff --git a/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.h b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.h new file mode 100644 index 0000000000..680bd94c09 --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RiaSumoSummary.h @@ -0,0 +1,85 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "RiaSumoDefines.h" + +#include +#include + +#include +#include +#include + +class RiaSumoConnector; +class QNetworkReply; + +//================================================================================================== +/// The summary data of a Sumo case: the vectors an ensemble has, their values, and the ensemble +/// parameters. Requests are made through RiaSumoConnector, which owns the connection and does the +/// transfers, and every call returns its result rather than leaving it in shared state. +//================================================================================================== +class RiaSumoSummary +{ +public: + explicit RiaSumoSummary( RiaSumoConnector& connector ); + + std::vector vectorNames( const SumoCaseId& caseId, const QString& ensembleName ); + + // The values of one summary vector, for all realizations, as a parquet blob. + QByteArray vectorData( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ); + + // The same for several vectors at once, returned by vector name. The blob id requests are issued as one + // concurrent group and so are the transfers, which matters because a vector that has not been aggregated + // yet is produced on demand by the request asking for it. + std::map vectorData( const SumoCaseId& caseId, const QString& ensembleName, const std::vector& vectorNames ); + + // The same again, but without waiting: all vectors are requested at once and onVectorReady is called for + // each as it arrives, on the thread the connector lives on. Empty contents mean that vector failed, and + // the callback is called exactly once per requested vector. + void vectorDataAsync( const SumoCaseId& caseId, + const QString& ensembleName, + const std::vector& vectorNames, + const std::function& onVectorReady ); + + // The ensemble parameters, as a parquet blob. + QByteArray parameterData( const SumoCaseId& caseId, const QString& ensembleName ); + + // The same without waiting. Like the vectors, the parameters are aggregated on demand by the service, so + // the first request for them can take a while and is not something to hold the user interface for. + void parameterDataAsync( const SumoCaseId& caseId, + const QString& ensembleName, + const std::function& onParametersReady ); + + QString vectorBlobId( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ); + QString parameterBlobId( const SumoCaseId& caseId, const QString& ensembleName ); + +private: + QString vectorBlobIdUrl( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ) const; + QString parameterBlobIdUrl( const SumoCaseId& caseId, const QString& ensembleName ) const; + QNetworkReply* makeParameterBlobIdRequest( const SumoCaseId& caseId, const QString& ensembleName ); + QNetworkReply* makeVectorBlobIdRequest( const SumoCaseId& caseId, const QString& ensembleName, const QString& vectorName ); + static QString blobIdFromReply( QNetworkReply* reply, const QString& vectorName ); + static QString logBlobId( const QString& blobId, const QString& vectorName ); + + static std::vector parseVectorNames( const QByteArray& body ); + +private: + RiaSumoConnector& m_connector; +}; diff --git a/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.cpp b/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.cpp new file mode 100644 index 0000000000..641aee7e48 --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.cpp @@ -0,0 +1,152 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024 Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "RifReaderSumoGridProperty.h" + +#include "RiaLogging.h" +#include "RiaSumoConnector.h" +#include "RiaSumoDefines.h" + +#include "RifRoffFileTools.h" + +#include +#include + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RifReaderSumoGridProperty::RifReaderSumoGridProperty( RiaSumoConnector* connector, + const QString& caseId, + const QString& ensembleName, + const QString& gridName, + int realization ) + : m_connector( connector ) + , m_caseId( caseId ) + , m_ensembleName( ensembleName ) + , m_gridName( gridName ) + , m_realization( realization ) + , m_caseData( nullptr ) +{ +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RifReaderSumoGridProperty::setStaticProperties( const std::vector& propertyNames ) +{ + m_staticProperties = propertyNames; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RifReaderSumoGridProperty::setDynamicProperties( const std::map>& propertyNameToTimestamps ) +{ + m_dynamicTimestamps = propertyNameToTimestamps; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RifReaderSumoGridProperty::open( const QString& /*fileName*/, RigEclipseCaseData* eclipseCase ) +{ + // The grid geometry is loaded elsewhere; only keep the case data for cell count and active cell masking. + m_caseData = eclipseCase; + return true; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RifReaderSumoGridProperty::staticResult( const QString& result, RiaDefines::PorosityModelType matrixOrFracture, std::vector* values ) +{ + if ( matrixOrFracture != RiaDefines::PorosityModelType::MATRIX_MODEL ) return false; + + // Only fetch properties this reader owns; other static results (e.g. computed DEPTH) are not on Sumo. + if ( std::find( m_staticProperties.begin(), m_staticProperties.end(), result ) == m_staticProperties.end() ) return false; + + return fetchAndDecode( result, "", values ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RifReaderSumoGridProperty::dynamicResult( const QString& result, + RiaDefines::PorosityModelType matrixOrFracture, + size_t stepIndex, + std::vector* values ) +{ + if ( matrixOrFracture != RiaDefines::PorosityModelType::MATRIX_MODEL ) return false; + + auto it = m_dynamicTimestamps.find( result ); + if ( it == m_dynamicTimestamps.end() || stepIndex >= it->second.size() ) return false; + + // The timestamp list is aligned with the case's common time steps. An empty entry means this property has + // no data at that time step, so report "no data" instead of fetching another step's values. + const QString& isoDateOrInterval = it->second[stepIndex]; + if ( isoDateOrInterval.isEmpty() ) return false; + + prefetchFromTimeStep( result, it->second, stepIndex ); + + return fetchAndDecode( result, isoDateOrInterval, values ); +} + +//-------------------------------------------------------------------------------------------------- +/// A displayed dynamic property is read one time step at a time, and each read is a blocking round trip to +/// Sumo. Fetch a window of the following time steps in one concurrent batch instead, so the reads that follow +/// are served from the connector's blob cache. The window bounds both the requests in flight and the data +/// pulled in for time steps that may never be displayed. +//-------------------------------------------------------------------------------------------------- +void RifReaderSumoGridProperty::prefetchFromTimeStep( const QString& propertyName, const std::vector& timestamps, size_t stepIndex ) +{ + if ( !m_connector ) return; + + const size_t batchSize = RiaSumoDefines::gridPropertyPrefetchBatchSize(); + + std::vector batch; + for ( size_t i = stepIndex; i < timestamps.size() && batch.size() < batchSize; i++ ) + { + // Skip the time steps this property has no data for, they are never downloaded. + if ( !timestamps[i].isEmpty() ) batch.push_back( timestamps[i] ); + } + + m_connector->grid().prefetchPropertyData( SumoCaseId( m_caseId ), m_ensembleName, m_gridName, m_realization, propertyName, batch ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RifReaderSumoGridProperty::fetchAndDecode( const QString& propertyName, const QString& isoDateOrInterval, std::vector* values ) +{ + if ( !m_connector || !m_caseData || !values ) return false; + + QByteArray contents = + m_connector->grid().propertyData( SumoCaseId( m_caseId ), m_ensembleName, m_gridName, m_realization, propertyName, isoDateOrInterval ); + + RiaLogging::debug( std::format( "Sumo grid property '{}' (time '{}'): downloaded {} bytes.", + propertyName.toStdString(), + isoDateOrInterval.toStdString(), + contents.size() ) ); + + if ( contents.isEmpty() ) return false; + + std::string buffer = contents.toStdString(); + std::istringstream stream( buffer, std::ios::binary ); + + return RifRoffFileTools::propertyValuesFromStream( stream, m_caseData, propertyName, values ); +} diff --git a/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.h b/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.h new file mode 100644 index 0000000000..cd594578fa --- /dev/null +++ b/ApplicationLibCode/Application/Tools/Cloud/RifReaderSumoGridProperty.h @@ -0,0 +1,71 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024 Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "RifReaderInterface.h" + +#include +#include +#include + +#include +#include + +class RiaSumoConnector; + +//================================================================================================== +// +// Lazily fetches grid cell properties for a single Sumo grid realization. ResInsight's result +// machinery calls staticResult()/dynamicResult() the first time a property is displayed; this reader +// then downloads the corresponding roff blob from Sumo and decodes the cell values. +// +//================================================================================================== +class RifReaderSumoGridProperty : public RifReaderInterface +{ +public: + RifReaderSumoGridProperty( RiaSumoConnector* connector, + const QString& caseId, + const QString& ensembleName, + const QString& gridName, + int realization ); + + void setStaticProperties( const std::vector& propertyNames ); + void setDynamicProperties( const std::map>& propertyNameToTimestamps ); + + bool open( const QString& fileName, RigEclipseCaseData* eclipseCase ) override; + + bool staticResult( const QString& result, RiaDefines::PorosityModelType matrixOrFracture, std::vector* values ) override; + bool dynamicResult( const QString& result, RiaDefines::PorosityModelType matrixOrFracture, size_t stepIndex, std::vector* values ) override; + +private: + bool fetchAndDecode( const QString& propertyName, const QString& isoDateOrInterval, std::vector* values ); + void prefetchFromTimeStep( const QString& propertyName, const std::vector& timestamps, size_t stepIndex ); + +private: + QPointer m_connector; + QString m_caseId; + QString m_ensembleName; + QString m_gridName; + int m_realization; + + RigEclipseCaseData* m_caseData; // set in open(); used for cell count and active cell masking + + std::vector m_staticProperties; + std::map> m_dynamicTimestamps; // property name -> sorted iso timestamps +}; diff --git a/ApplicationLibCode/Application/Tools/RiaLogging.cpp b/ApplicationLibCode/Application/Tools/RiaLogging.cpp index ee0b169a1f..de942253ca 100644 --- a/ApplicationLibCode/Application/Tools/RiaLogging.cpp +++ b/ApplicationLibCode/Application/Tools/RiaLogging.cpp @@ -240,6 +240,17 @@ void RiaLogging::appendLoggerInstance( std::unique_ptr loggerInstance sm_logger.push_back( std::move( loggerInstance ) ); } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiaLogging::flushPendingMessages() +{ + for ( const auto& logger : sm_logger ) + { + logger->flushPendingMessages(); + } +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/Application/Tools/RiaLogging.h b/ApplicationLibCode/Application/Tools/RiaLogging.h index b20fee415c..9d9ff48b86 100644 --- a/ApplicationLibCode/Application/Tools/RiaLogging.h +++ b/ApplicationLibCode/Application/Tools/RiaLogging.h @@ -52,6 +52,10 @@ class RiaLogger virtual void warning( const char* message ) = 0; virtual void info( const char* message ) = 0; virtual void debug( const char* message ) = 0; + + // Deliver messages a logger has accepted but not yet written out. A logger that has to hand messages from + // a worker thread over to another thread can otherwise let them arrive after messages logged later. + virtual void flushPendingMessages() {} }; //================================================================================================== @@ -73,6 +77,10 @@ class RiaLogging static void info( std::string_view message, std::string_view logKeyword = "" ); static void debug( std::string_view message, std::string_view logKeyword = "" ); + // Write out anything the loggers are holding, so messages logged from a worker thread appear before the + // messages the waiting thread logs once the worker is done. + static void flushPendingMessages(); + static std::chrono::time_point currentTime(); static void logElapsedTime( std::string_view message, const std::chrono::time_point& startTime ); diff --git a/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.cpp b/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.cpp index e7777e37b6..06330224fa 100644 --- a/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.cpp +++ b/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.cpp @@ -93,9 +93,11 @@ SimpleDialog::SimpleDialog( QWidget* parent ) //-------------------------------------------------------------------------------------------------- SimpleDialog::~SimpleDialog() { + // The connector belongs to RiaApplication and is shared with everything else reading from Sumo, so it + // must outlive this dialog. Only the connection made in createConnection is ours to undo. if ( m_sumoConnector ) { - m_sumoConnector->deleteLater(); + disconnect( m_sumoConnector, &RiaSumoConnector::tokenReady, this, &SimpleDialog::onTokenReady ); } } @@ -105,7 +107,10 @@ SimpleDialog::~SimpleDialog() void SimpleDialog::createConnection() { m_sumoConnector = RiaApplication::instance()->makeSumoConnector(); - connect( m_sumoConnector, &RiaSumoConnector::tokenReady, this, &SimpleDialog::onTokenReady ); + + // Authenticating more than once would otherwise leave one connection per attempt, and onTokenReady + // would be called once for each of them. + connect( m_sumoConnector, &RiaSumoConnector::tokenReady, this, &SimpleDialog::onTokenReady, Qt::UniqueConnection ); } //-------------------------------------------------------------------------------------------------- @@ -124,10 +129,9 @@ void SimpleDialog::onAssetsClicked() { if ( !isTokenValid() ) return; - m_sumoConnector->requestAssetsBlocking(); - m_sumoConnector->assets(); + const auto assets = m_sumoConnector->explore().assets(); - label->setText( "Requesting fields (see log for response" ); + label->setText( QString( "Received %1 assets" ).arg( assets.size() ) ); } //-------------------------------------------------------------------------------------------------- @@ -137,10 +141,10 @@ void SimpleDialog::onCasesClicked() { if ( !isTokenValid() ) return; - QString fieldName = "Drogon"; - m_sumoConnector->requestCasesForField( fieldName ); + QString fieldName = "Drogon"; + const auto cases = m_sumoConnector->explore().cases( fieldName ); - label->setText( "Requesting cases (see log for response" ); + label->setText( QString( "Received %1 cases" ).arg( cases.size() ) ); } //-------------------------------------------------------------------------------------------------- @@ -153,9 +157,9 @@ void SimpleDialog::onVectorNamesClicked() SumoCaseId caseId( "5b783aab-ce10-4b78-b129-baf8d8ce4baa" ); QString iteration = "iter-0"; - m_sumoConnector->requestVectorNamesForEnsemble( caseId, iteration ); + const auto vectorNames = m_sumoConnector->summary().vectorNames( caseId, iteration ); - label->setText( "Requesting vector names (see log for response" ); + label->setText( QString( "Received %1 vector names" ).arg( vectorNames.size() ) ); } //-------------------------------------------------------------------------------------------------- @@ -169,9 +173,9 @@ void SimpleDialog::onFindBlobIdClicked() QString iteration = "iter-0"; QString vectorName = "FOPT"; - m_sumoConnector->requestBlobIdForEnsemble( caseId, iteration, vectorName ); + m_blobId = m_sumoConnector->summary().vectorBlobId( caseId, iteration, vectorName ); - label->setText( "Requesting blob ID for vector name (see log for response" ); + label->setText( m_blobId.isEmpty() ? QString( "No blob ID received" ) : QString( "Blob ID: %1" ).arg( m_blobId ) ); } //-------------------------------------------------------------------------------------------------- @@ -181,16 +185,16 @@ void SimpleDialog::onParquetClicked() { if ( !isTokenValid() ) return; - if ( m_sumoConnector->blobIds().empty() ) + if ( m_blobId.isEmpty() ) { onFindBlobIdClicked(); } - if ( !m_sumoConnector->blobIds().empty() ) + if ( !m_blobId.isEmpty() ) { - m_sumoConnector->requestBlobDownload( m_sumoConnector->blobIds().back() ); + m_blobContents = m_sumoConnector->downloadBlobBlocking( m_blobId ); - label->setText( "Requesting blob ID for vector name (see log for response" ); + label->setText( QString( "Downloaded blob, %1 bytes" ).arg( m_blobContents.size() ) ); } } @@ -199,14 +203,10 @@ void SimpleDialog::onParquetClicked() //-------------------------------------------------------------------------------------------------- void SimpleDialog::onShowContentParquetClicked() { - if ( m_sumoConnector->blobContents().empty() ) return; - - auto blob = m_sumoConnector->blobContents().back(); - - auto content = blob.contents; + if ( m_blobContents.isEmpty() ) return; // TODO: show content using parquet reader - auto tableText = RifArrowTools::readFirstRowsOfTable( content ); + auto tableText = RifArrowTools::readFirstRowsOfTable( m_blobContents ); RiaLogging::info( tableText.toStdString() ); } @@ -220,9 +220,7 @@ void SimpleDialog::onRealizationsClicked() SumoCaseId caseId( "485041ce-ad72-48a3-ac8c-484c0ed95cf8" ); QString iteration = "iter-0"; - m_sumoConnector->requestRealizationIdsForEnsembleBlocking( caseId, iteration ); - - auto ids = m_sumoConnector->realizationIds(); + auto ids = m_sumoConnector->explore().realizationIds( caseId, iteration ); for ( const auto& id : ids ) { RiaLogging::info( id.toStdString() ); diff --git a/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.h b/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.h index 3b75d55f79..84dce40c4e 100644 --- a/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.h +++ b/ApplicationLibCode/Commands/ApplicationCommands/RicSumoDataFeature.h @@ -67,6 +67,11 @@ class SimpleDialog : public QDialog QPushButton* realizationIdsButton; QPointer m_sumoConnector; + + // The most recently resolved blob id, and the blob downloaded from it, kept here so the buttons that + // follow have something to work on. + QString m_blobId; + QByteArray m_blobContents; }; //================================================================================================== diff --git a/ApplicationLibCode/Commands/Sumo/CMakeLists_files.cmake b/ApplicationLibCode/Commands/Sumo/CMakeLists_files.cmake index 41f8eb1ee8..a3e4bff0a0 100644 --- a/ApplicationLibCode/Commands/Sumo/CMakeLists_files.cmake +++ b/ApplicationLibCode/Commands/Sumo/CMakeLists_files.cmake @@ -1,5 +1,6 @@ set(SOURCE_GROUP_SOURCE_FILES ${CMAKE_CURRENT_LIST_DIR}/RicCreateSumoEnsembleFeature.cpp + ${CMAKE_CURRENT_LIST_DIR}/RicCreateSumoGridEnsembleFeature.cpp ${CMAKE_CURRENT_LIST_DIR}/RicDeleteSumoTokenFeature.cpp ) diff --git a/ApplicationLibCode/Commands/Sumo/RicCreateSumoGridEnsembleFeature.cpp b/ApplicationLibCode/Commands/Sumo/RicCreateSumoGridEnsembleFeature.cpp new file mode 100644 index 0000000000..296ea56215 --- /dev/null +++ b/ApplicationLibCode/Commands/Sumo/RicCreateSumoGridEnsembleFeature.cpp @@ -0,0 +1,126 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "RicCreateSumoGridEnsembleFeature.h" + +#include "RiaDefines.h" +#include "RiaLogging.h" + +#include "RicNewViewFeature.h" + +#include "Rim3dView.h" +#include "RimEclipseCaseCollection.h" +#include "RimEclipseCaseEnsemble.h" +#include "RimEclipseViewCollection.h" +#include "RimOilField.h" +#include "RimProject.h" +#include "RimRoffCaseSumo.h" +#include "RimViewNameConfig.h" +#include "Sumo/RimSumoDataSource.h" + +#include "cafSelectionManagerTools.h" + +#include + +CAF_CMD_SOURCE_INIT( RicCreateSumoGridEnsembleFeature, "RicCreateSumoGridEnsembleFeature" ); + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RicCreateSumoGridEnsembleFeature::onActionTriggered( bool isChecked ) +{ + auto dataSources = caf::selectedObjectsByType(); + + for ( auto dataSource : dataSources ) + { + createGridEnsemble( dataSource ); + } +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RicCreateSumoGridEnsembleFeature::createGridEnsemble( RimSumoDataSource* dataSource ) +{ + if ( !dataSource ) return; + + const QString gridName = dataSource->selectedGridName(); + const std::vector realizationIds = dataSource->selectedRealizationIds(); + + if ( gridName.isEmpty() ) + { + RiaLogging::warning( "No grid selected. Unable to create grid ensemble from Sumo." ); + return; + } + + if ( realizationIds.empty() ) + { + RiaLogging::warning( "No realizations selected. Unable to create grid ensemble from Sumo." ); + return; + } + + RimProject* project = RimProject::current(); + if ( !project ) return; + + RimOilField* oilfield = project->activeOilField(); + if ( !oilfield ) return; + + auto eclipseCaseEnsemble = new RimEclipseCaseEnsemble; + eclipseCaseEnsemble->setName( QString( "%1 - %2" ).arg( dataSource->ensembleName(), gridName ) ); + eclipseCaseEnsemble->setDoComputeMobileVolumeWeightedMean( dataSource->doComputeMobileVolumeWeightedMean() ); + + for ( const QString& realizationId : realizationIds ) + { + bool ok = false; + int realization = realizationId.toInt( &ok ); + if ( !ok ) continue; + + if ( auto* gridCase = RimRoffCaseSumo::createFromDataSource( dataSource, gridName, realization ) ) + { + eclipseCaseEnsemble->addCase( gridCase ); + } + } + + if ( eclipseCaseEnsemble->cases().empty() ) + { + RiaLogging::warning( "No valid realizations selected. No grid ensemble created." ); + delete eclipseCaseEnsemble; + return; + } + + oilfield->analysisModels()->caseEnsembles.push_back( eclipseCaseEnsemble ); + oilfield->analysisModels()->updateConnectedEditors(); + + auto firstCase = eclipseCaseEnsemble->cases().front(); + if ( !firstCase ) return; + + auto view = RicNewViewFeature::addReservoirView( firstCase, nullptr, eclipseCaseEnsemble->viewCollection() ); + if ( view ) + { + view->nameConfig()->setAddCaseName( true ); + } +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RicCreateSumoGridEnsembleFeature::setupActionLook( QAction* actionToSetup ) +{ + actionToSetup->setText( "Create Grid Ensemble" + RiaDefines::betaFeaturePostfix() ); + actionToSetup->setIcon( QIcon( ":/CreateGridCaseGroup16x16.png" ) ); +} diff --git a/ApplicationLibCode/Commands/Sumo/RicCreateSumoGridEnsembleFeature.h b/ApplicationLibCode/Commands/Sumo/RicCreateSumoGridEnsembleFeature.h new file mode 100644 index 0000000000..05c5f4ede1 --- /dev/null +++ b/ApplicationLibCode/Commands/Sumo/RicCreateSumoGridEnsembleFeature.h @@ -0,0 +1,38 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "cafCmdFeature.h" + +class RimSumoDataSource; + +//================================================================================================== +/// +//================================================================================================== +class RicCreateSumoGridEnsembleFeature : public caf::CmdFeature +{ + CAF_CMD_HEADER_INIT; + +protected: + void onActionTriggered( bool isChecked ) override; + void setupActionLook( QAction* actionToSetup ) override; + +private: + static void createGridEnsemble( RimSumoDataSource* dataSource ); +}; diff --git a/ApplicationLibCode/FileInterface/RifRoffFileTools.cpp b/ApplicationLibCode/FileInterface/RifRoffFileTools.cpp index 14c0dc9f09..150de3fee0 100644 --- a/ApplicationLibCode/FileInterface/RifRoffFileTools.cpp +++ b/ApplicationLibCode/FileInterface/RifRoffFileTools.cpp @@ -108,6 +108,14 @@ bool RifRoffFileTools::openGridFile( const QString& fileName, RigEclipseCaseData return false; } + return openGridFile( stream, eclipseCase, errorMessages ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RifRoffFileTools::openGridFile( std::istream& stream, RigEclipseCaseData* eclipseCase, QString* errorMessages ) +{ auto getInt = []( auto values, const std::string& name ) { auto v = std::find_if( values.begin(), values.end(), [&name]( const auto& arg ) { return arg.first == name; } ); @@ -274,6 +282,7 @@ bool RifRoffFileTools::openGridFile( const QString& fileName, RigEclipseCaseData catch ( std::runtime_error& err ) { RiaLogging::error( std::format( "Roff file import failed: {}", err.what() ) ); + if ( errorMessages ) *errorMessages = QString::fromStdString( err.what() ); return false; } @@ -492,17 +501,26 @@ std::pair> RifRoffFileTools::createInputPropert { RiaLogging::info( std::format( "Reading properties from roff file: {}", fileName ) ); - std::string filename = fileName.toStdString(); - - std::map keywordMapping; - - std::ifstream stream( filename, std::ios::binary ); + std::ifstream stream( fileName.toStdString(), std::ios::binary ); if ( !stream.good() ) { RiaLogging::error( "Unable to open roff file" ); - return std::make_pair( false, keywordMapping ); + return std::make_pair( false, std::map{} ); } + return createInputProperties( stream, eclipseCaseData, fileName ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::pair> RifRoffFileTools::createInputProperties( std::istream& stream, + RigEclipseCaseData* eclipseCaseData, + const QString& sourceName, + RiaDefines::ResultCatType resultCategory ) +{ + std::map keywordMapping; + auto codeNamesAndValuesForKeyword = []( const std::string& keyword, roff::Reader& reader ) -> std::map { const std::string codeNamesKeyword = keyword + roff::Parser::postFixCodeNames(); @@ -545,7 +563,7 @@ std::pair> RifRoffFileTools::createInputPropert { if ( !appendZoneIndexPropertyFromSubgrids( eclipseCaseData, reader, keywordMapping ) ) { - RiaLogging::warning( std::format( "Unable to import ROFF subgrids zonation from {}", fileName ) ); + RiaLogging::warning( std::format( "Unable to import ROFF subgrids zonation from {}", sourceName ) ); } } else if ( eclipseCaseData->mainGrid()->cellCount() == keywordLength ) @@ -558,9 +576,9 @@ std::pair> RifRoffFileTools::createInputPropert newResultName = "ACTNUM"; } - if ( !appendNewInputPropertyResult( eclipseCaseData, newResultName, keyword, kind, reader ) ) + if ( !appendNewInputPropertyResult( eclipseCaseData, newResultName, keyword, kind, reader, resultCategory ) ) { - RiaLogging::error( std::format( "Unable to import result '{}' from {}", keyword, fileName ) ); + RiaLogging::error( std::format( "Unable to import result '{}' from {}", keyword, sourceName ) ); return std::make_pair( false, keywordMapping ); } @@ -604,6 +622,15 @@ std::pair> RifRoffFileTools::createInputPropert const auto codeNames = codeNamesAndValuesForKeyword( keyword, reader ); RicFaciesPropertiesImportTools::createColorLegendMatchDefaultRockColors( codeNames ); } + else + { + // Skipped: typically grid metadata, but a property array whose length does not match the grid + // cell count is also skipped here. Log it so a size mismatch does not fail silently. + RiaLogging::debug( std::format( "Skipping roff array '{}' (length {}), grid cell count is {}.", + keyword, + keywordLength, + eclipseCaseData->mainGrid()->cellCount() ) ); + } } } catch ( std::runtime_error& err ) @@ -612,6 +639,13 @@ std::pair> RifRoffFileTools::createInputPropert return std::make_pair( false, keywordMapping ); } + if ( keywordMapping.empty() ) + { + RiaLogging::warning( std::format( "No grid properties matching the grid cell count ({}) were imported from {}.", + eclipseCaseData->mainGrid()->cellCount(), + sourceName.toStdString() ) ); + } + return std::make_pair( true, keywordMapping ); } @@ -680,11 +714,90 @@ std::vector //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -bool RifRoffFileTools::appendNewInputPropertyResult( RigEclipseCaseData* caseData, - const QString& resultName, - const std::string& keyword, - roff::Token::Kind kind, - roff::Reader& reader ) +bool RifRoffFileTools::propertyValuesFromStream( std::istream& stream, + RigEclipseCaseData* eclipseCaseData, + const QString& propertyName, + std::vector* values ) +{ + if ( !eclipseCaseData || !eclipseCaseData->mainGrid() || !values ) return false; + + auto mainGrid = eclipseCaseData->mainGrid(); + + const int nx = static_cast( mainGrid->cellCountI() ); + const int ny = static_cast( mainGrid->cellCountJ() ); + const int nz = static_cast( mainGrid->cellCountK() ); + const size_t cellCount = mainGrid->cellCount(); + + try + { + roff::Reader reader( stream ); + reader.parse(); + + // Collect the arrays whose length matches the grid cell count. A Sumo property blob may contain more + // than one such array, so pick the one whose keyword matches the requested property name rather than + // blindly taking the first match (which could decode a different property). + std::vector> candidates; + for ( const auto& [keyword, kind] : reader.getNamedArrayTypes() ) + { + if ( reader.getArrayLength( keyword ) == cellCount ) candidates.push_back( { keyword, kind } ); + } + + if ( candidates.empty() ) return false; + + std::string candidateNames; + for ( const auto& [keyword, kind] : candidates ) + { + if ( !candidateNames.empty() ) candidateNames += ", "; + candidateNames += keyword; + } + + auto selected = candidates.front(); + for ( const auto& candidate : candidates ) + { + if ( QString::fromStdString( candidate.first ).compare( propertyName, Qt::CaseInsensitive ) == 0 ) + { + selected = candidate; + break; + } + } + + RiaLogging::debug( std::format( "Roff property '{}': arrays matching cell count [{}], using '{}'.", + propertyName.toStdString(), + candidateNames, + selected.first ) ); + + std::vector roffValues = readAndConvertToDouble( nx, ny, nz, selected.first, selected.second, reader ); + if ( roffValues.size() != cellCount ) return false; + + // Set better invalid value for inactive cells: roff file has -999. + auto activeCellInfo = eclipseCaseData->activeCellInfo( RiaDefines::PorosityModelType::MATRIX_MODEL ); + for ( size_t i = 0; i < cellCount; i++ ) + { + if ( !activeCellInfo->isActive( ReservoirCellIndex( mainGrid->reservoirCellIndex( i ) ) ) ) + { + roffValues[i] = HUGE_VAL; + } + } + + *values = std::move( roffValues ); + return true; + } + catch ( std::runtime_error& err ) + { + RiaLogging::error( std::format( "Roff property parsing failed: {}", err.what() ) ); + return false; + } +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RifRoffFileTools::appendNewInputPropertyResult( RigEclipseCaseData* caseData, + const QString& resultName, + const std::string& keyword, + roff::Token::Kind kind, + roff::Reader& reader, + RiaDefines::ResultCatType resultCategory ) { CAF_ASSERT( caseData ); @@ -707,7 +820,7 @@ bool RifRoffFileTools::appendNewInputPropertyResult( RigEclipseCaseData* caseDat } } - RigEclipseResultAddress resAddr( RiaDefines::ResultCatType::INPUT_PROPERTY, RifRoffFileTools::mapFromType( kind ), resultName ); + RigEclipseResultAddress resAddr( resultCategory, RifRoffFileTools::mapFromType( kind ), resultName ); caseData->results( RiaDefines::PorosityModelType::MATRIX_MODEL )->createResultEntry( resAddr, false ); auto newPropertyData = caseData->results( RiaDefines::PorosityModelType::MATRIX_MODEL )->modifiableCellScalarResultTimesteps( resAddr ); diff --git a/ApplicationLibCode/FileInterface/RifRoffFileTools.h b/ApplicationLibCode/FileInterface/RifRoffFileTools.h index 823a6f978b..48447ddac8 100644 --- a/ApplicationLibCode/FileInterface/RifRoffFileTools.h +++ b/ApplicationLibCode/FileInterface/RifRoffFileTools.h @@ -25,6 +25,7 @@ #include +#include #include #include @@ -52,10 +53,31 @@ class RifRoffFileTools : public cvf::Object static bool openGridFile( const QString& fileName, RigEclipseCaseData* eclipseCase, QString* errorMessages ); + // Parse a roff grid directly from an already opened binary stream. Used when the roff data does not + // originate from a file on disk (e.g. a blob downloaded from Sumo). + static bool openGridFile( std::istream& stream, RigEclipseCaseData* eclipseCase, QString* errorMessages ); + static std::pair> createInputProperties( const QString& fileName, RigEclipseCaseData* eclipseCase ); + // Read roff property data from an already opened binary stream (e.g. a blob downloaded from Sumo). The + // sourceName is only used for log messages. resultCategory selects the result category the properties are + // imported into (e.g. STATIC_NATIVE for static and DYNAMIC_NATIVE for time dependent properties). + static std::pair> + createInputProperties( std::istream& stream, + RigEclipseCaseData* eclipseCase, + const QString& sourceName, + RiaDefines::ResultCatType resultCategory = RiaDefines::ResultCatType::INPUT_PROPERTY ); + static bool hasGridData( const QString& filename ); + // Read a single grid property from an in-memory roff blob and return its values (with inactive cells + // masked), without registering it. Among the arrays matching the grid cell count, the one whose keyword + // matches propertyName (case-insensitive) is preferred; otherwise the first matching array is used. + static bool propertyValuesFromStream( std::istream& stream, + RigEclipseCaseData* eclipseCase, + const QString& propertyName, + std::vector* values ); + static size_t computeActiveCellMatrixIndex( std::vector& activeCells ); static std::vector computeZoneValuesFromSubgrids( const std::vector& nLayers, size_t nx, size_t ny, size_t nz ); @@ -81,11 +103,12 @@ class RifRoffFileTools : public cvf::Object static std::vector readAndConvertToDouble( int nx, int ny, int nz, const std::string& keyword, roff::Token::Kind kind, roff::Reader& reader ); - static bool appendNewInputPropertyResult( RigEclipseCaseData* caseData, - const QString& resultName, - const std::string& keyword, - roff::Token::Kind token, - roff::Reader& reader ); + static bool appendNewInputPropertyResult( RigEclipseCaseData* caseData, + const QString& resultName, + const std::string& keyword, + roff::Token::Kind token, + roff::Reader& reader, + RiaDefines::ResultCatType resultCategory ); static bool appendZoneIndexPropertyFromSubgrids( RigEclipseCaseData* caseData, roff::Reader& reader, std::map& keywordMapping ); diff --git a/ApplicationLibCode/ProjectDataModel/CMakeLists_files.cmake b/ApplicationLibCode/ProjectDataModel/CMakeLists_files.cmake index 8e4ccd7493..84b9d281c1 100644 --- a/ApplicationLibCode/ProjectDataModel/CMakeLists_files.cmake +++ b/ApplicationLibCode/ProjectDataModel/CMakeLists_files.cmake @@ -11,6 +11,7 @@ set(SOURCE_GROUP_SOURCE_FILES ${CMAKE_CURRENT_LIST_DIR}/RimEclipseInputPropertyCollection.cpp ${CMAKE_CURRENT_LIST_DIR}/RimEclipseInputCase.cpp ${CMAKE_CURRENT_LIST_DIR}/RimEclipseResultCase.cpp + ${CMAKE_CURRENT_LIST_DIR}/RimRoffCaseSumo.cpp ${CMAKE_CURRENT_LIST_DIR}/RimEclipseView.cpp ${CMAKE_CURRENT_LIST_DIR}/RimEclipseResultDefinition.cpp ${CMAKE_CURRENT_LIST_DIR}/RimEclipseCellColors.cpp diff --git a/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.cpp b/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.cpp index ef9988186f..bf18ccab7c 100644 --- a/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.cpp +++ b/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.cpp @@ -167,14 +167,21 @@ void RimCloudDataSourceCollection::fieldChangedByUi( const caf::PdmFieldHandle* if ( changedField == &m_sumoFieldName ) { + // What was picked below belonged to the asset that was just left, both the selection and the options + // it was chosen from. Forget the cached answers as well, or the case list would keep offering the + // cases of the previous asset. The editors are refreshed by the caller, which updates them as soon + // as this returns, so asking for that here would only fetch everything a second time. m_sumoCaseId = ""; - m_sumoEnsembleNames.v().clear(); + m_sumoEnsembleNames.setValue( {} ); - m_sumoConnector->requestCasesForFieldBlocking( m_sumoFieldName ); + clearCachedCases(); + clearCachedEnsembleNames(); } else if ( changedField == &m_sumoCaseId ) { - m_sumoEnsembleNames.v().clear(); + m_sumoEnsembleNames.setValue( {} ); + + clearCachedEnsembleNames(); } if ( changedField == &m_addEnsembles ) { @@ -214,12 +221,7 @@ QList RimCloudDataSourceCollection::calculateValueOption QList options; if ( fieldNeedingOptions == &m_sumoFieldName ) { - if ( m_sumoConnector->assets().empty() ) - { - m_sumoConnector->requestAssetsBlocking(); - } - - for ( const auto& asset : m_sumoConnector->assets() ) + for ( const auto& asset : cachedAssets() ) { if ( m_sumoFieldName().isEmpty() ) { @@ -231,24 +233,14 @@ QList RimCloudDataSourceCollection::calculateValueOption } else if ( fieldNeedingOptions == &m_sumoCaseId && !m_sumoFieldName().isEmpty() ) { - if ( m_sumoConnector->cases().empty() ) - { - m_sumoConnector->requestCasesForFieldBlocking( m_sumoFieldName ); - } - - for ( const auto& sumoCase : m_sumoConnector->cases() ) + for ( const auto& sumoCase : cachedCases( m_sumoFieldName ) ) { options.push_back( { sumoCase.name, sumoCase.caseId.get() } ); } } else if ( fieldNeedingOptions == &m_sumoEnsembleNames && !m_sumoCaseId().isEmpty() ) { - if ( m_sumoConnector->ensembleNamesForCase( SumoCaseId( m_sumoCaseId ) ).empty() ) - { - m_sumoConnector->requestEnsembleByCasesIdBlocking( SumoCaseId( m_sumoCaseId ) ); - } - - for ( const auto& name : m_sumoConnector->ensembleNamesForCase( SumoCaseId( m_sumoCaseId ) ) ) + for ( const auto& name : cachedEnsembleNames( SumoCaseId( m_sumoCaseId ) ) ) { options.push_back( { name, name } ); } @@ -257,6 +249,66 @@ QList RimCloudDataSourceCollection::calculateValueOption return options; } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +const std::vector& RimCloudDataSourceCollection::cachedAssets() +{ + if ( m_assets.empty() && m_sumoConnector ) + { + m_assets = m_sumoConnector->explore().assets(); + } + + return m_assets; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +const std::vector& RimCloudDataSourceCollection::cachedCases( const QString& assetName ) +{ + if ( m_casesAssetName != assetName && m_sumoConnector ) + { + m_cases = m_sumoConnector->explore().cases( assetName ); + m_casesAssetName = assetName; + } + + return m_cases; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +const std::vector& RimCloudDataSourceCollection::cachedEnsembleNames( const SumoCaseId& caseId ) +{ + if ( m_ensembleNamesCaseId != caseId.get() && m_sumoConnector ) + { + m_ensembleNames = m_sumoConnector->explore().ensembleNames( caseId ); + m_ensembleNamesCaseId = caseId.get(); + } + + return m_ensembleNames; +} + +//-------------------------------------------------------------------------------------------------- +/// Forget the cases Sumo answered with, so the next request for them asks again. Called when the asset they +/// belong to is left behind. +//-------------------------------------------------------------------------------------------------- +void RimCloudDataSourceCollection::clearCachedCases() +{ + m_casesAssetName.clear(); + m_cases.clear(); +} + +//-------------------------------------------------------------------------------------------------- +/// The same for the ensemble names, which belong to a case. +//-------------------------------------------------------------------------------------------------- +void RimCloudDataSourceCollection::clearCachedEnsembleNames() +{ + m_ensembleNamesCaseId.clear(); + m_ensembleNames.clear(); +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -402,7 +454,7 @@ std::vector RimCloudDataSourceCollection::addDataSources() } QString caseName; - for ( const auto& sumoCase : m_sumoConnector->cases() ) + for ( const auto& sumoCase : cachedCases( m_sumoFieldName ) ) { if ( sumoCase.caseId == sumoCaseId ) { @@ -411,11 +463,15 @@ std::vector RimCloudDataSourceCollection::addDataSources() } } - m_sumoConnector->requestRealizationIdsForEnsembleBlocking( sumoCaseId, ensembleName ); - m_sumoConnector->requestVectorNamesForEnsembleBlocking( sumoCaseId, ensembleName ); + const auto availableRealizationIds = m_sumoConnector->explore().realizationIds( sumoCaseId, ensembleName ); + const auto gridInfos = m_sumoConnector->grid().gridInfo( sumoCaseId, ensembleName ); + const auto vectorNames = m_sumoConnector->summary().vectorNames( sumoCaseId, ensembleName ); - auto availableRealizationIds = m_sumoConnector->realizationIds(); - auto vectorNames = m_sumoConnector->vectorNames(); + std::vector gridNames; + for ( const auto& gridInfo : gridInfos ) + { + gridNames.push_back( gridInfo.name ); + } auto dataSource = new RimSumoDataSource(); dataSource->setCaseId( sumoCaseId ); @@ -424,6 +480,7 @@ std::vector RimCloudDataSourceCollection::addDataSources() dataSource->setEnsembleName( ensembleName ); dataSource->setAvailableRealizationIds( availableRealizationIds ); dataSource->setVectorNames( vectorNames ); + dataSource->setGridNames( gridNames ); dataSource->updateName(); objectToSelect = dataSource; diff --git a/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.h b/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.h index 917848236b..2b48cca47f 100644 --- a/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.h +++ b/ApplicationLibCode/ProjectDataModel/Cloud/RimCloudDataSourceCollection.h @@ -58,6 +58,16 @@ class RimCloudDataSourceCollection : public caf::PdmObject static bool isCloudApiServerAvailable(); + // The option lists are rebuilt every time the property editor refreshes, so what Sumo answered is kept + // here and only asked for again when the selection it belongs to changes. + const std::vector& cachedAssets(); + const std::vector& cachedCases( const QString& assetName ); + const std::vector& cachedEnsembleNames( const SumoCaseId& caseId ); + + void clearCachedCases(); + + void clearCachedEnsembleNames(); + private: caf::PdmField m_authenticate; caf::PdmField m_sumoFieldName; @@ -73,4 +83,12 @@ class RimCloudDataSourceCollection : public caf::PdmObject caf::PdmField m_restartServer; QPointer m_sumoConnector; + + std::vector m_assets; + + QString m_casesAssetName; + std::vector m_cases; + + QString m_ensembleNamesCaseId; + std::vector m_ensembleNames; }; diff --git a/ApplicationLibCode/ProjectDataModel/Rim3dOverlayInfoConfig.cpp b/ApplicationLibCode/ProjectDataModel/Rim3dOverlayInfoConfig.cpp index faeb0c33d2..ace7be0e80 100644 --- a/ApplicationLibCode/ProjectDataModel/Rim3dOverlayInfoConfig.cpp +++ b/ApplicationLibCode/ProjectDataModel/Rim3dOverlayInfoConfig.cpp @@ -44,6 +44,7 @@ #include "RimCase.h" #include "RimCellEdgeColors.h" #include "RimEclipseCase.h" +#include "RimEclipseCaseEnsemble.h" #include "RimEclipseCellColors.h" #include "RimEclipseFaultColors.h" #include "RimEclipsePropertyFilterCollection.h" @@ -162,6 +163,10 @@ RigHistogramData Rim3dOverlayInfoConfig::histogramData() auto geoMechContourMap = dynamic_cast( geoMechView ); auto seismicView = dynamic_cast( m_viewDef.p() ); + // The mobile volume weighted mean requires MOBPORV (PORV, SWCR and MULTPV). Skip the calculation when the value + // is not displayed, as computing it can be expensive for cases backed by remote data. + m_histogramCalculator->setDoComputeMobileVolumeWeightedMean( m_showVolumeWeightedMean() ); + if ( eclipseContourMap ) return m_histogramCalculator->histogramData( eclipseContourMap ); else if ( geoMechContourMap ) @@ -754,6 +759,18 @@ void Rim3dOverlayInfoConfig::update3DInfo() { m_showVolumeWeightedMean = false; } + + if ( auto eclipseCase = reservoirView->eclipseCase() ) + { + // Ensembles can opt out of the mobile volume weighted mean, as the data it is derived from can be + // expensive to fetch. Used by Sumo grid ensembles, see RimSumoDataSource. + auto ensemble = eclipseCase->firstAncestorOrThisOfType(); + if ( ensemble && !ensemble->doComputeMobileVolumeWeightedMean() ) + { + m_showVolumeWeightedMean = false; + } + } + updateEclipse3DInfo( reservoirView ); } diff --git a/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.cpp b/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.cpp index f89690bc04..ceb6482943 100644 --- a/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.cpp +++ b/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.cpp @@ -51,10 +51,18 @@ RimEclipseCaseEnsemble::RimEclipseCaseEnsemble() CAF_PDM_InitFieldNoDefault( &m_viewCollection, "ViewCollection", "Views" ); m_viewCollection = new RimEclipseViewCollection; + // Limit the case dropdown of the ensemble views to the cases of this ensemble. + m_viewCollection->setEclipseCaseProvider( [this]() { return this->cases(); } ); + CAF_PDM_InitFieldNoDefault( &m_wellTargetMappings, "WellTargetMappings", "Well Target Mappings" ); CAF_PDM_InitFieldNoDefault( &m_statisticsContourMaps, "StatisticsContourMaps", "Statistics Contour maps" ); + // Set by the data source creating the ensemble. Defaults to true to keep the behavior of ensembles created + // before this field was introduced. + CAF_PDM_InitField( &m_doComputeMobileVolumeWeightedMean, "DoComputeMobileVolumeWeightedMean", true, "Compute Mobile Volume Weighted Mean" ); + m_doComputeMobileVolumeWeightedMean.uiCapability()->setUiHidden( true ); + setDeletable( true ); } @@ -70,6 +78,18 @@ RimEclipseCaseEnsemble::~RimEclipseCaseEnsemble() m_viewCollection = nullptr; } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimEclipseCaseEnsemble::initAfterRead() +{ + // The provider is a lambda and not serialized, so re-apply it after loading a project. + if ( m_viewCollection ) + { + m_viewCollection->setEclipseCaseProvider( [this]() { return this->cases(); } ); + } +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -274,6 +294,22 @@ std::vector RimEclipseCaseEnsemble::statisticsContourM return m_statisticsContourMaps.childrenByType(); } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RimEclipseCaseEnsemble::doComputeMobileVolumeWeightedMean() const +{ + return m_doComputeMobileVolumeWeightedMean(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimEclipseCaseEnsemble::setDoComputeMobileVolumeWeightedMean( bool enable ) +{ + m_doComputeMobileVolumeWeightedMean = enable; +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.h b/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.h index b4553f38a2..032420bcf2 100644 --- a/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.h +++ b/ApplicationLibCode/ProjectDataModel/RimEclipseCaseEnsemble.h @@ -75,7 +75,11 @@ class RimEclipseCaseEnsemble : public RimNamedObject, public RimReservoirGridEns void addStatisticsContourMap( RimStatisticsContourMap* statisticsContourMap ) override; std::vector statisticsContourMaps() const override; + bool doComputeMobileVolumeWeightedMean() const; + void setDoComputeMobileVolumeWeightedMean( bool enable ); + protected: + void initAfterRead() override; void appendMenuItems( caf::CmdFeatureMenuBuilder& menuBuilder ) const override; void defineUiOrdering( QString uiConfigName, caf::PdmUiOrdering& uiOrdering ) override; @@ -86,4 +90,5 @@ class RimEclipseCaseEnsemble : public RimNamedObject, public RimReservoirGridEns caf::PdmChildArrayField m_wellTargetMappings; caf::PdmChildArrayField m_statisticsContourMaps; caf::PdmPtrField m_selectedCase; + caf::PdmField m_doComputeMobileVolumeWeightedMean; }; diff --git a/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.cpp b/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.cpp index 8a245af5f2..f9b3e0d4ef 100644 --- a/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.cpp +++ b/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.cpp @@ -75,6 +75,7 @@ void caf::AppEnum::setUp() RimHistogramCalculator::RimHistogramCalculator() : m_isVisCellStatUpToDate( false ) , m_numBins( RigStatisticsDataCache::defaultNumBins() ) + , m_doComputeMobileVolumeWeightedMean( true ) { } @@ -141,6 +142,14 @@ void RimHistogramCalculator::applyCustomBinning( RigStatisticsDataCache* statist } } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimHistogramCalculator::setDoComputeMobileVolumeWeightedMean( bool enable ) +{ + m_doComputeMobileVolumeWeightedMean = enable; +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -272,7 +281,8 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* fldResults->p10p90ScalarValues( resAddr, timeStep, &histData.p10, &histData.p90 ); fldResults->meanScalarValue( resAddr, timeStep, &histData.mean ); fldResults->sumScalarValue( resAddr, timeStep, &histData.sum ); - fldResults->mobileVolumeWeightedMean( resAddr, timeStep, &histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) + fldResults->mobileVolumeWeightedMean( resAddr, timeStep, &histData.weightedMean ); histData.histogram = fldResults->scalarValuesHistogram( resAddr, timeStep ); @@ -290,7 +300,8 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* m_visibleCellStatistics->minMaxCellScalarValues( timeStep, histData.min, histData.max ); m_visibleCellStatistics->p10p90CellScalarValues( timeStep, histData.p10, histData.p90 ); m_visibleCellStatistics->sumCellScalarValues( timeStep, histData.sum ); - m_visibleCellStatistics->mobileVolumeWeightedMean( timeStep, histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) + m_visibleCellStatistics->mobileVolumeWeightedMean( timeStep, histData.weightedMean ); histData.histogram = m_visibleCellStatistics->cellScalarValuesHistogram( timeStep ); @@ -310,7 +321,7 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* fldResults->p10p90ScalarValues( resAddr, &histData.p10, &histData.p90 ); fldResults->meanScalarValue( resAddr, &histData.mean ); fldResults->sumScalarValue( resAddr, &histData.sum ); - fldResults->mobileVolumeWeightedMean( resAddr, &histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) fldResults->mobileVolumeWeightedMean( resAddr, &histData.weightedMean ); histData.histogram = fldResults->scalarValuesHistogram( resAddr ); @@ -327,7 +338,7 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* m_visibleCellStatistics->minMaxCellScalarValues( histData.min, histData.max ); m_visibleCellStatistics->p10p90CellScalarValues( histData.p10, histData.p90 ); m_visibleCellStatistics->sumCellScalarValues( histData.sum ); - m_visibleCellStatistics->mobileVolumeWeightedMean( histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) m_visibleCellStatistics->mobileVolumeWeightedMean( histData.weightedMean ); histData.histogram = m_visibleCellStatistics->cellScalarValuesHistogram(); @@ -348,7 +359,7 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* cellResults->p10p90CellScalarValues( eclResAddr, histData.p10, histData.p90 ); cellResults->meanCellScalarValues( eclResAddr, histData.mean ); cellResults->sumCellScalarValues( eclResAddr, histData.sum ); - cellResults->mobileVolumeWeightedMean( eclResAddr, histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) cellResults->mobileVolumeWeightedMean( eclResAddr, histData.weightedMean ); histData.histogram = cellResults->cellScalarValuesHistogram( eclResAddr ); statisticsCache = cellResults->statistics( eclResAddr ); @@ -360,7 +371,8 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* cellResults->p10p90CellScalarValues( eclResAddr, timeStep, histData.p10, histData.p90 ); cellResults->meanCellScalarValues( eclResAddr, timeStep, histData.mean ); cellResults->sumCellScalarValues( eclResAddr, timeStep, histData.sum ); - cellResults->mobileVolumeWeightedMean( eclResAddr, timeStep, histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) + cellResults->mobileVolumeWeightedMean( eclResAddr, timeStep, histData.weightedMean ); histData.histogram = cellResults->cellScalarValuesHistogram( eclResAddr, timeStep ); statisticsCache = cellResults->statistics( eclResAddr ); @@ -380,7 +392,7 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* m_visibleCellStatistics->minMaxCellScalarValues( histData.min, histData.max ); m_visibleCellStatistics->p10p90CellScalarValues( histData.p10, histData.p90 ); m_visibleCellStatistics->sumCellScalarValues( histData.sum ); - m_visibleCellStatistics->mobileVolumeWeightedMean( histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) m_visibleCellStatistics->mobileVolumeWeightedMean( histData.weightedMean ); histData.histogram = m_visibleCellStatistics->cellScalarValuesHistogram(); @@ -393,7 +405,7 @@ RigHistogramData RimHistogramCalculator::histogramData( RimEclipseView* m_visibleCellStatistics->minMaxCellScalarValues( timeStep, histData.min, histData.max ); m_visibleCellStatistics->p10p90CellScalarValues( timeStep, histData.p10, histData.p90 ); m_visibleCellStatistics->sumCellScalarValues( timeStep, histData.sum ); - m_visibleCellStatistics->mobileVolumeWeightedMean( timeStep, histData.weightedMean ); + if ( m_doComputeMobileVolumeWeightedMean ) m_visibleCellStatistics->mobileVolumeWeightedMean( timeStep, histData.weightedMean ); histData.histogram = m_visibleCellStatistics->cellScalarValuesHistogram( timeStep ); diff --git a/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.h b/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.h index 175c8ef541..15aebb8c31 100644 --- a/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.h +++ b/ApplicationLibCode/ProjectDataModel/RimHistogramCalculator.h @@ -63,6 +63,10 @@ class RimHistogramCalculator RigHistogramCalculator::OutOfRangeHandling outOfRangeHandling, std::optional> customBinRange ); + // The mobile volume weighted mean is derived from MOBPROV, which in turn requires PORV, SWCR and MULTPV. For + // cases backed by remote data these are expensive to fetch, so the calculation can be turned off. + void setDoComputeMobileVolumeWeightedMean( bool enable ); + RigHistogramData histogramData( RimEclipseContourMapView* contourMap ); RigHistogramData histogramData( RimGeoMechContourMapView* contourMap ); RigHistogramData histogramData( RimEclipseView* eclipseView, StatisticsCellRangeType cellRange, StatisticsTimeRangeType timeRange ); @@ -93,4 +97,5 @@ class RimHistogramCalculator RigHistogramCalculator::BinningMode m_binningMode = RigHistogramCalculator::BinningMode::LINEAR; RigHistogramCalculator::OutOfRangeHandling m_outOfRangeHandling = RigHistogramCalculator::OutOfRangeHandling::EXCLUDE; std::optional> m_customBinRange; + bool m_doComputeMobileVolumeWeightedMean; }; diff --git a/ApplicationLibCode/ProjectDataModel/RimMultiPlot.cpp b/ApplicationLibCode/ProjectDataModel/RimMultiPlot.cpp index c8f5864d0b..028cc03e8b 100644 --- a/ApplicationLibCode/ProjectDataModel/RimMultiPlot.cpp +++ b/ApplicationLibCode/ProjectDataModel/RimMultiPlot.cpp @@ -857,6 +857,10 @@ void RimMultiPlot::updatePlots() { if ( m_showWindow ) { + // The plots are loaded one after the other, so a source loading data remotely would make its requests + // one plot at a time. Give it the chance to load everything this window needs in one go first. + prefetchPlotData(); + for ( RimPlot* plot : plots() ) { plot->loadDataAndUpdate(); diff --git a/ApplicationLibCode/ProjectDataModel/RimMultiPlot.h b/ApplicationLibCode/ProjectDataModel/RimMultiPlot.h index d795e7f689..8c10e015f0 100644 --- a/ApplicationLibCode/ProjectDataModel/RimMultiPlot.h +++ b/ApplicationLibCode/ProjectDataModel/RimMultiPlot.h @@ -142,6 +142,11 @@ class RimMultiPlot : public RimPlotWindow, public RimTypedPlotCollection +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "RimRoffCaseSumo.h" + +#include "RiaApplication.h" +#include "RiaLogging.h" +#include "RiaPreferencesGrid.h" +#include "RiaResultNames.h" + +#include "Cloud/RiaSumoConnector.h" +#include "Cloud/RifReaderSumoGridProperty.h" + +#include "RifRoffFileTools.h" + +#include "RigCaseCellResultsData.h" +#include "RigEclipseCaseData.h" +#include "RigEclipseResultAddress.h" +#include "RigEclipseResultInfo.h" +#include "RigMainGrid.h" + +#include "RimReservoirCellResultsStorage.h" +#include "Sumo/RimSumoDataSource.h" + +#include "cafPdmObjectScriptingCapability.h" + +#include +#include + +#include +#include +#include +#include +#include + +CAF_PDM_SOURCE_INIT( RimRoffCaseSumo, "RimRoffCaseSumo" ); + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RimRoffCaseSumo::RimRoffCaseSumo() +{ + CAF_PDM_InitScriptableObject( "Sumo Grid Case", ":/Case48x48.png" ); + + CAF_PDM_InitFieldNoDefault( &m_sumoDataSource, "SumoDataSource", "Sumo Data Source" ); + m_sumoDataSource.uiCapability()->setUiHidden( true ); + + CAF_PDM_InitFieldNoDefault( &m_sumoCaseId, "SumoCaseId", "Sumo Case Id" ); + m_sumoCaseId.uiCapability()->setUiReadOnly( true ); + + CAF_PDM_InitFieldNoDefault( &m_ensembleName, "EnsembleName", "Ensemble Name" ); + m_ensembleName.uiCapability()->setUiReadOnly( true ); + + CAF_PDM_InitFieldNoDefault( &m_gridName, "GridName", "Grid Name" ); + m_gridName.uiCapability()->setUiReadOnly( true ); + + CAF_PDM_InitField( &m_realization, "Realization", -1, "Realization" ); + m_realization.uiCapability()->setUiReadOnly( true ); + + m_sumoConnector = RiaApplication::instance()->makeSumoConnector(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RimRoffCaseSumo::~RimRoffCaseSumo() +{ +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RimRoffCaseSumo* RimRoffCaseSumo::createFromDataSource( RimSumoDataSource* dataSource, const QString& gridName, int realization ) +{ + if ( !dataSource ) return nullptr; + + auto* gridCase = new RimRoffCaseSumo(); + gridCase->setSumoDataSource( dataSource ); + gridCase->setSumoCaseId( dataSource->caseId().get() ); + gridCase->setEnsembleName( dataSource->ensembleName() ); + gridCase->setGridName( gridName ); + gridCase->setRealization( realization ); + + // The grid is stored on Sumo, not on disk, so there is no real grid file name. Still assign a unique + // synthetic grid file name: an empty one collapses all custom case names onto a single + // path-variable key on project save (RiaProjectFileTools), overwriting every name with the last. + gridCase->setGridFileName( + QString( "sumo/%1/%2/realization-%3/%4.roff" ).arg( dataSource->caseId().get() ).arg( gridName ).arg( realization ).arg( gridName ) ); + + // Name the case using grid name, asset, ensemble and realization, e.g. "Geogrid_Drogon_iter-0_Real_0". + QString caseDisplayName = + QString( "%1_%2_%3_Real_%4" ).arg( gridName, dataSource->assetName(), dataSource->ensembleName() ).arg( realization ); + gridCase->setCustomCaseName( caseDisplayName ); + + return gridCase; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimRoffCaseSumo::setSumoDataSource( RimSumoDataSource* dataSource ) +{ + m_sumoDataSource = dataSource; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimRoffCaseSumo::setSumoCaseId( const QString& sumoCaseId ) +{ + m_sumoCaseId = sumoCaseId; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimRoffCaseSumo::setEnsembleName( const QString& ensembleName ) +{ + m_ensembleName = ensembleName; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimRoffCaseSumo::setGridName( const QString& gridName ) +{ + m_gridName = gridName; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimRoffCaseSumo::setRealization( int realization ) +{ + m_realization = realization; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RimRoffCaseSumo::gridName() const +{ + return m_gridName(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +int RimRoffCaseSumo::realization() const +{ + return m_realization(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RimRoffCaseSumo::openEclipseGridFile() +{ + if ( eclipseCaseData() ) + { + // Early exit if reservoir data is created + return true; + } + + if ( !m_sumoConnector ) + { + RiaLogging::error( "No Sumo connector available, unable to load grid from Sumo." ); + return false; + } + + setReservoirData( new RigEclipseCaseData( this ) ); + + if ( eclipseCaseData()->mainGrid()->cellCount() == 0 ) + { + QByteArray contents = m_sumoConnector->grid().gridData( SumoCaseId( m_sumoCaseId() ), m_ensembleName(), m_gridName(), m_realization() ); + if ( contents.isEmpty() ) + { + RiaLogging::error( + std::format( "Failed to download grid '{}' (realization {}) from Sumo.", m_gridName().toStdString(), m_realization() ) ); + return false; + } + + // The downloaded blob is a binary roff grid. Parse it directly from memory. + std::string buffer = contents.toStdString(); + std::istringstream stream( buffer, std::ios::binary ); + + QString errorMessages; + if ( RifRoffFileTools::openGridFile( stream, eclipseCaseData(), &errorMessages ) ) + { + eclipseCaseData()->mainGrid()->setFlipAxis( m_flipXAxis, m_flipYAxis ); + computeCachedData(); + } + else + { + RiaLogging::error( errorMessages.toStdString() ); + return false; + } + } + + results( RiaDefines::PorosityModelType::MATRIX_MODEL )->createPlaceholderResultEntries(); + + if ( RiaPreferencesGrid::current()->autoComputeDepthRelatedProperties() ) + { + eclipseCaseData()->computeDepthRelatedResults(); + } + + results( RiaDefines::PorosityModelType::MATRIX_MODEL )->computeCellVolumes(); + + // Make the Sumo grid properties available as cell results (fetched on demand when displayed). + registerSumoGridProperties(); + + // Rebuild the Data Sources result folders now that the result meta data is available. + updateResultAddressCollection(); + + return true; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RimRoffCaseSumo::locationOnDisc() const +{ + // The grid is stored on Sumo, not on disk. + return QString(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimRoffCaseSumo::defineUiOrdering( QString uiConfigName, caf::PdmUiOrdering& uiOrdering ) +{ + uiOrdering.add( &m_caseUserDescription ); + uiOrdering.add( &m_displayNameOption ); + uiOrdering.add( &m_caseId ); + + auto group = uiOrdering.addNewGroup( "Sumo" ); + group->add( &m_sumoCaseId ); + group->add( &m_ensembleName ); + group->add( &m_gridName ); + group->add( &m_realization ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimRoffCaseSumo::registerSumoGridProperties() +{ + if ( !m_sumoConnector || !eclipseCaseData() ) return; + + const auto gridPropertyInfos = + m_sumoConnector->grid().propertyInfo( SumoCaseId( m_sumoCaseId() ), m_ensembleName(), m_gridName(), m_realization() ); + + // Properties without a timestamp are static. Properties with a single timestamp are dynamic (one time + // step per timestamp). Time intervals (the iso string contains '/') are not supported and skipped. + std::vector staticPropertyNames; + std::map> dynamicPropertyTimestamps; // property name -> the timestamps it has + std::set allTimestamps; // union of timestamps across all properties + for ( const auto& info : gridPropertyInfos ) + { + if ( info.isoDateOrInterval.isEmpty() ) + { + staticPropertyNames.push_back( info.name ); + } + else if ( !info.isoDateOrInterval.contains( '/' ) ) + { + dynamicPropertyTimestamps[info.name].insert( info.isoDateOrInterval ); + allTimestamps.insert( info.isoDateOrInterval ); + } + } + + if ( staticPropertyNames.empty() && dynamicPropertyTimestamps.empty() ) return; + + // The Eclipse readers take the available phases from the INIT file. There is no INIT file here, so derive + // the phases from the saturation properties the case actually has. Without this the phase set stays empty, + // and RigCaseCellResultsData::defaultResult() skips its SOIL and SGAS branches - both are guarded by the + // phase set, while the SWAT branch is not - so a new view would always open on SWAT. + auto hasProperty = [&]( const QString& name ) + { return dynamicPropertyTimestamps.contains( name ) || std::ranges::find( staticPropertyNames, name ) != staticPropertyNames.end(); }; + + std::set availablePhases; + if ( hasProperty( RiaResultNames::soil() ) ) availablePhases.insert( RiaDefines::PhaseType::OIL_PHASE ); + if ( hasProperty( RiaResultNames::sgas() ) ) availablePhases.insert( RiaDefines::PhaseType::GAS_PHASE ); + if ( hasProperty( RiaResultNames::swat() ) ) availablePhases.insert( RiaDefines::PhaseType::WATER_PHASE ); + + if ( !availablePhases.empty() ) eclipseCaseData()->setAvailablePhases( availablePhases ); + + auto cellResults = results( RiaDefines::PorosityModelType::MATRIX_MODEL ); + if ( !cellResults ) return; + + auto reader = new RifReaderSumoGridProperty( m_sumoConnector, m_sumoCaseId(), m_ensembleName(), m_gridName(), m_realization() ); + reader->open( "", eclipseCaseData() ); + + // Register the property names as cell results so they are listed in the cell result editor. The values are + // not loaded here; the reader fetches them on demand the first time a property is displayed. + + for ( const auto& name : staticPropertyNames ) + { + RigEclipseResultAddress resultAddress( RiaDefines::ResultCatType::STATIC_NATIVE, RiaDefines::ResultDataType::FLOAT, name ); + cellResults->createResultEntry( resultAddress, false ); + } + reader->setStaticProperties( staticPropertyNames ); + + if ( !dynamicPropertyTimestamps.empty() ) + { + auto parseTimestamp = []( const QString& isoString ) -> QDateTime + { + QDateTime dateTime = QDateTime::fromString( isoString, Qt::ISODate ); + if ( !dateTime.isValid() ) + { + // Date-only string, e.g. "2018-01-01". + QDate date = QDate::fromString( isoString, Qt::ISODate ); + if ( date.isValid() ) dateTime = QDateTime( date, QTime( 0, 0, 0 ) ); + } + return dateTime; + }; + + // All dynamic results share one common, case-wide set of time steps so they use the same time step + // index space as the 3D view time slider. std::set is sorted, and ISO date strings sort chronologically. + std::vector commonTimestamps( allTimestamps.begin(), allTimestamps.end() ); + + std::vector dates; + std::vector reportNumbers; + std::vector daysSinceStart; + for ( int i = 0; i < static_cast( commonTimestamps.size() ); i++ ) + { + QDateTime date = parseTimestamp( commonTimestamps[i] ); + dates.push_back( date ); + reportNumbers.push_back( i ); + daysSinceStart.push_back( ( dates.front().isValid() && date.isValid() ) ? dates.front().daysTo( date ) : static_cast( i ) ); + } + auto commonTimeStepInfos = RigEclipseTimeStepInfo::createTimeStepInfos( dates, reportNumbers, daysSinceStart ); + + // For each property, build a list aligned with commonTimestamps. An empty entry marks a time step the + // property has no data for, so the reader reports "no data" there instead of another step's values. + std::map> readerDynamicTimestamps; + for ( const auto& [name, propertyTimestamps] : dynamicPropertyTimestamps ) + { + RigEclipseResultAddress resultAddress( RiaDefines::ResultCatType::DYNAMIC_NATIVE, RiaDefines::ResultDataType::FLOAT, name ); + cellResults->createResultEntry( resultAddress, false ); + cellResults->setTimeStepInfos( resultAddress, commonTimeStepInfos ); + + std::vector alignedTimestamps; + alignedTimestamps.reserve( commonTimestamps.size() ); + for ( const auto& timestamp : commonTimestamps ) + { + alignedTimestamps.push_back( propertyTimestamps.count( timestamp ) > 0 ? timestamp : QString() ); + } + readerDynamicTimestamps[name] = alignedTimestamps; + } + + reader->setDynamicProperties( readerDynamicTimestamps ); + } + + cellResults->setReaderInterface( reader ); +} diff --git a/ApplicationLibCode/ProjectDataModel/RimRoffCaseSumo.h b/ApplicationLibCode/ProjectDataModel/RimRoffCaseSumo.h new file mode 100644 index 0000000000..8d62e8cb3c --- /dev/null +++ b/ApplicationLibCode/ProjectDataModel/RimRoffCaseSumo.h @@ -0,0 +1,79 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024 Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "RimEclipseCase.h" + +#include "cafPdmField.h" +#include "cafPdmPtrField.h" + +#include +#include + +class RiaSumoConnector; +class RimSumoDataSource; + +//================================================================================================== +// +// Eclipse grid case backed by a roff grid stored on Sumo. The grid geometry is downloaded as a +// blob through RiaSumoConnector and parsed in memory, so there is no grid file on disk. +// +//================================================================================================== +class RimRoffCaseSumo : public RimEclipseCase +{ + CAF_PDM_HEADER_INIT; + +public: + RimRoffCaseSumo(); + ~RimRoffCaseSumo() override; + + // Create a grid case for a single realization of the given grid, linked back to the data source + // so the case can be updated when the data source realization filter changes. + static RimRoffCaseSumo* createFromDataSource( RimSumoDataSource* dataSource, const QString& gridName, int realization ); + + void setSumoDataSource( RimSumoDataSource* dataSource ); + void setSumoCaseId( const QString& sumoCaseId ); + void setEnsembleName( const QString& ensembleName ); + void setGridName( const QString& gridName ); + void setRealization( int realization ); + + QString gridName() const; + int realization() const; + + bool openEclipseGridFile() override; + + QString locationOnDisc() const override; + +protected: + void defineUiOrdering( QString uiConfigName, caf::PdmUiOrdering& uiOrdering ) override; + +private: + // Discover the available grid properties from Sumo, register them as cell results and attach a reader + // that fetches the property data on demand. Static properties only in this first version. + void registerSumoGridProperties(); + +private: + caf::PdmPtrField m_sumoDataSource; + caf::PdmField m_sumoCaseId; + caf::PdmField m_ensembleName; + caf::PdmField m_gridName; + caf::PdmField m_realization; + + QPointer m_sumoConnector; +}; diff --git a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryEnsemble.h b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryEnsemble.h index f610ea0b48..db10750384 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryEnsemble.h +++ b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryEnsemble.h @@ -83,6 +83,15 @@ class RimSummaryEnsemble : public caf::PdmObject virtual std::set ensembleSummaryAddresses() const; virtual std::set ensembleTimeSteps() const; + // Hint that these addresses are about to be read, given before the curves pull their values. Sources that + // fetch data remotely can use it to load everything in one go instead of one blocking request per address. + // Data is still loaded on demand, so this is an optimization only: not calling it changes nothing but speed. + virtual void prefetchSummaryData( const std::vector& resultAddresses ) {} + + // Whether any of these addresses is being loaded right now, for sources that load without waiting. A plot + // asks about the addresses of its own curves, so it can say it is still waiting for data. + virtual bool isSummaryDataPending( const std::vector& resultAddresses ) const { return false; } + void setEnsembleId( int ensembleId ); int ensembleId() const; bool hasEnsembleParameters() const; diff --git a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.cpp b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.cpp index 51ad0f97b8..3b12e83fe2 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.cpp +++ b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.cpp @@ -1588,6 +1588,32 @@ void RimSummaryMultiPlot::onPlotAdditionOrRemoval() RimMultiPlot::onPlotAdditionOrRemoval(); } +//-------------------------------------------------------------------------------------------------- +/// Gather what every plot in the window is about to read and hand it to each ensemble as one set. A plot +/// prefetches for itself as well, but the plots are loaded one at a time, so doing it here is what lets a +/// remote source load the whole window in one request group rather than one per plot. +//-------------------------------------------------------------------------------------------------- +void RimSummaryMultiPlot::prefetchPlotData() +{ + std::map> addressesByEnsemble; + + for ( RimSummaryPlot* plot : summaryPlots() ) + { + if ( !plot ) continue; + + for ( const auto& [ensemble, addresses] : plot->summaryAddressesByEnsemble() ) + { + auto& allAddresses = addressesByEnsemble[ensemble]; + allAddresses.insert( allAddresses.end(), addresses.begin(), addresses.end() ); + } + } + + for ( const auto& [ensemble, addresses] : addressesByEnsemble ) + { + ensemble->prefetchSummaryData( addresses ); + } +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.h b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.h index 9ac468526a..7fdb44f2ca 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.h +++ b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryMultiPlot.h @@ -123,6 +123,8 @@ class RimSummaryMultiPlot : public RimMultiPlot, public RimSummaryDataSourceStep void onPlotAdditionOrRemoval() override; + void prefetchPlotData() override; + private: void defineUiOrdering( QString uiConfigName, caf::PdmUiOrdering& uiOrdering ) override; void defineEditorAttribute( const caf::PdmFieldHandle* field, QString uiConfigName, caf::PdmUiEditorAttribute* attribute ) override; diff --git a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.cpp b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.cpp index 817d8af98b..16daec581a 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.cpp +++ b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.cpp @@ -62,6 +62,8 @@ #include "SummaryPlotCommands/RicSummaryPlotEditorUi.h" #include "Tools/RimPlotAxisTools.h" +#include "RiuAbstractOverlayContentFrame.h" +#include "RiuDraggableOverlayFrame.h" #include "RiuPlotAxis.h" #include "RiuPlotMainWindow.h" #include "RiuPlotMainWindowTools.h" @@ -1778,6 +1780,109 @@ void RimSummaryPlot::defineUiTreeOrdering( caf::PdmUiTreeOrdering& uiTreeOrderin uiTreeOrdering.skipRemainingChildren( true ); } +//-------------------------------------------------------------------------------------------------- +/// The addresses this plot's curves are about to read, grouped by the ensemble owning them. Used to tell a +/// source what is coming before any of it is read, see prefetchSummaryData(). +//-------------------------------------------------------------------------------------------------- +std::map> RimSummaryPlot::summaryAddressesByEnsemble() const +{ + std::map> addressesByEnsemble; + + auto addAddress = []( auto& addressesByEnsemble, RimSummaryEnsemble* ensemble, const RifEclipseSummaryAddress& address ) + { + if ( !ensemble || !address.isValid() ) return; + + addressesByEnsemble[ensemble].push_back( address ); + }; + + if ( m_summaryCurveCollection ) + { + for ( RimSummaryCurve* curve : m_summaryCurveCollection->curves() ) + { + if ( !curve ) continue; + + if ( auto summaryCase = curve->summaryCaseY() ) + { + addAddress( addressesByEnsemble, summaryCase->firstAncestorOrThisOfType(), curve->summaryAddressY() ); + } + + if ( auto summaryCase = curve->summaryCaseX() ) + { + addAddress( addressesByEnsemble, summaryCase->firstAncestorOrThisOfType(), curve->summaryAddressX() ); + } + } + } + + for ( RimEnsembleCurveSet* curveSet : m_ensembleCurveSetCollection->curveSets() ) + { + if ( !curveSet ) continue; + + addAddress( addressesByEnsemble, curveSet->summaryEnsemble(), curveSet->summaryAddressY() ); + } + + return addressesByEnsemble; +} + +//-------------------------------------------------------------------------------------------------- +/// Tell each ensemble in the plot which addresses its curves are about to read, before any of them read +/// one. Curves pull their values one at a time, so a source loading data remotely would otherwise make one +/// blocking request per curve; given the whole set up front it can load them together. +/// +/// This is a hint only. Every curve still loads its own data, and an ensemble that does not need the hint +/// ignores it. A plot in a plot window is usually covered by the window prefetching for all its plots at +/// once, in which case there is nothing left for this to do. +//-------------------------------------------------------------------------------------------------- +void RimSummaryPlot::prefetchSummaryData() +{ + for ( const auto& [ensemble, addresses] : summaryAddressesByEnsemble() ) + { + ensemble->prefetchSummaryData( addresses ); + } +} + +//-------------------------------------------------------------------------------------------------- +/// Say that this plot is waiting for data. A source can load without waiting, and then the curves are drawn +/// with what has arrived so far, leaving a plot looking finished when it is not. Called after every load, so +/// the frame appears when the data is asked for and goes away when the last of it has arrived. +//-------------------------------------------------------------------------------------------------- +void RimSummaryPlot::updateLoadingOverlayFrame() +{ + if ( !plotWidget() ) return; + + bool isWaitingForData = false; + for ( const auto& [ensemble, addresses] : summaryAddressesByEnsemble() ) + { + if ( ensemble->isSummaryDataPending( addresses ) ) + { + isWaitingForData = true; + break; + } + } + + if ( !isWaitingForData ) + { + if ( m_loadingOverlayFrame ) + { + plotWidget()->removeOverlayFrame( m_loadingOverlayFrame ); + delete m_loadingOverlayFrame; + m_loadingOverlayFrame = nullptr; + } + return; + } + + if ( !m_loadingOverlayFrame ) + { + m_loadingOverlayFrame = new RiuDraggableOverlayFrame( plotWidget()->getParentForOverlay(), plotWidget()->overlayMargins() ); + m_loadingOverlayFrame->setAnchorCorner( RiuDraggableOverlayFrame::AnchorCorner::TopLeft ); + + auto* spinnerFrame = new RiuSpinnerOverlayContentFrame( m_loadingOverlayFrame ); + m_loadingOverlayFrame->setContentFrame( spinnerFrame ); + spinnerFrame->setText( "Loading" ); + } + + plotWidget()->addOverlayFrame( m_loadingOverlayFrame ); +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -1788,6 +1893,8 @@ void RimSummaryPlot::onLoadDataAndUpdate() auto plotWindow = firstAncestorOrThisOfType(); if ( plotWindow == nullptr ) updateDockWindowVisibility(); + prefetchSummaryData(); + if ( m_summaryCurveCollection ) { m_summaryCurveCollection->loadDataAndUpdate( false ); @@ -1834,6 +1941,8 @@ void RimSummaryPlot::onLoadDataAndUpdate() updateAxes(); updateStackedCurveData(); + + updateLoadingOverlayFrame(); } //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.h b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.h index 04e39531c1..31a34833d9 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.h +++ b/ApplicationLibCode/ProjectDataModel/Summary/RimSummaryPlot.h @@ -56,6 +56,7 @@ class RimSummaryTimeAxisProperties; class RimPlotAxisPropertiesInterface; class RimPlotAxisProperties; class RiuSummaryQwtPlot; +class RiuDraggableOverlayFrame; class RimSummaryNameHelper; class RimSummaryPlotNameHelper; class RimPlotTemplateFileItem; @@ -150,6 +151,8 @@ class RimSummaryPlot : public RimPlot, public RimSummaryDataSourceStepping, publ void deleteAllSummaryCurves(); RimSummaryCurveCollection* summaryCurveCollection() const; + std::map> summaryAddressesByEnsemble() const; + void updatePlotTitle(); const RimSummaryNameHelper* activePlotTitleHelperAllCurves() const; @@ -250,6 +253,9 @@ class RimSummaryPlot : public RimPlot, public RimSummaryDataSourceStepping, publ void defineUiOrdering( QString uiConfigName, caf::PdmUiOrdering& uiOrdering ) override; void onLoadDataAndUpdate() override; + void prefetchSummaryData(); + void updateLoadingOverlayFrame(); + bool handleGlobalKeyEvent( QKeyEvent* keyEvent ) override; private slots: @@ -341,8 +347,9 @@ private slots: caf::PdmChildArrayField m_axisPropertiesArray; - QPointer m_summaryPlot; - std::unique_ptr m_plotInfoLabel; + QPointer m_summaryPlot; + QPointer m_loadingOverlayFrame; + std::unique_ptr m_plotInfoLabel; std::unique_ptr m_nameHelperAllCurves; caf::PdmChildField m_sourceStepping; diff --git a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp index 0fb675c8fe..9c19519c48 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp +++ b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.cpp @@ -30,13 +30,17 @@ #include "RifEclipseSummaryAddress.h" #include "Cloud/RimCloudDataSourceCollection.h" +#include "RimProject.h" #include "RimSummaryCaseMainCollection.h" #include "RimSummaryCaseSumo.h" +#include "RimSummaryPlot.h" #include "RimSumoDataSource.h" #include +#include #include +#include #include CAF_PDM_SOURCE_INIT( RimSummaryEnsembleSumo, "RimSummaryEnsembleSumo" ); @@ -93,6 +97,8 @@ RimSummaryEnsembleSumo::RimSummaryEnsembleSumo() setAsEnsemble( true ); m_sumoConnector = RiaApplication::instance()->makeSumoConnector(); + + m_lifetimeToken = std::make_shared( true ); } //-------------------------------------------------------------------------------------------------- @@ -180,53 +186,208 @@ void RimSummaryEnsembleSumo::updateName( const std::set& existingEnsemb //-------------------------------------------------------------------------------------------------- void RimSummaryEnsembleSumo::loadSummaryData( const RifEclipseSummaryAddress& resultAddress ) { - if ( resultAddress.isStatistics() ) return; + loadSummaryData( std::vector{ resultAddress } ); +} - // An address without a vector name has no blob to fetch. The special time address used as curve - // x-axis is one such address. Requesting it would produce a URL with an empty vector segment, which - // the service answers with 404, surfacing as a misleading "parquet file size is 0 bytes" error. - if ( resultAddress.vectorName().empty() ) return; +//-------------------------------------------------------------------------------------------------- +/// Load several vectors at once. Each vector is one parquet blob covering every realization, so the +/// addresses that are not cached yet are fetched as one concurrent batch rather than one after another. +/// That matters because a vector the service has not aggregated yet is produced on demand by the request +/// asking for it, so fetching serially costs the sum of those aggregations. +//-------------------------------------------------------------------------------------------------- +void RimSummaryEnsembleSumo::loadSummaryData( const std::vector& resultAddresses ) +{ + // Nothing is fetched while the caller waits. A curve asking for values it does not have yet gets none, + // the request is put on its way, and the curve is drawn again once it arrives. Waiting here instead + // stopped the application for as long as the service took, which for a vector it has not aggregated yet + // is a good while. + prefetchSummaryData( resultAddresses ); +} - if ( !m_sumoDataSource() ) return; +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimSummaryEnsembleSumo::loadEnsembleParameters() +{ + if ( !m_sumoDataSource() || !m_sumoConnector ) return; + + auto sumoCaseId = m_sumoDataSource->caseId(); + auto sumoEnsembleName = m_sumoDataSource->ensembleName(); + + auto parametersKey = ParquetKey{ sumoCaseId, sumoEnsembleName, "", true }; + if ( m_parquetTable.find( parametersKey ) != m_parquetTable.end() ) return; + if ( m_pendingVectors.find( parametersKey ) != m_pendingVectors.end() ) return; + + // Asked for without waiting, like the vectors. The service aggregates the parameters on demand too, so + // the first request for them can take a while, and it used to be made from inside the read of a curve + // value: dropping a vector into a plot stopped the application until the parameters had been fetched. + m_pendingVectors[parametersKey] = RifEclipseSummaryAddress(); + + std::weak_ptr isAlive = m_lifetimeToken; - auto resultText = QString::fromStdString( resultAddress.toEclipseTextAddress() ); + m_sumoConnector->summary().parameterDataAsync( sumoCaseId, + sumoEnsembleName, + [this, isAlive, parametersKey]( const QByteArray& contents ) + { + if ( isAlive.expired() ) return; + + onParameterDataReceived( parametersKey, contents ); + } ); +} + +//-------------------------------------------------------------------------------------------------- +/// The ensemble parameters have arrived. Called on the thread owning the user interface. +//-------------------------------------------------------------------------------------------------- +void RimSummaryEnsembleSumo::onParameterDataReceived( const ParquetKey& parquetKey, const QByteArray& contents ) +{ + auto it = m_pendingVectors.find( parquetKey ); + + // No longer wanted: the data source changed, or the cache was cleared, while this was on its way. + if ( it == m_pendingVectors.end() ) return; + + m_pendingVectors.erase( it ); + + RiaLogging::debug( std::format( "Load ensemble parameter sensitivities. Contents size: {}", contents.size() ) ); + + std::shared_ptr table = readParquetTable( contents, QString( "%1 parameter sensitivities" ).arg( parquetKey.ensembleId ) ); + + m_parquetTable[parquetKey] = table; + + distributeParametersDataToRealizations( table ); + + updatePlotsUsingThisEnsemble(); +} + +//-------------------------------------------------------------------------------------------------- +/// Ask for everything the plots are about to read, and return without waiting for any of it. Each vector is +/// taken in as it arrives, so a plot shows the vectors that are ready while the rest are still on their way +/// rather than staying blank until the slowest one is done. +/// +/// A vector still on its way is reported as having no data, and the curves using it are drawn empty. They are +/// redrawn when it arrives, see onVectorDataReceived. +//-------------------------------------------------------------------------------------------------- +void RimSummaryEnsembleSumo::prefetchSummaryData( const std::vector& resultAddresses ) +{ + if ( !m_sumoDataSource() || !m_sumoConnector ) return; auto sumoCaseId = m_sumoDataSource->caseId(); auto sumoEnsembleName = m_sumoDataSource->ensembleName(); - auto key = ParquetKey{ sumoCaseId, sumoEnsembleName, resultText, false }; - if ( m_parquetTable.find( key ) == m_parquetTable.end() ) + std::vector vectorNamesToFetch; + for ( const auto& resultAddress : resultAddresses ) { - auto contents = loadParquetData( key ); - RiaLogging::debug( std::format( "Load Summary Data. Contents size: {}", contents.size() ) ); + if ( resultAddress.isStatistics() ) continue; + if ( resultAddress.vectorName().empty() ) continue; - std::shared_ptr table = readParquetTable( contents, QString::fromStdString( resultAddress.uiText() ) ); - m_parquetTable[key] = table; + auto resultText = QString::fromStdString( resultAddress.toEclipseTextAddress() ); + auto key = ParquetKey{ sumoCaseId, sumoEnsembleName, resultText, false }; - distributeDataToRealizations( resultAddress, table ); + if ( m_parquetTable.find( key ) != m_parquetTable.end() ) continue; + if ( m_pendingVectors.find( key ) != m_pendingVectors.end() ) continue; + + m_pendingVectors[key] = resultAddress; + vectorNamesToFetch.push_back( resultText ); } - auto parametersKey = ParquetKey{ sumoCaseId, sumoEnsembleName, "", true }; - if ( m_parquetTable.find( parametersKey ) == m_parquetTable.end() ) - { - auto contents = m_sumoConnector->requestParametersParquetDataBlocking( sumoCaseId, sumoEnsembleName ); - RiaLogging::debug( std::format( "Load ensemble parameter sensitivities. Contents size: {}", contents.size() ) ); + // The parameters belong to the ensemble rather than to any one vector, and are wanted as soon as + // anything of it is read. Asked for here so they travel alongside the vectors. + loadEnsembleParameters(); - std::shared_ptr table = readParquetTable( contents, QString( "%1 parameter sensitivities" ).arg( sumoEnsembleName ) ); - m_parquetTable[parametersKey] = table; + if ( vectorNamesToFetch.empty() ) return; - distributeParametersDataToRealizations( table ); - } + std::weak_ptr isAlive = m_lifetimeToken; + + m_sumoConnector->summary().vectorDataAsync( sumoCaseId, + sumoEnsembleName, + vectorNamesToFetch, + [this, isAlive, sumoCaseId, sumoEnsembleName]( const QString& vectorName, + const QByteArray& contents ) + { + // The request outlived the ensemble that asked for it. + if ( isAlive.expired() ) return; + + onVectorDataReceived( ParquetKey{ sumoCaseId, sumoEnsembleName, vectorName, false }, + contents ); + } ); } //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- -QByteArray RimSummaryEnsembleSumo::loadParquetData( const ParquetKey& parquetKey ) +bool RimSummaryEnsembleSumo::isSummaryDataPending( const std::vector& resultAddresses ) const +{ + if ( m_pendingVectors.empty() || !m_sumoDataSource() ) return false; + + auto sumoCaseId = m_sumoDataSource()->caseId(); + auto sumoEnsembleName = m_sumoDataSource()->ensembleName(); + + for ( const auto& resultAddress : resultAddresses ) + { + auto resultText = QString::fromStdString( resultAddress.toEclipseTextAddress() ); + + if ( m_pendingVectors.contains( ParquetKey{ sumoCaseId, sumoEnsembleName, resultText, false } ) ) return true; + } + + return false; +} + +//-------------------------------------------------------------------------------------------------- +/// One requested vector has arrived. Called on the thread owning the user interface, once per vector. +//-------------------------------------------------------------------------------------------------- +void RimSummaryEnsembleSumo::onVectorDataReceived( const ParquetKey& parquetKey, const QByteArray& contents ) +{ + auto it = m_pendingVectors.find( parquetKey ); + + // No longer wanted: the data source changed, or the cache was cleared, while this was on its way. + if ( it == m_pendingVectors.end() ) return; + + const auto resultAddress = it->second; + m_pendingVectors.erase( it ); + + RiaLogging::debug( std::format( "Load Summary Data. Contents size: {}", contents.size() ) ); + + // Empty contents mean the request failed. The empty result is stored like any other, so a failure is not + // retried on every redraw, matching what a failed blocking load does. + std::shared_ptr table = readParquetTable( contents, QString::fromStdString( resultAddress.uiText() ) ); + + m_parquetTable[parquetKey] = table; + distributeDataToRealizations( resultAddress, table ); + + loadEnsembleParameters(); + + updatePlotsUsingThisEnsemble(); +} + +//-------------------------------------------------------------------------------------------------- +/// Redraw with what has arrived so far. The curves read their values again, those still waiting for data come +/// back empty, and the replot itself is coalesced by the redraw scheduler. +//-------------------------------------------------------------------------------------------------- +void RimSummaryEnsembleSumo::updatePlotsUsingThisEnsemble() { - if ( !m_sumoConnector ) return {}; + // Loading a plot can bring in the next vector, which asks for this update again. Finish the pass that is + // running and repeat it afterwards, rather than reloading plots from inside their own load. + if ( m_isUpdatingPlots ) + { + m_hasMissedPlotUpdate = true; + return; + } + + m_isUpdatingPlots = true; - return m_sumoConnector->requestParquetDataBlocking( SumoCaseId( parquetKey.caseId ), parquetKey.ensembleId, parquetKey.vectorName ); + do + { + m_hasMissedPlotUpdate = false; + + for ( RimSummaryPlot* summaryPlot : RimProject::current()->descendantsOfType() ) + { + if ( !summaryPlot->summaryAddressesByEnsemble().contains( this ) ) continue; + + summaryPlot->loadDataAndUpdate(); + summaryPlot->scheduleReplotIfVisible(); + } + } while ( m_hasMissedPlotUpdate ); + + m_isUpdatingPlots = false; } //-------------------------------------------------------------------------------------------------- @@ -654,6 +815,10 @@ void RimSummaryEnsembleSumo::clearCachedData() { m_resultAddresses.clear(); m_parquetTable.clear(); + + // Anything still on its way belongs to the data just thrown away. Forgetting it here makes those replies + // drop their contents on arrival, and lets the vectors be requested again if they are still wanted. + m_pendingVectors.clear(); } //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.h b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.h index c3b07b09dc..657f950626 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.h +++ b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSummaryEnsembleSumo.h @@ -65,12 +65,15 @@ class RimSummaryEnsembleSumo : public RimSummaryEnsemble void onRealizationSelectionChanged(); void loadSummaryData( const RifEclipseSummaryAddress& resultAddress ); + void loadSummaryData( const std::vector& resultAddresses ); std::string unitName( const RifEclipseSummaryAddress& resultAddress ); RiaDefines::EclipseUnitSystem unitSystem() const; std::set allResultAddresses() const; std::pair nameKeys() const override; void updateName( const std::set& existingEnsembleNames ) override; + void prefetchSummaryData( const std::vector& resultAddresses ) override; + bool isSummaryDataPending( const std::vector& resultAddresses ) const override; protected: void onLoadDataAndUpdate() override; @@ -83,13 +86,16 @@ class RimSummaryEnsembleSumo : public RimSummaryEnsemble void updateResultAddresses(); void clearCachedData(); - QByteArray loadParquetData( const ParquetKey& parquetKey ); - void distributeDataToRealizations( const RifEclipseSummaryAddress& resultAddress, std::shared_ptr table ); void buildMetaData(); void distributeParametersDataToRealizations( std::shared_ptr table ); void redistributeCachedDataToRealizations(); + void loadEnsembleParameters(); + + void onVectorDataReceived( const ParquetKey& parquetKey, const QByteArray& contents ); + void onParameterDataReceived( const ParquetKey& parquetKey, const QByteArray& contents ); + void updatePlotsUsingThisEnsemble(); static std::shared_ptr readParquetTable( const QByteArray& contents, const QString& messageTag ); @@ -100,4 +106,15 @@ class RimSummaryEnsembleSumo : public RimSummaryEnsemble std::set m_resultAddresses; std::map> m_parquetTable; + + // The vectors requested but not yet arrived, and the address each belongs to. A vector in here is not + // requested again, and is reported as having no data yet rather than being waited for. + std::map m_pendingVectors; + + // Held by the callbacks of requests still on their way. They check it before touching this object, so a + // reply arriving after the ensemble is gone is dropped instead of writing into freed memory. + std::shared_ptr m_lifetimeToken; + + bool m_isUpdatingPlots = false; + bool m_hasMissedPlotUpdate = false; }; diff --git a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.cpp b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.cpp index cb9be041e0..b093def4bb 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.cpp +++ b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.cpp @@ -20,6 +20,11 @@ #include "RiaStdStringTools.h" +#include "Rim3dOverlayInfoConfig.h" +#include "RimEclipseCaseEnsemble.h" +#include "RimEclipseView.h" +#include "RimEclipseViewCollection.h" +#include "RimRoffCaseSumo.h" #include "RimSummaryEnsembleSumo.h" #include "cafCmdFeatureMenuBuilder.h" @@ -29,7 +34,10 @@ #include -CAF_PDM_SOURCE_INIT( RimSumoDataSource, "RimSumoDataSource", "RimSummarySumoDataSource" ); +#include +#include + +CAF_PDM_SOURCE_INIT( RimSumoDataSource, "RimSumoDataSource", "RimSummarySumoDataSource", "RimSumoGridDataSource" ); //-------------------------------------------------------------------------------------------------- /// @@ -62,6 +70,19 @@ RimSumoDataSource::RimSumoDataSource() CAF_PDM_InitFieldNoDefault( &m_vectorNames, "VectorNames", "Vector Names" ); m_vectorNames.uiCapability()->setUiHidden( true ); + CAF_PDM_InitFieldNoDefault( &m_gridNames, "GridNames", "Grid Names" ); + m_gridNames.uiCapability()->setUiHidden( true ); + + CAF_PDM_InitFieldNoDefault( &m_selectedGridName, "GridName", "Grid Name" ); + + CAF_PDM_InitField( &m_doComputeMobileVolumeWeightedMean, + "DoComputeMobileVolumeWeightedMean", + false, + "Compute Mobile Volume Weighted Mean", + "", + "Show the mobile volume weighted mean in the 3D info box. This downloads the PORV and SWCR grid " + "properties for every realization." ); + setDeletable( true ); } @@ -198,6 +219,38 @@ void RimSumoDataSource::setVectorNames( const std::vector& vectorNames m_vectorNames = vectorNames; } +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +std::vector RimSumoDataSource::gridNames() const +{ + return m_gridNames(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RimSumoDataSource::setGridNames( const std::vector& gridNames ) +{ + m_gridNames = gridNames; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QString RimSumoDataSource::selectedGridName() const +{ + return m_selectedGridName(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +bool RimSumoDataSource::doComputeMobileVolumeWeightedMean() const +{ + return m_doComputeMobileVolumeWeightedMean(); +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -220,6 +273,7 @@ void RimSumoDataSource::updateName() void RimSumoDataSource::appendMenuItems( caf::CmdFeatureMenuBuilder& menuBuilder ) const { menuBuilder.addCmdFeature( "RicCreateSumoEnsembleFeature" ); + menuBuilder.addCmdFeature( "RicCreateSumoGridEnsembleFeature" ); } //-------------------------------------------------------------------------------------------------- @@ -270,6 +324,28 @@ void RimSumoDataSource::defineUiOrdering( QString uiConfigName, caf::PdmUiOrderi auto ensembleGroup = uiOrdering.addNewGroup( "Ensemble Selection" ); ensembleGroup->add( &m_realizationFilter ); ensembleGroup->add( &m_realizationFilterInfo ); + + auto gridGroup = uiOrdering.addNewGroup( "Grid Selection" ); + gridGroup->add( &m_selectedGridName ); + gridGroup->add( &m_doComputeMobileVolumeWeightedMean ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +QList RimSumoDataSource::calculateValueOptions( const caf::PdmFieldHandle* fieldNeedingOptions ) +{ + QList options; + + if ( fieldNeedingOptions == &m_selectedGridName ) + { + for ( const auto& gridName : m_gridNames() ) + { + options.push_back( caf::PdmOptionItemInfo( gridName, gridName ) ); + } + } + + return options; } //-------------------------------------------------------------------------------------------------- @@ -285,6 +361,10 @@ void RimSumoDataSource::fieldChangedByUi( const caf::PdmFieldHandle* changedFiel { onRealizationFilterChanged(); } + else if ( changedField == &m_doComputeMobileVolumeWeightedMean ) + { + updateGridCaseEnsembles(); + } } //-------------------------------------------------------------------------------------------------- @@ -299,6 +379,113 @@ void RimSumoDataSource::onRealizationFilterChanged() { ensemble->onRealizationSelectionChanged(); } + + // Update any grid case ensembles created from this data source. + updateGridCaseEnsembles(); +} + +//-------------------------------------------------------------------------------------------------- +/// Synchronize the realization grid cases (RimRoffCaseSumo) of every grid ensemble created from this +/// data source with the current realization filter: remove cases for deselected realizations and add +/// cases for newly selected ones. The grid name of each ensemble is preserved. +//-------------------------------------------------------------------------------------------------- +void RimSumoDataSource::updateGridCaseEnsembles() +{ + std::set selectedRealizations; + for ( const auto& realizationId : selectedRealizationIds() ) + { + bool ok = false; + int value = realizationId.toInt( &ok ); + if ( ok ) selectedRealizations.insert( value ); + } + + // Group the grid cases created from this data source by their owning ensemble. + std::map> casesByEnsemble; + for ( auto gridCase : objectsWithReferringPtrFieldsOfType() ) + { + if ( auto ensemble = gridCase->firstAncestorOrThisOfType() ) + { + casesByEnsemble[ensemble].push_back( gridCase ); + } + } + + for ( auto& [ensemble, gridCases] : casesByEnsemble ) + { + if ( gridCases.empty() ) continue; + + if ( ensemble->doComputeMobileVolumeWeightedMean() != m_doComputeMobileVolumeWeightedMean() ) + { + ensemble->setDoComputeMobileVolumeWeightedMean( m_doComputeMobileVolumeWeightedMean() ); + + // The 3D info box reads the setting when it is updated, so refresh it to apply the change immediately. + if ( auto viewColl = ensemble->viewCollection() ) + { + for ( auto view : viewColl->views() ) + { + if ( view->overlayInfoConfig() ) view->overlayInfoConfig()->update3DInfo(); + } + } + } + + // All cases in an ensemble share the same grid; keep it when adding new realizations. + const QString gridName = gridCases.front()->gridName(); + + std::set currentRealizations; + for ( auto gridCase : gridCases ) + { + currentRealizations.insert( gridCase->realization() ); + } + + // Add cases for newly selected realizations first, so a view displaying a removed realization + // can be repointed to one of these instead of being deleted. + for ( int realization : selectedRealizations ) + { + if ( currentRealizations.find( realization ) != currentRealizations.end() ) continue; + + if ( auto* gridCase = RimRoffCaseSumo::createFromDataSource( this, gridName, realization ) ) + { + ensemble->addCase( gridCase ); + } + } + + // Remove cases for realizations no longer selected. Remember any views that displayed them, so + // they can be repointed to a surviving case below (a deleted case auto-nulls the view's case, + // which would otherwise make the view disappear, e.g. when changing 30-40 to 31-40). + std::vector orphanedViews; + for ( auto gridCase : gridCases ) + { + if ( selectedRealizations.find( gridCase->realization() ) != selectedRealizations.end() ) continue; + + if ( auto viewColl = ensemble->viewCollection() ) + { + for ( auto view : viewColl->views() ) + { + if ( view->eclipseCase() == gridCase ) orphanedViews.push_back( view ); + } + } + + ensemble->removeCase( gridCase ); + delete gridCase; + } + + // Repoint orphaned views to a surviving case, or delete them if no realizations remain. + const auto remainingCases = ensemble->cases(); + for ( auto view : orphanedViews ) + { + if ( !remainingCases.empty() ) + { + view->setEclipseCase( remainingCases.front() ); + view->loadDataAndUpdate(); + } + else if ( auto viewColl = ensemble->viewCollection() ) + { + viewColl->removeView( view ); + delete view; + } + } + + ensemble->updateConnectedEditors(); + } } //-------------------------------------------------------------------------------------------------- diff --git a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.h b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.h index 19cc8e848c..4cac8ee611 100644 --- a/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.h +++ b/ApplicationLibCode/ProjectDataModel/Summary/Sumo/RimSumoDataSource.h @@ -25,12 +25,12 @@ //================================================================================================== // -// Common data source describing a single Sumo ensemble. Holds the information required by summary -// ensembles (asset, vector names). The available ensemble realizations - fetched from the -// realizations endpoint - are the source of truth, and the user selects a subset of them ("ensemble -// selection"). Consumers listen to the selected realization id subset. All values are populated by -// RimCloudDataSourceCollection, which owns the RiaSumoConnector; this object does not talk to the -// connector directly. +// Common data source describing a single Sumo ensemble. Holds the information required both for +// summary ensembles (vector names) and for grid case ensembles (asset, grid names). The available +// ensemble realizations - fetched from the realizations endpoint - are the source of truth, and the +// user selects a subset of them ("ensemble selection"). Both summary and grid consumers listen to +// the selected realization id subset. All values are populated by RimCloudDataSourceCollection, +// which owns the RiaSumoConnector; this object does not talk to the connector directly. // //================================================================================================== @@ -57,7 +57,7 @@ class RimSumoDataSource : public RimNamedObject std::vector availableRealizationIds() const; void setAvailableRealizationIds( const std::vector& realizationIds ); - // The subset of realizations matching the realization filter. Summary ensemble creation listens to this. + // The subset of realizations matching the realization filter. Both summary and grid creation listen to this. std::vector selectedRealizationIds() const; // Available summary vectors for the ensemble. Not shown in the UI, but used to populate the @@ -65,15 +65,24 @@ class RimSumoDataSource : public RimNamedObject std::vector vectorNames() const; void setVectorNames( const std::vector& vectorNames ); + std::vector gridNames() const; + void setGridNames( const std::vector& gridNames ); + + QString selectedGridName() const; + + bool doComputeMobileVolumeWeightedMean() const; + void updateName(); private: void appendMenuItems( caf::CmdFeatureMenuBuilder& menuBuilder ) const override; void defineEditorAttribute( const caf::PdmFieldHandle* field, QString uiConfigName, caf::PdmUiEditorAttribute* attribute ) override; void defineUiOrdering( QString uiConfigName, caf::PdmUiOrdering& uiOrdering ) override; + QList calculateValueOptions( const caf::PdmFieldHandle* fieldNeedingOptions ) override; void fieldChangedByUi( const caf::PdmFieldHandle* changedField, const QVariant& oldValue, const QVariant& newValue ) override; void onRealizationFilterChanged(); + void updateGridCaseEnsembles(); QString realizationFilterInfoText() const; QString availableRealizationsRangeText() const; @@ -90,4 +99,8 @@ class RimSumoDataSource : public RimNamedObject caf::PdmProxyValueField m_realizationFilterInfo; caf::PdmField> m_vectorNames; + + caf::PdmField> m_gridNames; + caf::PdmField m_selectedGridName; + caf::PdmField m_doComputeMobileVolumeWeightedMean; }; diff --git a/ApplicationLibCode/UnitTests/CMakeLists.txt b/ApplicationLibCode/UnitTests/CMakeLists.txt index 922bab36c9..d1732e946b 100644 --- a/ApplicationLibCode/UnitTests/CMakeLists.txt +++ b/ApplicationLibCode/UnitTests/CMakeLists.txt @@ -18,6 +18,7 @@ set(SOURCE_UNITTEST_FILES ${CMAKE_CURRENT_LIST_DIR}/RifOpmFlowDeckFile-Test.cpp ${CMAKE_CURRENT_LIST_DIR}/RifReaderEclipseOutput-Test.cpp ${CMAKE_CURRENT_LIST_DIR}/RifReaderEclipseSummary-Test.cpp + ${CMAKE_CURRENT_LIST_DIR}/RiaSumoBlobCache-Test.cpp ${CMAKE_CURRENT_LIST_DIR}/RigActiveCellInfo-Test.cpp ${CMAKE_CURRENT_LIST_DIR}/RigEclipseCaseDataTools-Test.cpp ${CMAKE_CURRENT_LIST_DIR}/RigEclipseCrossPlotDataExtractor-Test.cpp diff --git a/ApplicationLibCode/UnitTests/RiaSumoBlobCache-Test.cpp b/ApplicationLibCode/UnitTests/RiaSumoBlobCache-Test.cpp new file mode 100644 index 0000000000..158a2d6208 --- /dev/null +++ b/ApplicationLibCode/UnitTests/RiaSumoBlobCache-Test.cpp @@ -0,0 +1,201 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Copyright (C) 2024- Equinor ASA +// +// ResInsight is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// ResInsight is distributed in the hope that it will be useful, but WITHOUT ANY +// WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. +// +// See the GNU General Public License at +// for more details. +// +///////////////////////////////////////////////////////////////////////////////// + +#include "gtest/gtest.h" + +#include "Cloud/RiaSumoBlobCache.h" + +namespace +{ +QByteArray blobOfSize( int size, char fill = 'x' ) +{ + return QByteArray( size, fill ); +} +} // namespace + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, LookupOfMissingKeyReturnsEmpty ) +{ + RiaSumoBlobCache cache( 100 ); + + EXPECT_FALSE( cache.contains( "missing" ) ); + EXPECT_TRUE( cache.lookup( "missing" ).isEmpty() ); + EXPECT_EQ( size_t( 0 ), cache.entryCount() ); + EXPECT_EQ( size_t( 0 ), cache.sizeBytes() ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, InsertedBlobIsReturnedUnchanged ) +{ + RiaSumoBlobCache cache( 100 ); + + const QByteArray contents = blobOfSize( 10, 'a' ); + cache.insert( "key", contents ); + + EXPECT_TRUE( cache.contains( "key" ) ); + EXPECT_EQ( contents, cache.lookup( "key" ) ); + EXPECT_EQ( size_t( 1 ), cache.entryCount() ); + EXPECT_EQ( size_t( 10 ), cache.sizeBytes() ); +} + +//-------------------------------------------------------------------------------------------------- +/// An empty array is how a cache miss is reported, so it must never be stored as a value. +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, EmptyContentsAreNotCached ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "key", QByteArray() ); + + EXPECT_FALSE( cache.contains( "key" ) ); + EXPECT_EQ( size_t( 0 ), cache.entryCount() ); +} + +//-------------------------------------------------------------------------------------------------- +/// Caching a blob bigger than the whole limit would evict everything else to make room for it. +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, BlobLargerThanLimitIsNotCachedAndKeepsExistingEntries ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "small", blobOfSize( 10 ) ); + cache.insert( "huge", blobOfSize( 101 ) ); + + EXPECT_FALSE( cache.contains( "huge" ) ); + EXPECT_TRUE( cache.contains( "small" ) ); + EXPECT_EQ( size_t( 1 ), cache.entryCount() ); + EXPECT_EQ( size_t( 10 ), cache.sizeBytes() ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, BlobExactlyAtLimitIsCached ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "exact", blobOfSize( 100 ) ); + + EXPECT_TRUE( cache.contains( "exact" ) ); + EXPECT_EQ( size_t( 100 ), cache.sizeBytes() ); +} + +//-------------------------------------------------------------------------------------------------- +/// Exceeding the limit evicts from the back of the recency order, oldest first. +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, LeastRecentlyUsedIsEvictedFirst ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "a", blobOfSize( 40 ) ); + cache.insert( "b", blobOfSize( 40 ) ); + cache.insert( "c", blobOfSize( 40 ) ); // pushes the total to 120, so "a" has to go + + EXPECT_FALSE( cache.contains( "a" ) ); + EXPECT_TRUE( cache.contains( "b" ) ); + EXPECT_TRUE( cache.contains( "c" ) ); + EXPECT_EQ( size_t( 80 ), cache.sizeBytes() ); +} + +//-------------------------------------------------------------------------------------------------- +/// A lookup counts as use, so the blob it returns must survive the next eviction. +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, LookupRefreshesRecency ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "a", blobOfSize( 40 ) ); + cache.insert( "b", blobOfSize( 40 ) ); + + cache.lookup( "a" ); // "b" is now the least recently used + + cache.insert( "c", blobOfSize( 40 ) ); + + EXPECT_TRUE( cache.contains( "a" ) ); + EXPECT_FALSE( cache.contains( "b" ) ); + EXPECT_TRUE( cache.contains( "c" ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// Re-inserting a key must replace the entry rather than leave a stale recency entry behind, and the +/// accounted size must follow the new contents. +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, ReinsertingKeyReplacesEntryAndSize ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "key", blobOfSize( 10, 'a' ) ); + cache.insert( "key", blobOfSize( 30, 'b' ) ); + + EXPECT_EQ( size_t( 1 ), cache.entryCount() ); + EXPECT_EQ( size_t( 30 ), cache.sizeBytes() ); + EXPECT_EQ( blobOfSize( 30, 'b' ), cache.lookup( "key" ) ); + + // A stale recency entry would make this eviction drop the wrong key, or drop nothing at all. + cache.insert( "other", blobOfSize( 80 ) ); + + EXPECT_FALSE( cache.contains( "key" ) ); + EXPECT_TRUE( cache.contains( "other" ) ); + EXPECT_EQ( size_t( 1 ), cache.entryCount() ); + EXPECT_EQ( size_t( 80 ), cache.sizeBytes() ); +} + +//-------------------------------------------------------------------------------------------------- +/// One insert may have to evict several entries to get back within the limit. +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, InsertEvictsAsManyEntriesAsNeeded ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "a", blobOfSize( 20 ) ); + cache.insert( "b", blobOfSize( 20 ) ); + cache.insert( "c", blobOfSize( 20 ) ); + cache.insert( "big", blobOfSize( 90 ) ); + + EXPECT_TRUE( cache.contains( "big" ) ); + EXPECT_FALSE( cache.contains( "a" ) ); + EXPECT_FALSE( cache.contains( "b" ) ); + EXPECT_EQ( size_t( 1 ), cache.entryCount() ); + EXPECT_EQ( size_t( 90 ), cache.sizeBytes() ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +TEST( RiaSumoBlobCacheTest, ClearEmptiesTheCache ) +{ + RiaSumoBlobCache cache( 100 ); + + cache.insert( "a", blobOfSize( 20 ) ); + cache.insert( "b", blobOfSize( 20 ) ); + + cache.clear(); + + EXPECT_EQ( size_t( 0 ), cache.entryCount() ); + EXPECT_EQ( size_t( 0 ), cache.sizeBytes() ); + EXPECT_FALSE( cache.contains( "a" ) ); + + // The recency order must be cleared too, otherwise a later insert evicts against stale keys. + cache.insert( "c", blobOfSize( 20 ) ); + EXPECT_TRUE( cache.contains( "c" ) ); + EXPECT_EQ( size_t( 20 ), cache.sizeBytes() ); +} diff --git a/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.cpp b/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.cpp index eef813b43f..7e83d230e6 100644 --- a/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.cpp +++ b/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.cpp @@ -21,9 +21,11 @@ #include "RiaFontCache.h" #include "RiaPreferences.h" +#include #include #include #include +#include #include //-------------------------------------------------------------------------------------------------- @@ -101,3 +103,144 @@ void RiuTextOverlayContentFrame::updateLabelFont() font.setPointSize( caf::FontTools::pointSizeFromEnum( RiaPreferences::current()->defaultPlotFontSize() ) ); m_textLabel->setFont( font ); } + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +RiuSpinnerOverlayContentFrame::RiuSpinnerOverlayContentFrame( QWidget* parent /*= nullptr */ ) + : RiuAbstractOverlayContentFrame( parent ) +{ + QHBoxLayout* layout = new QHBoxLayout( this ); + + // Room for the spinner, which is painted rather than laid out: it has no content of its own to size it, + // and reserving the space keeps it from ending up under the text. + layout->setContentsMargins( 4 + spinnerSize() + spinnerMargin(), 4, 4, 4 ); + + m_textLabel = new QLabel; + layout->addWidget( m_textLabel ); + + m_animationTimer = new QTimer( this ); + m_animationTimer->setInterval( 50 ); + + QObject::connect( m_animationTimer, + &QTimer::timeout, + this, + [this]() + { + m_angleDegrees = ( m_angleDegrees + 30 ) % 360; + update(); + } ); + + updateLabelFont(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiuSpinnerOverlayContentFrame::setText( const QString& text ) +{ + m_textLabel->setText( text ); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +int RiuSpinnerOverlayContentFrame::spinnerSize() +{ + return 14; +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +int RiuSpinnerOverlayContentFrame::spinnerMargin() +{ + return 6; +} + +//-------------------------------------------------------------------------------------------------- +/// Animate only while on screen. A frame taken off a plot keeps its timer, and a timer left running would +/// wake the application up several times a second for something nobody is looking at. +//-------------------------------------------------------------------------------------------------- +void RiuSpinnerOverlayContentFrame::showEvent( QShowEvent* event ) +{ + RiuAbstractOverlayContentFrame::showEvent( event ); + + m_animationTimer->start(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiuSpinnerOverlayContentFrame::hideEvent( QHideEvent* event ) +{ + RiuAbstractOverlayContentFrame::hideEvent( event ); + + m_animationTimer->stop(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiuSpinnerOverlayContentFrame::paintEvent( QPaintEvent* event ) +{ + RiuAbstractOverlayContentFrame::paintEvent( event ); + + QPainter painter( this ); + drawSpinner( &painter, QPoint( 4, ( height() - spinnerSize() ) / 2 ) ); +} + +//-------------------------------------------------------------------------------------------------- +/// An arc left open at one end, turned a step further on every tick. The gap is what makes the turning +/// visible: a full circle would look the same at every angle. +//-------------------------------------------------------------------------------------------------- +void RiuSpinnerOverlayContentFrame::drawSpinner( QPainter* painter, const QPoint& topLeft ) const +{ + painter->save(); + painter->setRenderHint( QPainter::Antialiasing ); + + QPen pen( palette().color( QPalette::WindowText ), 2.0, Qt::SolidLine, Qt::RoundCap ); + painter->setPen( pen ); + painter->setBrush( Qt::NoBrush ); + + // Qt angles are in sixteenths of a degree and turn counterclockwise, so the sign makes the arc turn the + // way a clock does. Inset by the pen width, otherwise the stroke is drawn half outside the rectangle. + const QRect arcRect( topLeft.x() + 1, topLeft.y() + 1, spinnerSize() - 2, spinnerSize() - 2 ); + painter->drawArc( arcRect, -m_angleDegrees * 16, 300 * 16 ); + + painter->restore(); +} + +//-------------------------------------------------------------------------------------------------- +/// Drawn into snapshots as it looks at this moment. There is no animation in a still image, but leaving the +/// spinner out would make a plot that was still loading look finished. +//-------------------------------------------------------------------------------------------------- +void RiuSpinnerOverlayContentFrame::renderTo( QPainter* painter, const QRect& targetRect ) +{ + updateLabelFont(); + + painter->save(); + painter->translate( targetRect.topLeft() ); + + drawSpinner( painter, QPoint( 4, ( targetRect.height() - spinnerSize() ) / 2 ) ); + + painter->translate( contentsMargins().left(), contentsMargins().top() ); + painter->setFont( m_textLabel->font() ); + + QTextDocument td; + td.setDefaultFont( m_textLabel->font() ); + td.setHtml( m_textLabel->text() ); + td.drawContents( painter ); + + painter->restore(); +} + +//-------------------------------------------------------------------------------------------------- +/// +//-------------------------------------------------------------------------------------------------- +void RiuSpinnerOverlayContentFrame::updateLabelFont() +{ + QFont font = m_textLabel->font(); + font.setPointSize( caf::FontTools::pointSizeFromEnum( RiaPreferences::current()->defaultPlotFontSize() ) ); + m_textLabel->setFont( font ); +} diff --git a/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.h b/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.h index b11b0899b5..55da1c344b 100644 --- a/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.h +++ b/ApplicationLibCode/UserInterface/RiuAbstractOverlayContentFrame.h @@ -22,6 +22,7 @@ #include class QLabel; +class QTimer; class RiuAbstractOverlayContentFrame : public QFrame { @@ -51,3 +52,34 @@ class RiuTextOverlayContentFrame : public RiuAbstractOverlayContentFrame private: QPointer m_textLabel; }; + +//================================================================================================== +/// Says that something is going on, for work that finishes on its own and reports no progress along the +/// way. The animation is driven by a timer that only runs while the frame is visible, so a frame that has +/// been taken off a plot costs nothing. +//================================================================================================== +class RiuSpinnerOverlayContentFrame : public RiuAbstractOverlayContentFrame +{ + Q_OBJECT +public: + RiuSpinnerOverlayContentFrame( QWidget* parent = nullptr ); + + void setText( const QString& text ); + void renderTo( QPainter* painter, const QRect& targetRect ) override; + +protected: + void paintEvent( QPaintEvent* event ) override; + void showEvent( QShowEvent* event ) override; + void hideEvent( QHideEvent* event ) override; + +private: + void drawSpinner( QPainter* painter, const QPoint& topLeft ) const; + static int spinnerSize(); + static int spinnerMargin(); + void updateLabelFont(); + +private: + QPointer m_textLabel; + QTimer* m_animationTimer = nullptr; + int m_angleDegrees = 0; +}; diff --git a/ApplicationLibCode/UserInterface/RiuMessagePanel.cpp b/ApplicationLibCode/UserInterface/RiuMessagePanel.cpp index e7e7537ddb..a2eddc53ac 100644 --- a/ApplicationLibCode/UserInterface/RiuMessagePanel.cpp +++ b/ApplicationLibCode/UserInterface/RiuMessagePanel.cpp @@ -26,6 +26,7 @@ #include "cafStyleSheetTools.h" +#include #include #include #include @@ -205,6 +206,22 @@ void RiuMessagePanelLogger::debug( const char* message ) writeToMessagePanel( RILogLevel::RI_LL_DEBUG, message ); } +//-------------------------------------------------------------------------------------------------- +/// Deliver the messages handed over from other threads. Only the queued addMessage calls above are posted to +/// a panel, so this writes out the pending log messages and nothing else. Must be called from the thread +/// owning the panels. +//-------------------------------------------------------------------------------------------------- +void RiuMessagePanelLogger::flushPendingMessages() +{ + for ( auto& panel : m_messagePanels ) + { + if ( panel && panel->thread() == QThread::currentThread() ) + { + QCoreApplication::sendPostedEvents( panel, QEvent::MetaCall ); + } + } +} + //-------------------------------------------------------------------------------------------------- /// //-------------------------------------------------------------------------------------------------- @@ -219,13 +236,24 @@ void RiuMessagePanelLogger::writeToMessagePanel( RILogLevel messageLevel, const { if ( panel ) { - // Make sure we only output messages for the GUI-thread. - // We can loose some messages, but we avoid updating UI from a different thread that will cause asserts and - // potential crashes if ( panel->thread() == QThread::currentThread() ) { panel->addMessage( messageLevel, message ); } + else + { + // The panel can only be touched from the thread owning it. Hand the message over instead of + // dropping it, so messages logged from a worker thread still reach the panel. The text is + // copied into the queued call, as the caller owns the buffer. + const QString messageText = QString::fromUtf8( message ); + QMetaObject::invokeMethod( + panel, + [panel, messageLevel, messageText]() + { + if ( panel ) panel->addMessage( messageLevel, messageText ); + }, + Qt::QueuedConnection ); + } } } } diff --git a/ApplicationLibCode/UserInterface/RiuMessagePanel.h b/ApplicationLibCode/UserInterface/RiuMessagePanel.h index 5756d02ebf..5e472cff4d 100644 --- a/ApplicationLibCode/UserInterface/RiuMessagePanel.h +++ b/ApplicationLibCode/UserInterface/RiuMessagePanel.h @@ -70,6 +70,8 @@ class RiuMessagePanelLogger : public RiaLogger void info( const char* message ) override; void debug( const char* message ) override; + void flushPendingMessages() override; + private: void writeToMessagePanel( RILogLevel messageLevel, const char* message );