diff --git a/simulation-workflows/ACE3P.py b/simulation-workflows/ACE3P.py index 795c33f6018c46996d2bb5330f18b406aaaa387a..0ba96ec13ef0fdcc564a0fc09c8dcc5c2b138f39 100644 --- a/simulation-workflows/ACE3P.py +++ b/simulation-workflows/ACE3P.py @@ -78,6 +78,7 @@ class Export(smtk.operation.Operation): self.cumulus_output_folder = None self.nersc_job_folder = None self.nersc_input_folder = None + self.slurm_script = None success = ExportCMB(self) except Exception as err: self.log().addError('Exception => {}'.format(err)) @@ -95,14 +96,16 @@ class Export(smtk.operation.Operation): (self.cumulus_job_id, 'CumulusJobId'), (self.cumulus_output_folder, 'CumulusOutputFolder'), (self.nersc_job_folder, 'NerscJobFolder'), - (self.nersc_input_folder, 'NerscInputFolder') + (self.nersc_input_folder, 'NerscInputFolder'), + (self.slurm_script, 'SlurmScript') ] for t in result_fields: value, name = t if value is not None: item = result.find(name) if item is None: - self.log().addWarning('Op result missing {} item'.format(name)) + # self.log().addWarning('Op result missing {} item'.format(name)) + pass else: item.setIsEnabled(True) item.setValue(value) @@ -374,12 +377,24 @@ def ExportCMB(export_op): # Import nersc module (only when needed) from internal.writers import nersc reload(nersc) - # try: - # from internal.writers import nersc - # reload(nersc) - # except: - # scope.logger.addError('Failed to import girder_client') - # return False + + # Check "UseNewtInterface" item + newt_item = sim_item.find('UseNewtInterface') + if newt_item is not None and newt_item.isEnabled(): + # Generate slurm script only + command_list = nersc.create_slurm_commands(scope, sim_item) + command_string = '\n'.join(command_list) + + script_name = '{}.sh'.format(scope.solver_list[0]).lower() + script_path = os.path.join(scope.output_folder, script_name) + with open(script_path, 'w') as f: + f.write(command_string) + f.write('\n') + export_op.slurm_script = script_path + print('Wrote slurm command file', script_path) + completed = True + return completed + scope.cumulus = None scope.nersc_job_folder = None diff --git a/simulation-workflows/internal/ace3p-export.sbt b/simulation-workflows/internal/ace3p-export.sbt index 322640b9752e27cf1ae0ded134dee8a9c5efb84f..d2b624064c06736cb1102e5093c2d10b49a91a1d 100644 --- a/simulation-workflows/internal/ace3p-export.sbt +++ b/simulation-workflows/internal/ace3p-export.sbt @@ -64,6 +64,7 @@ <AttDef Type="export-result" BaseType="result"> <ItemDefinitions> <File Name="OutputFile" /> + <File Name="SlurmScript" /> <String Name="CumulusJobId" Optional="true" IsEnabledByDefault="false" /> <String Name="CumulusOutputFolder" Optional="true" IsEnabledByDefault="false" /> <String Name="NerscJobFolder" Optional="true" IsEnabledByDefault="false" /> diff --git a/simulation-workflows/internal/ace3p-submit.xml b/simulation-workflows/internal/ace3p-submit.xml index c2a57ac08e9f61dc2bcf5005736176dff5e94a7e..8094bd8b6914688b148d9fa350d63ef19a08b7dc 100644 --- a/simulation-workflows/internal/ace3p-submit.xml +++ b/simulation-workflows/internal/ace3p-submit.xml @@ -7,6 +7,7 @@ <Cat>Rf-Postprocess</Cat> </Categories> <ItemDefinitions> + <Void Name="UseNewtInterface" Label="Use NEWT interface" Optional="true" IsEnabledByDefault="false" AdvanceLevel="1" /> <String Name="JobName" Label="Job name" Version="0"> <BriefDescription>Label you can use to track your job</BriefDescription> <DefaultValue>ACE3P</DefaultValue> diff --git a/smtk/newt/examples/cxx/qtNerscFileBrowser.cxx b/smtk/newt/examples/cxx/qtNerscFileBrowser.cxx index e1605b41bfa8b793fd239799446dc53907668046..63c59f4fd99992a2ce68895f213f155b21d4b39b 100644 --- a/smtk/newt/examples/cxx/qtNerscFileBrowser.cxx +++ b/smtk/newt/examples/cxx/qtNerscFileBrowser.cxx @@ -54,6 +54,7 @@ int main(int argc, char* argv[]) // Display the file browser newt::qtNewtFileBrowserDialog* browser = new newt::qtNewtFileBrowserDialog(); + browser->hideApplyButton(); browser->setEnabled(false); browser->show(); diff --git a/smtk/newt/qtNewtFileBrowserDialog.cxx b/smtk/newt/qtNewtFileBrowserDialog.cxx index 831aea31296c0add4f6d40d7a7553e93e96e261c..d167e3b150c41348dddd99c0db4cf01a381ee853 100644 --- a/smtk/newt/qtNewtFileBrowserDialog.cxx +++ b/smtk/newt/qtNewtFileBrowserDialog.cxx @@ -15,6 +15,8 @@ #include <QAbstractButton> #include <QDebug> #include <QDialogButtonBox> +#include <QKeyEvent> +#include <QMargins> #include <QPushButton> #include <QVBoxLayout> @@ -25,6 +27,10 @@ qtNewtFileBrowserDialog::qtNewtFileBrowserDialog(QWidget* parentWidget) : QDialog(parentWidget) { QVBoxLayout* layout = new QVBoxLayout(this); + QMargins margins = layout->contentsMargins(); + margins.setTop(0); + layout->setContentsMargins(margins); + m_widget = new qtNewtFileBrowserWidget(this); layout->addWidget(m_widget); @@ -68,4 +74,22 @@ qtNewtFileBrowserDialog::qtNewtFileBrowserDialog(QWidget* parentWidget) QObject::connect( m_widget, &qtNewtFileBrowserWidget::pathCopied, this, &qtNewtFileBrowserDialog::pathCopied); } + +void qtNewtFileBrowserDialog::hideApplyButton() +{ + m_buttonBox->button(QDialogButtonBox::Apply)->hide(); +} + +void qtNewtFileBrowserDialog::keyPressEvent(QKeyEvent* keyEvent) +{ + // Prevent <Enter> key from closing the dialog + if (keyEvent->key() == Qt::Key_Enter || keyEvent->key() == Qt::Key_Return) + { + return; + } + + // (else) + QDialog::keyPressEvent(keyEvent); +} + } // namespace newt diff --git a/smtk/newt/qtNewtFileBrowserDialog.h b/smtk/newt/qtNewtFileBrowserDialog.h index 874fd8dab97d309055959a49421c7a5481858330..556649c34217986e5180da6b8c0712b0c41eea26 100644 --- a/smtk/newt/qtNewtFileBrowserDialog.h +++ b/smtk/newt/qtNewtFileBrowserDialog.h @@ -19,6 +19,7 @@ #include <QDialog> class QDialogButtonBox; +class QKeyEvent; namespace newt { @@ -37,6 +38,9 @@ public: qtNewtFileBrowserDialog(QWidget* parentWidget = nullptr); ~qtNewtFileBrowserDialog() = default; + // For apps that don't use the apply button + void hideApplyButton(); + signals: // Emitted when user clicks the dialog's "Apply" button. void applyPath(const QString& path); @@ -44,6 +48,9 @@ signals: // Emitted when user clicks the widget's "Copy" button. void pathCopied(const QString& path); +protected: + void keyPressEvent(QKeyEvent* e); + private: qtNewtFileBrowserWidget* m_widget = nullptr; QDialogButtonBox* m_buttonBox = nullptr; diff --git a/smtk/newt/qtNewtFileBrowserWidget.cxx b/smtk/newt/qtNewtFileBrowserWidget.cxx index 1329f2f5d3d2d8f436c65d2995821e73d618b4c1..867aafd06236ffe157b0d1e8b8c61a8db029424a 100644 --- a/smtk/newt/qtNewtFileBrowserWidget.cxx +++ b/smtk/newt/qtNewtFileBrowserWidget.cxx @@ -31,6 +31,7 @@ #include <QPlainTextEdit> #include <QPushButton> #include <QStandardItem> +#include <QStandardPaths> #include <QVBoxLayout> #include <Qt> #include <QtGlobal> @@ -88,6 +89,8 @@ public: case FileSystemItem::UNKNOWN: return "?"; } + + return "?"; } static bool lessThan(const FileSystemItem& left, const FileSystemItem& right) @@ -119,12 +122,14 @@ qtNewtFileBrowserWidget::qtNewtFileBrowserWidget(QWidget* parentWidget) this->ui->m_statusLabel->setText("Click Home or Scratch button"); this->ui->m_userLabel->setText("?"); this->ui->m_tableView->setModel(&m_model); + this->ui->m_progressBar->setValue(0); + this->ui->m_progressBar->hide(); if (m_newt->isLoggedIn()) { this->ui->m_userLabel->setText(m_newt->userName()); } - // Connect to signal from newt interface + // Connect to signals from newt interface QObject::connect( m_newt, &qtNewtInterface::loginComplete, this, &qtNewtFileBrowserWidget::onLogin); @@ -146,7 +151,13 @@ qtNewtFileBrowserWidget::qtNewtFileBrowserWidget(QWidget* parentWidget) this->ui->m_tableView, &QTableView::activated, this, &qtNewtFileBrowserWidget::onItemActivated); } -qtNewtFileBrowserWidget::~qtNewtFileBrowserWidget() {} +qtNewtFileBrowserWidget::~qtNewtFileBrowserWidget() +{ + if (!m_files.empty()) + { + qWarning() << __FILE__ << __LINE__ << "m_files.count()" << m_files.count(); + } +} void qtNewtFileBrowserWidget::onItemActivated(const QModelIndex& index) { @@ -322,7 +333,9 @@ void qtNewtFileBrowserWidget::onHomeClicked() return; } +#ifndef NDEBUG qDebug() << "Home path is" << text; +#endif this->m_homePath = text; this->gotoPath(text); reply->deleteLater(); @@ -364,6 +377,12 @@ void qtNewtFileBrowserWidget::onScratchClicked() void qtNewtFileBrowserWidget::contextMenuEvent(QContextMenuEvent* event) { + // If download in progress, don't display the context menu + if (this->ui->m_progressBar->isVisible()) + { + return; + } + // Get selected row auto pos = this->ui->m_tableView->viewport()->mapFromGlobal(event->globalPos()); int row = this->ui->m_tableView->rowAt(pos.y()); @@ -390,9 +409,15 @@ void qtNewtFileBrowserWidget::contextMenuEvent(QContextMenuEvent* event) FileSystemItem::Type fsType = static_cast<FileSystemItem::Type>(intValue); if (fsType == FileSystemItem::FILE) { + QAction* downloadFileAction = new QAction("Download File", this); + QObject::connect( + downloadFileAction, &QAction::triggered, [this, name]() { this->downloadFile(name); }); + menu->addAction(downloadFileAction); + QAction* viewFileAction = new QAction("View as Text", this); QObject::connect(viewFileAction, &QAction::triggered, [this, name]() { this->viewFile(name); }); menu->addAction(viewFileAction); + menu->exec(event->globalPos()); return; } @@ -409,6 +434,53 @@ void qtNewtFileBrowserWidget::contextMenuEvent(QContextMenuEvent* event) } } +void qtNewtFileBrowserWidget::downloadFile(const QString& remoteFileName) +{ + // Create QFile for writing to download directory + QString downloadDirectory = QStandardPaths::writableLocation(QStandardPaths::DownloadLocation); + QString localPath = downloadDirectory + "/" + remoteFileName; + QFile* file = new QFile(localPath); + if (!file->open(QIODevice::WriteOnly)) // truncates any existing file + { + delete file; + QString msg = QString("Unable to open local file for writing at %1").arg(localPath); + QMessageBox::critical(this, "Error Opening Local File", msg); + } + + QString remotePath = QString("%1/%2").arg(m_currentPath, remoteFileName); + QNetworkReply* reply = m_newt->requestFileDownload(MACHINE, remotePath); + + QObject::connect( + reply, &QNetworkReply::readyRead, [this, reply, file]() { file->write(reply->readAll()); }); + + QObject::connect( + reply, &QNetworkReply::downloadProgress, [this](qint64 bytesReceived, qint64 bytesTotal) { + this->ui->m_progressBar->setMaximum(static_cast<int>(bytesTotal)); + this->ui->m_progressBar->setValue(static_cast<int>(bytesReceived)); + }); + + QObject::connect(reply, &QNetworkReply::finished, [this, reply, localPath]() { + this->ui->m_progressBar->hide(); + QString msg; + if (reply->error() == 0) + { + msg = QString("Downloaded file %1").arg(localPath); + QMessageBox::information(this, "Download Complete", msg); + } + else + { + msg = QString("ERROR Downloading to file %1: %2").arg(localPath).arg(reply->error()); + QMessageBox::critical(this, "Download Error", msg); + } + reply->deleteLater(); + }); + + this->ui->m_progressBar->setRange(0, 100); + this->ui->m_progressBar->setValue(0); + this->ui->m_progressBar->show(); + qDebug() << "Downloading" << remotePath; +} + bool qtNewtFileBrowserWidget::getCommandReply(QNetworkReply* reply, QString& text) { QString content; @@ -465,8 +537,9 @@ void qtNewtFileBrowserWidget::handleNetworkError(QNetworkReply* reply) void qtNewtFileBrowserWidget::viewFile(const QString& name) { QString path = QString("%1/%2").arg(m_currentPath, name); - QNetworkReply* reply = m_newt->requestTextFile(MACHINE, path); + QNetworkReply* reply = m_newt->requestFileDownload(MACHINE, path); QObject::connect(reply, &QNetworkReply::finished, [this, reply, name]() { + this->ui->m_progressBar->hide(); if (reply->error()) { this->handleNetworkError(reply); @@ -487,6 +560,9 @@ void qtNewtFileBrowserWidget::viewFile(const QString& name) dialog.setMinimumSize(640, 640); dialog.exec(); }); + + this->ui->m_progressBar->setRange(0, 0); + this->ui->m_progressBar->show(); } void qtNewtFileBrowserWidget::onRequestNavigate(const QString& path) diff --git a/smtk/newt/qtNewtFileBrowserWidget.h b/smtk/newt/qtNewtFileBrowserWidget.h index b1ac789a0fbe900083768ab3fcfaa278455953a1..15e31708d2d92d1e49e504e24ffd9394aefc2390 100644 --- a/smtk/newt/qtNewtFileBrowserWidget.h +++ b/smtk/newt/qtNewtFileBrowserWidget.h @@ -18,11 +18,13 @@ #include <QIcon> #include <QModelIndex> +#include <QSet> #include <QStandardItemModel> #include <QString> #include <QWidget> class QContextMenuEvent; +class QFile; class QNetworkReply; namespace Ui @@ -53,6 +55,7 @@ signals: public slots: void onRequestNavigate(const QString& path); + protected slots: void onItemActivated(const QModelIndex& index); void onListFolderReply(); @@ -63,6 +66,7 @@ protected slots: protected: void contextMenuEvent(QContextMenuEvent* event); + void downloadFile(const QString& name); bool getCommandReply(QNetworkReply* reply, QString& text); void gotoPath(const QString& path); void goUpDirectory(); @@ -79,6 +83,7 @@ protected: QStandardItemModel m_model; qtNewtInterface* m_newt; + QSet<QFile*> m_files; // open file instances for download private: Ui::qtNewtFileBrowserWidget* ui; diff --git a/smtk/newt/qtNewtFileBrowserWidget.ui b/smtk/newt/qtNewtFileBrowserWidget.ui index 5cf86398ce9b83620d179f36d5e376c71ef6c425..06a25720546adcd1f45b3f88eacb6afedc6353cf 100644 --- a/smtk/newt/qtNewtFileBrowserWidget.ui +++ b/smtk/newt/qtNewtFileBrowserWidget.ui @@ -14,6 +14,9 @@ <string>NERSC File System</string> </property> <layout class="QVBoxLayout" name="verticalLayout"> + <property name="bottomMargin"> + <number>0</number> + </property> <item> <layout class="QHBoxLayout" name="horizontalLayout"> <item> @@ -81,19 +84,60 @@ <layout class="QHBoxLayout" name="horizontalLayout_2"> <item> <widget class="QLabel" name="m_statusLabel"> + <property name="minimumSize"> + <size> + <width>0</width> + <height>25</height> + </size> + </property> <property name="text"> <string>Status</string> </property> </widget> </item> + <item> + <spacer name="horizontalSpacer_2"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeType"> + <enum>QSizePolicy::Fixed</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>20</width> + <height>20</height> + </size> + </property> + </spacer> + </item> + <item> + <widget class="QProgressBar" name="m_progressBar"> + <property name="sizePolicy"> + <sizepolicy hsizetype="Expanding" vsizetype="Fixed"> + <horstretch>1</horstretch> + <verstretch>0</verstretch> + </sizepolicy> + </property> + <property name="value"> + <number>24</number> + </property> + <property name="invertedAppearance"> + <bool>false</bool> + </property> + </widget> + </item> <item> <spacer name="horizontalSpacer"> <property name="orientation"> <enum>Qt::Horizontal</enum> </property> + <property name="sizeType"> + <enum>QSizePolicy::Expanding</enum> + </property> <property name="sizeHint" stdset="0"> <size> - <width>40</width> + <width>20</width> <height>20</height> </size> </property> @@ -114,6 +158,9 @@ <property name="text"> <string>UserName</string> </property> + <property name="alignment"> + <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> + </property> </widget> </item> </layout> diff --git a/smtk/newt/qtNewtInterface.cxx b/smtk/newt/qtNewtInterface.cxx index 04b7883a8565a26f6fae014101dd3b98950d631c..ee9a6bc1af1dad5e9a7e74d477e5b10104a80909 100644 --- a/smtk/newt/qtNewtInterface.cxx +++ b/smtk/newt/qtNewtInterface.cxx @@ -10,6 +10,7 @@ #include "smtk/newt/qtNewtInterface.h" #include <QDebug> +#include <QFile> #include <QJsonDocument> #include <QJsonObject> #include <QNetworkAccessManager> @@ -94,38 +95,89 @@ void qtNewtInterface::mockLogin(const QString& username, const QString& newtSess emit this->loginComplete(username, 300); } -void qtNewtInterface::getCommandReply(QNetworkReply* reply, QString& result, QString& error) const +bool qtNewtInterface::parseReply(QNetworkReply* reply, QJsonObject& j, QString& error) const { - result.clear(); + j = QJsonObject(); error.clear(); QByteArray bytes = reply->readAll(); - // qDebug() << "getCommandReply received:" << bytes.constData(); + // qDebug() << "parseReply received:" << bytes.constData(); if (reply->error()) { error = this->getErrorMessage(reply, bytes); - return; + return false; } QJsonDocument jsonDoc = QJsonDocument::fromJson(bytes.constData()); if (!jsonDoc.isObject()) { - error = "Unexpected reply from NEWT - not a json object"; - result = bytes.constData(); - return; + error = "Unexpected reply from NEWT - not a json object: " + QString(bytes.constData()); + return false; } - const QJsonObject& jsonObj = jsonDoc.object(); - QString errorString = jsonObj.value("error").toString(); + j = jsonDoc.object(); + QString errorString = j.value("error").toString(); if (!errorString.isEmpty()) { error = QString("NEWT error: %1").arg(errorString); + return false; + } + + return true; +} + +bool qtNewtInterface::parseReply(QNetworkReply* reply, nlohmann::json& j, std::string& error) const +{ + j.clear(); + error.clear(); + QByteArray bytes = reply->readAll(); + + // qDebug() << "parseReply received:" << bytes.constData(); + if (reply->error()) + { + error = this->getErrorMessage(reply, bytes).toStdString(); + return false; + } + + j = nlohmann::json::parse(bytes.toStdString()); + if (!j.is_object()) + { + error = "Unexpected reply from NEWT - not a json object: " + std::string(bytes.constData()); + return false; + } + + auto iter = j.find("error"); + if (iter != j.end()) + { + std::string errorString = iter->get<std::string>(); + if (!errorString.empty()) + { + error = std::string("NEWT error: ") + errorString; + return false; + } + } + + return true; +} + +void qtNewtInterface::getCommandReply(QNetworkReply* reply, QString& result, QString& error) const +{ + result.clear(); + error.clear(); + + QJsonObject j; + if (!this->parseReply(reply, j, error)) + { + QByteArray bytes = reply->readAll(); result = bytes.constData(); return; } // Command result is in the "output" field - result = jsonObj.value("output").toString(); + if (j.contains("output")) + { + result = j.value("output").toString(); + } } QNetworkReply* qtNewtInterface::requestDirectoryList(const QString& machine, const QString& path) @@ -134,22 +186,67 @@ QNetworkReply* qtNewtInterface::requestDirectoryList(const QString& machine, con return this->sendGetRequest(url); } +QNetworkReply* qtNewtInterface::requestFileDownload( + const QString& machine, + const QString& remotePath) +{ + QString url = QString("%1/file/%2/%3?view=read").arg(NEWT_BASE_URL, machine, remotePath); + QNetworkRequest request(url); + QNetworkReply* reply = m_networkManager->get(request); + QObject::connect(reply, &QNetworkReply::sslErrors, this, &qtNewtInterface::onSslErrors); + return reply; +} + +QNetworkReply* qtNewtInterface::requestFileUpload( + const QString& localFilePath, + const QString& remotePath, + const QString& machine) +{ + QFile* file = new QFile(localFilePath); + if (!file->open(QIODevice::ReadOnly | QIODevice::Text)) + { + qWarning() << "Unable to open" << localFilePath; + delete file; + return nullptr; + } + + QString url = QString("%1/file/%2/%3").arg(NEWT_BASE_URL, machine, remotePath); + QNetworkRequest request(url); + request.setHeader(QNetworkRequest::ContentTypeHeader, "application/x-www-form-urlencoded"); + + QNetworkReply* reply = m_networkManager->post(request, file); + QObject::connect(reply, &QNetworkReply::sslErrors, this, &qtNewtInterface::onSslErrors); + file->setParent(reply); // to keep open until upload completes + return reply; +} + QNetworkReply* qtNewtInterface::requestHomePath(const QString& machine) { QString command("echo $HOME"); return this->sendCommand(command, machine); } -QNetworkReply* qtNewtInterface::requestScratchPath(const QString& machine) +QNetworkReply* qtNewtInterface::requestJobSubmit( + const QString& remoteFilePath, + const QString& machine) { - QString command("echo $SCRATCH"); - return this->sendCommand(command, machine); + QString url = QString("%1/queue/%2").arg(NEWT_BASE_URL, machine); + QNetworkRequest request(url); + request.setHeader(QNetworkRequest::ContentTypeHeader, "application/x-www-form-urlencoded"); + + QUrlQuery params; + params.addQueryItem("jobfile", remoteFilePath.toUtf8()); + + QNetworkReply* reply = m_networkManager->post(request, params.query().toUtf8()); + QObject::connect(reply, &QNetworkReply::sslErrors, this, &qtNewtInterface::onSslErrors); + + return reply; } -QNetworkReply* qtNewtInterface::requestTextFile(const QString& machine, const QString& path) +QNetworkReply* qtNewtInterface::requestScratchPath(const QString& machine) { - QString url = QString("%1/file/%2/%3?view=read").arg(NEWT_BASE_URL, machine, path); - return this->sendGetRequest(url); + QString command("echo $SCRATCH"); + return this->sendCommand(command, machine); } QNetworkReply* qtNewtInterface::sendCommand(const QString& command, const QString& machine) diff --git a/smtk/newt/qtNewtInterface.h b/smtk/newt/qtNewtInterface.h index fc49c1bd014eff7052b0fa859f5885646c11c3fa..38b2b5317f0eda5b8b6d8c94dad46b15c83ba200 100644 --- a/smtk/newt/qtNewtInterface.h +++ b/smtk/newt/qtNewtInterface.h @@ -15,11 +15,17 @@ #include "smtk/newt/Exports.h" #include <QByteArray> +#include <QJsonObject> #include <QList> #include <QObject> #include <QSslError> #include <QString> +#include <nlohmann/json.hpp> + +#include <string> + +class QFile; class QNetworkAccessManager; class QNetworkReply; @@ -47,6 +53,11 @@ public: // Backdoor login **for testing only** void mockLogin(const QString& username, const QString& newtSessionId); + // Parses reply to return output object and error message. + // Returns boolean indicating true if reply is OK + bool parseReply(QNetworkReply* reply, QJsonObject& j, QString& error) const; + bool parseReply(QNetworkReply* reply, nlohmann::json& j, std::string& error) const; + // Parses command reply to return output and/or error message void getCommandReply(QNetworkReply* reply, QString& result, QString& error) const; @@ -56,15 +67,28 @@ public: QNetworkReply* requestDirectoryList(const QString& machine, const QString& path); + // Start file download request. + QNetworkReply* requestFileDownload(const QString& machine, const QString& remotePath); + + // Start file upload. Check reply for status code 200 + // localFilePath is the file to upload + // remotePath is the path to the directory on the remote machine + // Returns nullptr if the local file is not found or cannot be read + QNetworkReply* requestFileUpload( + const QString& localFilePath, + const QString& remotePath, + const QString& machine); + // Start request to get user's $HOME path QNetworkReply* requestHomePath(const QString& machine); + // Start request to submit job to queue. + // remoteFilePath is the path to the slurm script on the target machine. + QNetworkReply* requestJobSubmit(const QString& remoteFilePath, const QString& machine); + // Start requrest to get user's $SCRATCH path QNetworkReply* requestScratchPath(const QString& machine); - // Start requires to download file - QNetworkReply* requestTextFile(const QString& machine, const QString& path); - // Start command request QNetworkReply* sendCommand(const QString& command, const QString& machine); @@ -83,6 +107,7 @@ protected: qtNewtInterface(QObject* parent = nullptr); QString getErrorMessage(QNetworkReply* reply, const QByteArray& bytes) const; + void onReadyRead(QNetworkReply* reply, QFile* file) const; QNetworkReply* sendGetRequest(const QString& url); bool m_isLoggedIn; diff --git a/smtk/simulation/ace3p/CMakeLists.txt b/smtk/simulation/ace3p/CMakeLists.txt index c1a497d1f05e4862a90c267fab54fb962ac92f10..b913bc46500136b05cff40f81ccdbf8167323cd3 100644 --- a/smtk/simulation/ace3p/CMakeLists.txt +++ b/smtk/simulation/ace3p/CMakeLists.txt @@ -36,6 +36,13 @@ foreach (operation ${operations}) list(APPEND ace3p_headers operations/${operation}.h) list(APPEND operation_dependencies ${header_name}) endforeach() + +# Additional dependencies for Export operation +list(APPEND operation_dependencies + "${CMAKE_SOURCE_DIR}/simulation-workflows/internal/ace3p-export.sbt" + "${CMAKE_SOURCE_DIR}/simulation-workflows/internal/ace3p-submit.xml" +) + add_custom_target(ace3p_generated_headers DEPENDS ${operation_dependencies}) add_library(smtkACE3P ${ace3p_sources}) diff --git a/smtk/simulation/ace3p/operations/Export.sbt b/smtk/simulation/ace3p/operations/Export.sbt index 08c7bb7a85ed16d1b9dede2e4e2247ccb4b1b8a8..79789880f8695a86356b208b8ac7df12075e14d8 100644 --- a/smtk/simulation/ace3p/operations/Export.sbt +++ b/smtk/simulation/ace3p/operations/Export.sbt @@ -60,6 +60,7 @@ <String Name="CumulusOutputFolder" Optional="true" IsEnabledByDefault="false" /> <String Name="NerscJobFolder" Optional="true" IsEnabledByDefault="false" /> <String Name="NerscInputFolder" Optional="true" IsEnabledByDefault="false" /> + <String Name="SlurmScript" Optional="true" IsEnabledByDefault="false" /> </ItemDefinitions> </AttDef> </Definitions>