Commit f7d6c928 authored by John Tourtellott's avatar John Tourtellott
Browse files

Update to use newt session id

parent 4ef4824d
<?xml version="1.0"?>
<SMTK_AttributeResource Version="3">
<Definitions>
<AttDef Type="ace3p-export" BaseType="operation" Label="Export to ACE3P" Version="1">
<AttDef Type="ace3p-export" BaseType="operation" Label="Export to ACE3P" Version="2">
<!-- Version 2 adds option for NEWT Session ID -->
<BriefDescription>
Write ACE3P input file for selected program.
</BriefDescription>
......@@ -13,7 +14,7 @@
<Resource Name="smtk::model::Resource" />
</Accepts>
</Resource>
<Resource Name="attributes" Label="Attributes">
<Resource Name="attributes" Label="Attributes" LockType="DoNotLock">
<Accepts>
<Resource Name="smtk::attribute::Resource"/>
</Accepts>
......@@ -159,16 +160,40 @@
<DefaultValue> </DefaultValue>
</String>
<String Name="CumulusHost" Label="Cumulus host" Version="0">
<DefaultValue>http://localhost:8080</DefaultValue>
<!-- DefaultValue>http://localhost:8080</DefaultValue
-->
<DefaultValue>http://app-girder.ace3p.dev-cattle.stable.spin.nersc.org:60002</DefaultValue>
</String>
<String Name="NERSCRepository" Label="Project repository" Version="0" />
<String Name="NERSCAccountName" Label="NERSC account name" Version="0" />
<String Name="NERSCAccountPassword" Label="NERSC account password"
Secure="true" Version="0" />
<String Name="NERSCCredentials" Label="NERSC Credentials" Version="0">
<ChildrenDefinitions>
<String Name="NEWTSessionId" Label="NEWT Session ID" Version="0" />
<String Name="NERSCAccountName" Label="NERSC Account name" Version="0" />
<String Name="NERSCAccountPassword" Label="Password"
Secure="true" Version="0" />
<String Name="NERSCMultfactorToken" Label="Multifactor token" Version="0" />
</ChildrenDefinitions>
<DiscreteInfo DefaultIndex="0">
<Structure>
<Value Enum="NEWT">newt_sessionid</Value>
<Items>
<Item>NEWTSessionId</Item>
</Items>
</Structure>
<Structure>
<Value Enum="Login">login</Value>
<Items>
<Item>NERSCAccountName</Item>
<Item>NERSCAccountPassword</Item>
<Item>NERSCMultfactorToken</Item>
</Items>
</Structure>
</DiscreteInfo>
</String>
<String Name="Machine" Label="NERSC Machine" Version="0">
<DiscreteInfo DefaultIndex="0">
<Value Enum="Cori (Cray XC40)">cori</Value>
<Value Enum="Edison (Cray XC30)">edison</Value>
</DiscreteInfo>
</String>
<Group Name="JobDirectory" Label="Job directory" Version="1" NumberOfRequiredGroups="1" Enabled="true">
......
......@@ -12,7 +12,7 @@
"""
Export functions for submitting jobs to NERSC
"""
print 'Loading nersc'
print('Loading nersc')
import imp
import os
......@@ -22,17 +22,17 @@ import traceback
import shutil
import girder_client
print 'girder_client', girder_client.__version__
print('girder_client', girder_client.__version__)
import requests
import smtk
import smtk.attribute
import cumulusclient
from . import cumulusclient
reload(cumulusclient) # for dev
import newtclient
from . import newtclient
reload(newtclient) # for dev
from cumulusclient import CumulusClient
from newtclient import NewtClient
from .cumulusclient import CumulusClient
from .newtclient import NewtClient
# ---------------------------------------------------------------------
def submit_ace3p(scope, sim_item):
......@@ -65,21 +65,26 @@ def submit_ace3p(scope, sim_item):
check_file(path, 'Cannot find simulation input file at %s')
# Start NERSC session
login_nersc(scope, sim_item)
scope.newt_sessionid = None
sessionid = login_nersc(scope, sim_item)
print('newt_sessionid', sessionid)
if sessionid is None or sessionid == "":
raise RuntimeError('ERROR: Unable to get valid session id')
scope.newt_sessionid = sessionid
# Initialize CumulusClient
scope.cumulus = create_cumulus_client(scope, sim_item)
# Create cluster
machine = get_string(sim_item, 'Machine')
print 'machine', machine
print('machine', machine)
if not machine:
raise Exception('Machine name not specified')
scope.cumulus.create_cluster(machine)
# Create run script
commands = create_slurm_commands(scope, sim_item)
print 'commands', commands
print('commands', commands)
script_name = '-'.join(scope.solver_list)
scope.cumulus.create_slurm_script(script_name, commands)
......@@ -89,12 +94,12 @@ def submit_ace3p(scope, sim_item):
# Submit job
submit_job(scope, sim_item)
print 'Submitted %s job, id %s' % (solver, scope.cumulus.job_id())
print('Submitted %s job, id %s' % (solver, scope.cumulus.job_id()))
except girder_client.HttpError as err:
print 'ERROR', err.responseText
print('ERROR', err.responseText)
raise
except Exception as ex:
print 'Exception', ex
print('Exception', ex)
traceback.print_exc()
raise
finally:
......@@ -108,7 +113,7 @@ def submit_ace3p(scope, sim_item):
def release_resources(scope):
'''
'''
print 'Releasing resources'
print('Releasing resources')
if scope.cumulus:
scope.cumulus.release_resources()
......@@ -117,35 +122,48 @@ def release_resources(scope):
# ---------------------------------------------------------------------
def login_nersc(scope, sim_item):
'''Logs into NERSC and gets session id
'''Returns NEWT session id, logging into NERSC if specified
Note that this method must *also* initialize NewtClient instance
'''
scope.newt_sessionid = None
nersc_url = 'https://newt.nersc.gov/newt'
# Check if sesson id provided
credentials_item = sim_item.find('NERSCCredentials')
credentials_type = credentials_item.value()
if credentials_type == 'newt_sessionid':
session_item = credentials_item.find('NEWTSessionId')
session_id = session_item.value()
scope.nersc = NewtClient(nersc_url, session_id)
return session_id
elif credentials_type != 'login':
raise RuntimeError('ERROR - Unrecognized credentials_type', credentials_type)
# Check user inputs
username = get_string(sim_item, 'NERSCAccountName')
print 'username', username
if not username:
raise Exception('ERROR: NERSC account name not specified')
print('username', username)
# if not username:
# raise Exception('ERROR: NERSC account name not specified')
password = get_string(sim_item, 'NERSCAccountPassword')
print 'password length', len(password)
if not password:
raise Exception('ERROR: NERSC account password not specified')
print('password length', len(password))
# if not password:
# raise Exception('ERROR: NERSC account password not specified')
mfa = get_string(sim_item, 'NERSCMultfactorToken')
nersc_url = 'https://newt.nersc.gov/newt'
scope.nersc = NewtClient(nersc_url)
r = scope.nersc.login(username, password)
scope.newt_sessionid = scope.nersc.get_sessionid()
print 'newt_sessionid', scope.newt_sessionid
r = scope.nersc.login(username, password+mfa)
return scope.nersc.get_sessionid()
# ---------------------------------------------------------------------
def create_cumulus_client(scope, sim_item):
'''Instantiates Cumulus client
'''
item = sim_item.find('CumulusHost')
cumulus_item = smtk.attribute.to_concrete(item)
cumulus_item = sim_item.find('CumulusHost')
# cumulus_item = smtk.attribute.to_concrete(item)
cumulus_host = cumulus_item.value(0)
if not cumulus_host:
raise Exception('ERROR: Cumulus host not specified')
......@@ -188,7 +206,7 @@ def create_slurm_commands(scope, sim_item):
command_list.append(convert_command)
elif '.gen' == ext:
# If model file is .gen, convert to ncdf
print 'Adding command to convert %s to netcdf' % scope.model_file
print('Adding command to convert %s to netcdf' % scope.model_file)
convert_command = \
('srun -n 1 %s/acdtool meshconvert %s') % (binary_folder, scope.model_file)
command_list.append(convert_command)
......@@ -275,7 +293,7 @@ def submit_job(scope, sim_item):
filesystem_item = jobdir_item.find('FileSystem')
if filesystem_item.value() == 'scratch':
scratch_dir = scope.nersc.get_scratch_folder(machine)
print 'Scratch filesystem root directory:', scratch_dir
print('Scratch filesystem root directory:', scratch_dir)
folder_item = filesystem_item.findChild(
'SubFolder', smtk.attribute.ACTIVE_CHILDREN)
folder = folder_item.value()
......@@ -356,18 +374,17 @@ def get_integer(group_item, name):
'''
item = group_item.find(name)
if not item:
print 'WARNING: item \"%s\" not found' % name
print('WARNING: item \"%s\" not found' % name)
return None
if not item.isEnabled():
return None
concrete_item = smtk.attribute.to_concrete(item)
if concrete_item.type() != smtk.attribute.Item.IntType:
print 'WARNING: item \"%s\" not an integer item' % name
if item.type() != smtk.attribute.Item.IntType:
print('WARNING: item \"%s\" not an integer item' % name)
return None
return concrete_item.value(0)
return item.value(0)
# ---------------------------------------------------------------------
def get_string(group_item, name):
......@@ -377,15 +394,14 @@ def get_string(group_item, name):
'''
item = group_item.find(name)
if not item:
print 'WARNING: item \"%s\" not found' % name
print('WARNING: item \"%s\" not found' % name)
return None
if not item.isEnabled():
return None
concrete_item = smtk.attribute.to_concrete(item)
if concrete_item.type() != smtk.attribute.Item.StringType:
print 'WARNING: item \"%s\" not a string item' % name
if item.type() != smtk.attribute.Item.StringType:
print('WARNING: item \"%s\" not a string item' % name)
return None
return concrete_item.value(0)
return item.value(0)
......@@ -9,7 +9,7 @@
# PURPOSE. See the above copyright notice for more information.
#
#=============================================================================
print 'Loading newtclient'
print('Loading newtclient')
import requests
......@@ -24,9 +24,9 @@ class NewtClient():
'''
# ---------------------------------------------------------------------
def __init__(self, base_url):
def __init__(self, base_url, session_id=None):
self._base_url = base_url
self._session_id = None
self._session_id = session_id
# ---------------------------------------------------------------------
def get_authentication_status(self):
......@@ -52,9 +52,9 @@ class NewtClient():
r = requests.post(url, data=credentials)
js = r.json()
if not js.get('auth'):
print 'NERSC url', url
print 'credentials', credentials
print 'NERSC response', r.text
print('NERSC url', url)
print('credentials', credentials)
print('NERSC response', r.text)
raise Exception('Login to NERSC failed')
self._session_id = js.get('newt_sessionid')
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment