Skip to content
GitLab
Commit b53eb206 authored by John Tourtellott's avatar John Tourtellott
Browse files

Finish updating for python3 

Addresses nuance where __file__ not defined for cmb:master
parent f7d6c928
Hide whitespace changes
Inline Side-by-side
simulation-workflows/ACE3P.py
View file @ b53eb206
......@@ -13,12 +13,15 @@
Export script for ACE3P workflows
"""
import datetime
import importlib
import logging
import os
import sys
from imp import reload
if sys.version_info[0] >= 3:
# Avoids deprecation warning
from importlib import reload
else:
from imp import reload
sys.dont_write_bytecode = True
import smtk
......@@ -28,7 +31,11 @@ import smtk.model
# Add the directory containing this file to the python module search list
import inspect
sys.path.insert(0, os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))))
source_file = os.path.abspath(inspect.getfile(inspect.currentframe()))
sys.path.insert(0, os.path.dirname(source_file))
# Make sure __file__ is set when using modelbuilder
__file__ = source_file
from internal.writers import utils, loading
from internal.writers import tem3pwriter # for testing only
......@@ -99,11 +106,12 @@ def ExportCMB(export_op):
# Get model resource
ref_item = scope.export_att.find('model')
scope.model_resource = ref_item.objectValue(0)
if scope.model_resource is None:
resource = ref_item.value(0)
if resource is None:
msg = 'ERROR - No model'
print(msg)
raise RuntimeError(msg)
scope.model_resource = smtk.model.Resource.CastTo(resource)
# Get input model (to be uploaded if submitting)
model_item = scope.export_att.findFile('MeshFile')
......
simulation-workflows/internal/__init__.py
View file @ b53eb206
import writers
from imp import reload
from . import writers
reload(writers)
simulation-workflows/internal/writers/basewriter.py
View file @ b53eb206
'''Common base class for output writers
'''
from imp import reload
import os
import cardformat
from . import cardformat
reload(cardformat)
from .cardformat import CardFormat
......
simulation-workflows/internal/writers/cardformat.py
View file @ b53eb206
......@@ -26,8 +26,8 @@ class CardFormat:
'''
item = att.itemAtPath(self.item_path, '/')
if item is None:
print 'ERROR: item not found for attribute %s path %s' % \
(att.name(), self.item_path)
print('ERROR: item not found for attribute %s path %s' % \
(att.name(), self.item_path))
return False
return self.write_item(scope, item, indent)
......@@ -90,7 +90,7 @@ class CardFormat:
'''
# Sanity check
if not hasattr(item, 'expression'):
print 'Invalid item type - does not support expressions', item.type()
print('Invalid item type - does not support expressions', item.type())
return False
exp_att = item.expression(0)
......
simulation-workflows/internal/writers/cumulusclient.py
View file @ b53eb206
......@@ -9,7 +9,7 @@
# PURPOSE. See the above copyright notice for more information.
#
#=============================================================================
print 'Loading cumulusclient'
print('Loading cumulusclient')
import json
import os
......@@ -75,7 +75,7 @@ class CumulusClient():
except Exception as ex:
# But just in case
self._private_folder_id = r[0]['_id']
print 'private_folder_id', self._private_folder_id
print('private_folder_id', self._private_folder_id)
# ---------------------------------------------------------------------
def job_id(self):
......@@ -111,7 +111,7 @@ class CumulusClient():
r = self._client.post('clusters', data=json.dumps(body))
self._cluster_id = r['_id']
print 'cluster_id', self._cluster_id
print('cluster_id', self._cluster_id)
# Reset the state of the cluster
body = {
......@@ -130,7 +130,7 @@ class CumulusClient():
break
elif r['status'] == 'error':
r = self._client.get('clusters/%s/log' % self._cluster_id)
print r
print(r)
raise Exception('ERROR creating cluster')
if sleeps > 9:
......@@ -147,7 +147,7 @@ class CumulusClient():
}
r = self._client.post('scripts', data=json.dumps(body))
self._script_id = r['_id']
print 'script_id', self._script_id
print('script_id', self._script_id)
# ---------------------------------------------------------------------
def create_job(self, job_name, tail=None):
......@@ -156,7 +156,7 @@ class CumulusClient():
# Create job folders
folder_name = uuid.uuid4().hex # unique name
self._job_folder_id = self.get_folder(self._private_folder_id, folder_name)
print 'Created job folder', folder_name
print('Created job folder', folder_name)
self._input_folder_id = self.get_folder(self._job_folder_id, 'input_files')
self._output_folder_id = self.get_folder(self._job_folder_id, 'output_files')
# Make sure job_name isn't null
......@@ -187,7 +187,7 @@ class CumulusClient():
job = self._client.post('jobs', data=json.dumps(body))
self._job_id = job['_id']
print 'Created job_id', self._job_id
print('Created job_id', self._job_id)
# ---------------------------------------------------------------------
def upload_inputs(self, files_to_upload, folders_to_upload=[]):
......@@ -248,10 +248,10 @@ class CumulusClient():
body['jobOutputDir'] = job_output_dir
print 'submit_job body:', body
print('submit_job body:', body)
url = 'clusters/%s/job/%s/submit' % (self._cluster_id, self._job_id)
self._client.put(url, data=json.dumps(body))
print 'Submitted job', self._job_id
print('Submitted job', self._job_id)
# ---------------------------------------------------------------------
def set_job_metadata(self, meta):
......@@ -297,12 +297,12 @@ class CumulusClient():
output = r['content']
if output and log_offset == 0:
print # end the user feedback dots
print() # end the user feedback dots
log_offset += len(output)
for l in output:
print l
print(l)
sys.stdout.flush()
......@@ -320,7 +320,7 @@ class CumulusClient():
self._client.downloadFolderRecursive(
self._output_folder_id, destination_folder)
print 'Downloaded files to %s' % destination_folder
print('Downloaded files to %s' % destination_folder)
# ---------------------------------------------------------------------
def release_resources(self):
......@@ -333,7 +333,7 @@ class CumulusClient():
'scripts': [self._script_id],
'folder': [self._job_folder]
}
for resource_type, id_list in resource_info.items():
for resource_type, id_list in list(resource_info.items()):
for resource_id in id_list:
if resource_id is not None:
url = '%s/%s' % (resource_type, resource_id)
......@@ -360,6 +360,6 @@ class CumulusClient():
r = self._client.createFolder(parent_id, name)
return r['_id']
except HttpError as e:
print e.responseText
print(e.responseText)
return None
simulation-workflows/internal/writers/loading.py
View file @ b53eb206
"""
"""
from imp import reload
from . import utils
import cardformat
from . import cardformat
reload(cardformat)
from .cardformat import CardFormat
......@@ -15,7 +16,7 @@ def write_loading(scope):
return
att_list.sort(key=lambda att:att.name())
print 'Write dipole and port mode excitation'
print('Write dipole and port mode excitation')
for att in att_list:
scope.output.write('\n')
scope.output.write('Loading:\n')
......
simulation-workflows/internal/writers/nersc.py
View file @ b53eb206
......@@ -14,7 +14,7 @@ Export functions for submitting jobs to NERSC
"""
print('Loading nersc')
import imp
from imp import reload
import os
import sys
import time
......@@ -65,12 +65,7 @@ def submit_ace3p(scope, sim_item):
check_file(path, 'Cannot find simulation input file at %s')
# Start NERSC session
scope.newt_sessionid = None
sessionid = login_nersc(scope, sim_item)
print('newt_sessionid', sessionid)
if sessionid is None or sessionid == "":
raise RuntimeError('ERROR: Unable to get valid session id')
scope.newt_sessionid = sessionid
login_nersc(scope, sim_item)
# Initialize CumulusClient
scope.cumulus = create_cumulus_client(scope, sim_item)
......@@ -122,48 +117,35 @@ def release_resources(scope):
# ---------------------------------------------------------------------
def login_nersc(scope, sim_item):
'''Returns NEWT session id, logging into NERSC if specified
'''Logs into NERSC and gets session id
Note that this method must *also* initialize NewtClient instance
'''
nersc_url = 'https://newt.nersc.gov/newt'
# Check if sesson id provided
credentials_item = sim_item.find('NERSCCredentials')
credentials_type = credentials_item.value()
if credentials_type == 'newt_sessionid':
session_item = credentials_item.find('NEWTSessionId')
session_id = session_item.value()
scope.nersc = NewtClient(nersc_url, session_id)
return session_id
elif credentials_type != 'login':
raise RuntimeError('ERROR - Unrecognized credentials_type', credentials_type)
scope.newt_sessionid = None
# Check user inputs
username = get_string(sim_item, 'NERSCAccountName')
print('username', username)
# if not username:
# raise Exception('ERROR: NERSC account name not specified')
if not username:
raise Exception('ERROR: NERSC account name not specified')
password = get_string(sim_item, 'NERSCAccountPassword')
print('password length', len(password))
# if not password:
# raise Exception('ERROR: NERSC account password not specified')
mfa = get_string(sim_item, 'NERSCMultfactorToken')
if not password:
raise Exception('ERROR: NERSC account password not specified')
nersc_url = 'https://newt.nersc.gov/newt'
scope.nersc = NewtClient(nersc_url)
r = scope.nersc.login(username, password+mfa)
return scope.nersc.get_sessionid()
r = scope.nersc.login(username, password)
scope.newt_sessionid = scope.nersc.get_sessionid()
print('newt_sessionid', scope.newt_sessionid)
# ---------------------------------------------------------------------
def create_cumulus_client(scope, sim_item):
'''Instantiates Cumulus client
'''
cumulus_item = sim_item.find('CumulusHost')
# cumulus_item = smtk.attribute.to_concrete(item)
item = sim_item.find('CumulusHost')
cumulus_item = smtk.attribute.to_concrete(item)
cumulus_host = cumulus_item.value(0)
if not cumulus_host:
raise Exception('ERROR: Cumulus host not specified')
......@@ -380,11 +362,12 @@ def get_integer(group_item, name):
if not item.isEnabled():
return None
if item.type() != smtk.attribute.Item.IntType:
concrete_item = smtk.attribute.to_concrete(item)
if concrete_item.type() != smtk.attribute.Item.IntType:
print('WARNING: item \"%s\" not an integer item' % name)
return None
return item.value(0)
return concrete_item.value(0)
# ---------------------------------------------------------------------
def get_string(group_item, name):
......@@ -400,8 +383,9 @@ def get_string(group_item, name):
if not item.isEnabled():
return None
if item.type() != smtk.attribute.Item.StringType:
concrete_item = smtk.attribute.to_concrete(item)
if concrete_item.type() != smtk.attribute.Item.StringType:
print('WARNING: item \"%s\" not a string item' % name)
return None
return item.value(0)
return concrete_item.value(0)
simulation-workflows/internal/writers/newtclient.py
View file @ b53eb206
......@@ -24,9 +24,9 @@ class NewtClient():
'''
# ---------------------------------------------------------------------
def __init__(self, base_url, session_id=None):
def __init__(self, base_url):
self._base_url = base_url
self._session_id = session_id
self._session_id = None
# ---------------------------------------------------------------------
def get_authentication_status(self):
......
simulation-workflows/internal/writers/tem3pwriter.py
View file @ b53eb206
'''Writer for TEM3P input files
'''
import csv
from imp import reload
import logging
import os
import string
......@@ -10,7 +11,7 @@ import smtk
from . import basewriter, cardformat
reload(basewriter)
reload(cardformat)
from cardformat import CardFormat
from .cardformat import CardFormat
from . import utils
class Tem3PWriter(basewriter.BaseWriter):
......@@ -59,7 +60,7 @@ class Tem3PWriter(basewriter.BaseWriter):
# Write thermal section
categories = scope.sim_atts.analysisCategories('TEM3P Thermal Nonlinear')
scope.categories = list(categories)
print 'Using categories: %s' % scope.categories
print('Using categories: %s' % scope.categories)
self.solver = 'tem3p-thermal-nonlinear'
self.name_prefix = 'Thermal'
self.write_command()
......@@ -132,7 +133,7 @@ class Tem3PWriter(basewriter.BaseWriter):
'''Write the meshdump section
'''
if self.solver in ['tem3p-eigen', 'tem3p-elastic']:
print 'Writing MeshDump section'
print('Writing MeshDump section')
meshdump_att = self.get_attribute('MeshDump')
self.start_command('MeshDump', indent=' ')
self.write_standard_items(meshdump_att, skip_list=['Source'], indent=' ')
......@@ -159,7 +160,7 @@ class Tem3PWriter(basewriter.BaseWriter):
'''Writes the solver specification(s)
'''
print 'Writing solver'
print('Writing solver')
if self.solver == 'tem3p-eigen':
self.write_standard_instance_att('EigenSolver')
else:
......@@ -183,7 +184,7 @@ class Tem3PWriter(basewriter.BaseWriter):
def write_elastic_material(self):
''''''
print 'Writing elastic materials'
print('Writing elastic materials')
att_list = self.scope.sim_atts.findAttributes('TEM3PElasticMaterial')
if not att_list:
return
......@@ -193,8 +194,8 @@ class Tem3PWriter(basewriter.BaseWriter):
# Separate command for each model entity (requried?)
ent_idlist = utils.get_entity_ids(self.scope, att.associations())
if not ent_idlist:
print 'Warning: attribute %s not associated to any model entities' % \
att.name()
print('Warning: attribute %s not associated to any model entities' % \
att.name())
continue
type_item = att.findString('Material')
......@@ -224,19 +225,19 @@ class Tem3PWriter(basewriter.BaseWriter):
'''Writes ThermalConductivity
'''
print 'Writing thermal materials'
print('Writing thermal materials')
att_list = self.scope.sim_atts.findAttributes('TEM3PThermalMaterial')
if not att_list:
return
att_list.sort(key=lambda att:att.name())
print att_list
print(att_list)
for att in att_list:
# Separate command for each model entity (requried?)
ent_idlist = utils.get_entity_ids(self.scope, att.associations())
if not ent_idlist:
print 'Warning: attribute %s not associated to any model entities' % \
att.name()
print('Warning: attribute %s not associated to any model entities' % \
att.name())
continue
material_tuple = self.parse_thermal_conductivity(
......@@ -253,7 +254,7 @@ class Tem3PWriter(basewriter.BaseWriter):
def write_boundary_condition(self):
''''''
print 'Writing boundary conditions'
print('Writing boundary conditions')
if self.is_elastic_solver():
att_type = 'TEM3PMechanicalBC'
elif self.is_thermal_solver():
......@@ -278,8 +279,8 @@ class Tem3PWriter(basewriter.BaseWriter):
# Separate command for each model entity (requried?)
ent_idlist = utils.get_entity_ids(self.scope, att.associations())
if not ent_idlist:
print 'Warning: attribute %s not associated to any model entities' % \
att.name()
print('Warning: attribute %s not associated to any model entities' % \
att.name())
continue
for ent_id in ent_idlist:
......@@ -359,7 +360,7 @@ class Tem3PWriter(basewriter.BaseWriter):
relative_path = os.path.join(source_dir, os.pardir, 'non-linear-material', surface_res_value)
path = os.path.abspath(relative_path)
if not os.path.exists(path):
print 'Material file not found at:', path
print('Material file not found at:', path)
raise Exception('Unabled to find standard materials file %s' % surface_res_value)
self.scope.output.write(' SurfaceResistance: %s\n' % surface_res_value)
self.scope.files_to_upload.add(path)
......@@ -384,7 +385,7 @@ class Tem3PWriter(basewriter.BaseWriter):
Uses brute force to assign material numbers
'''
print 'Writing shells'
print('Writing shells')
att_list = self.scope.sim_atts.findAttributes('ThermalShell')
if not att_list:
return
......@@ -399,13 +400,13 @@ class Tem3PWriter(basewriter.BaseWriter):
# Separate command for each model entity (requried?)
ent_idlist = utils.get_entity_ids(self.scope, att.associations())
if not ent_idlist:
print 'Warning: attribute %s not associated to any model entities' % \
att.name()
print('Warning: attribute %s not associated to any model entities' % \
att.name())
continue
material_tuple = self.parse_thermal_conductivity(
att, 'ConstantThermalConductivity', 'GeneralThermalConductivity')
print 'material_tuple:', material_tuple
print('material_tuple:', material_tuple)
material_id = material_dict.get(material_tuple)
if material_id is None:
material_id = new_material_id
......@@ -422,8 +423,8 @@ class Tem3PWriter(basewriter.BaseWriter):
self.finish_command()
# Traverse material_dict and write out "material" sections
for material_tuple in sorted(material_dict.iterkeys()):
print 'mat_tuple', material_tuple
for material_tuple in sorted(material_dict.keys()):
print('mat_tuple', material_tuple)
mat_type, mat_value = material_tuple
material_id = material_dict.get(material_tuple)
mat_value_string = mat_value if mat_type == 'Function' else '%g' % mat_value
......@@ -435,7 +436,7 @@ class Tem3PWriter(basewriter.BaseWriter):
def write_heat_source(self):
''''''
print 'Writing heat sources'
print('Writing heat sources')
att_list = self.scope.sim_atts.findAttributes('HeatSource')
if not att_list:
return
......@@ -445,8 +446,8 @@ class Tem3PWriter(basewriter.BaseWriter):
# Separate command for each model entity (requried?)
ent_idlist = utils.get_entity_ids(self.scope, att.associations())
if not ent_idlist:
print 'Warning: attribute %s not associated to any model entities' % \
att.name()
print('Warning: attribute %s not associated to any model entities' % \
att.name())
continue
for ent_id in ent_idlist:
......@@ -482,7 +483,7 @@ class Tem3PWriter(basewriter.BaseWriter):
def write_harmonic_analysis(self):
print 'Writing harmonic analysis'
print('Writing harmonic analysis')
self.write_standard_instance_att('HarmonicAnalysis', silent_list=['Frequency'])
......@@ -531,7 +532,7 @@ class Tem3PWriter(basewriter.BaseWriter):
relative_path = os.path.join(source_dir, os.pardir, 'non-linear-material', tc_choice)
path = os.path.abspath(relative_path)
if not os.path.exists(path):
print 'Material file not found at:', path
print('Material file not found at:', path)
raise Exception('Unabled to find standard materials file %s' % tc_choice)
return('Function', tc_choice)
......@@ -544,7 +545,7 @@ class Tem3PWriter(basewriter.BaseWriter):
'''
if self.scope.symlink is None:
print 'setting scope.symlink: %s' % remote_path
print('setting scope.symlink: %s' % remote_path)
self.scope.symlink = remote_path
return True
......
simulation-workflows/internal/writers/track3pwriter.py
View file @ b53eb206
'''Writer for Track3P input files
'''
from imp import reload
import os
import smtk
......@@ -9,7 +10,7 @@ from smtk import attribute
from . import basewriter, cardformat
reload(basewriter)
reload(cardformat)
from cardformat import CardFormat
from .cardformat import CardFormat
from . import utils
class Track3PWriter(basewriter.BaseWriter):
......@@ -164,15 +165,15 @@ class Track3PWriter(basewriter.BaseWriter):
'''
att = self.get_attribute(att_type)
if not att:
print 'Did not find attribute type %s' % att_type
print('Did not find attribute type %s' % att_type)
return
if item_path is None:
item_path = att_type
item = att.itemAtPath(item_path)
if not item:
print 'Did not find item at path \"%s\"" under attribute %s' % \
(att_type, item_path)
print('Did not find item at path \"%s\"" under attribute %s' % \
(att_type, item_path))
return
if item.type() == smtk.attribute.Item.VoidType:
......@@ -240,7 +241,7 @@ class Track3PWriter(basewriter.BaseWriter):
selector_att = self.get_attribute('Track3PModelSelectorAtt')
selector_item = selector_att.findString('Track3PModelSelectorItem')
track3p_sim = selector_item.value(0)
print ('%s simulation' % track3p_sim)
print('%s simulation' % track3p_sim)
att_list = self.scope.sim_atts.findAttributes('Track3PMaterial')
......@@ -282,7 +283,7 @@ class Track3PWriter(basewriter.BaseWriter):
for d,keyword in [(volume_dict,'SolidBlockId'), (surface_dict,'BoundarySurfaceID')]:
# Convert id list to string and build new dict of <string, att>
sorted_dict = dict()
for att, idlist in d.iteritems():
for att, idlist in d.items():
idlist.sort()
string_list = [str(i) for i in idlist]
id_string = ', '.join(string_list)
......@@ -290,7 +291,7 @@ class Track3PWriter(basewriter.BaseWriter):
#print '{}'.format(sorted_dict)
# Traverse sorted_dict and write materials
for id_string, att in sorted_dict.iteritems():
for id_string, att in sorted_dict.items():
if 'Primary' in att.type():
self.start_command('Material')
self.scope.output.write(' Type: Primary\n')
......@@ -403,7 +404,7 @@ class Track3PWriter(basewriter.BaseWriter):
boundary_item = sey_item.find('BoundarySurface')
id_list = utils.get_entity_ids(self.scope, boundary_item)
if not id_list:
print 'No BoundarySurface entity'
print('No BoundarySurface entity')
break
id_string = ','.join(str(id) for id in id_list)
self.scope.output.write(' BoundarySurfaceID%d: %s\n' % \
......