Commit b5c5b451 authored by John Tourtellott's avatar John Tourtellott
Browse files

Add SALI client and simple put & query scripts

parent b3d7ce77
# Virtual env file
sali/
sali/*
# Files used to store girder url and apikey
*.key
"""
Script to query girder instance for items corresponding to simulation assets.
SALI = Simulation Asset Location Index
"""
import argparse
import hashlib
import json
import os
import socket
import requests
import girder_client
from girder_client import GirderClient, HttpError
class SALIClient():
""""""
def __init__(self, girder_url, apikey):
""""""
self._girder_client = None
self._girder_url = girder_url
self._sali_folder_id = None
self._session = requests.Session()
# Initialize girder client
url = '%s/api/v1' % self._girder_url
self._girder_client = GirderClient(apiUrl=url)
self._girder_client.authenticate(apiKey=apikey)
# Get user's private folder
user = self._girder_client.get('user/me')
#print('user', user)
user_id = user['_id']
gen = self._girder_client.listFolder(user_id, 'user', name='Private')
private_folder = self._next_item(gen)
if private_folder is None:
raise RuntimeError('Failed to find Private folder for this user')
private_folder_id = private_folder['_id']
# print('private_folder_id', private_folder_id)
# Get SALI folder (create if needed)
gen = self._girder_client.listFolder(private_folder_id, name='SALI')
sali_folder = self._next_item(gen)
if sali_folder is None:
# Create folder now
sali_folder = self._girder_client.createFolder(private_folder_id, 'SALI',
description='Simulation Asset Location Index - first prototype')
print('Created SALI folder, id {}'.format(self._sali_folder['_id']))
self._sali_folder_id = sali_folder['_id']
def query(self, local_location, remote_machine='cori', return_all_metadata=False):
"""Checks for item in the SALI folder with the given model name."""
filename = os.path.basename(local_location)
# print('query name:', filename)
gen = self._girder_client.listItem(self._sali_folder_id, name=filename)
item = self._next_item(gen)
if return_all_metadata:
return item.get('meta')
# (else)
remote_location = item.get('meta').get(remote_machine, {}).get('path')
return remote_location
def put(self, local_location, remote_machine, remote_location, can_replace=False):
"""Adds item to SALI folder."""
# Make sure that input file exists
if not os.path.exists(local_location):
raise RuntimeError('Local file not found at {}'.format(local_location))
filename = os.path.basename(local_location)
# TODO Check for current item
# Get md5 and hostname
md5 = None
with open(local_location, 'rb') as fp:
md5 = hashlib.md5(fp.read()).hexdigest()
if md5 is None:
raise RuntimeError('Error getting model file md5')
print('md5:', md5)
hostname = socket.gethostname()
print('hostname:', hostname)
source = dict(hostname=hostname, path=local_location, md5=md5)
remote = dict(path=remote_location)
metadata = dict(sources=[source])
metadata[remote_machine] = remote
item = self._girder_client.createItem(
self._sali_folder_id, filename, reuseExisting=True, metadata=metadata)
print('createItem returned', item)
def _next_item(self, gen):
"""Returns next item from generator, or None if empty"""
try:
entry = next(gen)
except StopIteration:
return None
return entry
"""
Script to add simulation asset location to SALI.
SALI = Simulation Asset Location Index
"""
import argparse
import json
from client import sali_client
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Add remote asset location to SALI')
parser.add_argument('keyfile', help='json file with girder url and apikey strings')
parser.add_argument('modelfile', help='model file (path)')
parser.add_argument('cori_path', help='path on Cori file system to save')
parser.add_argument('-r', action='store_true', help='can replace existing cori_path')
args = parser.parse_args()
# Read keyfile
with open(args.keyfile) as key_fp:
js = json.load(key_fp)
girder_url = js['url']
apikey = js['apikey']
if girder_url is None:
raise RuntimeError('key file missing girder_url')
if apikey is None:
raise RuntimeError('key file missing apikey')
sali = sali_client.SALIClient(girder_url, apikey)
result = sali.put(args.modelfile, 'cori', args.cori_path, can_replace=args.r)
print('finis')
"""
Script to query girder instance for items corresponding to simulation assets.
SALI = Simulation Asset Location Index
"""
import argparse
import json
import os
from client import sali_client
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Query SALI for remote asset locations')
parser.add_argument('keyfile', help='json file with girder url and apikey strings')
parser.add_argument('modelfile', help='model file')
args = parser.parse_args()
# print(args)
# Read keyfile
with open(args.keyfile) as fp:
js = json.load(fp)
girder_url = js['url']
apikey = js['apikey']
if girder_url is None:
raise RuntimeError('key file missing girder_url')
if apikey is None:
raise RuntimeError('key file missing apikey')
# Make sure model file exists
if not os.path.exists(args.modelfile):
raise RuntimeError('Model file not found at {}'.format(args.modelfile))
sali = sali_client.SALIClient(girder_url, apikey)
# Call query for the input model file
result = sali.query(args.modelfile)
print('query result:', result)
print('finis')
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment