Skip to content

GitLab

Commit ae2b9c12 authored by Andrea Lorenzon's avatar Andrea Lorenzon 💬
Browse files

refactoring and linting

parent cb68ab5a
Hide whitespace changes
Inline Side-by-side
app.py
View file @ ae2b9c12
import sys
import os
sys.path.insert(0,'./modules')
sys.path.insert(0, './modules')
if not os.path.exists("./data"):
try:
......@@ -13,31 +13,31 @@ if not os.path.exists("./data"):
except Exception as e:
print("Could not find nor create data directory: ", e)
from flask import Flask
import settings
from h5ceric import extractDatabase, plot_dispatcher
from h5decorators import nocache
from modules.h5ceric import extract_dataset, plot_dispatcher
from modules.h5decorators import nocache
deploymentType = settings.deploy
app = Flask(__name__)
########################## helper function ##################
def splitURL(path):
def split_url(path):
"""helper function to split a flask path
This helper function accept as input a string, in the format "<fileName>,dataset=<datasetName>",
This helper function accept as input a string, in the format "<file_name>,dataset=<dataset_name>",
and split it to return the two variables in a tuple
:param path: the path accessed in flask
:return: a tuple (fileName, datasetName) with
:return: a tuple (file_name, dataset_name) with
"""
pathArray = path.split(",dataset=")
path_array = path.split(",dataset=")
fileName = pathArray[0] # path of the file folder
datasetName = pathArray[-1] # filename
return fileName, datasetName
file_name = path_array[0] # path of the file folder
dataset_name = path_array[-1] # filename
return file_name, dataset_name
############################# homepage ###########################
......@@ -51,41 +51,44 @@ def hello():
:return: it renders the template templates/index.html
"""
from jsTreeCeric import JSON_from_folder
folderDictionary = JSON_from_folder(settings.fileFolder[settings.deploy])
from modules.jsTreeCeric import JSON_from_folder
folder_dictionary = JSON_from_folder(settings.fileFolder[settings.deploy])
import json
folderTree = json.dumps(folderDictionary)
folder_tree = json.dumps(folder_dictionary)
h5Tree = json.dumps([])
h5_tree = json.dumps([])
from flask import render_template
return render_template("index.html", treeJsonFromFlask=folderTree, h5Tree=h5Tree, investigation_id="0")
return render_template("index.html", treeJsonFromFlask=folder_tree, h5Tree=h5_tree, investigation_id="0")
############################# VUO homepage ###########################
@app.route('/VUO')
@nocache
def helloVUO():
def hello_vuo():
""" main flask web page renderer for VUO
:return: it renders the template templates/index.html
"""
import requests, VUO, json
import requests
import json
from modules.VUO import vuo_api, get_session_cookie
from flask import request, render_template
apiserver = VUO.VUOAPI('userfoldertree')
api_server = vuo_api('userfoldertree')
payload = {"investigation_id": str(request.args['investigation_id']),
"sessionCookie": str(VUO.getSessionCookie())}
r = requests.get(apiserver, params=payload)
"sessionCookie": str(get_session_cookie())}
r = requests.get(api_server, params=payload)
r.encoding = "ascii"
print ("r:"+str(r.text))
print("r:" + str(r.text))
mes = json.loads(r.text)['message']
print(str(mes))
ascii = json.dumps(mes)
h5Tree = json.dumps([])
message = json.dumps(mes)
h5_tree = json.dumps([])
return render_template("index.html",
treeJsonFromFlask=ascii,
h5Tree=h5Tree,
treeJsonFromFlask=message,
h5Tree=h5_tree,
investigation_id=request.args['investigation_id'])
......@@ -99,9 +102,9 @@ def tunnel(route):
"""
import requests
from flask import request, jsonify
from VUO import VUOAPI
URL = VUOAPI(route) # compose VUO API target
r = requests.get(URL, params=request.args)
from modules.VUO import vuo_api
url = vuo_api(route) # compose VUO API target
r = requests.get(url, params=request.args)
return jsonify(r.json())
......@@ -117,78 +120,81 @@ def openhdf5vuo(path):
:param path: a URL route path
:return: a JSON, formatted as jsTree input, that list the content of the hdf5 file located in <path>
"""
import requests, VUO, json
import requests
import json
from modules.VUO import vuo_api, get_session_cookie
from flask import request
apiserver = VUO.VUOAPI('hdf5tree')
from settings import debug
apiserver = vuo_api('hdf5tree')
print("vuo/openhdf5 request args"+str(request.args))
invID = str(request.args.get('investigation_id',""))
cookie = str(VUO.getSessionCookie())
file =str(path)
debug and print("vuo/openhdf5 request args" + str(request.args))
inv_id = str(request.args.get('investigation_id', ""))
cookie = str(get_session_cookie())
file = str(path)
payload = {"investigation_id": invID,
payload = {"investigation_id": inv_id,
"sessionCookie": cookie,
"file":file}
print("apiserver ="+apiserver)
print("payload = "+str(payload))
print("sending a request to the api")
"file": file}
debug and print("apiserver =" + apiserver)
debug and print("payload = " + str(payload))
debug and print("sending a request to the api")
r = requests.get(apiserver, params=payload)
r.encoding = "ascii"
print("\n\nl'api ha risposto: "+str(r.json()))
debug and print("\n\nl'api ha risposto: " + str(r.json()))
converted = convert(r.json())
print("\n\n converted: "+str(converted))
ascii = json.dumps(convert(r.json()))
return ascii
debug and print("\n\n converted: " + str(converted))
message = json.dumps(convert(r.json()))
return message
######################### VUO bokeh visualization ######################
@app.route('/VUO/visualize/')
@nocache
def send_jsVUO():
def send_js_vuo():
"""plots a hdf5 dataset
:return: plot the dataset with Bokeh, and renders the template template.html with the plot
"""
import requests, VUO, json
import requests
import json
import numpy
from modules.VUO import get_session_cookie, vuo_api
from flask import request, render_template
invID = request.args.get("investigation_id", None)
from modules.h5ceric import DatabaseNotFound
cookie = str(VUO.getSessionCookie())
inv_id = request.args.get("investigation_id", None)
cookie = str(get_session_cookie())
filename = request.args.get('filename', None)
ds_name = request.args.get('dataset', None)
z = request.args.get('z', None)
apiserver = VUO.VUOAPI('file_info')
apiserver = vuo_api('file_info')
payload = {"file": filename,
"sessionCookie":cookie,
"investigation_id":invID}
"sessionCookie": cookie,
"investigation_id": inv_id}
info = requests.get(apiserver,params=payload)
info = requests.get(apiserver, params=payload)
fullpath = json.loads(info.text)['full_path']
print("visualize file info dict=" + str(json.loads(info.text)))
# TO DO check it's an hdf5
from h5ceric import DatabaseNotFound
# get dataset as np.array
try:
arr = extractDatabase("", fullpath, ds_name)
arr = extract_dataset("", fullpath, ds_name)
except OSError:
return render_template("missing_file.html")
except DatabaseNotFound:
return render_template("missing_dataset.html")
# create the plot
import numpy
# dispatch the plot creation
if isinstance(arr, numpy.ndarray):
p = plot_dispatcher(arr, ds_name, z)
......@@ -204,39 +210,39 @@ def send_jsVUO():
# embed in html
return render_template('template.html', script=script, div=div)
######################### get file info from VUO ######################
@app.route('/VUO/file_info')
@nocache
def VUOfile_info():
def vuofile_info():
"""
return mimetype and URL of a file
:param path: file path
:return: a JSON with "mime_type" and "url" fields.
"""
from flask import request
import requests, VUO, json
import requests
import json
from modules.VUO import vuo_api, get_session_cookie
from settings import debug
file = request.args.get('file', '')
print("file -> "+file)
debug and print("file -> " + file)
apiserver = VUO.VUOAPI('file_info')
apiserver = vuo_api('file_info')
payload = {'file': file,
'sessionCookie': VUO.getSessionCookie(),
'sessionCookie': get_session_cookie(),
'investigation_id': request.args.get('investigation_id', '')}
# print("payload", payload)
r = requests.get(apiserver, params=payload)
r.encoding = "ascii"
# print("r is",r.text)
converted = convert(r.json())
print("\n\n converted: " + str(converted))
ascii = json.dumps(convert(r.json()))
debug and print("\n\n converted: " + str(converted))
message = json.dumps(convert(r.json()))
return message
return ascii
######################### read file content ######################
@app.route('/get_file/<path:path>')
......@@ -275,7 +281,7 @@ def display_report():
import h5py
from flask import request
report = request.args.get('report', None)
reportData = json.loads(report)
report_data = json.loads(report)
html = """<!DOCTYPE html>\n
<html lang="en">\n
<head>\n
......@@ -287,14 +293,14 @@ def display_report():
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css" integrity="sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh" crossorigin="anonymous">\n
"""
for filename, content in reportData.items():
for filename, content in report_data.items():
html += '<h4>Filename:{}</h4>'.format(filename)
for item in content:
# get type of item["dataset"] in item["filename"]
if filename.split(".")[-1] in ["nxs", "h5", "hdf5", "nexus"]:
with h5py.File(settings.fileFolder[deploymentType]+"/"+filename,'r') as f:
with h5py.File(settings.fileFolder[deploymentType] + "/" + filename, 'r') as f:
# dispatch plotting javascript
if len(f[item['dataset']].shape) == 0:
......@@ -305,47 +311,51 @@ def display_report():
html += '</ul>\n'
# if it's a 2D dataset...
elif len(f[item['dataset']].shape)==2:
elif len(f[item['dataset']].shape) == 2:
html += '<ul>\n'
html += '<li><strong>2D Dataset:</strong> '+str(item["dataset"])+'</li>\n'
html += '<li><strong>Image:</strong><br>\n<img src="'+ str(item["image"]).replace(" ", "+") + '"></li>\n'
html += '<li><strong>2D Dataset:</strong> ' + str(item["dataset"]) + '</li>\n'
html += '<li><strong>Image:</strong><br>\n<img src="' + str(item["image"]).replace(" ",
"+") + '"></li>\n'
html += '</ul>\n'
# if it's a 3D dataset, get the current slice.
elif len(f[item['dataset']].shape)==3:
elif len(f[item['dataset']].shape) == 3:
html += '<ul>\n'
html += '<li><strong>3D Dataset:</strong> '+str(item["dataset"])+'</li>\n'
html += '<li><strong>Z:</strong> '+str(item["z"])+'</li>\n'
html += '<li><strong>Image:</strong><br>\n<img src="'+str(item["image"]).replace(" ", "+") + '"></li>\n'
html += '<li><strong>3D Dataset:</strong> ' + str(item["dataset"]) + '</li>\n'
html += '<li><strong>Z:</strong> ' + str(item["z"]) + '</li>\n'
html += '<li><strong>Image:</strong><br>\n<img src="' + str(item["image"]).replace(" ",
"+") + '"></li>\n'
html += '</ul>\n'
else:
displayCode = ''
if "image" in item["mime"]:
displayCode = '<img class="img-responsive" style="height: 75vh; max-width: 100%;" src="' + item["url"]+'" >'
elif "video" in item["mime"]:
displayCode = '<video class="img-responsive" style="height: 75vh; max-width: 100%;" video-element" controls><source type="'+item['mime']+'" src="'+item['url']+'"></video>'
elif "text" in item["mime"]:
displayCode = '<iframe src="'+item["url"]+'" style="height: 75vh; max-width: 100%;">'
html += '<ul>\n'
html += displayCode
html += '</ul>\n'
display_code = ''
if "image" in item["mime"]:
display_code = '<img class="img-responsive" style="height: 75vh; max-width: 100%;" src="' + item[
"url"] + '" >'
elif "video" in item["mime"]:
display_code = '<video class="img-responsive" style="height: 75vh; max-width: 100%;" video-element" controls><source type="' + \
item['mime'] + '" src="' + item['url'] + '"></video>'
elif "text" in item["mime"]:
display_code = '<iframe src="' + item["url"] + '" style="height: 75vh; max-width: 100%;">'
html += '<ul>\n'
html += display_code
html += '</ul>\n'
html += '</body>\n</html>\n'
# embed in html
return html
def convert(input):
if isinstance(input, dict):
return {convert(key): convert(value) for key, value in input.items()}
elif isinstance(input, list):
return [convert(element) for element in input]
elif isinstance(input, str):
return str(input)
def convert(whatever):
if isinstance(whatever, dict):
return {convert(key): convert(value) for key, value in whatever.items()}
elif isinstance(whatever, list):
return [convert(element) for element in whatever]
elif isinstance(whatever, str):
return str(whatever)
else:
return input
return whatever
# With debug=True, Flask server will auto-reload
......
modules/VUO.py
View file @ ae2b9c12
vuoApiServer = "users-nuvola-dev.elettra.eu:5000"
def getSessionCookie():
def get_session_cookie():
"""
get session from cookie, or use settings.sessionCookie hardcoded one if not in browser
:return: the cookie
......@@ -21,19 +21,19 @@ def getSessionCookie():
return settings.sessionCookie
def VUOAPI(route):
def vuo_api(route):
"""
compose the URL of a complete VUO API local tunnel call
compose the url of a complete VUO API local tunnel call
:param str route: the required route
:return: str host:port/VUOAPI/route
"""
vuoServerTemplate = "http://{}/{}"
URL = vuoServerTemplate.format(vuoApiServer, route)
print(URL)
return URL
vuo_server_template = "http://{}/{}"
url = vuo_server_template.format(vuoApiServer, route)
print(url)
return url
def makePayloadFromDict(d):
def make_payload_from_dict(d):
"""
converts a dictionary in a string "key=value&key=value..."
:param d: a dict
......
modules/h5ceric.py
View file @ ae2b9c12
......@@ -112,7 +112,7 @@ class DatabaseNotFound(Exception):
pass
def extractDatabase(folder, filename, required_dataset):
def extract_dataset(folder, filename, required_dataset):
"""Retrieve numpy arrays from hdf5 datasets.
:param folder: use settings.folder, it's the absolute path of the hdf5 folder
......@@ -121,14 +121,15 @@ def extractDatabase(folder, filename, required_dataset):
:return: numpy.array with the required data
"""
import os
fullpath = os.path.join(folder, filename)
from settings import debug
import numpy as np
fullpath = os.path.join(folder, filename)
with h5py.File(fullpath, "r") as f:
# print("opened file "+fullpath)
for datasetname, dataset in h5py_dataset_iterator(f):
# print("checking if "+datasetname+" matches required dataset")
if required_dataset == datasetname:
debug and print("opened file "+fullpath)
for dataset_name, dataset in h5py_dataset_iterator(f):
debug and print("checking if "+dataset_name+" matches required dataset")
if required_dataset == dataset_name:
return np.array(dataset)
raise OSError
......@@ -148,32 +149,31 @@ def plot_dispatcher(arr, ds_name, pos=0):
# plot single data
if len(arr.shape) == 0:
from hdf5BokehPlots import plotSingleValues
return plotSingleValues(arr, ds_name)
from hdf5BokehPlots import plot_single_values
return plot_single_values(arr, ds_name)
### plot 1D ###
if len(arr.shape) == 1:
from hdf5BokehPlots import plot1dBokeh
return plot1dBokeh(arr, ds_name)
from hdf5BokehPlots import plot_1d_bokeh
return plot_1d_bokeh(arr, ds_name)
### plot 2D ###
if len(arr.shape) == 2:
from math import isclose
if isclose(arr.shape[0], arr.shape[1], rel_tol=0.3):
from hdf5BokehPlots import plot2dBokeh
return plot2dBokeh(arr, ds_name)
from hdf5BokehPlots import plot_2d_bokeh
return plot_2d_bokeh(arr, ds_name)
else:
from hdf5BokehPlots import plot2dSeries
from hdf5BokehPlots import plot_2d_series
import numpy as np
import settings
if arr.shape[0] > settings.maxPointInHexPlot:
arr = arr[np.random.choice(arr.shape[0], settings.maxPointInHexPlot, replace=False), :]
return plot2dSeries(arr[:, 0], arr[:, 1], ds_name)
return plot_2d_series(arr[:, 0], arr[:, 1], ds_name)
### plot 3+D ###
# slice array if order > 3: this needs to be fixed for higher dimensions
if len(arr.shape) >= 3:
from hdf5BokehPlots import plot3dBokeh
return plot3dBokeh(arr, ds_name, pos)
from hdf5BokehPlots import plot_3d_bokeh
return plot_3d_bokeh(arr, ds_name, pos)
modules/h5decorators.py
View file @ ae2b9c12
from datetime import datetime
from functools import update_wrapper, wraps
from flask import make_response
......
modules/hdf5BokehPlots.py
View file @ ae2b9c12
def plotSingleValues(arr, ds_name):
def plot_single_values(arr, ds_name):
"""
create a canvas with a single value (will be showed in plot title).
......@@ -20,7 +20,7 @@ def plotSingleValues(arr, ds_name):
return {"plot": p}
def plot1dBokeh(arr, ds_name):
def plot_1d_bokeh(arr, ds_name):
""" creates a bokeh plot from a 1D time series
creates a bokeh plot from a 1D time series
......@@ -29,14 +29,13 @@ def plot1dBokeh(arr, ds_name):
:param ds_name: string, plot title
:return: a bokeh figure
"""
import numpy as np
from bokeh.plotting import figure
from bokeh.models import DataRange1d
import numpy # linspace
from bokeh.models import HoverTool
size_x = arr.shape[0]
x = numpy.linspace(0, len(arr), size_x)
x = np.linspace(0, len(arr), size_x)
x_range = DataRange1d(range_padding=0, range_padding_units='percent')
# generate and return bokeh plot
......@@ -47,11 +46,9 @@ def plot1dBokeh(arr, ds_name):
sizing_mode='stretch_both'
)
p.grid.grid_line_width = 0.2
p.line(x, arr, color="navy", alpha=0.5)
# hovertools section
from bokeh.models import HoverTool
# Format the tooltip
tooltips = [
......@@ -65,7 +62,7 @@ def plot1dBokeh(arr, ds_name):
return {"plot": p}
def plot2dBokeh(arr, ds_name):
def plot_2d_bokeh(arr, ds_name):
""" creates a bokeh plot from a 2D figure
creates a bokeh plot from a 2D figure
......@@ -91,7 +88,6 @@ def plot2dBokeh(arr, ds_name):
)
p.grid.grid_line_width = 0.2
p.image(image=[arr], x=0, y=0, dw=size_x, dh=size_y,
palette="Viridis256",
level="image"
......@@ -99,7 +95,7 @@ def plot2dBokeh(arr, ds_name):
return {"plot": p}
def plot2dSeries(X,Y, ds_name):
def plot_2d_series(X, Y, ds_name):
""" creates a bokeh plot from a 2D figure
creates a bokeh plot from a 2D figure
......@@ -108,55 +104,6 @@ def plot2dSeries(X,Y, ds_name):
:param ds_name: string, plot title
:return: a bokeh figure
"""
# from bokeh.plotting import figure, gridplot
# from bokeh.models import DataRange1d
# from bokeh.models import HoverTool
#
# size_x, size_y = X.shape, Y.shape
#
# x_range = DataRange1d(range_padding=0, range_padding_units='percent')
# y_range = DataRange1d(range_padding=0, range_padding_units='percent')
#
# # generate and return bokeh plot
# p = figure(title=ds_name,
# toolbar_location="above",
# x_range=x_range, y_range=y_range,
# match_aspect=True,
# sizing_mode='stretch_both'
# )
#
# p.grid.grid_line_width = 0.2
#
# p = figure(title="Hexbin for max 5K points", match_aspect=True,
# tools="pan,wheel_zoom,box_zoom,reset,", background_fill_color='#440154')
# p.grid.visible = False
# import numpy as np
#
# nHexs = 30
# hexsize = max(np.amax(X)-np.amin(X), np.amax(Y)-np.amin(Y))/nHexs
#
# r, bins = p.hexbin(X, Y, size=hexsize, hover_color="pink", hover_alpha=0.8)
#
# p.circle(X, Y, color="white", size=1, alpha=0.2)
#
# p.add_tools(HoverTool(
# tooltips=[("count", "@c"), ("(q,r)", "(@q, @r)")],
# mode="mouse", point_policy="follow_mouse", renderers=[r]
# ))
#
# h1 = figure(title="hist1", match_aspect=True,
# tools="pan,wheel_zoom,box_zoom,reset", background_fill_color='#440154',
# x_range=x_range, y_range=y_range, plot_width=150)
#
#
# histY,edgesY= np.histogram(Y, density=True, bins=50)
# h1.grid.visible = False
# h1.quad(top=histY, bottom=0, left=edgesY[:-1], right=edgesY[1:],
# fill_color="#036564", line_color="#033649",\