Commit 12a8ff7a authored by Andrea Lorenzon's avatar Andrea Lorenzon 💬
Browse files

refactoring and linting

parent 12034fae
Pipeline #1570 failed with stages
in 24 seconds
import sys
sys.path.insert(0, './modules')
import os
from typing import *
from flask import Flask
import settings
from h5ceric import extractDatabase, plot_dispatcher
from h5decorators import nocache
from modules.h5ceric import extract_dataset, plot_dispatcher
from modules.h5decorators import nocache
import flask
if not os.path.exists("./data"):
......@@ -19,28 +20,26 @@ if not os.path.exists("./data"):
except Exception as e:
print("Could not find nor create data directory: ", e)
deploymentType = settings.deploy
app = Flask(__name__)
########################## helper function ##################
def splitURL(path: str) -> Tuple[str, str]:
def split_url(path: str) -> Tuple[str, str]:
"""helper function to split a flask path
This helper function accept as input a string, in the format "<fileName>,dataset=<datasetName>",
This helper function accept as input a string, in the format "<file_name>,dataset=<dataset_name>",
and split it to return the two variables in a tuple
:param path: the path accessed in flask
:return: a tuple (fileName, datasetName) with
:return: a tuple (file_name, dataset_name) with
"""
pathArray = path.split(",dataset=")
path_array = path.split(",dataset=")
fileName = pathArray[0] # path of the file folder
datasetName = pathArray[-1] # filename
return fileName, datasetName
file_name = path_array[0] # path of the file folder
dataset_name = path_array[-1] # filename
return file_name, dataset_name
############################# homepage ###########################
......@@ -56,15 +55,15 @@ def hello() -> Any:
print("start folder: ", settings.fileFolder[deploymentType])
print("deployment type: ", settings.deploy)
from jsTreeCeric import JSON_from_folder
folderDictionary = JSON_from_folder(settings.fileFolder[deploymentType])
# print("folder dict : ", folderDictionary)
from modules.jsTreeCeric import JSON_from_folder
folder_dictionary = JSON_from_folder(settings.fileFolder[deploymentType])
# print("folder dict : ", folder_dictionary)
import json
folderTree = json.dumps(folderDictionary)
folder_tree = json.dumps(folder_dictionary)
h5Tree = json.dumps([])
h5_tree = json.dumps([])
from flask import render_template
return render_template("index.html", treeJsonFromFlask=folderTree, h5Tree=h5Tree)
return render_template("index.html", treeJsonFromFlask=folder_tree, h5Tree=h5_tree)
########################### hdf5 tree ############################
......@@ -79,11 +78,11 @@ def openhdf5(path: str) -> str:
:param path: a URL route path
:return: a JSON, formatted as jsTree input, that list the content of the hdf5 file located in <path>
"""
from h5ceric import list_datasets
from modules.h5ceric import list_datasets
import json
h5Tree = json.dumps(list_datasets(settings.fileFolder[deploymentType] + path))
return h5Tree
h5_tree = json.dumps(list_datasets(settings.fileFolder[deploymentType] + path))
return h5_tree
######################### bokeh visualization ######################
......@@ -95,7 +94,7 @@ def send_js() -> Any:
:return: plot the dataset with Bokeh, and renders the template template.html with the plot
"""
from flask import render_template, request
from h5ceric import DatabaseNotFound
from modules.h5ceric import DatabaseNotFound
# get dataset as np.array
......@@ -104,7 +103,7 @@ def send_js() -> Any:
z = request.args.get('z', None)
try:
arr = extractDatabase(settings.fileFolder[deploymentType], filename, ds_name)
arr = extract_dataset(settings.fileFolder[deploymentType], filename, ds_name)
except OSError:
return render_template("missing_file.html")
except DatabaseNotFound:
......@@ -189,7 +188,7 @@ def display_report() -> str:
import h5py
from flask import request
report = request.args.get('report', None)
reportData = json.loads(report)
report_data = json.loads(report)
html = """<!DOCTYPE html>\n
<html lang="en">\n
<head>\n
......@@ -201,7 +200,7 @@ def display_report() -> str:
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css" integrity="sha384-Vkoo8x4CGsO3+Hhxv8T/Q5PaXtkKtu6ug5TOeNV6gBiFeWPGFN9MuhOf23Q9Ifjh" crossorigin="anonymous">\n
"""
for filename, content in reportData.items():
for filename, content in report_data.items():
html += '<h4>Filename:{}</h4>'.format(filename)
for item in content:
......@@ -236,17 +235,17 @@ def display_report() -> str:
html += '</ul>\n'
else:
displayCode = ''
display_code = ''
if "image" in item["mime"]:
displayCode = '<img class="img-responsive" style="height: 75vh; max-width: 100%;" src="' + item[
display_code = '<img class="img-responsive" style="height: 75vh; max-width: 100%;" src="' + item[
"url"] + '" >'
elif "video" in item["mime"]:
displayCode = '<video class="img-responsive" style="height: 75vh; max-width: 100%;" video-element" controls><source type="' + \
display_code = '<video class="img-responsive" style="height: 75vh; max-width: 100%;" video-element" controls><source type="' + \
item['mime'] + '" src="' + item['url'] + '"></video>'
elif "text" in item["mime"]:
displayCode = '<iframe src="' + item["url"] + '" style="height: 75vh; max-width: 100%;">'
display_code = '<iframe src="' + item["url"] + '" style="height: 75vh; max-width: 100%;">'
html += '<ul>\n'
html += displayCode
html += display_code
html += '</ul>\n'
......
__name__ = "h5ceric"
import collections
from typing import Dict, Any, Union
import h5py
import numpy
############################ helper function: hdf5 generator from iterator ############################
def traverse_datasets(hdf_file: h5py.File):
""" Generator for hdf5 folder traversal
......@@ -20,7 +19,6 @@ def traverse_datasets(hdf_file: h5py.File):
yield path
############################ debug tools: hdf5 dataset printer ############################
def list_datasets(name: str, prefix: str = '') -> Dict:
"""List all datasets in an hdf5 file
......@@ -40,20 +38,20 @@ def list_datasets(name: str, prefix: str = '') -> Dict:
:param h5file: hdf5 file
:returns output: a dict to be converted in JSON for JSTree
"""
partialTree = []
partial_tree = []
for keys in list(h5file.keys()):
if isinstance(h5file[keys], str):
partialTree.append(h5file[keys])
partial_tree.append(h5file[keys])
if isinstance(h5file[keys], h5py.Dataset):
partialTree.append({'text': h5file[keys].name,
'icon': settings.datasetIcon})
partial_tree.append({'text': h5file[keys].name,
'icon': settings.datasetIcon})
elif isinstance(h5file[keys], h5py.Group):
partialTree.append({'text': h5file[keys].name,
'children': iterate_hdf5(h5file[keys]),
'icon': settings.hdf5Icon})
return partialTree
partial_tree.append({'text': h5file[keys].name,
'children': iterate_hdf5(h5file[keys]),
'icon': settings.hdf5Icon})
return partial_tree
if isinstance(name, str):
try:
......@@ -67,8 +65,6 @@ def list_datasets(name: str, prefix: str = '') -> Dict:
return output
############################ debug tools: hdf5 dataset string repr ############################
def dict_datasets(name: str) -> None:
"""Prints hdf5 datasets.
......@@ -84,9 +80,7 @@ def dict_datasets(name: str) -> None:
print("{: >20}{: >20}{: >20}".format(*text))
############################ helper: hdf5 iterator ############################
def h5py_dataset_iterator(file: Union[h5py.File,h5py.Group], prefix: object = '') -> object:
def h5py_dataset_iterator(file: Union[h5py.File, h5py.Group], prefix: object = '') -> collections.Iterable:
"""Hdf5 dataset iterator
Recursively iterates a dataset, returns an iterator over all datasets
......@@ -105,15 +99,12 @@ def h5py_dataset_iterator(file: Union[h5py.File,h5py.Group], prefix: object = ''
yield from h5py_dataset_iterator(items, path)
############################ dataset extraction ############################
class DatabaseNotFound(Exception):
"""generic exception for trying to access a non-existent dataset in a hfd5 file"""
pass
def extractDatabase(folder: str, filename: str, required_dataset: str):
def extract_dataset(folder: str, filename: str, required_dataset: str):
"""Retrieve numpy arrays from hdf5 datasets.
:param folder: use settings.folder, it's the absolute path of the hdf5 folder
......@@ -126,15 +117,13 @@ def extractDatabase(folder: str, filename: str, required_dataset: str):
with h5py.File(fullpath, "r") as f:
# print("opened file "+fullpath)
for datasetname, dataset in h5py_dataset_iterator(f):
# print("checking if "+datasetname+" matches required dataset")
if required_dataset == datasetname:
for dataset_name, dataset in h5py_dataset_iterator(f):
# print("checking if "+dataset_name+" matches required dataset")
if required_dataset == dataset_name:
return np.array(dataset)
raise OSError
########################### bokeh plot creation ############################
def plot_dispatcher(arr: numpy.ndarray, ds_name: str, pos: int = 0):
"""Creates a bokeh plot from a numpy 1D or 2D array
......@@ -148,32 +137,32 @@ def plot_dispatcher(arr: numpy.ndarray, ds_name: str, pos: int = 0):
# plot single data
if len(arr.shape) == 0:
from hdf5BokehPlots import plotSingleValues
return plotSingleValues(arr, ds_name)
from hdf5BokehPlots import plot_single_values
return plot_single_values(arr, ds_name)
### plot 1D ###
# plot 1D #
if len(arr.shape) == 1:
from hdf5BokehPlots import plot1dBokeh
return plot1dBokeh(arr, ds_name)
from hdf5BokehPlots import plot_1d_bokeh
return plot_1d_bokeh(arr, ds_name)
### plot 2D ###
# plot 2D #
if len(arr.shape) == 2:
from math import isclose
if isclose(arr.shape[0], arr.shape[1], rel_tol=0.3):
from hdf5BokehPlots import plot2dBokeh
return plot2dBokeh(arr, ds_name)
from hdf5BokehPlots import plot_2d_bokeh
return plot_2d_bokeh(arr, ds_name)
else:
from hdf5BokehPlots import plot2dSeries
from hdf5BokehPlots import plot_2d_series
import numpy as np
import settings
if arr.shape[0] > settings.maxPointInHexPlot:
arr = arr[np.random.choice(arr.shape[0], settings.maxPointInHexPlot, replace=False), :]
return plot2dSeries(arr[:, 0], arr[:, 1], ds_name)
return plot_2d_series(arr[:, 0], arr[:, 1], ds_name)
### plot 3+D ###
# plot 3+D #
# slice array if order > 3: this needs to be fixed for higher dimensions
if len(arr.shape) >= 3:
from hdf5BokehPlots import plot3dBokeh
return plot3dBokeh(arr, ds_name, pos)
from hdf5BokehPlots import plot_3d_bokeh
return plot_3d_bokeh(arr, ds_name, pos)
import numpy
def plotSingleValues(arr: numpy.ndarray, ds_name: str) -> dict:
def plot_single_values(arr: numpy.ndarray, ds_name: str) -> dict:
"""
create a canvas with a single value (will be showed in plot title).
......@@ -23,7 +23,7 @@ def plotSingleValues(arr: numpy.ndarray, ds_name: str) -> dict:
return {"plot": p}
def plot1dBokeh(arr: numpy.ndarray, ds_name: str) -> dict:
def plot_1d_bokeh(arr: numpy.ndarray, ds_name: str) -> dict:
""" creates a bokeh plot from a 1D time series
creates a bokeh plot from a 1D time series
......@@ -68,7 +68,7 @@ def plot1dBokeh(arr: numpy.ndarray, ds_name: str) -> dict:
return {"plot": p}
def plot2dBokeh(arr: numpy.ndarray, ds_name: str) -> dict:
def plot_2d_bokeh(arr: numpy.ndarray, ds_name: str) -> dict:
""" creates a bokeh plot from a 2D figure
creates a bokeh plot from a 2D figure
......@@ -103,79 +103,27 @@ def plot2dBokeh(arr: numpy.ndarray, ds_name: str) -> dict:
return {"plot": p}
def plot2dSeries(X: numpy.ndarray, Y: numpy.ndarray, ds_name: str) -> dict:
def plot_2d_series(x: numpy.ndarray, y: numpy.ndarray, ds_name='') -> dict:
""" creates a bokeh plot from a 2D figure
:param Y: values of the series
:param X: x axis of the serie (time?)
:param arr: 2D numpy ndarray
:param ds_name: string, plot title
:param y: values of the series
:param x: x axis of the serie (time?)
:return: a bokeh figure
"""
# from bokeh.plotting import figure, gridplot
# from bokeh.models import DataRange1d
# from bokeh.models import HoverTool
#
# size_x, size_y = X.shape, Y.shape
#
# x_range = DataRange1d(range_padding=0, range_padding_units='percent')
# y_range = DataRange1d(range_padding=0, range_padding_units='percent')
#
# # generate and return bokeh plot
# p = figure(title=ds_name,
# toolbar_location="above",
# x_range=x_range, y_range=y_range,
# match_aspect=True,
# sizing_mode='stretch_both'
# )
#
# p.grid.grid_line_width = 0.2
#
# p = figure(title="Hexbin for max 5K points", match_aspect=True,
# tools="pan,wheel_zoom,box_zoom,reset,", background_fill_color='#440154')
# p.grid.visible = False
# import numpy as np
#
# nHexs = 30
# hexsize = max(np.amax(X)-np.amin(X), np.amax(Y)-np.amin(Y))/nHexs
#
# r, bins = p.hexbin(X, Y, size=hexsize, hover_color="pink", hover_alpha=0.8)
#
# p.circle(X, Y, color="white", size=1, alpha=0.2)
#
# p.add_tools(HoverTool(
# tooltips=[("count", "@c"), ("(q,r)", "(@q, @r)")],
# mode="mouse", point_policy="follow_mouse", renderers=[r]
# ))
#
# h1 = figure(title="hist1", match_aspect=True,
# tools="pan,wheel_zoom,box_zoom,reset", background_fill_color='#440154',
# x_range=x_range, y_range=y_range, plot_width=150)
#
#
# histY,edgesY= np.histogram(Y, density=True, bins=50)
# h1.grid.visible = False
# h1.quad(top=histY, bottom=0, left=edgesY[:-1], right=edgesY[1:],
# fill_color="#036564", line_color="#033649",\
# )
# q= gridplot(children=([[p,h1],[None,None]]))
#
#
import holoviews as hv
hv.extension('bokeh')
import numpy as np
arr = np.stack((X, Y), axis=0).T
arr = np.stack((x, y), axis=0).T
points = hv.Points(arr)
points = points.hist(num_bins=51, dimension=['x', 'y'])
# print(type(points.right))
points.main.opts(width=500, height=500, size=0.8)
# print(dir(points.top))
return {"plot": hv.render(points, backend='bokeh')}
def plot3dBokeh(arr: numpy.ndarray, ds_name:str, y: int) -> dict:
def plot_3d_bokeh(arr: numpy.ndarray, ds_name: str, y: int) -> dict:
""" creates a bokeh plot from a 3D matrix
creates a bokeh plot from a 3D matrix
......
import os
# deploy type ["local", "VUO", "K8S"]
deploy= "local"
deploy = "local"
# paths
fileFolderPath = {"local" : os.getenv('H5NUVOLA_PATH', "/home/andrealorenzon/projects/h5nuvola2/"),
"VUO" : "/",
"K8S" : os.getenv('H5NUVOLA_PATH', "./"),
"rafec" : os.getenv('H5NUVOLA_PATH', "/")}
fileFolderPath = {"local": os.getenv('H5NUVOLA_PATH', "/home/andrealorenzon/projects/h5nuvola2/"),
"VUO": "/",
"K8S": os.getenv('H5NUVOLA_PATH', "./"),
"rafec": os.getenv('H5NUVOLA_PATH', "/")}
fileFolderName = {"local" : "hdf5_test_files",
"VUO" : "",
"K8S" : "data",
"rafec" : "/"}
fileFolderName = {"local": "hdf5_test_files",
"VUO": "",
"K8S": "data",
"rafec": "/"}
fileFolder = {}
for k in fileFolderPath.keys():
fileFolder[k] = fileFolderPath[k] + fileFolderName[k] + "/"
# server settings
host = {"local" : "localhost",
"VUO" : "users-nuvola-dev.elettra.eu",
"K8S" : "0.0.0.0",
"rafec" : "localhost"}
host = {"local": "localhost",
"VUO": "users-nuvola-dev.elettra.eu",
"K8S": "0.0.0.0",
"rafec": "localhost"}
port = 5001
debug = True
......@@ -36,4 +37,4 @@ h5nuvolaLogoIcon = "static/logo/h5nuvola_logo_91_80px_30dpi.png"
plotPalette_1D = "Viridis256"
plotPalette_3D = "Viridis256"
maxPointInHexPlot = 10000 # max n. of point for 2d series hexplot (beamPlot)
maxPointInHexPlot = 10000 # max n. of point for 2d series hexplot (beamPlot)
......@@ -22,7 +22,6 @@
<!-- FontAwesome -->
<link rel="stylesheet" href="static/libs/fontawesome-free-5.13.0-web/css/fontawesome.min.css">
<link rel="stylesheet" href="static/libs/fontawesome-free-5.13.0-web/css/all.css">
<link rel="stylesheet" href="static/libs/fontawesome-free-5.13.0-web/css/all.css">
<!-- Bokeh -->
<script src="https://cdn.bokeh.org/bokeh/release/bokeh-2.2.1.min.js"></script>
<!-- Bokeh widgets (slider) -->
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment