diff --git a/requirements.txt b/requirements.txt
index 9604d1e..9a12d84 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,18 +1,87 @@
-astrodbkit>=2.2
-astropy>=5.2.0,<6
-bokeh==3.0.2
-flask>=3.0,<4
-flask-cors>=4.0,<5
-flask-wtf>=1.2,<2
-markdown2>=2.4,<3
-multiprocess>=0.70,<1
-numpy>=1.24,<2
-pandas>=2.0,<2.1
-pysqlite3>=0.5,<1
-pytest>=8.0,<9
-requests>=2.31,<3
-specutils>=1.12,<2
-sqlalchemy>=2.0,<3
-tqdm>=4.66,<5
-werkzeug>=3.0,<4
-wtforms>=3.1,<4
+antiorm==1.2.1
+asdf==4.1.0
+asdf-astropy==0.7.0
+asdf_coordinates_schemas==0.3.0
+asdf_standard==1.1.1
+asdf_transform_schemas==0.5.0
+asdf_wcs_schemas==0.4.0
+astrodbkit==2.2
+astropy==5.3.4
+astroquery==0.4.9.post1
+attrs==25.1.0
+backports.tarfile==1.2.0
+beautifulsoup4==4.13.3
+blinker==1.9.0
+bokeh==3.7.3
+certifi==2025.1.31
+cffi==1.17.1
+charset-normalizer==3.4.1
+click==8.1.8
+contourpy==1.3.1
+cryptography==44.0.0
+db==0.1.1
+db-sqlite3==0.0.1
+dill==0.3.9
+distlib==0.3.9
+distro==1.9.0
+filelock==3.16.1
+Flask==3.1.0
+Flask-Cors==4.0.2
+Flask-WTF==1.2.2
+greenlet==3.1.1
+gwcs==0.21.0
+html5lib==1.1
+idna==3.10
+importlib_metadata==8.6.1
+iniconfig==2.0.0
+iniparse==0.5
+itsdangerous==2.2.0
+jaraco.classes==3.4.0
+jaraco.context==6.0.1
+jaraco.functools==4.1.0
+jeepney==0.8.0
+Jinja2==3.1.5
+jmespath==1.0.1
+keyring==25.6.0
+markdown2==2.5.3
+MarkupSafe==3.0.2
+more-itertools==10.6.0
+multiprocess==0.70.17
+narwhals==1.41.0
+ndcube==2.3.0
+numpy==1.26.4
+packaging==24.2
+pandas==2.0.3
+pathlib2==2.3.7.post1
+pillow==11.1.0
+platformdirs==4.3.6
+pluggy==1.5.0
+pycparser==2.22
+pyerfa==2.0.1.5
+PyMySQL==1.1.1
+pyOpenSSL==24.3.0
+pysqlite3==0.5.4
+pytest==8.3.4
+python-dateutil==2.9.0.post0
+pytz==2024.2
+pyvo==1.6
+PyYAML==6.0.2
+requests==2.32.3
+scipy==1.15.1
+SecretStorage==3.3.3
+semantic-version==2.10.0
+six==1.17.0
+soupsieve==2.6
+specutils==1.19.0
+SQLAlchemy==2.0.38
+tornado==6.4.2
+tqdm==4.67.1
+typing_extensions==4.12.2
+tzdata==2025.1
+urllib3==2.3.0
+virtualenv==20.28.0
+webencodings==0.5.1
+Werkzeug==3.1.3
+WTForms==3.2.1
+xyzservices==2025.1.0
+zipp==3.21.0
diff --git a/simple_app/app_simple.py b/simple_app/app_simple.py
index 7b041d4..8f336aa 100644
--- a/simple_app/app_simple.py
+++ b/simple_app/app_simple.py
@@ -45,8 +45,8 @@ def search():
if (query := form.search.data) is None: # content in main searchbar
query = ''
- curdoc().template_variables['query'] = query # add query to bokeh curdoc
- curdoc().template_variables['ref_query'] = ref_query # add query to bokeh curdoc
+ session['query'] = query # add query to bokeh curdoc
+ session['ref_query'] = ref_query # add query to bokeh curdoc
db = SimpleDB(db_file) # open database
# object search function
@@ -96,7 +96,7 @@ def coordinate_query():
if (query := form.query.data) is None:
query = ''
- curdoc().template_variables['query'] = query
+ session['query'] = query
db = SimpleDB(db_file)
# parse query into ra, dec, radius
@@ -127,7 +127,7 @@ def full_text_search():
query = ''
limmaxrows = True
- curdoc().template_variables['query'] = query
+ session['query'] = query
db = SimpleDB(db_file)
# search through the tables using the given query
@@ -160,14 +160,14 @@ def raw_query():
if (query := form.sqlfield.data) is None:
query = ''
- curdoc().template_variables['query'] = query
+ session['query'] = query
# attempt to query the database
try:
results: Optional[pd.DataFrame] = db.sql_query(query, fmt='pandas')
# catch any broken queries (should not activate as will be caught by validation)
- except (ResourceClosedError, OperationalError, IndexError, SqliteWarning, BadSQLError):
+ except (ResourceClosedError, OperationalError, IndexError, SqliteWarning, BadSQLError, ProgrammingError):
results = pd.DataFrame()
results = reference_handle(results, db_file, True)
@@ -188,7 +188,7 @@ def solo_result(query: str):
query: str
The query -- full match to a main ID
"""
- curdoc().template_variables['query'] = query
+ session['query'] = query
db = SimpleDB(db_file)
# search database for given object
@@ -199,6 +199,7 @@ def solo_result(query: str):
except KeyError:
abort(404, f'"{query}" does match any result in SIMPLE!')
return
+
everything = Inventory(resultdict, db_file)
# create camd and spectra plots
@@ -283,7 +284,7 @@ def create_file_for_download(key: str):
key: str
The dataframe string
"""
- query = curdoc().template_variables['query']
+ query: str = session.get('query')
db = SimpleDB(db_file)
# search for a given object and a given key
@@ -305,7 +306,7 @@ def create_files_for_solo_download():
"""
Creates and downloads all dataframes from solo results
"""
- query = curdoc().template_variables['query']
+ query = session.get('query')
db = SimpleDB(db_file)
# search for a given object
@@ -328,7 +329,7 @@ def create_spectra_files_for_download():
"""
Downloads the spectra files and zips together
"""
- query = curdoc().template_variables['query']
+ query = session.get('query')
db = SimpleDB(db_file)
# search for a given object and specifically its spectra
@@ -346,13 +347,35 @@ def create_spectra_files_for_download():
abort(400, 'Could not download fits')
+@app_simple.route('/write_multi_spectra', methods=['GET'])
+def create_multi_spectra_files_for_download():
+ """
+ Downloads the spectra files and zips together
+ """
+ query = session.get('query')
+ db = SimpleDB(db_file)
+
+ # search for the spectra within a given free search
+ resultdict: Dict[str, pd.DataFrame] = db.search_string(query, fmt='pandas', verbose=False)
+ spectra_df: pd.DataFrame = resultdict['Spectra']
+
+ # write all spectra for object to zipped file
+ zipped = write_spec_files(spectra_df.access_url.values)
+ if zipped is not None:
+ response = Response(zipped, mimetype='application/zip')
+ response = control_response(response, app_type='zip')
+ return response
+
+ abort(400, 'Could not download fits')
+
+
@app_simple.route('/write_filt', methods=['GET'])
def create_file_for_filtered_download():
"""
Creates and downloads the shown dataframe when in filtered search
"""
- query = curdoc().template_variables['query']
- refquery = curdoc().template_variables['ref_query']
+ query = session.get('query')
+ refquery = session.get('ref_query')
db = SimpleDB(db_file)
# search for a given object and a full text search at the same time
@@ -375,7 +398,7 @@ def create_file_for_coordinate_download():
"""
Creates and downloads the shown dataframe when in coordinate search
"""
- query = curdoc().template_variables['query']
+ query = session.get('query')
db = SimpleDB(db_file)
# query the database for given coordinates and parse those coordinates
@@ -401,12 +424,12 @@ def create_file_for_full_download(key: str):
key: str
The dataframe string
"""
- query = curdoc().template_variables['query']
+ query = session.get('query')
db = SimpleDB(db_file)
# search database with a free-text search and specific table
resultdict: Dict[str, pd.DataFrame] = db.search_string(query, fmt='pandas', verbose=False)
-
+ print(query, key, resultdict.keys())
# write to csv
if key in resultdict:
results: pd.DataFrame = resultdict[key]
@@ -422,7 +445,7 @@ def create_files_for_multi_download():
"""
Creates and downloads all dataframes from full results
"""
- query = curdoc().template_variables['query']
+ query = session.get('query')
db = SimpleDB(db_file)
# search with full-text search
@@ -439,7 +462,7 @@ def create_file_for_sql_download():
"""
Creates and downloads the shown dataframe when in sql query
"""
- query = curdoc().template_variables['query']
+ query = session.get('query')
db = SimpleDB(db_file)
# query database via sql
diff --git a/simple_app/plots.py b/simple_app/plots.py
index 4416458..50aff99 100644
--- a/simple_app/plots.py
+++ b/simple_app/plots.py
@@ -333,14 +333,14 @@ def sky_plot() -> figure:
_p_sky = figure(title='Sky Plot', outer_height=500,
active_scroll='wheel_zoom', active_drag='box_zoom',
tools='pan,wheel_zoom,box_zoom,box_select,reset',
- sizing_mode='stretch_width', x_range=[-180, 180], y_range=[-90, 90])
+ sizing_mode='stretch_width', x_range=Range1d(-180, 180), y_range=Range1d(-90, 90))
# background for skyplot
_p_sky.ellipse(x=0, y=0, width=360, height=180, color='#444444', name='background')
# scatter plot for sky plot
- circle = _p_sky.circle(source=full_cds, x='ra_projected', y='dec_projected',
- size=6, name='circle', color='ghostwhite')
+ circle = _p_sky.scatter(marker='circle', source=full_cds, x='ra_projected', y='dec_projected',
+ size=6, name='circle', color='ghostwhite')
# bokeh tools for sky plot
this_hover = HoverTool(renderers=[circle, ], tooltips=tooltips)
@@ -367,7 +367,8 @@ def colour_colour_plot() -> Tuple[figure, Toggle, Toggle, Select, Select]:
_p_colour_colour = bokeh_formatter(_p_colour_colour)
# scatter plot for colour-colour
- full_plot = _p_colour_colour.circle(x=x_full_name, y=y_full_name, source=full_cds, size=6, color=cmap)
+ full_plot = _p_colour_colour.scatter(marker='circle', x=x_full_name, y=y_full_name, source=full_cds, size=6,
+ color=cmap)
# colour bar for colour-colour plot
cbar = ColorBar(color_mapper=cmap['transform'], label_standoff=12,
@@ -395,6 +396,8 @@ def colour_colour_plot() -> Tuple[figure, Toggle, Toggle, Select, Select]:
args={'full_plot': full_plot, 'full_data': full_cds.data,
'y_button': _button_y_flip, 'y_axis': _p_colour_colour.yaxis[0],
'y_range': _p_colour_colour.y_range}))
+ _p_colour_colour.js_on_event('reset', CustomJS(args=dict(dropdown_x=_dropdown_x, dropdown_y=_dropdown_y),
+ code=js_callbacks.reset_dropdown))
return _p_colour_colour, _button_x_flip, _button_y_flip, _dropdown_x, _dropdown_y
def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select, Select]:
@@ -413,7 +416,8 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select,
_p_camd = bokeh_formatter(_p_camd)
# scatter plot for camd
- full_mag_plot = _p_camd.circle(x=x_full_name, y=y_full_name, source=full_cds, size=6, color=cmap)
+ full_mag_plot = _p_camd.scatter(marker='circle', x=x_full_name, y=y_full_name, source=full_cds, size=6,
+ color=cmap)
# colour bar for camd
cbar = ColorBar(color_mapper=cmap['transform'], label_standoff=12,
@@ -439,6 +443,8 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select,
args={'full_plot': full_mag_plot,
'full_data': full_cds.data, 'y_button': _button_mag_y_flip,
'y_axis': _p_camd.yaxis[0], 'y_range': _p_camd.y_range}))
+ _p_camd.js_on_event('reset', CustomJS(args=dict(dropdown_x=_dropdown_mag_x, dropdown_y=_dropdown_mag_y),
+ code=js_callbacks.reset_dropdown))
return _p_camd, _button_mag_x_flip, _button_mag_y_flip, _dropdown_mag_x, _dropdown_mag_y
# gather all necessary data including parallaxes, spectral types and bands
@@ -491,8 +497,8 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select,
absmagnames = absmagnames[~np.isin(absmagnames, bad_cols)]
absmag_shown_name = [name_simplifier(mag) for mag in absmagnames]
dropdown_menu_mag = [*zip(absmagnames, absmag_shown_name)]
- y_full_name = absmagnames[0]
- y_shown_name = absmag_shown_name[0]
+ y_full_name = absmagnames[1]
+ y_shown_name = absmag_shown_name[1]
# camd plot
p_camd, button_mag_x_flip, button_mag_y_flip, dropdown_mag_x, dropdown_mag_y = colour_absolute_magnitude_diagram()
@@ -650,13 +656,13 @@ def camd_plot(query: str, everything: Inventory, all_bands: np.ndarray, all_resu
# scatter plot for given object
this_cds = ColumnDataSource(data=this_photometry)
- this_plot = p.square(x=x_full_name, y=y_full_name, source=this_cds,
- color=cmap, size=20) # plot for this object
+ this_plot = p.scatter(marker='square', x=x_full_name, y=y_full_name, source=this_cds,
+ color=cmap, size=20) # plot for this object
# scatter plot for all data
cds_full = ColumnDataSource(data=all_results_full) # bokeh cds object
- full_plot = p.circle(x=x_full_name, y=y_full_name, source=cds_full,
- color=cmap, alpha=0.5, size=6) # plot all objects
+ full_plot = p.scatter(marker='circle', x=x_full_name, y=y_full_name, source=cds_full,
+ color=cmap, alpha=0.5, size=6) # plot all objects
full_plot.level = 'underlay' # put full plot underneath this plot
# colour bar
@@ -683,6 +689,8 @@ def camd_plot(query: str, everything: Inventory, all_bands: np.ndarray, all_resu
args={'full_plot': full_plot, 'this_plot': this_plot,
'full_data': cds_full.data, 'y_button': button_y_flip,
'y_axis': p.yaxis[0], 'y_range': p.y_range}))
+ p.js_on_event('reset', CustomJS(args=dict(dropdown_x=dropdown_x, dropdown_y=dropdown_y),
+ code=js_callbacks.reset_dropdown))
# creating bokeh layout and html
plots = column(p, row(dropdown_x,
@@ -705,6 +713,6 @@ def main_plots():
if __name__ == '__main__':
- ARGS, DB_FILE, PHOTOMETRIC_FILTERS, ALL_RESULTS, ALL_RESULTS_FULL, VERSION_STR,\
+ ARGS, DB_FILE, PHOTOMETRIC_FILTERS, ALL_RESULTS, ALL_RESULTS_FULL, VERSION_STR, \
ALL_PHOTO, ALL_BANDS, ALL_PLX, ALL_SPTS = main_utils()
NIGHTSKYTHEME, JSCALLBACKS = main_plots()
diff --git a/simple_app/simple_callbacks.js b/simple_app/simple_callbacks.js
index e5475a4..ae9f703 100644
--- a/simple_app/simple_callbacks.js
+++ b/simple_app/simple_callbacks.js
@@ -289,4 +289,15 @@ function reset_slider (spectra_slide) {
spectra_slide.value = [0.81, 0.82];
spectra_slide.change.emit();
+}
+
+function reset_dropdown(dropdown_x, dropdown_y) {
+
+ // resetting the dropdown columns when reset button pressed
+ dropdown_x.value = dropdown_x.options[0][0];
+ dropdown_y.value = dropdown_y.options[1][0];
+
+ dropdown_x.change.emit();
+ dropdown_y.change.emit();
+
}
\ No newline at end of file
diff --git a/simple_app/simports.py b/simple_app/simports.py
index cb9c89d..6c5614e 100644
--- a/simple_app/simports.py
+++ b/simple_app/simports.py
@@ -9,7 +9,7 @@
from astropy.table import Table # tables in astropy
from bokeh.embed import components # converting python bokeh to javascript
from bokeh.layouts import row, column # bokeh displaying nicely
-from bokeh.models import ColumnDataSource, Range1d, CustomJS,\
+from bokeh.models import ColumnDataSource, Range1d, CustomJS, \
Select, Toggle, TapTool, OpenURL, HoverTool, Span, RangeSlider, Label, ColorBar, FixedTicker # bokeh models
from bokeh.palettes import Colorblind8, Turbo256 # plotting palettes
from bokeh.plotting import figure, curdoc # bokeh plotting
@@ -17,7 +17,7 @@
from bokeh.themes import built_in_themes, Theme # appearance of bokeh glyphs
from bokeh.transform import linear_cmap # making colour maps
from flask import (Flask, render_template, jsonify, send_from_directory, redirect, url_for,
- Response, abort, request) # website
+ Response, abort, request, session) # website
from flask_cors import CORS # cross origin fix (aladin mostly)
from flask_wtf import FlaskForm # web forms
from markdown2 import markdown # using markdown formatting
@@ -25,7 +25,7 @@
import pandas as pd # running dataframes
import pytest # testing
from specutils import Spectrum1D # spectrum objects
-from sqlalchemy.exc import ResourceClosedError, OperationalError # errors from sqlalchemy
+from sqlalchemy.exc import ResourceClosedError, OperationalError, ProgrammingError # errors from sqlalchemy
from sqlite3 import Warning as SqliteWarning # errors from sqlite
from tqdm import tqdm # progress bars
from werkzeug.exceptions import HTTPException # underlying http
@@ -42,6 +42,6 @@
from shutil import copy # copying files
import sys # system arguments
from time import strftime, localtime # time stuff for naming files
-from typing import Tuple, Optional, List, Union, Dict # type hinting (good in IDEs)
+from typing import Tuple, Optional, List, Union, Dict, Generator # type hinting (good in IDEs)
from urllib.parse import quote # handling strings into url friendly form
from zipfile import ZipFile # zipping files together
diff --git a/simple_app/utils.py b/simple_app/utils.py
index 2be5c8b..528aac5 100644
--- a/simple_app/utils.py
+++ b/simple_app/utils.py
@@ -46,7 +46,6 @@ def __init__(self, connection_string):
connection_arguments={'check_same_thread': False})
-
class Inventory:
"""
For use in the solo result page where the inventory of an object is queried, grabs also the RA & Dec
@@ -89,7 +88,7 @@ def __init__(self, d_result: Dict[str, List[Dict[str, List[Union[str, float, int
return
@staticmethod
- def spectra_handle(df: pd.DataFrame, drop_source: bool = True):
+ def spectra_handle(df: pd.DataFrame, drop_source: bool = True, multi_obj: bool = False):
"""
Handles spectra, converting files to links
@@ -99,6 +98,8 @@ def spectra_handle(df: pd.DataFrame, drop_source: bool = True):
The table for the spectra
drop_source: bool
Switch to keep source in the dataframe
+ multi_obj: bool
+ Switch on multiple objects being looked at or just one individual object
Returns
-------
@@ -122,7 +123,11 @@ def spectra_handle(df: pd.DataFrame, drop_source: bool = True):
df.drop(columns=drop_cols, inplace=True, errors='ignore')
# editing dataframe with nicely formatted columns
- df['download'] = url_links
+ if multi_obj:
+ href_path = 'write_multi_spectra'
+ else:
+ href_path = 'write_spectra'
+ df[f'download'] = url_links
df['observation_date'] = df['observation_date'].dt.date
return df
@@ -204,7 +209,7 @@ class CoordQueryForm(CSRFOverride):
query = StringField('Query by coordinate within radius:', id='mainsearchfield') # searchbar
submit = SubmitField('Query', id='querybutton') # clicker button to send request
- def __init__(self, *args, **kwargs):
+ def __init__(self, *args, **kwargs: str):
super(CoordQueryForm, self).__init__(*args, **kwargs)
self.db_file: str = kwargs['db_file']
return
@@ -343,7 +348,7 @@ class SQLForm(FlaskForm):
sqlfield = TextAreaField('Enter SQL query here:', id='rawsqlarea', render_kw={'rows': '4'})
submit = SubmitField('Query', id='querybutton')
- def __init__(self, *args, **kwargs):
+ def __init__(self, *args, **kwargs: str):
super(SQLForm, self).__init__(*args, **kwargs)
self.db_file: str = kwargs['db_file']
return
@@ -383,7 +388,7 @@ def validate_sqlfield(self, field):
_: Optional[pd.DataFrame] = db.sql_query(query, fmt='pandas')
# catch these expected errors
- except (ResourceClosedError, OperationalError, IndexError, SqliteWarning, BadSQLError) as e:
+ except (ResourceClosedError, OperationalError, IndexError, SqliteWarning, BadSQLError, ProgrammingError) as e:
raise ValidationError('Invalid SQL: ' + str(e))
# any unexpected errors
@@ -401,13 +406,15 @@ class JSCallbacks:
button_flip = ''
normalisation_slider = ''
reset_slider = ''
+ reset_dropdown = ''
def __init__(self):
"""
Loads simple_callbacks and unpacks the js functions within, to the python variables into instance attributes
"""
# open js functions script
- js_func_names = ('dropdown_x_js', 'dropdown_y_js', 'button_flip', 'normalisation_slider', 'reset_slider')
+ js_func_names = ('dropdown_x_js', 'dropdown_y_js', 'button_flip', 'normalisation_slider',
+ 'reset_slider', 'reset_dropdown')
with open('simple_app/simple_callbacks.js', 'r') as func_call:
which_var = ''
out_string = """"""
@@ -613,6 +620,7 @@ def replacer(val: int) -> str:
return this_new_phot
+# noinspection PyTypeChecker
def parse_photometry(photometry_df: pd.DataFrame, all_bands: np.ndarray, multi_source: bool = False) -> pd.DataFrame:
"""
Parses the photometry dataframe handling multiple references for same magnitude
@@ -951,6 +959,8 @@ def one_df_query(results: pd.DataFrame, table_id: Optional[str] = None, limit_ma
source_links = []
for source in results.source.values:
+ if not isinstance(source, str):
+ source = source[0]
url_link = quote(source)
source_link = f'{source}'
source_links.append(source_link)
@@ -1001,6 +1011,8 @@ def multi_df_query(results: Dict[str, pd.DataFrame], db_file: str,
# wrapping the one_df_query method for each table
for table_name, df in results.items():
+ if table_name == 'Spectra':
+ df = Inventory.spectra_handle(df, False, True)
df = reference_handle(df, db_file)
stringed_df = one_df_query(df, table_name.lower() + 'table', limit_max_rows)
d_results[table_name] = stringed_df
@@ -1111,7 +1123,7 @@ def control_response(response: Response, key: str = '', app_type: str = 'csv') -
return response
-def write_file(results: pd.DataFrame) -> str:
+def write_file(results: pd.DataFrame) -> Generator:
"""
Creates a csv file ready for download on a line by line basis