diff --git a/requirements.txt b/requirements.txt index 9604d1e..9a12d84 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,87 @@ -astrodbkit>=2.2 -astropy>=5.2.0,<6 -bokeh==3.0.2 -flask>=3.0,<4 -flask-cors>=4.0,<5 -flask-wtf>=1.2,<2 -markdown2>=2.4,<3 -multiprocess>=0.70,<1 -numpy>=1.24,<2 -pandas>=2.0,<2.1 -pysqlite3>=0.5,<1 -pytest>=8.0,<9 -requests>=2.31,<3 -specutils>=1.12,<2 -sqlalchemy>=2.0,<3 -tqdm>=4.66,<5 -werkzeug>=3.0,<4 -wtforms>=3.1,<4 +antiorm==1.2.1 +asdf==4.1.0 +asdf-astropy==0.7.0 +asdf_coordinates_schemas==0.3.0 +asdf_standard==1.1.1 +asdf_transform_schemas==0.5.0 +asdf_wcs_schemas==0.4.0 +astrodbkit==2.2 +astropy==5.3.4 +astroquery==0.4.9.post1 +attrs==25.1.0 +backports.tarfile==1.2.0 +beautifulsoup4==4.13.3 +blinker==1.9.0 +bokeh==3.7.3 +certifi==2025.1.31 +cffi==1.17.1 +charset-normalizer==3.4.1 +click==8.1.8 +contourpy==1.3.1 +cryptography==44.0.0 +db==0.1.1 +db-sqlite3==0.0.1 +dill==0.3.9 +distlib==0.3.9 +distro==1.9.0 +filelock==3.16.1 +Flask==3.1.0 +Flask-Cors==4.0.2 +Flask-WTF==1.2.2 +greenlet==3.1.1 +gwcs==0.21.0 +html5lib==1.1 +idna==3.10 +importlib_metadata==8.6.1 +iniconfig==2.0.0 +iniparse==0.5 +itsdangerous==2.2.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.1.0 +jeepney==0.8.0 +Jinja2==3.1.5 +jmespath==1.0.1 +keyring==25.6.0 +markdown2==2.5.3 +MarkupSafe==3.0.2 +more-itertools==10.6.0 +multiprocess==0.70.17 +narwhals==1.41.0 +ndcube==2.3.0 +numpy==1.26.4 +packaging==24.2 +pandas==2.0.3 +pathlib2==2.3.7.post1 +pillow==11.1.0 +platformdirs==4.3.6 +pluggy==1.5.0 +pycparser==2.22 +pyerfa==2.0.1.5 +PyMySQL==1.1.1 +pyOpenSSL==24.3.0 +pysqlite3==0.5.4 +pytest==8.3.4 +python-dateutil==2.9.0.post0 +pytz==2024.2 +pyvo==1.6 +PyYAML==6.0.2 +requests==2.32.3 +scipy==1.15.1 +SecretStorage==3.3.3 +semantic-version==2.10.0 +six==1.17.0 +soupsieve==2.6 +specutils==1.19.0 +SQLAlchemy==2.0.38 +tornado==6.4.2 +tqdm==4.67.1 +typing_extensions==4.12.2 +tzdata==2025.1 +urllib3==2.3.0 +virtualenv==20.28.0 +webencodings==0.5.1 +Werkzeug==3.1.3 +WTForms==3.2.1 +xyzservices==2025.1.0 +zipp==3.21.0 diff --git a/simple_app/app_simple.py b/simple_app/app_simple.py index 7b041d4..8f336aa 100644 --- a/simple_app/app_simple.py +++ b/simple_app/app_simple.py @@ -45,8 +45,8 @@ def search(): if (query := form.search.data) is None: # content in main searchbar query = '' - curdoc().template_variables['query'] = query # add query to bokeh curdoc - curdoc().template_variables['ref_query'] = ref_query # add query to bokeh curdoc + session['query'] = query # add query to bokeh curdoc + session['ref_query'] = ref_query # add query to bokeh curdoc db = SimpleDB(db_file) # open database # object search function @@ -96,7 +96,7 @@ def coordinate_query(): if (query := form.query.data) is None: query = '' - curdoc().template_variables['query'] = query + session['query'] = query db = SimpleDB(db_file) # parse query into ra, dec, radius @@ -127,7 +127,7 @@ def full_text_search(): query = '' limmaxrows = True - curdoc().template_variables['query'] = query + session['query'] = query db = SimpleDB(db_file) # search through the tables using the given query @@ -160,14 +160,14 @@ def raw_query(): if (query := form.sqlfield.data) is None: query = '' - curdoc().template_variables['query'] = query + session['query'] = query # attempt to query the database try: results: Optional[pd.DataFrame] = db.sql_query(query, fmt='pandas') # catch any broken queries (should not activate as will be caught by validation) - except (ResourceClosedError, OperationalError, IndexError, SqliteWarning, BadSQLError): + except (ResourceClosedError, OperationalError, IndexError, SqliteWarning, BadSQLError, ProgrammingError): results = pd.DataFrame() results = reference_handle(results, db_file, True) @@ -188,7 +188,7 @@ def solo_result(query: str): query: str The query -- full match to a main ID """ - curdoc().template_variables['query'] = query + session['query'] = query db = SimpleDB(db_file) # search database for given object @@ -199,6 +199,7 @@ def solo_result(query: str): except KeyError: abort(404, f'"{query}" does match any result in SIMPLE!') return + everything = Inventory(resultdict, db_file) # create camd and spectra plots @@ -283,7 +284,7 @@ def create_file_for_download(key: str): key: str The dataframe string """ - query = curdoc().template_variables['query'] + query: str = session.get('query') db = SimpleDB(db_file) # search for a given object and a given key @@ -305,7 +306,7 @@ def create_files_for_solo_download(): """ Creates and downloads all dataframes from solo results """ - query = curdoc().template_variables['query'] + query = session.get('query') db = SimpleDB(db_file) # search for a given object @@ -328,7 +329,7 @@ def create_spectra_files_for_download(): """ Downloads the spectra files and zips together """ - query = curdoc().template_variables['query'] + query = session.get('query') db = SimpleDB(db_file) # search for a given object and specifically its spectra @@ -346,13 +347,35 @@ def create_spectra_files_for_download(): abort(400, 'Could not download fits') +@app_simple.route('/write_multi_spectra', methods=['GET']) +def create_multi_spectra_files_for_download(): + """ + Downloads the spectra files and zips together + """ + query = session.get('query') + db = SimpleDB(db_file) + + # search for the spectra within a given free search + resultdict: Dict[str, pd.DataFrame] = db.search_string(query, fmt='pandas', verbose=False) + spectra_df: pd.DataFrame = resultdict['Spectra'] + + # write all spectra for object to zipped file + zipped = write_spec_files(spectra_df.access_url.values) + if zipped is not None: + response = Response(zipped, mimetype='application/zip') + response = control_response(response, app_type='zip') + return response + + abort(400, 'Could not download fits') + + @app_simple.route('/write_filt', methods=['GET']) def create_file_for_filtered_download(): """ Creates and downloads the shown dataframe when in filtered search """ - query = curdoc().template_variables['query'] - refquery = curdoc().template_variables['ref_query'] + query = session.get('query') + refquery = session.get('ref_query') db = SimpleDB(db_file) # search for a given object and a full text search at the same time @@ -375,7 +398,7 @@ def create_file_for_coordinate_download(): """ Creates and downloads the shown dataframe when in coordinate search """ - query = curdoc().template_variables['query'] + query = session.get('query') db = SimpleDB(db_file) # query the database for given coordinates and parse those coordinates @@ -401,12 +424,12 @@ def create_file_for_full_download(key: str): key: str The dataframe string """ - query = curdoc().template_variables['query'] + query = session.get('query') db = SimpleDB(db_file) # search database with a free-text search and specific table resultdict: Dict[str, pd.DataFrame] = db.search_string(query, fmt='pandas', verbose=False) - + print(query, key, resultdict.keys()) # write to csv if key in resultdict: results: pd.DataFrame = resultdict[key] @@ -422,7 +445,7 @@ def create_files_for_multi_download(): """ Creates and downloads all dataframes from full results """ - query = curdoc().template_variables['query'] + query = session.get('query') db = SimpleDB(db_file) # search with full-text search @@ -439,7 +462,7 @@ def create_file_for_sql_download(): """ Creates and downloads the shown dataframe when in sql query """ - query = curdoc().template_variables['query'] + query = session.get('query') db = SimpleDB(db_file) # query database via sql diff --git a/simple_app/plots.py b/simple_app/plots.py index 4416458..50aff99 100644 --- a/simple_app/plots.py +++ b/simple_app/plots.py @@ -333,14 +333,14 @@ def sky_plot() -> figure: _p_sky = figure(title='Sky Plot', outer_height=500, active_scroll='wheel_zoom', active_drag='box_zoom', tools='pan,wheel_zoom,box_zoom,box_select,reset', - sizing_mode='stretch_width', x_range=[-180, 180], y_range=[-90, 90]) + sizing_mode='stretch_width', x_range=Range1d(-180, 180), y_range=Range1d(-90, 90)) # background for skyplot _p_sky.ellipse(x=0, y=0, width=360, height=180, color='#444444', name='background') # scatter plot for sky plot - circle = _p_sky.circle(source=full_cds, x='ra_projected', y='dec_projected', - size=6, name='circle', color='ghostwhite') + circle = _p_sky.scatter(marker='circle', source=full_cds, x='ra_projected', y='dec_projected', + size=6, name='circle', color='ghostwhite') # bokeh tools for sky plot this_hover = HoverTool(renderers=[circle, ], tooltips=tooltips) @@ -367,7 +367,8 @@ def colour_colour_plot() -> Tuple[figure, Toggle, Toggle, Select, Select]: _p_colour_colour = bokeh_formatter(_p_colour_colour) # scatter plot for colour-colour - full_plot = _p_colour_colour.circle(x=x_full_name, y=y_full_name, source=full_cds, size=6, color=cmap) + full_plot = _p_colour_colour.scatter(marker='circle', x=x_full_name, y=y_full_name, source=full_cds, size=6, + color=cmap) # colour bar for colour-colour plot cbar = ColorBar(color_mapper=cmap['transform'], label_standoff=12, @@ -395,6 +396,8 @@ def colour_colour_plot() -> Tuple[figure, Toggle, Toggle, Select, Select]: args={'full_plot': full_plot, 'full_data': full_cds.data, 'y_button': _button_y_flip, 'y_axis': _p_colour_colour.yaxis[0], 'y_range': _p_colour_colour.y_range})) + _p_colour_colour.js_on_event('reset', CustomJS(args=dict(dropdown_x=_dropdown_x, dropdown_y=_dropdown_y), + code=js_callbacks.reset_dropdown)) return _p_colour_colour, _button_x_flip, _button_y_flip, _dropdown_x, _dropdown_y def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select, Select]: @@ -413,7 +416,8 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select, _p_camd = bokeh_formatter(_p_camd) # scatter plot for camd - full_mag_plot = _p_camd.circle(x=x_full_name, y=y_full_name, source=full_cds, size=6, color=cmap) + full_mag_plot = _p_camd.scatter(marker='circle', x=x_full_name, y=y_full_name, source=full_cds, size=6, + color=cmap) # colour bar for camd cbar = ColorBar(color_mapper=cmap['transform'], label_standoff=12, @@ -439,6 +443,8 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select, args={'full_plot': full_mag_plot, 'full_data': full_cds.data, 'y_button': _button_mag_y_flip, 'y_axis': _p_camd.yaxis[0], 'y_range': _p_camd.y_range})) + _p_camd.js_on_event('reset', CustomJS(args=dict(dropdown_x=_dropdown_mag_x, dropdown_y=_dropdown_mag_y), + code=js_callbacks.reset_dropdown)) return _p_camd, _button_mag_x_flip, _button_mag_y_flip, _dropdown_mag_x, _dropdown_mag_y # gather all necessary data including parallaxes, spectral types and bands @@ -491,8 +497,8 @@ def colour_absolute_magnitude_diagram() -> Tuple[figure, Toggle, Toggle, Select, absmagnames = absmagnames[~np.isin(absmagnames, bad_cols)] absmag_shown_name = [name_simplifier(mag) for mag in absmagnames] dropdown_menu_mag = [*zip(absmagnames, absmag_shown_name)] - y_full_name = absmagnames[0] - y_shown_name = absmag_shown_name[0] + y_full_name = absmagnames[1] + y_shown_name = absmag_shown_name[1] # camd plot p_camd, button_mag_x_flip, button_mag_y_flip, dropdown_mag_x, dropdown_mag_y = colour_absolute_magnitude_diagram() @@ -650,13 +656,13 @@ def camd_plot(query: str, everything: Inventory, all_bands: np.ndarray, all_resu # scatter plot for given object this_cds = ColumnDataSource(data=this_photometry) - this_plot = p.square(x=x_full_name, y=y_full_name, source=this_cds, - color=cmap, size=20) # plot for this object + this_plot = p.scatter(marker='square', x=x_full_name, y=y_full_name, source=this_cds, + color=cmap, size=20) # plot for this object # scatter plot for all data cds_full = ColumnDataSource(data=all_results_full) # bokeh cds object - full_plot = p.circle(x=x_full_name, y=y_full_name, source=cds_full, - color=cmap, alpha=0.5, size=6) # plot all objects + full_plot = p.scatter(marker='circle', x=x_full_name, y=y_full_name, source=cds_full, + color=cmap, alpha=0.5, size=6) # plot all objects full_plot.level = 'underlay' # put full plot underneath this plot # colour bar @@ -683,6 +689,8 @@ def camd_plot(query: str, everything: Inventory, all_bands: np.ndarray, all_resu args={'full_plot': full_plot, 'this_plot': this_plot, 'full_data': cds_full.data, 'y_button': button_y_flip, 'y_axis': p.yaxis[0], 'y_range': p.y_range})) + p.js_on_event('reset', CustomJS(args=dict(dropdown_x=dropdown_x, dropdown_y=dropdown_y), + code=js_callbacks.reset_dropdown)) # creating bokeh layout and html plots = column(p, row(dropdown_x, @@ -705,6 +713,6 @@ def main_plots(): if __name__ == '__main__': - ARGS, DB_FILE, PHOTOMETRIC_FILTERS, ALL_RESULTS, ALL_RESULTS_FULL, VERSION_STR,\ + ARGS, DB_FILE, PHOTOMETRIC_FILTERS, ALL_RESULTS, ALL_RESULTS_FULL, VERSION_STR, \ ALL_PHOTO, ALL_BANDS, ALL_PLX, ALL_SPTS = main_utils() NIGHTSKYTHEME, JSCALLBACKS = main_plots() diff --git a/simple_app/simple_callbacks.js b/simple_app/simple_callbacks.js index e5475a4..ae9f703 100644 --- a/simple_app/simple_callbacks.js +++ b/simple_app/simple_callbacks.js @@ -289,4 +289,15 @@ function reset_slider (spectra_slide) { spectra_slide.value = [0.81, 0.82]; spectra_slide.change.emit(); +} + +function reset_dropdown(dropdown_x, dropdown_y) { + + // resetting the dropdown columns when reset button pressed + dropdown_x.value = dropdown_x.options[0][0]; + dropdown_y.value = dropdown_y.options[1][0]; + + dropdown_x.change.emit(); + dropdown_y.change.emit(); + } \ No newline at end of file diff --git a/simple_app/simports.py b/simple_app/simports.py index cb9c89d..6c5614e 100644 --- a/simple_app/simports.py +++ b/simple_app/simports.py @@ -9,7 +9,7 @@ from astropy.table import Table # tables in astropy from bokeh.embed import components # converting python bokeh to javascript from bokeh.layouts import row, column # bokeh displaying nicely -from bokeh.models import ColumnDataSource, Range1d, CustomJS,\ +from bokeh.models import ColumnDataSource, Range1d, CustomJS, \ Select, Toggle, TapTool, OpenURL, HoverTool, Span, RangeSlider, Label, ColorBar, FixedTicker # bokeh models from bokeh.palettes import Colorblind8, Turbo256 # plotting palettes from bokeh.plotting import figure, curdoc # bokeh plotting @@ -17,7 +17,7 @@ from bokeh.themes import built_in_themes, Theme # appearance of bokeh glyphs from bokeh.transform import linear_cmap # making colour maps from flask import (Flask, render_template, jsonify, send_from_directory, redirect, url_for, - Response, abort, request) # website + Response, abort, request, session) # website from flask_cors import CORS # cross origin fix (aladin mostly) from flask_wtf import FlaskForm # web forms from markdown2 import markdown # using markdown formatting @@ -25,7 +25,7 @@ import pandas as pd # running dataframes import pytest # testing from specutils import Spectrum1D # spectrum objects -from sqlalchemy.exc import ResourceClosedError, OperationalError # errors from sqlalchemy +from sqlalchemy.exc import ResourceClosedError, OperationalError, ProgrammingError # errors from sqlalchemy from sqlite3 import Warning as SqliteWarning # errors from sqlite from tqdm import tqdm # progress bars from werkzeug.exceptions import HTTPException # underlying http @@ -42,6 +42,6 @@ from shutil import copy # copying files import sys # system arguments from time import strftime, localtime # time stuff for naming files -from typing import Tuple, Optional, List, Union, Dict # type hinting (good in IDEs) +from typing import Tuple, Optional, List, Union, Dict, Generator # type hinting (good in IDEs) from urllib.parse import quote # handling strings into url friendly form from zipfile import ZipFile # zipping files together diff --git a/simple_app/utils.py b/simple_app/utils.py index 2be5c8b..528aac5 100644 --- a/simple_app/utils.py +++ b/simple_app/utils.py @@ -46,7 +46,6 @@ def __init__(self, connection_string): connection_arguments={'check_same_thread': False}) - class Inventory: """ For use in the solo result page where the inventory of an object is queried, grabs also the RA & Dec @@ -89,7 +88,7 @@ def __init__(self, d_result: Dict[str, List[Dict[str, List[Union[str, float, int return @staticmethod - def spectra_handle(df: pd.DataFrame, drop_source: bool = True): + def spectra_handle(df: pd.DataFrame, drop_source: bool = True, multi_obj: bool = False): """ Handles spectra, converting files to links @@ -99,6 +98,8 @@ def spectra_handle(df: pd.DataFrame, drop_source: bool = True): The table for the spectra drop_source: bool Switch to keep source in the dataframe + multi_obj: bool + Switch on multiple objects being looked at or just one individual object Returns ------- @@ -122,7 +123,11 @@ def spectra_handle(df: pd.DataFrame, drop_source: bool = True): df.drop(columns=drop_cols, inplace=True, errors='ignore') # editing dataframe with nicely formatted columns - df['download'] = url_links + if multi_obj: + href_path = 'write_multi_spectra' + else: + href_path = 'write_spectra' + df[f'download'] = url_links df['observation_date'] = df['observation_date'].dt.date return df @@ -204,7 +209,7 @@ class CoordQueryForm(CSRFOverride): query = StringField('Query by coordinate within radius:', id='mainsearchfield') # searchbar submit = SubmitField('Query', id='querybutton') # clicker button to send request - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs: str): super(CoordQueryForm, self).__init__(*args, **kwargs) self.db_file: str = kwargs['db_file'] return @@ -343,7 +348,7 @@ class SQLForm(FlaskForm): sqlfield = TextAreaField('Enter SQL query here:', id='rawsqlarea', render_kw={'rows': '4'}) submit = SubmitField('Query', id='querybutton') - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs: str): super(SQLForm, self).__init__(*args, **kwargs) self.db_file: str = kwargs['db_file'] return @@ -383,7 +388,7 @@ def validate_sqlfield(self, field): _: Optional[pd.DataFrame] = db.sql_query(query, fmt='pandas') # catch these expected errors - except (ResourceClosedError, OperationalError, IndexError, SqliteWarning, BadSQLError) as e: + except (ResourceClosedError, OperationalError, IndexError, SqliteWarning, BadSQLError, ProgrammingError) as e: raise ValidationError('Invalid SQL: ' + str(e)) # any unexpected errors @@ -401,13 +406,15 @@ class JSCallbacks: button_flip = '' normalisation_slider = '' reset_slider = '' + reset_dropdown = '' def __init__(self): """ Loads simple_callbacks and unpacks the js functions within, to the python variables into instance attributes """ # open js functions script - js_func_names = ('dropdown_x_js', 'dropdown_y_js', 'button_flip', 'normalisation_slider', 'reset_slider') + js_func_names = ('dropdown_x_js', 'dropdown_y_js', 'button_flip', 'normalisation_slider', + 'reset_slider', 'reset_dropdown') with open('simple_app/simple_callbacks.js', 'r') as func_call: which_var = '' out_string = """""" @@ -613,6 +620,7 @@ def replacer(val: int) -> str: return this_new_phot +# noinspection PyTypeChecker def parse_photometry(photometry_df: pd.DataFrame, all_bands: np.ndarray, multi_source: bool = False) -> pd.DataFrame: """ Parses the photometry dataframe handling multiple references for same magnitude @@ -951,6 +959,8 @@ def one_df_query(results: pd.DataFrame, table_id: Optional[str] = None, limit_ma source_links = [] for source in results.source.values: + if not isinstance(source, str): + source = source[0] url_link = quote(source) source_link = f'{source}' source_links.append(source_link) @@ -1001,6 +1011,8 @@ def multi_df_query(results: Dict[str, pd.DataFrame], db_file: str, # wrapping the one_df_query method for each table for table_name, df in results.items(): + if table_name == 'Spectra': + df = Inventory.spectra_handle(df, False, True) df = reference_handle(df, db_file) stringed_df = one_df_query(df, table_name.lower() + 'table', limit_max_rows) d_results[table_name] = stringed_df @@ -1111,7 +1123,7 @@ def control_response(response: Response, key: str = '', app_type: str = 'csv') - return response -def write_file(results: pd.DataFrame) -> str: +def write_file(results: pd.DataFrame) -> Generator: """ Creates a csv file ready for download on a line by line basis