From 52b90b2736288e0db08526f82d66d15b3b737969 Mon Sep 17 00:00:00 2001 From: Kelle Cruz Date: Wed, 30 Jul 2025 20:02:53 +0000 Subject: [PATCH 1/3] upgrading to specutils 2.1. still broken. --- requirements.txt | 4 ++-- simple_app/plots.py | 2 +- simple_app/simports.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9604d1e..5299d28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ astrodbkit>=2.2 -astropy>=5.2.0,<6 +astropy>=7.1 bokeh==3.0.2 flask>=3.0,<4 flask-cors>=4.0,<5 @@ -11,7 +11,7 @@ pandas>=2.0,<2.1 pysqlite3>=0.5,<1 pytest>=8.0,<9 requests>=2.31,<3 -specutils>=1.12,<2 +specutils>=2.1 sqlalchemy>=2.0,<3 tqdm>=4.66,<5 werkzeug>=3.0,<4 diff --git a/simple_app/plots.py b/simple_app/plots.py index 4416458..4981857 100644 --- a/simple_app/plots.py +++ b/simple_app/plots.py @@ -173,7 +173,7 @@ def normalise() -> np.ndarray: # checking each spectra in table for spec in t_spectra: - spectrum: Spectrum1D = spec['access_url'] + spectrum = Spectrum.read(spec['access_url']) # checking spectrum has good units and not only NaNs or 0s try: diff --git a/simple_app/simports.py b/simple_app/simports.py index cb9c89d..6f59d2e 100644 --- a/simple_app/simports.py +++ b/simple_app/simports.py @@ -24,7 +24,7 @@ import numpy as np # numerical python import pandas as pd # running dataframes import pytest # testing -from specutils import Spectrum1D # spectrum objects +from specutils import Spectrum # spectrum objects from sqlalchemy.exc import ResourceClosedError, OperationalError # errors from sqlalchemy from sqlite3 import Warning as SqliteWarning # errors from sqlite from tqdm import tqdm # progress bars From 7d5b038c155cbdb86acc288db0530e74a8664e4e Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Mon, 8 Dec 2025 14:14:07 -0500 Subject: [PATCH 2/3] Fix for JWST spectra handling --- requirements.txt | 2 +- simple_app/plots.py | 9 +++-- simple_app/simports.py | 81 +++++++++++++++++++++++++++--------------- 3 files changed, 60 insertions(+), 32 deletions(-) diff --git a/requirements.txt b/requirements.txt index e21dc2b..eefbb7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ numpy==1.26.4 pandas==2.0.3 pytest==8.3.4 requests==2.32.4 -specutils==2.0.0 +specutils==2.2.0 SQLAlchemy==2.0.38 tqdm==4.67.1 Werkzeug==3.1.4 diff --git a/simple_app/plots.py b/simple_app/plots.py index 77843db..282fc0e 100644 --- a/simple_app/plots.py +++ b/simple_app/plots.py @@ -152,7 +152,7 @@ def normalise() -> np.ndarray: db = SimpleDB(db_file) # open database t_spectra: Table = db.query(db.Spectra).\ filter(db.Spectra.c.source == query).\ - table(spectra=['access_url']) + table() # do not use spectra=['access_url'] here, it will try to read the spectra as Spectrum objects # initialise plot n_fail, fail_string_list = 0, [] @@ -173,7 +173,12 @@ def normalise() -> np.ndarray: # checking each spectra in table for spec in t_spectra: - spectrum = Spectrum.read(spec['access_url']) + try: + # Manually convert the spectrum to a Spectrum object + spectrum = Spectrum.read(spec['access_url'], cache=False) + except Exception as e: + print(f"Unable to read {spec['access_url']} as Spectrum. Error: {e}") + continue # checking spectrum has good units and not only NaNs or 0s try: diff --git a/simple_app/simports.py b/simple_app/simports.py index db3f5df..b7b5368 100644 --- a/simple_app/simports.py +++ b/simple_app/simports.py @@ -1,46 +1,69 @@ """ Importing all packages """ -# external packages + +import argparse # parsing the arguments given with file +import os # operating system +import sys # system arguments +from copy import deepcopy # memory control +from difflib import get_close_matches # for redirecting bad file paths +from io import BufferedIOBase, BytesIO, StringIO # writing files without saving to disk +from shutil import copy # copying files +from sqlite3 import Warning as SqliteWarning # errors from sqlite +from time import localtime, strftime # time stuff for naming files +from typing import Dict, Generator, List, Optional, Tuple, Union # type hinting (good in IDEs) +from urllib.parse import quote # handling strings into url friendly form +from zipfile import ZipFile # zipping files together + +import astropy.units as u # units +import multiprocess as mp # multiprocessing for efficiency +import numpy as np # numerical python +import pandas as pd # running dataframes +import pytest # testing +import requests # accessing internet from astrodbkit.astrodb import Database # used for pulling out database and querying from astropy.coordinates import SkyCoord # coordinates from astropy.io import fits # handling fits files -import astropy.units as u # units from astropy.table import Table # tables in astropy from bokeh.embed import components # converting python bokeh to javascript -from bokeh.layouts import row, column # bokeh displaying nicely -from bokeh.models import ColumnDataSource, Range1d, CustomJS, \ - Select, Toggle, TapTool, OpenURL, HoverTool, Span, RangeSlider, Label, ColorBar, FixedTicker # bokeh models +from bokeh.layouts import column, row # bokeh displaying nicely +from bokeh.models import ( + ColorBar, + ColumnDataSource, + CustomJS, + FixedTicker, + HoverTool, + Label, + OpenURL, + Range1d, + RangeSlider, + Select, # bokeh models + Span, + TapTool, + Toggle, +) from bokeh.palettes import Colorblind8, Turbo256 # plotting palettes -from bokeh.plotting import figure, curdoc # bokeh plotting +from bokeh.plotting import curdoc, figure # bokeh plotting from bokeh.resources import CDN # resources for webpage -from bokeh.themes import built_in_themes, Theme # appearance of bokeh glyphs +from bokeh.themes import Theme, built_in_themes # appearance of bokeh glyphs from bokeh.transform import linear_cmap # making colour maps -from flask import (Flask, render_template, jsonify, send_from_directory, redirect, url_for, - Response, abort, request, session) # website +from flask import ( + Flask, + Response, # website + abort, + jsonify, + redirect, + render_template, + request, + send_from_directory, + session, + url_for, +) from flask_cors import CORS # cross origin fix (aladin mostly) from flask_wtf import FlaskForm # web forms from markdown2 import markdown # using markdown formatting -import numpy as np # numerical python -import pandas as pd # running dataframes -import pytest # testing -from sqlalchemy.exc import ResourceClosedError, OperationalError, ProgrammingError # errors from sqlalchemy -from sqlite3 import Warning as SqliteWarning # errors from sqlite +from specutils import Spectrum +from sqlalchemy.exc import OperationalError, ProgrammingError, ResourceClosedError # errors from sqlalchemy from tqdm import tqdm # progress bars from werkzeug.exceptions import HTTPException # underlying http from wtforms import StringField, SubmitField, TextAreaField, ValidationError # web forms - -# internal packages -import argparse # parsing the arguments given with file -from copy import deepcopy # memory control -from difflib import get_close_matches # for redirecting bad file paths -from io import StringIO, BytesIO, BufferedIOBase # writing files without saving to disk -import multiprocess as mp # multiprocessing for efficiency -import os # operating system -import requests # accessing internet -from shutil import copy # copying files -import sys # system arguments -from time import strftime, localtime # time stuff for naming files -from typing import Tuple, Optional, List, Union, Dict, Generator # type hinting (good in IDEs) -from urllib.parse import quote # handling strings into url friendly form -from zipfile import ZipFile # zipping files together From b6ed6df8dc5171da75ba566247180f57971b9617 Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Mon, 8 Dec 2025 14:16:22 -0500 Subject: [PATCH 3/3] using cache --- simple_app/plots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simple_app/plots.py b/simple_app/plots.py index 282fc0e..971f5a7 100644 --- a/simple_app/plots.py +++ b/simple_app/plots.py @@ -175,7 +175,7 @@ def normalise() -> np.ndarray: for spec in t_spectra: try: # Manually convert the spectrum to a Spectrum object - spectrum = Spectrum.read(spec['access_url'], cache=False) + spectrum = Spectrum.read(spec['access_url'], cache=True) except Exception as e: print(f"Unable to read {spec['access_url']} as Spectrum. Error: {e}") continue