Convert and ingest WISE-1810

.gitignore

@@ -11,4 +11,3 @@
 *.egg-info
 build/
 simple/version.py
-

data/reference/Instruments.json

@@ -509,5 +509,12 @@
         "telescope": "Lick Shane 3m",
         "description": null,
         "reference": null
+    },
+    {
+        "instrument": "EMIR",
+        "mode": "Missing",
+        "telescope": "GTC",
+        "description": "Espectrografo Multiobjeto Infra-Rojo",
+        "reference": null
     }
 ]

data/reference/Telescopes.json

@@ -126,7 +126,7 @@
     },
     {
         "telescope": "GTC",
-        "description": null,
+        "description": "Gran Telescopio de Canarias",
         "reference": null
     },
     {

data/source/cwisep_j181006.00-101001.1.json

@@ -94,6 +94,34 @@
             "reference": "Lodi22"
         }
     ],
+    "Spectra": [
+        {
+            "access_url": "https://bdnyc.s3.us-east-1.amazonaws.com/data_target_WISE1810_comb_Jun2021_YJ_STD_bb.fits",
+            "original_spectrum": null,
+            "local_spectrum": null,
+            "regime": "nir",
+            "telescope": "GTC",
+            "instrument": "EMIR",
+            "mode": "Missing",
+            "observation_date": "2021-06-01T00:00:00",
+            "comments": null,
+            "reference": "Lodi22",
+            "other_references": null
+        },
+        {
+            "access_url": "https://bdnyc.s3.us-east-1.amazonaws.com/WISE1810m10_OB0001_R1000R_06Sept2020.fits",
+            "original_spectrum": null,
+            "local_spectrum": null,
+            "regime": "optical",
+            "telescope": "GTC",
+            "instrument": "OSIRIS",
+            "mode": "Missing",
+            "observation_date": "2020-09-06T00:00:00",
+            "comments": null,
+            "reference": "Lodi22",
+            "other_references": null
+        }
+    ],
     "SpectralTypes": [
         {
             "spectral_type_string": "esdT0",

scripts/ingests/WISE_1810/ingest_WISE1810.py

@@ -0,0 +1,90 @@
+import logging
+from astrodb_utils import load_astrodb, AstroDBError
+from astrodb_utils.instruments import ingest_instrument
+from simple import REFERENCE_TABLES
+from simple.utils.spectra import ingest_spectrum
+from datetime import datetime
+
+# set up logging for ASTRODB
+astrodb_utils_logger = logging.getLogger("astrodb_utils")
+astrodb_utils_logger.setLevel(logging.INFO)
+
+# set up logging for this ingest script
+logger = logging.getLogger("astrodb_utils.WISE_1810")
+logger.setLevel(logging.INFO)
+
+# Load Database
+recreate_db = True
+save_db = True
+
+SCHEMA_PATH = "simple/schema.yaml"
+db = load_astrodb(
+    "SIMPLE.sqlite",
+    recreatedb=recreate_db,
+    reference_tables=REFERENCE_TABLES,
+    felis_schema=SCHEMA_PATH
+)
+
+# Ingest Instruments ----
+def add_instrument():
+    """
+    Telescope: GTC (existed)
+    Instrument: EMIR (ingestion needed)
+    """
+    try:
+        ingest_instrument(
+            db,
+            telescope="GTC", 
+            instrument="EMIR",
+            mode="Missing",
+            raise_error=True
+        )
+        print ("Instrument added successfully")
+
+
+    except AstroDBError as e:
+        logger.error(f"Error adding instruments: {e}")
+
+# convert obs date format
+format_str = "%Y-%m-%d %H:%M:%S0000"
+
+# same sources with different instruments and FITS file
+spectra_data = [{
+        "access_url": "https://bdnyc.s3.us-east-1.amazonaws.com/WISE1810m10_OB0001_R1000R_06Sept2020.fits",
+        "regime": "optical",
+        "instrument": 'OSIRIS',
+        "observation_date": datetime.strptime("2020-09-06 00:00:000000", format_str)
+    },
+    {
+        "access_url": "https://bdnyc.s3.us-east-1.amazonaws.com/data_target_WISE1810_comb_Jun2021_YJ_STD_bb.fits",
+        "regime": "nir",
+        "instrument": 'EMIR',
+        "observation_date":datetime.strptime("2021-06-01 00:00:000000", format_str)
+    }
+]
+
+# Ingest Spectra ----
+def add_spectra():
+    for data in spectra_data:
+        try:
+            ingest_spectrum(
+                db,
+                source="CWISEP J181006.00-101001.1",
+                spectrum=data["access_url"],
+                regime=data["regime"],
+                mode="Missing",
+                telescope="GTC",
+                instrument=data["instrument"],
+                obs_date=data["observation_date"],
+                reference="Lodi22"
+            )
+            logger.info(f"Successfully ingested spectrum for CWISEP J181006.00-101001.1 with {data['instrument']}")
+        except AstroDBError as e:
+            logger.error(f"Error ingesting spectrum: {e}")
+
+# Run ingestion function
+add_instrument()
+add_spectra()
+
+if save_db: 
+    db.save_database(directory="data/")

scripts/spectra_convert/WISE1810_txt.py

@@ -0,0 +1,93 @@
+import astropy.units as u
+from astropy.io.fits import getheader, Header
+from astropy.io import fits
+from astrodb_utils.spectra import check_spectrum_plottable
+from astrodb_utils.fits import add_wavelength_keywords
+from specutils import Spectrum1D
+import numpy as np
+from astroquery.simbad import Simbad
+import os
+
+
+file= ["/Users/guanying/SIMPLE_Archive/SIMPLE-db/scripts/spectra_convert/data_target_WISE1810_comb_Jun2021_YJ_STD_bb.txt",
+       "/Users/guanying/SIMPLE_Archive/SIMPLE-db/scripts/spectra_convert/WISE1810m10_OB0001_R1000R_06Sept2020.txt"]
+
+for filename in file:
+
+    # Read the data, there are 2 columns: wavelength and flux
+    data = np.loadtxt(filename, comments="#")
+    if "data_target_" in filename:
+        print("Reading", filename)
+
+        # select the range limit from tested file handle_WISE1810_txt.ipynb
+        wave = data[224:1380, 0] * u.AA
+        flux = data[224:1380, 1] * (u.erg / u.cm**2 / u.s / u.AA)
+
+        # create spectrum object
+        spectrum = Spectrum1D(spectral_axis=wave, flux=flux)
+
+        # convert spectrum
+        header = Header()
+        header.set('DATE-OBS', "2021-06-01T00:00:00")
+        header.set('INSTRUME', "EMIR")
+
+
+    else:
+        print(f"Reading {filename}\n")
+
+        wave = data[:1870, 0] * u.AA
+        flux = data[:1870, 1] * (u.erg / u.cm**2 / u.s / u.AA)
+
+        # create spectrum object
+        spectrum = Spectrum1D(spectral_axis=wave, flux=flux)
+
+        # convert spectrum
+        header = Header()
+        header.set('DATE-OBS', "2020-09-06T00:00:00")
+        header.set('INSTRUME', "OSIRIS")
+
+
+    # --- modify the following header to both spectrum --- ##
+    header.set('OBJECT', "CWISEP J181006.00-101001.1")
+    header.set('BUNIT', "erg / (cm2 s Angstrom)")
+    header.set('TELESCOP', "GTC")
+    header.set('VOREF', "2022A&A...663A..84L")
+    header.set('TITLE', "Physical properties and trigonometric distance of the peculiar dwarf WISE J181005.5 101002.3")
+    header.set("AUTHOR", "N. Lodieu, et al.")
+    header.set("CONTRIB1", "Guan Ying Goh, converted to SIMPLE format")
+
+
+    # get RA and DEC from Simbad
+    try:
+        result = Simbad.query_object("CWISEP J181006.00-101001.1")
+        header["RA_TARG"] = result[0]["ra"]
+        header["DEC_TARG"] = result[0]["dec"]
+    except Exception as e:
+        print(f"Error getting ra/deg: {e}")
+
+    header.set('RA_TARG', 272.52575) # got from simbad
+    header.set('DEC_TARG', -10.16675)
+
+    spectrum.meta["header"] = header
+
+    # add Spectrum to FITS file
+    output_file = os.path.splitext(filename)[0] + ".fits"
+    spectrum.write(output_file, format="tabular-fits", overwrite=True)
+    print(f"Wrote to fits format")
+
+output_dir = os.path.dirname(output_file)
+new_files = os.listdir(output_dir)
+new_files = [f for f in new_files if f.endswith('.fits')]
+
+# check header and spectrum
+for file in new_files:
+    new_file_path = os.path.join(output_dir, file)
+    spectrum = Spectrum1D.read(new_file_path, format="tabular-fits")
+
+    header = spectrum.meta["header"]
+    add_wavelength_keywords(header, spectrum.spectral_axis)
+    spectrum.meta["header"] = header
+    spectrum.write(new_file_path, format="tabular-fits", overwrite=True)
+
+    if check_spectrum_plottable(spectrum, show_plot=True):
+        print(f"{file} is plottable.")

tests/test_data_spectra.py

@@ -4,14 +4,14 @@
 
 def test_spectra_count(db):
     n_spectra = db.query(db.Spectra).count()
-    assert n_spectra == 1606, f"found {n_spectra} sources"
+    assert n_spectra == 1608, f"found {n_spectra} sources"
 
 
 @pytest.mark.parametrize(
     ("regime", "n_spectra"),
     [
-        ("optical", 743),
-        ("nir", 636),
+        ("optical", 744),
+        ("nir", 637),
         ("mir", 227),
         ("unknown", 0),
     ],