From 7311fc1d80f164319b43a62161f3977c24d21324 Mon Sep 17 00:00:00 2001 From: Dhanur Sharma Date: Tue, 8 Apr 2025 11:11:04 -0500 Subject: [PATCH 1/2] Added science_at_gsfc to the list of collections for TDAMM --- sde_collections/models/collection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sde_collections/models/collection.py b/sde_collections/models/collection.py index 097c5ce7..a604e60a 100644 --- a/sde_collections/models/collection.py +++ b/sde_collections/models/collection.py @@ -694,6 +694,7 @@ def queue_necessary_classifications(self): tdamm_collections = [ "imagine_the_universe", "physics_of_the_cosmos", + "science_at_gsfc", "stsci_space_telescope_science_institute", ] if self.config_folder in tdamm_collections: From e774665d98f8bd173fedadb35d93d07ea30067b2 Mon Sep 17 00:00:00 2001 From: Dhanur Sharma Date: Tue, 8 Apr 2025 13:51:54 -0500 Subject: [PATCH 2/2] Added useage docstring to export_urls_to_csv --- .../management/commands/export_urls_to_csv.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/sde_collections/management/commands/export_urls_to_csv.py b/sde_collections/management/commands/export_urls_to_csv.py index 6e5c332d..d01f9d2a 100644 --- a/sde_collections/management/commands/export_urls_to_csv.py +++ b/sde_collections/management/commands/export_urls_to_csv.py @@ -2,6 +2,39 @@ docker-compose -f local.yml run --rm django python manage.py export_urls_to_csv \ --output physics_of_the_cosmos.csv --collections physics_of_the_cosmos + This management command allows you to export URLs from DumpUrl, DeltaUrl, or CuratedUrl models + to a CSV file for analysis or backup purposes. The export can be filtered by collections and + can optionally include full text content. + + Examples: + # Export all CuratedUrls (default behavior) + docker-compose -f local.yml run --rm django python manage.py export_urls_to_csv + + # Export DeltaUrls for specific collections + docker-compose -f local.yml run --rm django python manage.py export_urls_to_csv \ + --model DeltaUrl --collections collection1 collection2 + + # Export with full text content included + docker-compose -f local.yml run --rm django python manage.py export_urls_to_csv \ + --full_text + + # Specify custom output filename + docker-compose -f local.yml run --rm django python manage.py export_urls_to_csv \ + --output custom_name.csv + + # Export in larger batches for performance + docker-compose -f local.yml run --rm django python manage.py export_urls_to_csv \ + --batch-size 5000 + + All exports are saved to the 'csv_exports' directory which is created if it doesn't exist. + + Available options: + --model: Model to export (DumpUrl, DeltaUrl, or CuratedUrl) + --collections: List of collection config_folders to filter by. Separated by spaces. + --output: Output filename (saved in csv_exports directory) + --batch-size: Number of records to process in each batch + --full_text: Include full text content in the export + """ import csv