diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fdc59e8..f334069b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -137,3 +137,14 @@ For each PR made, an entry should be added to this changelog. It should contain - Description: The feedback form API was throwing CORS errors and to rectify that, we need to add the apt https link for sde-lrm. - Changes: - Added `https://sde-lrm.nasa-impact.net` to `CORS_ALLOWED_ORIGINS` in the base settings. + +- 1252-document-type-filter-not-working-in-delta-urls-page + - Description: Fixed document type filtering functionality in the "Document Type Patterns" tab in Delta URLs page. + - Changes: + - Added a new event listener to the Document Type Patterns dropdown to trigger the filtering of the table results based on the selected value. + +- 1251-column-sorting-issue-curated-urls-count-sorts-by-delta-urls-count + - Description: Fixed incorrect sorting behavior in Collections table where sorting by Curated URLs column was not working as expected. + - Changes: + - Added `data-order` attribute to URL count columns for proper numeric sorting + - Updated SearchPane comparisons to use `@data-order` values instead of string-based loose equality checks to ensure correct numeric filtering diff --git a/config/celery.py b/config/celery.py index 1ab83cb9..465e6fb7 100644 --- a/config/celery.py +++ b/config/celery.py @@ -2,7 +2,6 @@ import os from celery import Celery -from celery.schedules import crontab # Set the default Django settings module os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") @@ -14,11 +13,3 @@ # Load task modules from all registered Django app configs app.autodiscover_tasks() - -app.conf.beat_schedule = { - "process-inference-queue": { - "task": "inference.tasks.process_inference_job_queue", - # Only run between 6pm and 7am - "schedule": crontab(minute="*/5", hour="18-23,0-6"), - }, -} diff --git a/inference/__init__.py b/inference/__init__.py index e69de29b..d1fe6f78 100644 --- a/inference/__init__.py +++ b/inference/__init__.py @@ -0,0 +1 @@ +default_app_config = "inference.apps.InferenceConfig" diff --git a/inference/apps.py b/inference/apps.py index 99523aba..657933cb 100644 --- a/inference/apps.py +++ b/inference/apps.py @@ -5,3 +5,6 @@ class InferenceConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "inference" verbose_name = "Inference" + + def ready(self): + import inference.signals # noqa F401 diff --git a/inference/signals.py b/inference/signals.py new file mode 100644 index 00000000..70d0dd39 --- /dev/null +++ b/inference/signals.py @@ -0,0 +1,24 @@ +from django.db.models.signals import post_migrate +from django.dispatch import receiver + + +@receiver(post_migrate) +def create_periodic_tasks(sender, **kwargs): + if sender.name == "inference": + from django_celery_beat.models import CrontabSchedule, PeriodicTask + + # Create schedule for every 5 minutes between 6pm-7am + crontab, _ = CrontabSchedule.objects.get_or_create( + minute="*/5", + hour="18-23,0-6", + day_of_week="*", + day_of_month="*", + month_of_year="*", + ) + + # Create the periodic task if it doesn't exist + PeriodicTask.objects.get_or_create( + crontab=crontab, + name="Process inference queue (6pm-7am)", + task="inference.tasks.process_inference_job_queue", + ) diff --git a/inference/tests/test_batch.py b/inference/tests/test_batch.py index cce016e7..b6d10963 100644 --- a/inference/tests/test_batch.py +++ b/inference/tests/test_batch.py @@ -36,7 +36,7 @@ def mock_url_large(self): """Returns a mock URL object with large text content""" url = Mock() url.id = 2 - url.scraped_text = "X" * 10010 # Exceeds default max size + url.scraped_text = "X" * 12000 # Exceeds default max size url.scraped_title = "Large Content Page" url.url = "https://example.com/large-page" return url @@ -191,7 +191,7 @@ def test_iter_url_batches_mix_normal_and_oversized(self, processor): # Normal URL url1 = Mock(id=1, scraped_text="X" * 2000, scraped_title="Title 1", url="https://example.com/1") # Oversized URL - url2 = Mock(id=2, scraped_text="X" * 11000, scraped_title="Title 2", url="https://example.com/2") + url2 = Mock(id=2, scraped_text="X" * 15000, scraped_title="Title 2", url="https://example.com/2") # Another normal URL url3 = Mock(id=3, scraped_text="X" * 3000, scraped_title="Title 3", url="https://example.com/3") diff --git a/sde_indexing_helper/static/js/collection_list.js b/sde_indexing_helper/static/js/collection_list.js index 78fd4894..7eb5c7bf 100644 --- a/sde_indexing_helper/static/js/collection_list.js +++ b/sde_indexing_helper/static/js/collection_list.js @@ -138,43 +138,47 @@ let table = $("#collection_table").DataTable({ { label: "0 URLs", value: function (rowData, rowIdx) { - return $(rowData[COLUMNS.DELTA_URLS]).text() == 0; + return parseInt(rowData[COLUMNS.DELTA_URLS]['@data-order']) === 0; }, }, { label: "1 solo URL", value: function (rowData, rowIdx) { - return $(rowData[COLUMNS.DELTA_URLS]).text() == 1; + return parseInt(rowData[COLUMNS.DELTA_URLS]['@data-order']) === 1; }, }, { label: "1 to 100 URLs", value: function (rowData, rowIdx) { - return $(rowData[COLUMNS.DELTA_URLS]).text() <= 100 && $(rowData[COLUMNS.DELTA_URLS]).text() > 1; + const value = parseInt(rowData[COLUMNS.DELTA_URLS]['@data-order']); + return value > 1 && value <= 100; }, }, { label: "100 to 1,000 URLs", value: function (rowData, rowIdx) { - return $(rowData[COLUMNS.DELTA_URLS]).text() <= 1000 && $(rowData[COLUMNS.DELTA_URLS]).text() > 100; + const value = parseInt(rowData[COLUMNS.DELTA_URLS]['@data-order']); + return value > 100 && value <= 1000; }, }, { label: "1,000 to 10,000 URLs", value: function (rowData, rowIdx) { - return $(rowData[COLUMNS.DELTA_URLS]).text() <= 10000 && $(rowData[COLUMNS.DELTA_URLS]).text() > 1000; + const value = parseInt(rowData[COLUMNS.DELTA_URLS]['@data-order']); + return value > 1000 && value <= 10000; }, }, { label: "10,000 to 100,000 URLs", value: function (rowData, rowIdx) { - return $(rowData[COLUMNS.DELTA_URLS]).text() <= 100000 && $(rowData[COLUMNS.DELTA_URLS]).text() > 10000; + const value = parseInt(rowData[COLUMNS.DELTA_URLS]['@data-order']); + return value > 10000 && value <= 100000; }, }, { label: "Over 100,000 URLs", value: function (rowData, rowIdx) { - return $(rowData[COLUMNS.DELTA_URLS]).text() > 100000; + return parseInt(rowData[COLUMNS.DELTA_URLS]['@data-order']) > 100000; }, }, ], @@ -189,43 +193,47 @@ let table = $("#collection_table").DataTable({ { label: "0 URLs", value: function (rowData, rowIdx) { - return $(rowData[COLUMNS.CURATED_URLS]).text() == 0; + return parseInt(rowData[COLUMNS.CURATED_URLS]['@data-order']) === 0; }, }, { label: "1 solo URL", value: function (rowData, rowIdx) { - return $(rowData[COLUMNS.CURATED_URLS]).text() == 1; + return parseInt(rowData[COLUMNS.CURATED_URLS]['@data-order']) === 1; }, }, { label: "1 to 100 URLs", value: function (rowData, rowIdx) { - return $(rowData[COLUMNS.CURATED_URLS]).text() <= 100 && $(rowData[COLUMNS.CURATED_URLS]).text() > 1; + const value = parseInt(rowData[COLUMNS.CURATED_URLS]['@data-order']); + return value > 1 && value <= 100; }, }, { label: "100 to 1,000 URLs", value: function (rowData, rowIdx) { - return $(rowData[COLUMNS.CURATED_URLS]).text() <= 1000 && $(rowData[COLUMNS.CURATED_URLS]).text() > 100; + const value = parseInt(rowData[COLUMNS.CURATED_URLS]['@data-order']); + return value > 100 && value <= 1000; }, }, { label: "1,000 to 10,000 URLs", value: function (rowData, rowIdx) { - return $(rowData[COLUMNS.CURATED_URLS]).text() <= 10000 && $(rowData[COLUMNS.CURATED_URLS]).text() > 1000; + const value = parseInt(rowData[COLUMNS.CURATED_URLS]['@data-order']); + return value > 1000 && value <= 10000; }, }, { label: "10,000 to 100,000 URLs", value: function (rowData, rowIdx) { - return $(rowData[COLUMNS.CURATED_URLS]).text() <= 100000 && $(rowData[COLUMNS.CURATED_URLS]).text() > 10000; + const value = parseInt(rowData[COLUMNS.CURATED_URLS]['@data-order']); + return value > 10000 && value <= 100000; }, }, { label: "Over 100,000 URLs", value: function (rowData, rowIdx) { - return $(rowData[COLUMNS.CURATED_URLS]).text() > 100000; + return parseInt(rowData[COLUMNS.CURATED_URLS]['@data-order']) > 100000; }, }, ], diff --git a/sde_indexing_helper/static/js/delta_url_list.js b/sde_indexing_helper/static/js/delta_url_list.js index 85a92093..c961a981 100644 --- a/sde_indexing_helper/static/js/delta_url_list.js +++ b/sde_indexing_helper/static/js/delta_url_list.js @@ -881,6 +881,11 @@ function initializeDataTable() { $("#deltaDocTypeMatchPatternFilter").on("beforeinput", function (val) { document_type_patterns_table.columns(0).search(this.value).draw(); }); + + $("#document-type-patterns-dropdown-2").on("change", function () { + document_type_patterns_table.columns(2).search(this.value).draw(); + }); + } var division_patterns_table = $("#division_patterns_table").DataTable({ diff --git a/sde_indexing_helper/templates/sde_collections/collection_list.html b/sde_indexing_helper/templates/sde_collections/collection_list.html index 738ece8f..1930022e 100644 --- a/sde_indexing_helper/templates/sde_collections/collection_list.html +++ b/sde_indexing_helper/templates/sde_collections/collection_list.html @@ -151,14 +151,14 @@