Staging deployment - Fix Celery schedule and DocumentType filter

CHANGELOG.md

@@ -137,3 +137,14 @@ For each PR made, an entry should be added to this changelog. It should contain
   - Description: The feedback form API was throwing CORS errors and to rectify that, we need to add the apt https link for sde-lrm.
   - Changes:
     - Added `https://sde-lrm.nasa-impact.net` to `CORS_ALLOWED_ORIGINS` in the base settings.
+
+- 1252-document-type-filter-not-working-in-delta-urls-page
+  - Description: Fixed document type filtering functionality in the "Document Type Patterns" tab in Delta URLs page.
+  - Changes:
+    - Added a new event listener to the Document Type Patterns dropdown to trigger the filtering of the table results based on the selected value.
+
+- 1251-column-sorting-issue-curated-urls-count-sorts-by-delta-urls-count
+  - Description: Fixed incorrect sorting behavior in Collections table where sorting by Curated URLs column was not working as expected.
+  - Changes:
+    - Added `data-order` attribute to URL count columns for proper numeric sorting
+    - Updated SearchPane comparisons to use `@data-order` values instead of string-based loose equality checks to ensure correct numeric filtering

config/celery.py

@@ -2,7 +2,6 @@
 import os
 
 from celery import Celery
-from celery.schedules import crontab
 
 # Set the default Django settings module
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
@@ -14,11 +13,3 @@
 
 # Load task modules from all registered Django app configs
 app.autodiscover_tasks()
-
-app.conf.beat_schedule = {
-    "process-inference-queue": {
-        "task": "inference.tasks.process_inference_job_queue",
-        # Only run between 6pm and 7am
-        "schedule": crontab(minute="*/5", hour="18-23,0-6"),
-    },
-}

inference/__init__.py

@@ -0,0 +1 @@
+default_app_config = "inference.apps.InferenceConfig"

inference/apps.py

@@ -5,3 +5,6 @@ class InferenceConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
     name = "inference"
     verbose_name = "Inference"
+
+    def ready(self):
+        import inference.signals  # noqa F401

inference/signals.py

@@ -0,0 +1,24 @@
+from django.db.models.signals import post_migrate
+from django.dispatch import receiver
+
+
+@receiver(post_migrate)
+def create_periodic_tasks(sender, **kwargs):
+    if sender.name == "inference":
+        from django_celery_beat.models import CrontabSchedule, PeriodicTask
+
+        # Create schedule for every 5 minutes between 6pm-7am
+        crontab, _ = CrontabSchedule.objects.get_or_create(
+            minute="*/5",
+            hour="18-23,0-6",
+            day_of_week="*",
+            day_of_month="*",
+            month_of_year="*",
+        )
+
+        # Create the periodic task if it doesn't exist
+        PeriodicTask.objects.get_or_create(
+            crontab=crontab,
+            name="Process inference queue (6pm-7am)",
+            task="inference.tasks.process_inference_job_queue",
+        )

inference/tests/test_batch.py

@@ -36,7 +36,7 @@ def mock_url_large(self):
         """Returns a mock URL object with large text content"""
         url = Mock()
         url.id = 2
-        url.scraped_text = "X" * 10010  # Exceeds default max size
+        url.scraped_text = "X" * 12000  # Exceeds default max size
         url.scraped_title = "Large Content Page"
         url.url = "https://example.com/large-page"
         return url
@@ -191,7 +191,7 @@ def test_iter_url_batches_mix_normal_and_oversized(self, processor):
         # Normal URL
         url1 = Mock(id=1, scraped_text="X" * 2000, scraped_title="Title 1", url="https://example.com/1")
         # Oversized URL
-        url2 = Mock(id=2, scraped_text="X" * 11000, scraped_title="Title 2", url="https://example.com/2")
+        url2 = Mock(id=2, scraped_text="X" * 15000, scraped_title="Title 2", url="https://example.com/2")
         # Another normal URL
         url3 = Mock(id=3, scraped_text="X" * 3000, scraped_title="Title 3", url="https://example.com/3")
 

sde_indexing_helper/static/js/collection_list.js

@@ -138,43 +138,47 @@ let table = $("#collection_table").DataTable({
           {
             label: "0 URLs",
             value: function (rowData, rowIdx) {
-              return $(rowData[COLUMNS.DELTA_URLS]).text() == 0;
+              return parseInt(rowData[COLUMNS.DELTA_URLS]['@data-order']) === 0;
             },
           },
           {
             label: "1 solo URL",
             value: function (rowData, rowIdx) {
-              return $(rowData[COLUMNS.DELTA_URLS]).text() == 1;
+              return parseInt(rowData[COLUMNS.DELTA_URLS]['@data-order']) === 1;
             },
           },
           {
             label: "1 to 100 URLs",
             value: function (rowData, rowIdx) {
-              return $(rowData[COLUMNS.DELTA_URLS]).text() <= 100 && $(rowData[COLUMNS.DELTA_URLS]).text() > 1;
+              const value = parseInt(rowData[COLUMNS.DELTA_URLS]['@data-order']);
+              return value > 1 && value <= 100;
             },
           },
           {
             label: "100 to 1,000 URLs",
             value: function (rowData, rowIdx) {
-              return $(rowData[COLUMNS.DELTA_URLS]).text() <= 1000 && $(rowData[COLUMNS.DELTA_URLS]).text() > 100;
+              const value = parseInt(rowData[COLUMNS.DELTA_URLS]['@data-order']);
+              return value > 100 && value <= 1000;
             },
           },
           {
             label: "1,000 to 10,000 URLs",
             value: function (rowData, rowIdx) {
-              return $(rowData[COLUMNS.DELTA_URLS]).text() <= 10000 && $(rowData[COLUMNS.DELTA_URLS]).text() > 1000;
+              const value = parseInt(rowData[COLUMNS.DELTA_URLS]['@data-order']);
+              return value > 1000 && value <= 10000;
             },
           },
           {
             label: "10,000 to 100,000 URLs",
             value: function (rowData, rowIdx) {
-              return $(rowData[COLUMNS.DELTA_URLS]).text() <= 100000 && $(rowData[COLUMNS.DELTA_URLS]).text() > 10000;
+              const value = parseInt(rowData[COLUMNS.DELTA_URLS]['@data-order']);
+              return value > 10000 && value <= 100000;
             },
           },
           {
             label: "Over 100,000 URLs",
             value: function (rowData, rowIdx) {
-              return $(rowData[COLUMNS.DELTA_URLS]).text() > 100000;
+              return parseInt(rowData[COLUMNS.DELTA_URLS]['@data-order']) > 100000;
             },
           },
         ],
@@ -189,43 +193,47 @@ let table = $("#collection_table").DataTable({
           {
             label: "0 URLs",
             value: function (rowData, rowIdx) {
-              return $(rowData[COLUMNS.CURATED_URLS]).text() == 0;
+              return parseInt(rowData[COLUMNS.CURATED_URLS]['@data-order']) === 0;
             },
           },
           {
             label: "1 solo URL",
             value: function (rowData, rowIdx) {
-              return $(rowData[COLUMNS.CURATED_URLS]).text() == 1;
+              return parseInt(rowData[COLUMNS.CURATED_URLS]['@data-order']) === 1;
             },
           },
           {
             label: "1 to 100 URLs",
             value: function (rowData, rowIdx) {
-              return $(rowData[COLUMNS.CURATED_URLS]).text() <= 100 && $(rowData[COLUMNS.CURATED_URLS]).text() > 1;
+              const value = parseInt(rowData[COLUMNS.CURATED_URLS]['@data-order']);
+              return value > 1 && value <= 100;
             },
           },
           {
             label: "100 to 1,000 URLs",
             value: function (rowData, rowIdx) {
-              return $(rowData[COLUMNS.CURATED_URLS]).text() <= 1000 && $(rowData[COLUMNS.CURATED_URLS]).text() > 100;
+              const value = parseInt(rowData[COLUMNS.CURATED_URLS]['@data-order']);
+              return value > 100 && value <= 1000;
             },
           },
           {
             label: "1,000 to 10,000 URLs",
             value: function (rowData, rowIdx) {
-              return $(rowData[COLUMNS.CURATED_URLS]).text() <= 10000 && $(rowData[COLUMNS.CURATED_URLS]).text() > 1000;
+              const value = parseInt(rowData[COLUMNS.CURATED_URLS]['@data-order']);
+              return value > 1000 && value <= 10000;
             },
           },
           {
             label: "10,000 to 100,000 URLs",
             value: function (rowData, rowIdx) {
-              return $(rowData[COLUMNS.CURATED_URLS]).text() <= 100000 && $(rowData[COLUMNS.CURATED_URLS]).text() > 10000;
+              const value = parseInt(rowData[COLUMNS.CURATED_URLS]['@data-order']);
+              return value > 10000 && value <= 100000;
             },
           },
           {
             label: "Over 100,000 URLs",
             value: function (rowData, rowIdx) {
-              return $(rowData[COLUMNS.CURATED_URLS]).text() > 100000;
+              return parseInt(rowData[COLUMNS.CURATED_URLS]['@data-order']) > 100000;
             },
           },
         ],

sde_indexing_helper/static/js/delta_url_list.js

@@ -881,6 +881,11 @@ function initializeDataTable() {
   $("#deltaDocTypeMatchPatternFilter").on("beforeinput", function (val) {
     document_type_patterns_table.columns(0).search(this.value).draw();
   });
+
+  $("#document-type-patterns-dropdown-2").on("change", function () {
+    document_type_patterns_table.columns(2).search(this.value).draw();
+  });
+
 }
 
 var division_patterns_table = $("#division_patterns_table").DataTable({

sde_indexing_helper/templates/sde_collections/collection_list.html

@@ -151,14 +151,14 @@ <h2 class="title">Welcome back!</h2>
                     <td class="whiteText noBorder">{{ collection.get_division_display }}</td>
 
                     <!-- Delta URLs Column - Shows count and links if >= 0 -->
-                    <td class="noBorder centerAlign">
+                    <td class="noBorder centerAlign" data-order="{{ collection.num_delta_urls }}">
                         <a href=" {% if collection.num_delta_urls >= 0 %} {% url 'sde_collections:delta_urls' collection.pk %} {% endif %} "
                            class="btn btn-sm {% if collection.num_delta_urls >= 0 %}btn-primary {% else %}disabled{% endif %}candidateCount"
                            role="button">{{ collection.num_delta_urls|intcomma }}</a>
                     </td>
 
                     <!-- Curated URLs Column - Shows count and links if >= 0 -->
-                    <td class="noBorder centerAlign">
+                    <td class="noBorder centerAlign" data-order="{{ collection.num_curated_urls }}">
                         <a href=" {% if collection.num_curated_urls >= 0 %} {% url 'sde_collections:delta_urls' collection.pk %} {% endif %} "
                            class="btn btn-sm {% if collection.num_curated_urls >= 0 %}btn-primary {% else %}disabled{% endif %}candidateCount"
                            role="button">{{ collection.num_curated_urls|intcomma }}</a>