From 15f86adeff592fb4bebf9b28d5a5192d9333639a Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 21 Oct 2025 20:51:35 +0000 Subject: [PATCH 01/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index b405ffc..f24f5f7 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-10-21 20:47:06 UTC -**Git Commit:** e46b76948e36c2eb4fdebcd64af12bef03300382 +**Test Date:** 2025-10-21 20:51:35 UTC +**Git Commit:** 43a0849199fb8f67f011baa6ee4ab44e67d05774 **Branch:** dev -**Workflow Run:** 18697165949 +**Workflow Run:** 18697277881 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 1.5526 seconds -- **VFB_00101567 Query Time**: 1.1230 seconds -- **Total Query Time**: 2.6756 seconds +- **FBbt_00003748 Query Time**: 2.2578 seconds +- **VFB_00101567 Query Time**: 0.6434 seconds +- **Total Query Time**: 2.9012 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-10-21 20:47:06 UTC* +*Last updated: 2025-10-21 20:51:35 UTC* From 25d81a9cb3953cb780f1ff9f0116a11d2fd1e4c4 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Tue, 4 Nov 2025 20:22:38 +0000 Subject: [PATCH 02/70] Implement NeuronsPartHere query and corresponding test script --- src/vfbquery/solr_result_cache.py | 14 +- src/vfbquery/vfb_queries.py | 211 ++++++++++++++++++++++++++++++ test_neurons_part_here.py | 84 ++++++++++++ 3 files changed, 308 insertions(+), 1 deletion(-) create mode 100644 test_neurons_part_here.py diff --git a/src/vfbquery/solr_result_cache.py b/src/vfbquery/solr_result_cache.py index c377ee1..086da9e 100644 --- a/src/vfbquery/solr_result_cache.py +++ b/src/vfbquery/solr_result_cache.py @@ -635,7 +635,19 @@ def wrapper(*args, **kwargs): result = func(*args, **kwargs) # Cache the result asynchronously to avoid blocking - if result: + # Handle DataFrame, dict, and other result types properly + result_is_valid = False + if result is not None: + if hasattr(result, 'empty'): # DataFrame + result_is_valid = not result.empty + elif isinstance(result, dict): + result_is_valid = bool(result) + elif isinstance(result, (list, str)): + result_is_valid = len(result) > 0 + else: + result_is_valid = True + + if result_is_valid: # Validate result before caching for term_info if query_type == 'term_info': if (result and isinstance(result, dict) and diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 2147b86..d4314a6 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -658,6 +658,15 @@ def term_info_parse_object(results, short_form): q = NeuronInputsTo_to_schema(termInfo["Name"], {"neuron_short_form": vfbTerm.term.core.short_form}) queries.append(q) + # NeuronsPartHere query - for Class+Anatomy terms (synaptic neuropils, etc.) + # Matches XMI criteria: Class + Synaptic_neuropil, or other anatomical regions + if contains_all_tags(termInfo["SuperTypes"], ["Class"]) and ( + "Synaptic_neuropil" in termInfo["SuperTypes"] or + "Anatomy" in termInfo["SuperTypes"] + ): + q = NeuronsPartHere_to_schema(termInfo["Name"], {"short_form": vfbTerm.term.core.short_form}) + queries.append(q) + # Add Publications to the termInfo object if vfbTerm.pubs and len(vfbTerm.pubs) > 0: publications = [] @@ -824,6 +833,30 @@ def ListAllAvailableImages_to_schema(name, take_default): return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) +def NeuronsPartHere_to_schema(name, take_default): + """ + Schema for NeuronsPartHere query. + Finds neuron classes that have some part overlapping with the specified anatomical region. + + Matching criteria from XMI: + - Class + Synaptic_neuropil (types.1 + types.5) + - Additional type matches for comprehensive coverage + + Query chain: Owlery subclass query ā†’ process ā†’ SOLR + OWL query: "Neuron and overlaps some $ID" + """ + query = "NeuronsPartHere" + label = f"Neurons with some part in {name}" + function = "get_neurons_with_part_in" + takes = { + "short_form": {"$and": ["Class", "Anatomy"]}, + "default": take_default, + } + preview = 5 + preview_columns = ["id", "label", "tags", "thumbnail"] + + return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) + def serialize_solr_output(results): # Create a copy of the document and remove Solr-specific fields doc = dict(results.docs[0]) @@ -1544,6 +1577,184 @@ def contains_all_tags(lst: List[str], tags: List[str]) -> bool: """ return all(tag in lst for tag in tags) +@with_solr_cache('neurons_part_here') +def get_neurons_with_part_in(short_form: str, return_dataframe=True, limit: int = -1): + """ + Retrieves neuron classes that have some part overlapping with the specified anatomical region. + + This implements the NeuronsPartHere query from the VFB XMI specification. + Query chain (from XMI): Owlery (Index 1) ā†’ Process ā†’ SOLR (Index 3) + OWL query: "'Neuron' that 'overlaps' some ''" + + :param short_form: short form of the anatomical region (Class) + :param return_dataframe: Returns pandas dataframe if true, otherwise returns formatted dict + :param limit: maximum number of results to return (default -1, returns all results) + :return: Neuron classes with parts in the specified region + """ + + try: + # Step 1: Query Owlery for neuron classes that overlap this anatomical region + # This uses the OWL reasoner to find all neuron subclasses matching the pattern + neuron_class_ids = vc.vfb.oc.get_subclasses( + query=f"'Neuron' that 'overlaps' some '{short_form}'", + query_by_label=True, + verbose=False + ) + + if not neuron_class_ids: + # No neurons found - return empty results + if return_dataframe: + return pd.DataFrame() + return { + "headers": _get_neurons_part_here_headers(), + "rows": [], + "count": 0 + } + + # Apply limit if specified (before SOLR query to save processing) + if limit != -1 and limit > 0: + neuron_class_ids = neuron_class_ids[:limit] + + total_count = len(neuron_class_ids) + + # Step 2: Query SOLR directly for just the anat_query field + # For Class terms (neuron classes), the field is 'anat_query' not 'anat_image_query' + # This matches the original VFBquery pattern and contains all result row metadata + # This is much faster than loading full term_info for each neuron + rows = [] + for neuron_id in neuron_class_ids: + try: + # Query SOLR with fl=anat_query to get only the result table data + # This is the same field used in the original VFBquery implementation + results = vfb_solr.search( + q=f'id:{neuron_id}', + fl='anat_query', + rows=1 + ) + + if results.hits > 0 and results.docs and 'anat_query' in results.docs[0]: + # Parse the anat_query JSON string + anat_query_str = results.docs[0]['anat_query'][0] + anat_data = json.loads(anat_query_str) + + # Extract core term information + term_core = anat_data.get('term', {}).get('core', {}) + neuron_short_form = term_core.get('short_form', neuron_id) + + # Extract label (prefer symbol over label, matching Neo4j behavior) + label_text = term_core.get('label', 'Unknown') + if term_core.get('symbol') and len(term_core.get('symbol', '')) > 0: + label_text = term_core.get('symbol') + # Decode URL-encoded strings from SOLR + from urllib.parse import unquote + label_text = unquote(label_text) + + # Extract tags from unique_facets + tags = '|'.join(term_core.get('unique_facets', [])) + + # Extract thumbnail from anatomy_channel_image if available + thumbnail = '' + anatomy_images = anat_data.get('anatomy_channel_image', []) + if anatomy_images and len(anatomy_images) > 0: + # Get the first anatomy channel image (example instance) + first_img = anatomy_images[0] + channel_image = first_img.get('channel_image', {}) + image_info = channel_image.get('image', {}) + thumbnail_url = image_info.get('image_thumbnail', '') + + if thumbnail_url: + # Convert to HTTPS and use non-transparent version + thumbnail_url = thumbnail_url.replace('http://', 'https://').replace('thumbnailT.png', 'thumbnail.png') + + # Format thumbnail markdown with template info + template_anatomy = image_info.get('template_anatomy', {}) + if template_anatomy: + template_label = template_anatomy.get('symbol') or template_anatomy.get('label', '') + template_label = unquote(template_label) + # Get the anatomy info for alt text + anatomy_info = first_img.get('anatomy', {}) + anatomy_label = anatomy_info.get('symbol') or anatomy_info.get('label', label_text) + anatomy_label = unquote(anatomy_label) + alt_text = f"{anatomy_label} aligned to {template_label}" + thumbnail = f"[![{alt_text}]({thumbnail_url} '{alt_text}')]({neuron_short_form})" + + # Extract source information from xrefs if available + source = '' + source_id = '' + xrefs = anat_data.get('xrefs', []) + if xrefs and len(xrefs) > 0: + # Get the first data source xref + for xref in xrefs: + if xref.get('is_data_source', False): + site_info = xref.get('site', {}) + site_label = site_info.get('symbol') or site_info.get('label', '') + site_short_form = site_info.get('short_form', '') + if site_label and site_short_form: + source = f"[{site_label}]({site_short_form})" + + accession = xref.get('accession', '') + link_base = xref.get('link_base', '') + if accession and link_base: + source_id = f"[{accession}]({link_base}{accession})" + break + + # Build row matching expected format + row = { + 'id': neuron_short_form, + 'label': f"[{label_text}]({neuron_short_form})", + 'tags': tags, + 'source': source, + 'source_id': source_id, + 'thumbnail': thumbnail + } + rows.append(row) + + except Exception as e: + print(f"Error fetching SOLR data for {neuron_id}: {e}") + continue + + # Convert to DataFrame if requested + if return_dataframe: + df = pd.DataFrame(rows) + # Apply markdown encoding + columns_to_encode = ['label', 'thumbnail'] + df = encode_markdown_links(df, columns_to_encode) + return df + + # Convert to expected format with proper headers + formatted_results = { + "headers": _get_neurons_part_here_headers(), + "rows": rows, + "count": total_count + } + + return formatted_results + + except Exception as e: + print(f"Error in get_neurons_with_part_in: {e}") + import traceback + traceback.print_exc() + # Return empty results with proper structure + if return_dataframe: + return pd.DataFrame() + return { + "headers": _get_neurons_part_here_headers(), + "rows": [], + "count": 0 + } + +def _get_neurons_part_here_headers(): + """Return standard headers for get_neurons_with_part_in results""" + return { + "id": {"title": "Add", "type": "selection_id", "order": -1}, + "label": {"title": "Name", "type": "markdown", "order": 0, "sort": {0: "Asc"}}, + "tags": {"title": "Tags", "type": "tags", "order": 2}, + "source": {"title": "Data Source", "type": "metadata", "order": 3}, + "source_id": {"title": "Data Source ID", "type": "metadata", "order": 4}, + "thumbnail": {"title": "Thumbnail", "type": "markdown", "order": 9} + } + + def fill_query_results(term_info): for query in term_info['Queries']: # print(f"Query Keys:{query.keys()}") diff --git a/test_neurons_part_here.py b/test_neurons_part_here.py new file mode 100644 index 0000000..4a37e53 --- /dev/null +++ b/test_neurons_part_here.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +""" +Test script for NeuronsPartHere query implementation. +Tests with medulla [FBbt_00003748] which should return 471 results per the screenshot. +""" + +import sys +import os + +# Add src to path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) + +from vfbquery.vfb_queries import get_neurons_with_part_in + +def test_neurons_part_here(): + """Test NeuronsPartHere query with medulla""" + + print("="*80) + print("Testing NeuronsPartHere query with medulla [FBbt_00003748]") + print("Expected: 471 results (from screenshot)") + print("="*80) + print() + + # Test with medulla - should return 471 results + medulla_id = "FBbt_00003748" + + try: + print(f"Querying neurons with parts in medulla ({medulla_id})...") + print() + + # Get results as dataframe + results_df = get_neurons_with_part_in(medulla_id, return_dataframe=True, limit=-1) + + if results_df is not None and not results_df.empty: + count = len(results_df) + print(f"āœ“ SUCCESS: Found {count} neuron classes") + print() + + # Show first few results + print("First 5 results:") + print("-" * 80) + for idx, row in results_df.head(5).iterrows(): + print(f" {idx+1}. {row.get('label', 'N/A')[:60]}") + print(f" ID: {row.get('id', 'N/A')}") + print(f" Tags: {row.get('tags', 'N/A')[:60]}") + print() + + # Verify count matches expected + if count == 471: + print("āœ“āœ“ PERFECT MATCH: Got exactly 471 results as expected!") + elif count > 450 and count < 500: + print(f"āš  CLOSE: Got {count} results (expected 471)") + print(" This might be due to data updates in VFB") + else: + print(f"āš  WARNING: Expected 471 results but got {count}") + + print() + print("=" * 80) + print("QUERY SUCCESSFUL") + print("=" * 80) + return True + + else: + print("āœ— FAILED: No results returned") + print() + print("=" * 80) + print("QUERY FAILED - No results") + print("=" * 80) + return False + + except Exception as e: + print(f"āœ— ERROR: {type(e).__name__}: {e}") + print() + import traceback + traceback.print_exc() + print() + print("=" * 80) + print("QUERY FAILED - Exception occurred") + print("=" * 80) + return False + +if __name__ == "__main__": + success = test_neurons_part_here() + sys.exit(0 if success else 1) From ced69038bd3b0bf61d4bfca045ad060916f2db26 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 4 Nov 2025 20:23:40 +0000 Subject: [PATCH 03/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index f24f5f7..85811a1 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-10-21 20:51:35 UTC -**Git Commit:** 43a0849199fb8f67f011baa6ee4ab44e67d05774 +**Test Date:** 2025-11-04 20:23:40 UTC +**Git Commit:** 25d81a9cb3953cb780f1ff9f0116a11d2fd1e4c4 **Branch:** dev -**Workflow Run:** 18697277881 +**Workflow Run:** 19081787198 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 2.2578 seconds -- **VFB_00101567 Query Time**: 0.6434 seconds -- **Total Query Time**: 2.9012 seconds +- **FBbt_00003748 Query Time**: 1.1860 seconds +- **VFB_00101567 Query Time**: 1.2064 seconds +- **Total Query Time**: 2.3924 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-10-21 20:51:35 UTC* +*Last updated: 2025-11-04 20:23:40 UTC* From ca748cc9eeaf26034544e7805e27a712cd75d797 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Tue, 4 Nov 2025 20:42:19 +0000 Subject: [PATCH 04/70] Increase preview results to 10 in NeuronsPartHere query and add comprehensive test suite for query functionality --- src/test/test_neurons_part_here.py | 204 +++++++++++++++++++++++++++++ src/vfbquery/vfb_queries.py | 2 +- test_neurons_part_here.py | 84 ------------ 3 files changed, 205 insertions(+), 85 deletions(-) create mode 100644 src/test/test_neurons_part_here.py delete mode 100644 test_neurons_part_here.py diff --git a/src/test/test_neurons_part_here.py b/src/test/test_neurons_part_here.py new file mode 100644 index 0000000..e535639 --- /dev/null +++ b/src/test/test_neurons_part_here.py @@ -0,0 +1,204 @@ +import unittest +import sys +import os + +# Add src to path for imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'src')) + +from vfbquery.vfb_queries import get_neurons_with_part_in, get_term_info + + +class NeuronsPartHereTest(unittest.TestCase): + """Test suite for NeuronsPartHere query implementation""" + + def setUp(self): + """Set up test fixtures""" + self.medulla_id = 'FBbt_00003748' + # Expected count based on VFB data (as of test creation) + # Allowing tolerance for data updates + self.expected_count = 471 + self.count_tolerance = 5 # Allow Ā±5 for data updates + + def test_neurons_part_here_returns_results(self): + """Test that NeuronsPartHere query returns results for medulla""" + print("\n" + "=" * 80) + print("Testing NeuronsPartHere query - Basic functionality") + print("=" * 80) + + results_df = get_neurons_with_part_in( + self.medulla_id, + return_dataframe=True, + limit=-1 + ) + + self.assertIsNotNone(results_df, "Results should not be None") + self.assertGreater(len(results_df), 0, "Should return at least one result") + + print(f"āœ“ Query returned {len(results_df)} neuron classes") + + def test_neurons_part_here_result_count(self): + """Test that NeuronsPartHere returns expected number of results for medulla""" + print("\n" + "=" * 80) + print(f"Testing NeuronsPartHere result count (expected ~{self.expected_count})") + print("=" * 80) + + results_df = get_neurons_with_part_in( + self.medulla_id, + return_dataframe=True, + limit=-1 + ) + + actual_count = len(results_df) + count_diff = abs(actual_count - self.expected_count) + + print(f"Expected: {self.expected_count} results") + print(f"Actual: {actual_count} results") + print(f"Difference: {count_diff}") + + # Allow some tolerance for data updates + self.assertLessEqual( + count_diff, + self.count_tolerance, + f"Result count {actual_count} differs from expected {self.expected_count} by more than {self.count_tolerance}" + ) + + if count_diff > 0: + print(f"āš  Count differs by {count_diff} (within tolerance of {self.count_tolerance})") + else: + print(f"āœ“ Exact count match: {actual_count}") + + def test_neurons_part_here_result_structure(self): + """Test that results have the expected structure with required columns""" + print("\n" + "=" * 80) + print("Testing NeuronsPartHere result structure") + print("=" * 80) + + results_df = get_neurons_with_part_in( + self.medulla_id, + return_dataframe=True, + limit=5 + ) + + # Check required columns + required_columns = ['id', 'label', 'tags', 'thumbnail'] + for col in required_columns: + self.assertIn(col, results_df.columns, f"Column '{col}' should be present") + + print(f"āœ“ All required columns present: {', '.join(required_columns)}") + + # Check that we have data in the columns + first_row = results_df.iloc[0] + self.assertIsNotNone(first_row['id'], "ID should not be None") + self.assertIsNotNone(first_row['label'], "Label should not be None") + + print(f"āœ“ Sample result: {first_row['label']}") + + def test_neurons_part_here_has_examples(self): + """Test that neuron class results include example images (thumbnails)""" + print("\n" + "=" * 80) + print("Testing NeuronsPartHere includes example images") + print("=" * 80) + + results_df = get_neurons_with_part_in( + self.medulla_id, + return_dataframe=True, + limit=10 + ) + + # Count how many results have thumbnails + has_thumbnails = results_df['thumbnail'].notna().sum() + total_results = len(results_df) + + print(f"Results with thumbnails: {has_thumbnails}/{total_results}") + + # At least some results should have thumbnails (example instances) + self.assertGreater( + has_thumbnails, + 0, + "At least some neuron classes should have example images" + ) + + # Show example thumbnails + sample_with_thumbnail = results_df[results_df['thumbnail'].notna()].iloc[0] + print(f"\nāœ“ Example with thumbnail:") + print(f" {sample_with_thumbnail['label']}") + print(f" Thumbnail: {sample_with_thumbnail['thumbnail'][:100]}...") + + def test_neurons_part_here_preview_in_term_info(self): + """Test that NeuronsPartHere query appears with preview results in term_info""" + print("\n" + "=" * 80) + print("Testing NeuronsPartHere preview results in term_info") + print("=" * 80) + + term_info = get_term_info(self.medulla_id, preview=True) + + self.assertIsNotNone(term_info, "term_info should not be None") + self.assertIn('Queries', term_info, "term_info should have Queries") + + # Find NeuronsPartHere query + neurons_part_here_query = None + for query in term_info.get('Queries', []): + if query.get('query') == 'NeuronsPartHere': + neurons_part_here_query = query + break + + self.assertIsNotNone( + neurons_part_here_query, + "NeuronsPartHere query should be present in term_info" + ) + + print(f"āœ“ NeuronsPartHere query found") + print(f" Label: {neurons_part_here_query.get('label', 'Unknown')}") + print(f" Preview limit: {neurons_part_here_query.get('preview', 0)}") + + # Check preview results + preview_results = neurons_part_here_query.get('preview_results', {}) + preview_rows = preview_results.get('rows', []) + + self.assertGreater( + len(preview_rows), + 0, + "Preview results should be populated" + ) + + print(f" Preview results: {len(preview_rows)} items") + + # Check that preview results include thumbnails + with_thumbnails = sum(1 for row in preview_rows if row.get('thumbnail', '')) + print(f" Results with example images: {with_thumbnails}/{len(preview_rows)}") + + self.assertGreater( + with_thumbnails, + 0, + "At least some preview results should have example images" + ) + + print(f"\nāœ“ Preview includes example images") + + def test_neurons_part_here_limit_parameter(self): + """Test that the limit parameter works correctly""" + print("\n" + "=" * 80) + print("Testing NeuronsPartHere limit parameter") + print("=" * 80) + + limit = 10 + results_df = get_neurons_with_part_in( + self.medulla_id, + return_dataframe=True, + limit=limit + ) + + actual_count = len(results_df) + + self.assertLessEqual( + actual_count, + limit, + f"Result count {actual_count} should not exceed limit {limit}" + ) + + print(f"āœ“ Limit parameter working: requested {limit}, got {actual_count}") + + +if __name__ == '__main__': + # Run tests with verbose output + unittest.main(verbosity=2) diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index d4314a6..fec80ef 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -852,7 +852,7 @@ def NeuronsPartHere_to_schema(name, take_default): "short_form": {"$and": ["Class", "Anatomy"]}, "default": take_default, } - preview = 5 + preview = 10 # Show 10 preview results with example images preview_columns = ["id", "label", "tags", "thumbnail"] return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) diff --git a/test_neurons_part_here.py b/test_neurons_part_here.py deleted file mode 100644 index 4a37e53..0000000 --- a/test_neurons_part_here.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script for NeuronsPartHere query implementation. -Tests with medulla [FBbt_00003748] which should return 471 results per the screenshot. -""" - -import sys -import os - -# Add src to path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) - -from vfbquery.vfb_queries import get_neurons_with_part_in - -def test_neurons_part_here(): - """Test NeuronsPartHere query with medulla""" - - print("="*80) - print("Testing NeuronsPartHere query with medulla [FBbt_00003748]") - print("Expected: 471 results (from screenshot)") - print("="*80) - print() - - # Test with medulla - should return 471 results - medulla_id = "FBbt_00003748" - - try: - print(f"Querying neurons with parts in medulla ({medulla_id})...") - print() - - # Get results as dataframe - results_df = get_neurons_with_part_in(medulla_id, return_dataframe=True, limit=-1) - - if results_df is not None and not results_df.empty: - count = len(results_df) - print(f"āœ“ SUCCESS: Found {count} neuron classes") - print() - - # Show first few results - print("First 5 results:") - print("-" * 80) - for idx, row in results_df.head(5).iterrows(): - print(f" {idx+1}. {row.get('label', 'N/A')[:60]}") - print(f" ID: {row.get('id', 'N/A')}") - print(f" Tags: {row.get('tags', 'N/A')[:60]}") - print() - - # Verify count matches expected - if count == 471: - print("āœ“āœ“ PERFECT MATCH: Got exactly 471 results as expected!") - elif count > 450 and count < 500: - print(f"āš  CLOSE: Got {count} results (expected 471)") - print(" This might be due to data updates in VFB") - else: - print(f"āš  WARNING: Expected 471 results but got {count}") - - print() - print("=" * 80) - print("QUERY SUCCESSFUL") - print("=" * 80) - return True - - else: - print("āœ— FAILED: No results returned") - print() - print("=" * 80) - print("QUERY FAILED - No results") - print("=" * 80) - return False - - except Exception as e: - print(f"āœ— ERROR: {type(e).__name__}: {e}") - print() - import traceback - traceback.print_exc() - print() - print("=" * 80) - print("QUERY FAILED - Exception occurred") - print("=" * 80) - return False - -if __name__ == "__main__": - success = test_neurons_part_here() - sys.exit(0 if success else 1) From 3ec07112563d414cf07ca12cd2421f524d00047c Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 4 Nov 2025 20:45:51 +0000 Subject: [PATCH 05/70] Update performance test results [skip ci] --- performance.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/performance.md b/performance.md index 85811a1..f2b833b 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-04 20:23:40 UTC -**Git Commit:** 25d81a9cb3953cb780f1ff9f0116a11d2fd1e4c4 +**Test Date:** 2025-11-04 20:45:51 UTC +**Git Commit:** a2ac3daf9e395ffcb8bd56e08da97040d1a6f467 **Branch:** dev -**Workflow Run:** 19081787198 +**Workflow Run:** 19082267123 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 1.1860 seconds -- **VFB_00101567 Query Time**: 1.2064 seconds -- **Total Query Time**: 2.3924 seconds +- **FBbt_00003748 Query Time**: 122.3027 seconds +- **VFB_00101567 Query Time**: 0.7956 seconds +- **Total Query Time**: 123.0983 seconds -šŸŽ‰ **Result**: All performance thresholds met! +āš ļø **Result**: Some performance thresholds exceeded or test failed --- -*Last updated: 2025-11-04 20:23:40 UTC* +*Last updated: 2025-11-04 20:45:51 UTC* From e301f6e6f61f34c6c0dde95e83056da59cb8b30a Mon Sep 17 00:00:00 2001 From: Rob Court Date: Tue, 4 Nov 2025 21:14:03 +0000 Subject: [PATCH 06/70] Add comprehensive reference documentation for VFB queries --- VFB_QUERIES_REFERENCE.md | 706 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 706 insertions(+) create mode 100644 VFB_QUERIES_REFERENCE.md diff --git a/VFB_QUERIES_REFERENCE.md b/VFB_QUERIES_REFERENCE.md new file mode 100644 index 0000000..5746e4b --- /dev/null +++ b/VFB_QUERIES_REFERENCE.md @@ -0,0 +1,706 @@ +# VFB Queries - Comprehensive Reference + +**Last Updated**: November 4, 2025 +**Purpose**: Track all VFB queries from the XMI specification and their conversion status in VFBquery Python implementation + +--- + +## Table of Contents + +1. [Overview](#overview) +2. [Query Information Sources](#query-information-sources) +3. [Query Matching Criteria System](#query-matching-criteria-system) +4. [All VFB Queries - Complete List](#all-vfb-queries---complete-list) +5. [Conversion Status Summary](#conversion-status-summary) +6. [Implementation Patterns](#implementation-patterns) +7. [Next Steps](#next-steps) + +--- + +## Overview + +VFB queries are defined in the XMI specification and expose various ways to query the Virtual Fly Brain knowledge base. Each query: + +- Has a unique identifier (e.g., `NeuronsPartHere`, `ComponentsOf`) +- Targets specific entity types via matching criteria +- Chains through data sources: Owlery (OWL reasoning) ā†’ Neo4j ā†’ SOLR +- Returns structured results with preview capability + +--- + +## Query Information Sources + +### 1. XMI Specification +**Location**: `https://raw.githubusercontent.com/VirtualFlyBrain/geppetto-vfb/master/model/vfb.xmi` + +**What it contains**: +- Complete query definitions (SimpleQuery, CompoundQuery, CompoundRefQuery) +- Query chains (Owlery ā†’ Neo4j ā†’ SOLR processing) +- Matching criteria (which entity types each query applies to) +- Cypher queries for Neo4j +- OWL queries for Owlery reasoning + +### 2. Python Implementation +**Location**: `src/vfbquery/vfb_queries.py` + +**What it contains**: +- Query schema functions (e.g., `NeuronsPartHere_to_schema()`) +- Query execution functions (e.g., `get_neurons_with_part_in()`) +- Result processing and formatting +- SOLR caching integration + +### 3. Schema Documentation +**Location**: `schema.md` + +**What it contains**: +- JSON schema structure for term info +- Query result format specifications +- Preview column definitions +- Example outputs + +### 4. Test Suite +**Location**: `src/test/test_neurons_part_here.py` (example) + +**What it contains**: +- Query functionality tests +- Expected result validation +- Preview result verification +- Performance benchmarks + +--- + +## Query Matching Criteria System + +Queries are conditionally applied based on entity type. The XMI uses a library reference system: + +### Entity Type References (from XMI) +``` +//@libraries.3/@types.0 = Individual (base) +//@libraries.3/@types.1 = Class (base) +//@libraries.3/@types.2 = Neuron (Individual) +//@libraries.3/@types.3 = Tract_or_nerve +//@libraries.3/@types.4 = Clone +//@libraries.3/@types.5 = Synaptic_neuropil +//@libraries.3/@types.16 = pub (Publication) +//@libraries.3/@types.20 = Template +//@libraries.3/@types.22 = Cluster +//@libraries.3/@types.23 = Synaptic_neuropil_domain +//@libraries.3/@types.24 = DataSet +//@libraries.3/@types.25 = NBLAST +//@libraries.3/@types.26 = Visual_system (Anatomy) +//@libraries.3/@types.27 = Expression_pattern +//@libraries.3/@types.28 = Nervous_system (Anatomy) +//@libraries.3/@types.30 = License +//@libraries.3/@types.36 = Term_reference +//@libraries.3/@types.37 = Intersectional_expression_pattern +//@libraries.3/@types.41 = Dataset (Individual) +//@libraries.3/@types.42 = Connected_neuron +//@libraries.3/@types.43 = Region_connectivity +//@libraries.3/@types.44 = NBLAST_exp +//@libraries.3/@types.45 = NeuronBridge +//@libraries.3/@types.46 = Expression_pattern_fragment +//@libraries.3/@types.47 = scRNAseq +//@libraries.3/@types.48 = Gene +//@libraries.3/@types.49 = User_upload (NBLAST) +//@libraries.3/@types.50 = Neuroblast +``` + +### Matching Criteria Examples + +**NeuronsPartHere**: +```xml + + + +``` +Applies to: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_neuropil_domain + +--- + +## All VFB Queries - Complete List + +### āœ… CONVERTED - Queries with Python Implementation + +#### 1. **NeuronsPartHere** āœ… +- **ID**: `NeuronsPartHere` +- **Name**: "Neurons with any part here" +- **Description**: "Neurons with some part in $NAME" +- **Matching Criteria**: + - Class + Synaptic_neuropil + - Class + Visual_system + - Class + Synaptic_neuropil_domain +- **Query Chain**: Owlery subclass ā†’ Process ā†’ SOLR +- **OWL Query**: `object= and some <$ID>` +- **Python Function**: `get_neurons_with_part_in()` +- **Schema Function**: `NeuronsPartHere_to_schema()` +- **Preview**: 10 results (id, label, tags, thumbnail) +- **Status**: āœ… **FULLY IMPLEMENTED** (Nov 4, 2025) +- **Test Coverage**: `test_neurons_part_here.py` (6 tests, 100% passing) + +#### 2. **ListAllAvailableImages** āœ… +- **ID**: `ListAllAvailableImages` +- **Name**: "List all available images for class with examples" +- **Description**: "List all available images of $NAME" +- **Matching Criteria**: Class + Anatomy +- **Query Chain**: Neo4j ā†’ Process Images ā†’ SOLR +- **Python Function**: `get_instances()` +- **Schema Function**: `ListAllAvailableImages_to_schema()` +- **Preview**: 5 results (id, label, tags, thumbnail) +- **Status**: āœ… **FULLY IMPLEMENTED** + +#### 3. **SimilarMorphologyTo** āœ… (Partial) +- **ID**: `SimilarMorphologyTo` / `has_similar_morphology_to` +- **Name**: "NBLAST similarity neo Query" +- **Description**: "Neurons with similar morphology to $NAME [NBLAST mean score]" +- **Matching Criteria**: Individual + Neuron + NBLAST +- **Query Chain**: Neo4j NBLAST query ā†’ Process +- **Python Function**: `get_similar_neurons()` (exists but may need enhancement) +- **Schema Function**: `SimilarMorphologyTo_to_schema()` +- **Preview**: 5 results (id, score, name, tags, thumbnail) +- **Status**: āœ… **IMPLEMENTED** (may need preview enhancement) + +#### 4. **NeuronInputsTo** āœ… (Partial) +- **ID**: `NeuronInputsTo` +- **Name**: "Neuron inputs query" +- **Description**: "Find neurons with synapses into $NAME" +- **Matching Criteria**: Individual + Neuron +- **Python Function**: `get_individual_neuron_inputs()` +- **Schema Function**: `NeuronInputsTo_to_schema()` +- **Preview**: -1 (all results, ribbon format) +- **Preview Columns**: Neurotransmitter, Weight +- **Status**: āœ… **IMPLEMENTED** (ribbon format) + +--- + +### šŸ”¶ PARTIALLY CONVERTED - Schema Exists, Implementation Incomplete + +#### 5. **ComponentsOf** šŸ”¶ +- **ID**: `ComponentsOf` +- **Name**: "Components of" +- **Description**: "Components of $NAME" +- **Matching Criteria**: Class + Clone +- **Query Chain**: Owlery Part of ā†’ Process ā†’ SOLR +- **OWL Query**: `object= some <$ID>` +- **Status**: šŸ”¶ **SCHEMA EXISTS** - needs full implementation + +#### 6. **PartsOf** šŸ”¶ +- **ID**: `PartsOf` +- **Name**: "Parts of" +- **Description**: "Parts of $NAME" +- **Matching Criteria**: Class (any) +- **Query Chain**: Owlery Part of ā†’ Process ā†’ SOLR +- **Status**: šŸ”¶ **SCHEMA EXISTS** - needs full implementation + +--- + +### āŒ NOT CONVERTED - XMI Only + +#### 7. **ExpressionOverlapsHere** āŒ +- **ID**: `ExpressionOverlapsHere` +- **Name**: "Expression overlapping what anatomy" +- **Description**: "Anatomy $NAME is expressed in" +- **Matching Criteria**: + - Class + Expression_pattern + - Class + Expression_pattern_fragment +- **Query Chain**: Neo4j ep_2_anat query ā†’ Process +- **Cypher Query**: Complex pattern matching for expression patterns +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 8. **TransgeneExpressionHere** āŒ +- **ID**: `TransgeneExpressionHere` +- **Name**: "Expression overlapping selected anatomy" +- **Description**: "Reports of transgene expression in $NAME" +- **Matching Criteria**: + - Class + Nervous_system + Anatomy + - Class + Nervous_system + Neuron +- **Query Chain**: Multi-step Owlery and Neo4j queries +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 9. **NeuronClassesFasciculatingHere** āŒ +- **ID**: `NeuronClassesFasciculatingHere` / `AberNeuronClassesFasciculatingHere` +- **Name**: "Neuron classes fasciculating here" +- **Description**: "Neurons fasciculating in $NAME" +- **Matching Criteria**: Class + Tract_or_nerve +- **Query Chain**: Owlery ā†’ Process ā†’ SOLR +- **OWL Query**: `object= and some <$ID>` +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 10. **ImagesNeurons** āŒ +- **ID**: `ImagesNeurons` +- **Name**: "Images of neurons with some part here" +- **Description**: "Images of neurons with some part in $NAME" +- **Matching Criteria**: + - Class + Synaptic_neuropil + - Class + Synaptic_neuropil_domain +- **Query Chain**: Owlery instances ā†’ Process ā†’ SOLR +- **OWL Query**: `object= and some <$ID>` (instances, not classes) +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 11. **NeuronsSynaptic** āŒ +- **ID**: `NeuronsSynaptic` +- **Name**: "Neurons Synaptic" +- **Description**: "Neurons with synaptic terminals in $NAME" +- **Matching Criteria**: + - Class + Synaptic_neuropil + - Class + Visual_system + - Class + Synaptic_neuropil_domain +- **Query Chain**: Owlery ā†’ Process ā†’ SOLR +- **OWL Query**: `object= and some <$ID>` +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 12. **NeuronsPresynapticHere** āŒ +- **ID**: `NeuronsPresynapticHere` +- **Name**: "Neurons Presynaptic" +- **Description**: "Neurons with presynaptic terminals in $NAME" +- **Matching Criteria**: Class + Synaptic_neuropil, Visual_system, Synaptic_neuropil_domain +- **Query Chain**: Owlery ā†’ Process ā†’ SOLR +- **OWL Query**: `object= and some <$ID>` +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 13. **NeuronsPostsynapticHere** āŒ +- **ID**: `NeuronsPostsynapticHere` +- **Name**: "Neurons Postsynaptic" +- **Description**: "Neurons with postsynaptic terminals in $NAME" +- **Matching Criteria**: Class + Synaptic_neuropil, Visual_system, Synaptic_neuropil_domain +- **Query Chain**: Owlery ā†’ Process ā†’ SOLR +- **OWL Query**: `object= and some <$ID>` +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 14. **PaintedDomains** āŒ +- **ID**: `PaintedDomains` / `domainsForTempId` +- **Name**: "Show all painted domains for template" +- **Description**: "List all painted anatomy available for $NAME" +- **Matching Criteria**: Template + Individual +- **Query Chain**: Neo4j domains query ā†’ Process +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 15. **DatasetImages** āŒ +- **ID**: `DatasetImages` / `imagesForDataSet` +- **Name**: "Show all images for a dataset" +- **Description**: "List all images included in $NAME" +- **Matching Criteria**: DataSet + Individual +- **Query Chain**: Neo4j ā†’ Process ā†’ SOLR +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 16. **TractsNervesInnervatingHere** āŒ +- **ID**: `TractsNervesInnervatingHere` / `innervatesX` +- **Name**: "Tracts/nerves innervating synaptic neuropil" +- **Description**: "Tracts/nerves innervating $NAME" +- **Matching Criteria**: + - Class + Synaptic_neuropil + - Class + Synaptic_neuropil_domain +- **Query Chain**: Owlery ā†’ Process ā†’ SOLR +- **OWL Query**: `object= and some <$ID>` +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 17. **LineageClonesIn** āŒ +- **ID**: `LineageClonesIn` / `lineageClones` +- **Name**: "Lineage clones found here" +- **Description**: "Lineage clones found in $NAME" +- **Matching Criteria**: + - Class + Synaptic_neuropil + - Class + Synaptic_neuropil_domain +- **Query Chain**: Owlery ā†’ Process ā†’ SOLR +- **OWL Query**: `object= and some <$ID>` +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 18. **AllAlignedImages** āŒ +- **ID**: `AllAlignedImages` / `imagesForTempQuery` +- **Name**: "Show all images aligned to template" +- **Description**: "List all images aligned to $NAME" +- **Matching Criteria**: Template + Individual +- **Query Chain**: Neo4j ā†’ Neo4j Pass ā†’ SOLR +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 19. **SubclassesOf** āŒ +- **ID**: `SubclassesOf` / `subclasses` +- **Name**: "Subclasses of" +- **Description**: "Subclasses of $NAME" +- **Matching Criteria**: Class (any) +- **Query Chain**: Owlery ā†’ Process ā†’ SOLR +- **OWL Query**: `object=<$ID>` +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 20. **AlignedDatasets** āŒ +- **ID**: `AlignedDatasets` / `template_2_datasets_ids` +- **Name**: "Show all datasets aligned to template" +- **Description**: "List all datasets aligned to $NAME" +- **Matching Criteria**: Template + Individual +- **Query Chain**: Neo4j ā†’ Neo4j Pass ā†’ SOLR ā†’ Process +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 21. **AllDatasets** āŒ +- **ID**: `AllDatasets` / `all_datasets_ids` +- **Name**: "Show all datasets" +- **Description**: "List all datasets" +- **Matching Criteria**: Template +- **Query Chain**: Neo4j ā†’ Neo4j Pass ā†’ SOLR ā†’ Process +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 22. **neuron_region_connectivity_query** āŒ +- **ID**: `ref_neuron_region_connectivity_query` / `compound_neuron_region_connectivity_query` +- **Name**: "Show connectivity to regions from Neuron X" +- **Description**: "Show connectivity per region for $NAME" +- **Matching Criteria**: Region_connectivity +- **Query Chain**: Neo4j compound query ā†’ Process +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 23. **neuron_neuron_connectivity_query** āŒ +- **ID**: `ref_neuron_neuron_connectivity_query` / `compound_neuron_neuron_connectivity_query` +- **Name**: "Show connectivity to neurons from Neuron X" +- **Description**: "Show neurons connected to $NAME" +- **Matching Criteria**: Connected_neuron +- **Query Chain**: Neo4j compound query ā†’ Process +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 24. **SimilarMorphologyToPartOf** āŒ +- **ID**: `SimilarMorphologyToPartOf` / `has_similar_morphology_to_part_of` +- **Name**: "NBLASTexp similarity neo Query" +- **Description**: "Expression patterns with some similar morphology to $NAME [NBLAST mean score]" +- **Matching Criteria**: Individual + Neuron + NBLAST_exp +- **Query Chain**: Neo4j NBLAST exp query ā†’ Process +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 25. **TermsForPub** āŒ +- **ID**: `TermsForPub` / `neoTermIdsRefPub` +- **Name**: "has_reference_to_pub" +- **Description**: "List all terms that reference $NAME" +- **Matching Criteria**: Individual + Publication +- **Query Chain**: Neo4j ā†’ Neo4j Pass ā†’ SOLR +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 26. **SimilarMorphologyToPartOfexp** āŒ +- **ID**: `SimilarMorphologyToPartOfexp` +- **Name**: "has_similar_morphology_to_part_of_exp" +- **Description**: "Neurons with similar morphology to part of $NAME [NBLAST mean score]" +- **Matching Criteria**: + - Individual + Expression_pattern + NBLAST_exp + - Individual + Expression_pattern_fragment + NBLAST_exp +- **Query Chain**: Neo4j NBLAST exp query ā†’ Process +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 27. **SimilarMorphologyToNB** āŒ +- **ID**: `SimilarMorphologyToNB` / `has_similar_morphology_to_nb` +- **Name**: "NeuronBridge similarity neo Query" +- **Description**: "Neurons that overlap with $NAME [NeuronBridge]" +- **Matching Criteria**: NeuronBridge + Individual + Expression_pattern +- **Query Chain**: Neo4j NeuronBridge query ā†’ Process +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 28. **SimilarMorphologyToNBexp** āŒ +- **ID**: `SimilarMorphologyToNBexp` / `has_similar_morphology_to_nb_exp` +- **Name**: "NeuronBridge similarity neo Query (expression)" +- **Description**: "Expression patterns that overlap with $NAME [NeuronBridge]" +- **Matching Criteria**: NeuronBridge + Individual + Neuron +- **Query Chain**: Neo4j NeuronBridge query ā†’ Process +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 29. **anatScRNAseqQuery** āŒ +- **ID**: `anatScRNAseqQuery` / `anat_scRNAseq_query_compound` +- **Name**: "anat_scRNAseq_query" +- **Description**: "Single cell transcriptomics data for $NAME" +- **Matching Criteria**: Class + Nervous_system + scRNAseq +- **Query Chain**: Owlery ā†’ Owlery Pass ā†’ Neo4j scRNAseq query +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 30. **clusterExpression** āŒ +- **ID**: `clusterExpression` / `cluster_expression_query_compound` +- **Name**: "cluster_expression" +- **Description**: "Genes expressed in $NAME" +- **Matching Criteria**: Individual + Cluster +- **Query Chain**: Neo4j cluster expression query ā†’ Process +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 31. **scRNAdatasetData** āŒ +- **ID**: `scRNAdatasetData` / `dataset_scRNAseq_query_compound` +- **Name**: "Show all Clusters for a scRNAseq dataset" +- **Description**: "List all Clusters for $NAME" +- **Matching Criteria**: DataSet + scRNAseq +- **Query Chain**: Neo4j dataset scRNAseq query ā†’ Process +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 32. **expressionCluster** āŒ +- **ID**: `expressionCluster` / `expression_cluster_query_compound` +- **Name**: "expression_cluster" +- **Description**: "scRNAseq clusters expressing $NAME" +- **Matching Criteria**: Class + Gene + scRNAseq +- **Query Chain**: Neo4j expression cluster query ā†’ Process +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 33. **SimilarMorphologyToUserData** āŒ +- **ID**: `SimilarMorphologyToUserData` / `has_similar_morphology_to_userdata` +- **Name**: "User data NBLAST similarity" +- **Description**: "Neurons with similar morphology to your upload $NAME [NBLAST mean score]" +- **Matching Criteria**: User_upload + Individual +- **Query Chain**: SOLR cached user NBLAST query ā†’ Process +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 34. **ImagesThatDevelopFrom** āŒ +- **ID**: `ImagesThatDevelopFrom` / `imagesDevelopsFromNeuroblast` +- **Name**: "Show all images that develops_from X" +- **Description**: "List images of neurons that develop from $NAME" +- **Matching Criteria**: Class + Neuroblast +- **Query Chain**: Owlery instances ā†’ Owlery Pass ā†’ SOLR +- **OWL Query**: `object= and some <$ID>` +- **Status**: āŒ **NOT IMPLEMENTED** + +#### 35. **epFrag** āŒ +- **ID**: `epFrag` +- **Name**: "Images of expression pattern fragments" +- **Description**: "Images of fragments of $NAME" +- **Matching Criteria**: Class + Expression_pattern +- **Query Chain**: Owlery individual parts ā†’ Process ā†’ SOLR +- **OWL Query**: `object= some <$ID>` (instances) +- **Status**: āŒ **NOT IMPLEMENTED** + +--- + +## Conversion Status Summary + +### Statistics +- **Total VFB Queries**: 35 +- **āœ… Fully Implemented**: 4 (11%) +- **šŸ”¶ Partially Implemented**: 2 (6%) +- **āŒ Not Implemented**: 29 (83%) + +### Implementation Priority Categories + +#### High Priority (Common Use Cases) +1. āŒ **NeuronsSynaptic** - synaptic terminal queries are very common +2. āŒ **NeuronsPresynapticHere** - presynaptic connectivity is essential +3. āŒ **NeuronsPostsynapticHere** - postsynaptic connectivity is essential +4. āŒ **ExpressionOverlapsHere** - expression pattern queries are frequent +5. āŒ **ComponentsOf** - anatomical hierarchy navigation +6. āŒ **PartsOf** - anatomical hierarchy navigation + +#### Medium Priority (Specialized Queries) +7. āŒ **neuron_region_connectivity_query** - connectivity analysis +8. āŒ **neuron_neuron_connectivity_query** - circuit analysis +9. āŒ **SubclassesOf** - ontology navigation +10. āŒ **anatScRNAseqQuery** - transcriptomics integration +11. āŒ **clusterExpression** - gene expression analysis + +#### Lower Priority (Advanced/Specialized) +- NeuronBridge queries (27, 28) +- User data NBLAST (33) +- Dataset-specific queries (14, 15, 20, 21, 31) +- Template-specific queries (14, 19, 20) +- Lineage queries (17, 34) + +--- + +## Implementation Patterns + +### Pattern 1: Owlery-Based Class Queries (Most Common) + +**Example**: NeuronsPartHere (āœ… implemented) + +```python +def get_neurons_with_part_in(short_form: str, limit: int = None): + """ + Query neurons that have some part overlapping with anatomical region. + + Steps: + 1. Owlery subclass query (OWL reasoning) + 2. Process IDs + 3. SOLR lookup for full details + """ + # 1. Owlery query + owlery_url = f"http://owl.virtualflybrain.org/kbs/vfb/subclasses" + owl_query = f"object= and some " + + # 2. Get class IDs from Owlery + class_ids = owlery_request(owlery_url, owl_query) + + # 3. Lookup in SOLR + results = solr_lookup(class_ids, limit=limit) + + return results +``` + +**Applies to**: NeuronsSynaptic, NeuronsPresynapticHere, NeuronsPostsynapticHere, ComponentsOf, PartsOf, SubclassesOf + +### Pattern 2: Neo4j Complex Queries + +**Example**: ExpressionOverlapsHere (āŒ not implemented) + +```python +def get_expression_overlaps(short_form: str): + """ + Query expression patterns overlapping anatomy. + + Uses Neo4j Cypher query with complex pattern matching: + - Match expression patterns + - Follow relationships through anonymous individuals + - Collect publication references + - Retrieve example images + """ + # Neo4j Cypher query (from XMI) + cypher = """ + MATCH (ep:Expression_pattern:Class)<-[ar:overlaps|part_of]-(anoni:Individual)-[:INSTANCEOF]->(anat:Class) + WHERE ep.short_form in [$id] + WITH anoni, anat, ar + OPTIONAL MATCH (p:pub {short_form: []+ar.pub[0]}) + ... + """ + + # Execute and process results + results = neo4j_query(cypher, id=short_form) + return process_results(results) +``` + +**Applies to**: ExpressionOverlapsHere, TransgeneExpressionHere, neuron_region_connectivity_query, neuron_neuron_connectivity_query + +### Pattern 3: Owlery Instance Queries + +**Example**: ImagesNeurons (āŒ not implemented) + +```python +def get_neuron_images_in(short_form: str): + """ + Get individual neuron instances (not classes) with part in region. + + Uses Owlery instances endpoint instead of subclasses. + """ + # Owlery instances query + owlery_url = f"http://owl.virtualflybrain.org/kbs/vfb/instances" + owl_query = f"object= and some " + + # Rest is similar to Pattern 1 + ... +``` + +**Applies to**: ImagesNeurons, epFrag, ImagesThatDevelopFrom + +### Pattern 4: SOLR Cached Queries + +**Example**: anatScRNAseqQuery, clusterExpression (āŒ not implemented) + +```python +def get_cluster_expression(short_form: str): + """ + Retrieve cached scRNAseq cluster data from SOLR. + + Uses pre-cached Neo4j query results stored in SOLR. + """ + # Query SOLR for cached field + results = vfb_solr.search(f'id:{short_form}', fl='cluster_expression_query') + + # Process cached JSON + return process_cached_query(results.docs[0]['cluster_expression_query']) +``` + +**Applies to**: anatScRNAseqQuery, clusterExpression, scRNAdatasetData, expressionCluster, SimilarMorphologyToUserData + +--- + +## Next Steps + +### Immediate Next Query: **NeuronsSynaptic** + +**Why this one?** +1. High-value query for neuroscience research +2. Uses same Pattern 1 (Owlery-based) as NeuronsPartHere āœ… +3. We have a working template from NeuronsPartHere +4. Clear matching criteria and well-defined OWL query +5. Moderate complexity - good learning progression + +**Implementation checklist**: +- [ ] Create `NeuronsSynaptic_to_schema()` function +- [ ] Create `get_neurons_with_synapses_in()` function +- [ ] Add query matching logic in `get_term_info()` +- [ ] Create comprehensive test suite +- [ ] Update documentation + +### Recommended Query Order + +**Phase 1: Owlery Pattern Queries** (Easiest, similar to NeuronsPartHere) +1. āœ… NeuronsPartHere (DONE) +2. **NeuronsSynaptic** (NEXT) +3. NeuronsPresynapticHere +4. NeuronsPostsynapticHere +5. ComponentsOf +6. PartsOf +7. SubclassesOf + +**Phase 2: Neo4j Pattern Queries** (Medium difficulty) +8. ExpressionOverlapsHere +9. TransgeneExpressionHere +10. neuron_region_connectivity_query +11. neuron_neuron_connectivity_query + +**Phase 3: Instance & Specialized Queries** (More complex) +12. ImagesNeurons +13. anatScRNAseqQuery +14. clusterExpression +15. ... (others as needed) + +--- + +## Implementation Template + +Use this template for each new query: + +```python +# 1. Schema function +def QueryName_to_schema(name, take_default): + """ + Schema for QueryName. + [Description of what the query does] + + Matching criteria from XMI: + - [List matching criteria] + + Query chain: [Describe data source chain] + OWL/Cypher query: [Quote the actual query] + """ + query = "QueryName" + label = f"[Human readable description with {name}]" + function = "function_name" + takes = { + "short_form": {"$and": ["Type1", "Type2"]}, + "default": take_default, + } + preview = 10 # or -1 for all + preview_columns = ["id", "label", ...] # columns to show + + return Query(query=query, label=label, function=function, + takes=takes, preview=preview, preview_columns=preview_columns) + +# 2. Execution function +@with_solr_cache('query_name') +def function_name(short_form: str, limit: int = None): + """ + Execute the QueryName query. + + :param short_form: Term ID to query + :param limit: Optional result limit + :return: Query results as DataFrame or dict + """ + # Implementation here + pass + +# 3. Add to term_info matching logic +# In get_term_info(), add conditional: +if is_type(vfbTerm, ["Type1", "Type2"]): + q = QueryName_to_schema(termInfo["Name"], {"short_form": vfbTerm.term.core.short_form}) + termInfo["Queries"].append(q) + +# 4. Create test file +# src/test/test_query_name.py with comprehensive tests +``` + +--- + +## Resources + +- **XMI Spec**: https://raw.githubusercontent.com/VirtualFlyBrain/geppetto-vfb/master/model/vfb.xmi +- **Owlery API**: http://owl.virtualflybrain.org/kbs/vfb/ +- **SOLR API**: https://solr.virtualflybrain.org/solr/vfb_json/select +- **Neo4j**: http://pdb.v4.virtualflybrain.org/db/neo4j/tx +- **VFB Ontology**: http://purl.obolibrary.org/obo/fbbt.owl + +--- + +**Last Reviewed**: November 4, 2025 +**Maintainer**: VFBquery Development Team From 63c0e45928c299f4c51378a160a04a6932cd6dd2 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 4 Nov 2025 21:17:18 +0000 Subject: [PATCH 07/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index f2b833b..55fcb84 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-04 20:45:51 UTC -**Git Commit:** a2ac3daf9e395ffcb8bd56e08da97040d1a6f467 +**Test Date:** 2025-11-04 21:17:18 UTC +**Git Commit:** 15fe2548b7ee727a66a3d5e67da770f705f629e3 **Branch:** dev -**Workflow Run:** 19082267123 +**Workflow Run:** 19083014772 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 122.3027 seconds -- **VFB_00101567 Query Time**: 0.7956 seconds -- **Total Query Time**: 123.0983 seconds +- **FBbt_00003748 Query Time**: 115.7751 seconds +- **VFB_00101567 Query Time**: 0.8872 seconds +- **Total Query Time**: 116.6623 seconds āš ļø **Result**: Some performance thresholds exceeded or test failed --- -*Last updated: 2025-11-04 20:45:51 UTC* +*Last updated: 2025-11-04 21:17:18 UTC* From 6fe0da0b91f9bcc97c80be3bdf0c7cf2003296fa Mon Sep 17 00:00:00 2001 From: Rob Court Date: Tue, 4 Nov 2025 22:24:13 +0000 Subject: [PATCH 08/70] Implement new Owlery-based queries and corresponding tests - Added NeuronsSynaptic, NeuronsPresynapticHere, NeuronsPostsynapticHere, ComponentsOf, PartsOf, and SubclassesOf queries to vfb_queries.py. - Enhanced term_info_parse_object to include new queries based on anatomical regions and classes. - Created a comprehensive test suite for the new queries in test_new_owlery_queries.py, ensuring they return expected results and are included in term_info. - Updated existing functions to maintain consistency and improve error handling. --- VFB_QUERIES_REFERENCE.md | 202 ++++++----- src/test/test_new_owlery_queries.py | 282 ++++++++++++++++ src/vfbquery/vfb_queries.py | 498 +++++++++++++++++++++++----- 3 files changed, 822 insertions(+), 160 deletions(-) create mode 100644 src/test/test_new_owlery_queries.py diff --git a/VFB_QUERIES_REFERENCE.md b/VFB_QUERIES_REFERENCE.md index 5746e4b..ddd4094 100644 --- a/VFB_QUERIES_REFERENCE.md +++ b/VFB_QUERIES_REFERENCE.md @@ -121,23 +121,113 @@ Applies to: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_n ### āœ… CONVERTED - Queries with Python Implementation +--- + +### āœ… FULLY CONVERTED - Complete Implementation + #### 1. **NeuronsPartHere** āœ… - **ID**: `NeuronsPartHere` -- **Name**: "Neurons with any part here" -- **Description**: "Neurons with some part in $NAME" +- **Name**: "Neuron Classes with some part in a region" +- **Description**: "Neuron classes with some part overlapping $NAME" - **Matching Criteria**: - Class + Synaptic_neuropil - - Class + Visual_system - - Class + Synaptic_neuropil_domain -- **Query Chain**: Owlery subclass ā†’ Process ā†’ SOLR -- **OWL Query**: `object= and some <$ID>` + - Class + Anatomy (broader match) +- **Query Chain**: Owlery subclass query ā†’ Process ā†’ SOLR +- **OWL Query**: `'Neuron' that 'overlaps' some '{short_form}'` - **Python Function**: `get_neurons_with_part_in()` - **Schema Function**: `NeuronsPartHere_to_schema()` +- **Cache Key**: `'neurons_part_here'` +- **Preview**: 10 results with images (id, label, tags, thumbnail, source) +- **Status**: āœ… **FULLY IMPLEMENTED** with tests + +#### 2. **NeuronsSynaptic** āœ… +- **ID**: `NeuronsSynaptic` +- **Name**: "Neurons with synaptic terminals in region" +- **Description**: "Neuron classes with synaptic terminals in $NAME" +- **Matching Criteria**: + - Class + Synaptic_neuropil + - Class + Visual_system + - Class + Synaptic_neuropil_domain +- **Query Chain**: Owlery subclass query ā†’ Process ā†’ SOLR +- **OWL Query**: `'Neuron' that 'has synaptic terminals in' some '{short_form}'` +- **Python Function**: `get_neurons_with_synapses_in()` +- **Schema Function**: `NeuronsSynaptic_to_schema()` +- **Cache Key**: `'neurons_synaptic'` +- **Preview**: 10 results (id, label, tags, thumbnail) +- **Status**: āœ… **FULLY IMPLEMENTED** with term_info integration + +#### 3. **NeuronsPresynapticHere** āœ… +- **ID**: `NeuronsPresynapticHere` +- **Name**: "Neurons with presynaptic terminals in region" +- **Description**: "Neuron classes with presynaptic terminals in $NAME" +- **Matching Criteria**: + - Class + Synaptic_neuropil + - Class + Visual_system + - Class + Synaptic_neuropil_domain +- **Query Chain**: Owlery subclass query ā†’ Process ā†’ SOLR +- **OWL Query**: `'Neuron' that 'has presynaptic terminal in' some '{short_form}'` +- **Python Function**: `get_neurons_with_presynaptic_terminals_in()` +- **Schema Function**: `NeuronsPresynapticHere_to_schema()` +- **Cache Key**: `'neurons_presynaptic'` +- **Preview**: 10 results (id, label, tags, thumbnail) +- **Status**: āœ… **FULLY IMPLEMENTED** with term_info integration + +#### 4. **NeuronsPostsynapticHere** āœ… +- **ID**: `NeuronsPostsynapticHere` +- **Name**: "Neurons with postsynaptic terminals in region" +- **Description**: "Neuron classes with postsynaptic terminals in $NAME" +- **Matching Criteria**: + - Class + Synaptic_neuropil + - Class + Visual_system + - Class + Synaptic_neuropil_domain +- **Query Chain**: Owlery subclass query ā†’ Process ā†’ SOLR +- **OWL Query**: `'Neuron' that 'has postsynaptic terminal in' some '{short_form}'` +- **Python Function**: `get_neurons_with_postsynaptic_terminals_in()` +- **Schema Function**: `NeuronsPostsynapticHere_to_schema()` +- **Cache Key**: `'neurons_postsynaptic'` +- **Preview**: 10 results (id, label, tags, thumbnail) +- **Status**: āœ… **FULLY IMPLEMENTED** with term_info integration + +#### 5. **ComponentsOf** āœ… +- **ID**: `ComponentsOf` +- **Name**: "Components of" +- **Description**: "Components of $NAME" +- **Matching Criteria**: Class + Clone +- **Query Chain**: Owlery Part of ā†’ Process ā†’ SOLR +- **OWL Query**: `'part of' some '{short_form}'` +- **Python Function**: `get_components_of()` +- **Schema Function**: `ComponentsOf_to_schema()` +- **Cache Key**: `'components_of'` +- **Preview**: 10 results (id, label, tags, thumbnail) +- **Status**: āœ… **FULLY IMPLEMENTED** with term_info integration + +#### 6. **PartsOf** āœ… +- **ID**: `PartsOf` +- **Name**: "Parts of" +- **Description**: "Parts of $NAME" +- **Matching Criteria**: Class (any) +- **Query Chain**: Owlery Part of ā†’ Process ā†’ SOLR +- **OWL Query**: `'part of' some '{short_form}'` +- **Python Function**: `get_parts_of()` +- **Schema Function**: `PartsOf_to_schema()` +- **Cache Key**: `'parts_of'` +- **Preview**: 10 results (id, label, tags, thumbnail) +- **Status**: āœ… **FULLY IMPLEMENTED** with term_info integration + +#### 7. **SubclassesOf** āœ… +- **ID**: `SubclassesOf` +- **Name**: "Subclasses of" +- **Description**: "Subclasses of $NAME" +- **Matching Criteria**: Class (any) +- **Query Chain**: Owlery subclasses query ā†’ Process ā†’ SOLR +- **OWL Query**: `'{short_form}'` (direct class query) +- **Python Function**: `get_subclasses_of()` +- **Schema Function**: `SubclassesOf_to_schema()` +- **Cache Key**: `'subclasses_of'` - **Preview**: 10 results (id, label, tags, thumbnail) -- **Status**: āœ… **FULLY IMPLEMENTED** (Nov 4, 2025) -- **Test Coverage**: `test_neurons_part_here.py` (6 tests, 100% passing) +- **Status**: āœ… **FULLY IMPLEMENTED** with term_info integration -#### 2. **ListAllAvailableImages** āœ… +#### 8. **ListAllAvailableImages** āœ… - **ID**: `ListAllAvailableImages` - **Name**: "List all available images for class with examples" - **Description**: "List all available images of $NAME" @@ -148,7 +238,7 @@ Applies to: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_n - **Preview**: 5 results (id, label, tags, thumbnail) - **Status**: āœ… **FULLY IMPLEMENTED** -#### 3. **SimilarMorphologyTo** āœ… (Partial) +#### 9. **SimilarMorphologyTo** āœ… (Partial) - **ID**: `SimilarMorphologyTo` / `has_similar_morphology_to` - **Name**: "NBLAST similarity neo Query" - **Description**: "Neurons with similar morphology to $NAME [NBLAST mean score]" @@ -159,7 +249,7 @@ Applies to: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_n - **Preview**: 5 results (id, score, name, tags, thumbnail) - **Status**: āœ… **IMPLEMENTED** (may need preview enhancement) -#### 4. **NeuronInputsTo** āœ… (Partial) +#### 10. **NeuronInputsTo** āœ… (Partial) - **ID**: `NeuronInputsTo` - **Name**: "Neuron inputs query" - **Description**: "Find neurons with synapses into $NAME" @@ -172,30 +262,9 @@ Applies to: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_n --- -### šŸ”¶ PARTIALLY CONVERTED - Schema Exists, Implementation Incomplete - -#### 5. **ComponentsOf** šŸ”¶ -- **ID**: `ComponentsOf` -- **Name**: "Components of" -- **Description**: "Components of $NAME" -- **Matching Criteria**: Class + Clone -- **Query Chain**: Owlery Part of ā†’ Process ā†’ SOLR -- **OWL Query**: `object= some <$ID>` -- **Status**: šŸ”¶ **SCHEMA EXISTS** - needs full implementation - -#### 6. **PartsOf** šŸ”¶ -- **ID**: `PartsOf` -- **Name**: "Parts of" -- **Description**: "Parts of $NAME" -- **Matching Criteria**: Class (any) -- **Query Chain**: Owlery Part of ā†’ Process ā†’ SOLR -- **Status**: šŸ”¶ **SCHEMA EXISTS** - needs full implementation - ---- - ### āŒ NOT CONVERTED - XMI Only -#### 7. **ExpressionOverlapsHere** āŒ +#### 11. **ExpressionOverlapsHere** āŒ - **ID**: `ExpressionOverlapsHere` - **Name**: "Expression overlapping what anatomy" - **Description**: "Anatomy $NAME is expressed in" @@ -236,37 +305,7 @@ Applies to: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_n - **OWL Query**: `object= and some <$ID>` (instances, not classes) - **Status**: āŒ **NOT IMPLEMENTED** -#### 11. **NeuronsSynaptic** āŒ -- **ID**: `NeuronsSynaptic` -- **Name**: "Neurons Synaptic" -- **Description**: "Neurons with synaptic terminals in $NAME" -- **Matching Criteria**: - - Class + Synaptic_neuropil - - Class + Visual_system - - Class + Synaptic_neuropil_domain -- **Query Chain**: Owlery ā†’ Process ā†’ SOLR -- **OWL Query**: `object= and some <$ID>` -- **Status**: āŒ **NOT IMPLEMENTED** - -#### 12. **NeuronsPresynapticHere** āŒ -- **ID**: `NeuronsPresynapticHere` -- **Name**: "Neurons Presynaptic" -- **Description**: "Neurons with presynaptic terminals in $NAME" -- **Matching Criteria**: Class + Synaptic_neuropil, Visual_system, Synaptic_neuropil_domain -- **Query Chain**: Owlery ā†’ Process ā†’ SOLR -- **OWL Query**: `object= and some <$ID>` -- **Status**: āŒ **NOT IMPLEMENTED** - -#### 13. **NeuronsPostsynapticHere** āŒ -- **ID**: `NeuronsPostsynapticHere` -- **Name**: "Neurons Postsynaptic" -- **Description**: "Neurons with postsynaptic terminals in $NAME" -- **Matching Criteria**: Class + Synaptic_neuropil, Visual_system, Synaptic_neuropil_domain -- **Query Chain**: Owlery ā†’ Process ā†’ SOLR -- **OWL Query**: `object= and some <$ID>` -- **Status**: āŒ **NOT IMPLEMENTED** - -#### 14. **PaintedDomains** āŒ +#### 12. **PaintedDomains** āŒ - **ID**: `PaintedDomains` / `domainsForTempId` - **Name**: "Show all painted domains for template" - **Description**: "List all painted anatomy available for $NAME" @@ -312,16 +351,7 @@ Applies to: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_n - **Query Chain**: Neo4j ā†’ Neo4j Pass ā†’ SOLR - **Status**: āŒ **NOT IMPLEMENTED** -#### 19. **SubclassesOf** āŒ -- **ID**: `SubclassesOf` / `subclasses` -- **Name**: "Subclasses of" -- **Description**: "Subclasses of $NAME" -- **Matching Criteria**: Class (any) -- **Query Chain**: Owlery ā†’ Process ā†’ SOLR -- **OWL Query**: `object=<$ID>` -- **Status**: āŒ **NOT IMPLEMENTED** - -#### 20. **AlignedDatasets** āŒ +#### 19. **AlignedDatasets** āŒ - **ID**: `AlignedDatasets` / `template_2_datasets_ids` - **Name**: "Show all datasets aligned to template" - **Description**: "List all datasets aligned to $NAME" @@ -459,24 +489,32 @@ Applies to: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_n ### Statistics - **Total VFB Queries**: 35 -- **āœ… Fully Implemented**: 4 (11%) +- **āœ… Fully Implemented**: 10 (29%) - **šŸ”¶ Partially Implemented**: 2 (6%) -- **āŒ Not Implemented**: 29 (83%) +- **āŒ Not Implemented**: 23 (66%) + +### Recently Implemented (This Session) +- āœ… **NeuronsSynaptic** - neurons with synaptic terminals in region +- āœ… **NeuronsPresynapticHere** - neurons with presynaptic terminals in region +- āœ… **NeuronsPostsynapticHere** - neurons with postsynaptic terminals in region +- āœ… **ComponentsOf** - components of anatomical structures +- āœ… **PartsOf** - parts of anatomical structures +- āœ… **SubclassesOf** - subclasses of a class ### Implementation Priority Categories #### High Priority (Common Use Cases) -1. āŒ **NeuronsSynaptic** - synaptic terminal queries are very common -2. āŒ **NeuronsPresynapticHere** - presynaptic connectivity is essential -3. āŒ **NeuronsPostsynapticHere** - postsynaptic connectivity is essential +1. āœ… **NeuronsSynaptic** - synaptic terminal queries are very common (COMPLETED) +2. āœ… **NeuronsPresynapticHere** - presynaptic connectivity is essential (COMPLETED) +3. āœ… **NeuronsPostsynapticHere** - postsynaptic connectivity is essential (COMPLETED) 4. āŒ **ExpressionOverlapsHere** - expression pattern queries are frequent -5. āŒ **ComponentsOf** - anatomical hierarchy navigation -6. āŒ **PartsOf** - anatomical hierarchy navigation +5. āœ… **ComponentsOf** - anatomical hierarchy navigation (COMPLETED) +6. āœ… **PartsOf** - anatomical hierarchy navigation (COMPLETED) #### Medium Priority (Specialized Queries) 7. āŒ **neuron_region_connectivity_query** - connectivity analysis 8. āŒ **neuron_neuron_connectivity_query** - circuit analysis -9. āŒ **SubclassesOf** - ontology navigation +9. āœ… **SubclassesOf** - ontology navigation (COMPLETED) 10. āŒ **anatScRNAseqQuery** - transcriptomics integration 11. āŒ **clusterExpression** - gene expression analysis diff --git a/src/test/test_new_owlery_queries.py b/src/test/test_new_owlery_queries.py new file mode 100644 index 0000000..522abaf --- /dev/null +++ b/src/test/test_new_owlery_queries.py @@ -0,0 +1,282 @@ +""" +Tests for newly implemented Owlery-based queries: +- NeuronsSynaptic +- NeuronsPresynapticHere +- NeuronsPostsynapticHere +- ComponentsOf +- PartsOf +- SubclassesOf +""" + +import unittest +from vfbquery.vfb_queries import ( + get_neurons_with_synapses_in, + get_neurons_with_presynaptic_terminals_in, + get_neurons_with_postsynaptic_terminals_in, + get_components_of, + get_parts_of, + get_subclasses_of, + get_term_info +) + + +class TestNeuronsSynaptic(unittest.TestCase): + """Tests for NeuronsSynaptic query""" + + def setUp(self): + self.medulla_id = 'FBbt_00003748' # medulla - synaptic neuropil + + def test_neurons_synaptic_returns_results(self): + """Test that NeuronsSynaptic query returns results for medulla""" + result = get_neurons_with_synapses_in( + self.medulla_id, + return_dataframe=False + ) + + self.assertIsNotNone(result) + self.assertIn('headers', result) + self.assertIn('rows', result) + self.assertIn('count', result) + + def test_neurons_synaptic_has_expected_columns(self): + """Test that result has expected column structure""" + result = get_neurons_with_synapses_in( + self.medulla_id, + return_dataframe=False + ) + + headers = result['headers'] + self.assertIn('id', headers) + self.assertIn('label', headers) + self.assertIn('tags', headers) + self.assertIn('thumbnail', headers) + + def test_neurons_synaptic_in_term_info(self): + """Test that NeuronsSynaptic appears in term_info queries""" + term_info = get_term_info(self.medulla_id, preview=True) + + self.assertIn('Queries', term_info) # Note: Capital 'Q' + query_labels = [q['label'] for q in term_info['Queries']] + + # Check if our query is present + expected_label = f"Neurons with synaptic terminals in {term_info['Name']}" + self.assertIn(expected_label, query_labels) + + +class TestNeuronsPresynapticHere(unittest.TestCase): + """Tests for NeuronsPresynapticHere query""" + + def setUp(self): + self.medulla_id = 'FBbt_00003748' # medulla - synaptic neuropil + + def test_neurons_presynaptic_returns_results(self): + """Test that NeuronsPresynapticHere query returns results for medulla""" + result = get_neurons_with_presynaptic_terminals_in( + self.medulla_id, + return_dataframe=False + ) + + self.assertIsNotNone(result) + self.assertIn('headers', result) + self.assertIn('rows', result) + self.assertIn('count', result) + + def test_neurons_presynaptic_has_expected_columns(self): + """Test that result has expected column structure""" + result = get_neurons_with_presynaptic_terminals_in( + self.medulla_id, + return_dataframe=False + ) + + headers = result['headers'] + self.assertIn('id', headers) + self.assertIn('label', headers) + self.assertIn('tags', headers) + self.assertIn('thumbnail', headers) + + def test_neurons_presynaptic_in_term_info(self): + """Test that NeuronsPresynapticHere appears in term_info queries""" + term_info = get_term_info(self.medulla_id, preview=True) + + self.assertIn('Queries', term_info) # Note: Capital 'Q' + query_labels = [q['label'] for q in term_info['Queries']] + + # Check if our query is present + expected_label = f"Neurons with presynaptic terminals in {term_info['Name']}" + self.assertIn(expected_label, query_labels) + + +class TestNeuronsPostsynapticHere(unittest.TestCase): + """Tests for NeuronsPostsynapticHere query""" + + def setUp(self): + self.medulla_id = 'FBbt_00003748' # medulla - synaptic neuropil + + def test_neurons_postsynaptic_returns_results(self): + """Test that NeuronsPostsynapticHere query returns results for medulla""" + result = get_neurons_with_postsynaptic_terminals_in( + self.medulla_id, + return_dataframe=False + ) + + self.assertIsNotNone(result) + self.assertIn('headers', result) + self.assertIn('rows', result) + self.assertIn('count', result) + + def test_neurons_postsynaptic_has_expected_columns(self): + """Test that result has expected column structure""" + result = get_neurons_with_postsynaptic_terminals_in( + self.medulla_id, + return_dataframe=False + ) + + headers = result['headers'] + self.assertIn('id', headers) + self.assertIn('label', headers) + self.assertIn('tags', headers) + self.assertIn('thumbnail', headers) + + def test_neurons_postsynaptic_in_term_info(self): + """Test that NeuronsPostsynapticHere appears in term_info queries""" + term_info = get_term_info(self.medulla_id, preview=True) + + self.assertIn('Queries', term_info) # Note: Capital 'Q' + query_labels = [q['label'] for q in term_info['Queries']] + + # Check if our query is present + expected_label = f"Neurons with postsynaptic terminals in {term_info['Name']}" + self.assertIn(expected_label, query_labels) + + +class TestComponentsOf(unittest.TestCase): + """Tests for ComponentsOf query""" + + def setUp(self): + self.clone_id = 'FBbt_00110369' # adult SLPpm4 lineage clone + + def test_components_of_returns_results(self): + """Test that ComponentsOf query returns results for clone""" + result = get_components_of( + self.clone_id, + return_dataframe=False + ) + + self.assertIsNotNone(result) + self.assertIn('headers', result) + self.assertIn('rows', result) + self.assertIn('count', result) + + def test_components_of_has_expected_columns(self): + """Test that result has expected column structure""" + result = get_components_of( + self.clone_id, + return_dataframe=False + ) + + headers = result['headers'] + self.assertIn('id', headers) + self.assertIn('label', headers) + self.assertIn('tags', headers) + self.assertIn('thumbnail', headers) + + def test_components_of_in_term_info(self): + """Test that ComponentsOf appears in term_info queries""" + term_info = get_term_info(self.clone_id, preview=True) + + self.assertIn('Queries', term_info) # Note: Capital 'Q' + query_labels = [q['label'] for q in term_info['Queries']] + + # Check if our query is present + expected_label = f"Components of {term_info['Name']}" + self.assertIn(expected_label, query_labels) + + +class TestPartsOf(unittest.TestCase): + """Tests for PartsOf query""" + + def setUp(self): + self.medulla_id = 'FBbt_00003748' # medulla - any Class + + def test_parts_of_returns_results(self): + """Test that PartsOf query returns results for medulla""" + result = get_parts_of( + self.medulla_id, + return_dataframe=False + ) + + self.assertIsNotNone(result) + self.assertIn('headers', result) + self.assertIn('rows', result) + self.assertIn('count', result) + + def test_parts_of_has_expected_columns(self): + """Test that result has expected column structure""" + result = get_parts_of( + self.medulla_id, + return_dataframe=False + ) + + headers = result['headers'] + self.assertIn('id', headers) + self.assertIn('label', headers) + self.assertIn('tags', headers) + self.assertIn('thumbnail', headers) + + def test_parts_of_in_term_info(self): + """Test that PartsOf appears in term_info queries""" + term_info = get_term_info(self.medulla_id, preview=True) + + self.assertIn('Queries', term_info) # Note: Capital 'Q' + query_labels = [q['label'] for q in term_info['Queries']] + + # Check if our query is present + expected_label = f"Parts of {term_info['Name']}" + self.assertIn(expected_label, query_labels) + + +class TestSubclassesOf(unittest.TestCase): + """Tests for SubclassesOf query""" + + def setUp(self): + self.medulla_id = 'FBbt_00003748' # medulla - any Class + + def test_subclasses_of_returns_results(self): + """Test that SubclassesOf query returns results for medulla""" + result = get_subclasses_of( + self.medulla_id, + return_dataframe=False + ) + + self.assertIsNotNone(result) + self.assertIn('headers', result) + self.assertIn('rows', result) + self.assertIn('count', result) + + def test_subclasses_of_has_expected_columns(self): + """Test that result has expected column structure""" + result = get_subclasses_of( + self.medulla_id, + return_dataframe=False + ) + + headers = result['headers'] + self.assertIn('id', headers) + self.assertIn('label', headers) + self.assertIn('tags', headers) + self.assertIn('thumbnail', headers) + + def test_subclasses_of_in_term_info(self): + """Test that SubclassesOf appears in term_info queries""" + term_info = get_term_info(self.medulla_id, preview=True) + + self.assertIn('Queries', term_info) # Note: Capital 'Q' + query_labels = [q['label'] for q in term_info['Queries']] + + # Check if our query is present + expected_label = f"Subclasses of {term_info['Name']}" + self.assertIn(expected_label, query_labels) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index fec80ef..7655cad 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -667,6 +667,54 @@ def term_info_parse_object(results, short_form): q = NeuronsPartHere_to_schema(termInfo["Name"], {"short_form": vfbTerm.term.core.short_form}) queries.append(q) + # NeuronsSynaptic query - for synaptic neuropils and visual systems + # Matches XMI criteria: Class + (Synaptic_neuropil OR Visual_system OR Synaptic_neuropil_domain) + if contains_all_tags(termInfo["SuperTypes"], ["Class"]) and ( + "Synaptic_neuropil" in termInfo["SuperTypes"] or + "Visual_system" in termInfo["SuperTypes"] or + "Synaptic_neuropil_domain" in termInfo["SuperTypes"] + ): + q = NeuronsSynaptic_to_schema(termInfo["Name"], {"short_form": vfbTerm.term.core.short_form}) + queries.append(q) + + # NeuronsPresynapticHere query - for synaptic neuropils and visual systems + # Matches XMI criteria: Class + (Synaptic_neuropil OR Visual_system OR Synaptic_neuropil_domain) + if contains_all_tags(termInfo["SuperTypes"], ["Class"]) and ( + "Synaptic_neuropil" in termInfo["SuperTypes"] or + "Visual_system" in termInfo["SuperTypes"] or + "Synaptic_neuropil_domain" in termInfo["SuperTypes"] + ): + q = NeuronsPresynapticHere_to_schema(termInfo["Name"], {"short_form": vfbTerm.term.core.short_form}) + queries.append(q) + + # NeuronsPostsynapticHere query - for synaptic neuropils and visual systems + # Matches XMI criteria: Class + (Synaptic_neuropil OR Visual_system OR Synaptic_neuropil_domain) + if contains_all_tags(termInfo["SuperTypes"], ["Class"]) and ( + "Synaptic_neuropil" in termInfo["SuperTypes"] or + "Visual_system" in termInfo["SuperTypes"] or + "Synaptic_neuropil_domain" in termInfo["SuperTypes"] + ): + q = NeuronsPostsynapticHere_to_schema(termInfo["Name"], {"short_form": vfbTerm.term.core.short_form}) + queries.append(q) + + # ComponentsOf query - for clones + # Matches XMI criteria: Class + Clone + if contains_all_tags(termInfo["SuperTypes"], ["Class", "Clone"]): + q = ComponentsOf_to_schema(termInfo["Name"], {"short_form": vfbTerm.term.core.short_form}) + queries.append(q) + + # PartsOf query - for any Class + # Matches XMI criteria: Class (any) + if contains_all_tags(termInfo["SuperTypes"], ["Class"]): + q = PartsOf_to_schema(termInfo["Name"], {"short_form": vfbTerm.term.core.short_form}) + queries.append(q) + + # SubclassesOf query - for any Class + # Matches XMI criteria: Class (any) + if contains_all_tags(termInfo["SuperTypes"], ["Class"]): + q = SubclassesOf_to_schema(termInfo["Name"], {"short_form": vfbTerm.term.core.short_form}) + queries.append(q) + # Add Publications to the termInfo object if vfbTerm.pubs and len(vfbTerm.pubs) > 0: publications = [] @@ -857,6 +905,156 @@ def NeuronsPartHere_to_schema(name, take_default): return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) + +def NeuronsSynaptic_to_schema(name, take_default): + """ + Schema for NeuronsSynaptic query. + Finds neuron classes that have synaptic terminals in the specified anatomical region. + + Matching criteria from XMI: + - Class + Synaptic_neuropil + - Class + Visual_system + - Class + Synaptic_neuropil_domain + + Query chain: Owlery subclass query ā†’ process ā†’ SOLR + OWL query: "Neuron and has_synaptic_terminals_in some $ID" + """ + query = "NeuronsSynaptic" + label = f"Neurons with synaptic terminals in {name}" + function = "get_neurons_with_synapses_in" + takes = { + "short_form": {"$and": ["Class", "Anatomy"]}, + "default": take_default, + } + preview = 10 + preview_columns = ["id", "label", "tags", "thumbnail"] + + return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) + + +def NeuronsPresynapticHere_to_schema(name, take_default): + """ + Schema for NeuronsPresynapticHere query. + Finds neuron classes that have presynaptic terminals in the specified anatomical region. + + Matching criteria from XMI: + - Class + Synaptic_neuropil + - Class + Visual_system + - Class + Synaptic_neuropil_domain + + Query chain: Owlery subclass query ā†’ process ā†’ SOLR + OWL query: "Neuron and has_presynaptic_terminal_in some $ID" + """ + query = "NeuronsPresynapticHere" + label = f"Neurons with presynaptic terminals in {name}" + function = "get_neurons_with_presynaptic_terminals_in" + takes = { + "short_form": {"$and": ["Class", "Anatomy"]}, + "default": take_default, + } + preview = 10 + preview_columns = ["id", "label", "tags", "thumbnail"] + + return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) + + +def NeuronsPostsynapticHere_to_schema(name, take_default): + """ + Schema for NeuronsPostsynapticHere query. + Finds neuron classes that have postsynaptic terminals in the specified anatomical region. + + Matching criteria from XMI: + - Class + Synaptic_neuropil + - Class + Visual_system + - Class + Synaptic_neuropil_domain + + Query chain: Owlery subclass query ā†’ process ā†’ SOLR + OWL query: "Neuron and has_postsynaptic_terminal_in some $ID" + """ + query = "NeuronsPostsynapticHere" + label = f"Neurons with postsynaptic terminals in {name}" + function = "get_neurons_with_postsynaptic_terminals_in" + takes = { + "short_form": {"$and": ["Class", "Anatomy"]}, + "default": take_default, + } + preview = 10 + preview_columns = ["id", "label", "tags", "thumbnail"] + + return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) + + +def ComponentsOf_to_schema(name, take_default): + """ + Schema for ComponentsOf query. + Finds components (parts) of the specified anatomical class. + + Matching criteria from XMI: + - Class + Clone + + Query chain: Owlery part_of query ā†’ process ā†’ SOLR + OWL query: "part_of some $ID" + """ + query = "ComponentsOf" + label = f"Components of {name}" + function = "get_components_of" + takes = { + "short_form": {"$and": ["Class", "Anatomy"]}, + "default": take_default, + } + preview = 10 + preview_columns = ["id", "label", "tags", "thumbnail"] + + return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) + + +def PartsOf_to_schema(name, take_default): + """ + Schema for PartsOf query. + Finds parts of the specified anatomical class. + + Matching criteria from XMI: + - Class (any) + + Query chain: Owlery part_of query ā†’ process ā†’ SOLR + OWL query: "part_of some $ID" + """ + query = "PartsOf" + label = f"Parts of {name}" + function = "get_parts_of" + takes = { + "short_form": {"$and": ["Class"]}, + "default": take_default, + } + preview = 10 + preview_columns = ["id", "label", "tags", "thumbnail"] + + return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) + + +def SubclassesOf_to_schema(name, take_default): + """ + Schema for SubclassesOf query. + Finds subclasses of the specified class. + + Matching criteria from XMI: + - Class (any) + + Query chain: Owlery subclasses query ā†’ process ā†’ SOLR + OWL query: Direct subclasses of $ID + """ + query = "SubclassesOf" + label = f"Subclasses of {name}" + function = "get_subclasses_of" + takes = { + "short_form": {"$and": ["Class"]}, + "default": take_default, + } + preview = 10 + preview_columns = ["id", "label", "tags", "thumbnail"] + + return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) + def serialize_solr_output(results): # Create a copy of the document and remove Solr-specific fields doc = dict(results.docs[0]) @@ -1584,69 +1782,227 @@ def get_neurons_with_part_in(short_form: str, return_dataframe=True, limit: int This implements the NeuronsPartHere query from the VFB XMI specification. Query chain (from XMI): Owlery (Index 1) ā†’ Process ā†’ SOLR (Index 3) - OWL query: "'Neuron' that 'overlaps' some ''" + OWL query (from XMI): object= and some <$ID> + Where: FBbt_00005106 = neuron, RO_0002131 = overlaps :param short_form: short form of the anatomical region (Class) :param return_dataframe: Returns pandas dataframe if true, otherwise returns formatted dict :param limit: maximum number of results to return (default -1, returns all results) :return: Neuron classes with parts in the specified region """ + owl_query = f" and some <{short_form}>" + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, + solr_field='anat_query', include_source=True) + + +@with_solr_cache('neurons_synaptic') +def get_neurons_with_synapses_in(short_form: str, return_dataframe=True, limit: int = -1): + """ + Retrieves neuron classes that have synaptic terminals in the specified anatomical region. + + This implements the NeuronsSynaptic query from the VFB XMI specification. + Query chain (from XMI): Owlery ā†’ Process ā†’ SOLR + OWL query (from XMI): object= and some <$ID> + Where: FBbt_00005106 = neuron, RO_0002130 = has synaptic terminals in + Matching criteria: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_neuropil_domain + + :param short_form: short form of the anatomical region (Class) + :param return_dataframe: Returns pandas dataframe if true, otherwise returns formatted dict + :param limit: maximum number of results to return (default -1, returns all results) + :return: Neuron classes with synaptic terminals in the specified region + """ + owl_query = f" and some <{short_form}>" + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query') + + +@with_solr_cache('neurons_presynaptic') +def get_neurons_with_presynaptic_terminals_in(short_form: str, return_dataframe=True, limit: int = -1): + """ + Retrieves neuron classes that have presynaptic terminals in the specified anatomical region. + + This implements the NeuronsPresynapticHere query from the VFB XMI specification. + Query chain (from XMI): Owlery ā†’ Process ā†’ SOLR + OWL query (from XMI): object= and some <$ID> + Where: FBbt_00005106 = neuron, RO_0002113 = has presynaptic terminal in + Matching criteria: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_neuropil_domain + + :param short_form: short form of the anatomical region (Class) + :param return_dataframe: Returns pandas dataframe if true, otherwise returns formatted dict + :param limit: maximum number of results to return (default -1, returns all results) + :return: Neuron classes with presynaptic terminals in the specified region + """ + owl_query = f" and some <{short_form}>" + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query') + + +@with_solr_cache('neurons_postsynaptic') +def get_neurons_with_postsynaptic_terminals_in(short_form: str, return_dataframe=True, limit: int = -1): + """ + Retrieves neuron classes that have postsynaptic terminals in the specified anatomical region. + + This implements the NeuronsPostsynapticHere query from the VFB XMI specification. + Query chain (from XMI): Owlery ā†’ Process ā†’ SOLR + OWL query (from XMI): object= and some <$ID> + Where: FBbt_00005106 = neuron, RO_0002110 = has postsynaptic terminal in + Matching criteria: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_neuropil_domain + + :param short_form: short form of the anatomical region (Class) + :param return_dataframe: Returns pandas dataframe if true, otherwise returns formatted dict + :param limit: maximum number of results to return (default -1, returns all results) + :return: Neuron classes with postsynaptic terminals in the specified region + """ + owl_query = f" and some <{short_form}>" + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query') + + +@with_solr_cache('components_of') +def get_components_of(short_form: str, return_dataframe=True, limit: int = -1): + """ + Retrieves components (parts) of the specified anatomical class. + + This implements the ComponentsOf query from the VFB XMI specification. + Query chain (from XMI): Owlery Part of ā†’ Process ā†’ SOLR + OWL query (from XMI): object= some <$ID> + Where: BFO_0000050 = part of + Matching criteria: Class + Clone + + :param short_form: short form of the anatomical class + :param return_dataframe: Returns pandas dataframe if true, otherwise returns formatted dict + :param limit: maximum number of results to return (default -1, returns all results) + :return: Components of the specified class + """ + owl_query = f" some <{short_form}>" + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query') + + +@with_solr_cache('parts_of') +def get_parts_of(short_form: str, return_dataframe=True, limit: int = -1): + """ + Retrieves parts of the specified anatomical class. + + This implements the PartsOf query from the VFB XMI specification. + Query chain (from XMI): Owlery Part of ā†’ Process ā†’ SOLR + OWL query (from XMI): object= some <$ID> + Where: BFO_0000050 = part of + Matching criteria: Class (any) + + :param short_form: short form of the anatomical class + :param return_dataframe: Returns pandas dataframe if true, otherwise returns formatted dict + :param limit: maximum number of results to return (default -1, returns all results) + :return: Parts of the specified class + """ + owl_query = f" some <{short_form}>" + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query') + + +@with_solr_cache('subclasses_of') +def get_subclasses_of(short_form: str, return_dataframe=True, limit: int = -1): + """ + Retrieves subclasses of the specified class. + + This implements the SubclassesOf query from the VFB XMI specification. + Query chain (from XMI): Owlery ā†’ Process ā†’ SOLR + OWL query: Direct subclasses of '' + Matching criteria: Class (any) + + :param short_form: short form of the class + :param return_dataframe: Returns pandas dataframe if true, otherwise returns formatted dict + :param limit: maximum number of results to return (default -1, returns all results) + :return: Subclasses of the specified class + """ + # For subclasses, we query the class itself (Owlery subclasses endpoint handles this) + owl_query = f"'{short_form}'" + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query') + +def _get_neurons_part_here_headers(): + """Return standard headers for get_neurons_with_part_in results""" + return { + "id": {"title": "Add", "type": "selection_id", "order": -1}, + "label": {"title": "Name", "type": "markdown", "order": 0, "sort": {0: "Asc"}}, + "tags": {"title": "Tags", "type": "tags", "order": 2}, + "source": {"title": "Data Source", "type": "metadata", "order": 3}, + "source_id": {"title": "Data Source ID", "type": "metadata", "order": 4}, + "thumbnail": {"title": "Thumbnail", "type": "markdown", "order": 9} + } + + +def _get_standard_query_headers(): + """Return standard headers for most query results (no source/source_id)""" + return { + "id": {"title": "Add", "type": "selection_id", "order": -1}, + "label": {"title": "Name", "type": "markdown", "order": 0, "sort": {0: "Asc"}}, + "tags": {"title": "Tags", "type": "tags", "order": 2}, + "thumbnail": {"title": "Thumbnail", "type": "markdown", "order": 9} + } + + +def _owlery_query_to_results(owl_query_string: str, short_form: str, return_dataframe: bool = True, + limit: int = -1, solr_field: str = 'anat_query', + include_source: bool = False): + """ + Shared helper function for Owlery-based queries. + This implements the common pattern: + 1. Query Owlery for class IDs matching an OWL pattern + 2. Fetch details from SOLR for each class + 3. Format results as DataFrame or dict + + :param owl_query_string: OWL query in VFB label syntax (e.g., "'Neuron' that 'overlaps' some 'FBbt_00003748'") + :param short_form: The anatomical region or entity short form + :param return_dataframe: Returns pandas DataFrame if True, otherwise returns formatted dict + :param limit: Maximum number of results to return (default -1 for all) + :param solr_field: SOLR field to query (default 'anat_query' for Class, 'anat_image_query' for Individuals) + :param include_source: Whether to include source and source_id columns + :return: Query results + """ try: - # Step 1: Query Owlery for neuron classes that overlap this anatomical region - # This uses the OWL reasoner to find all neuron subclasses matching the pattern - neuron_class_ids = vc.vfb.oc.get_subclasses( - query=f"'Neuron' that 'overlaps' some '{short_form}'", + # Step 1: Query Owlery for classes matching the OWL pattern + class_ids = vc.vfb.oc.get_subclasses( + query=owl_query_string, query_by_label=True, verbose=False ) - if not neuron_class_ids: - # No neurons found - return empty results + if not class_ids: + # No results found - return empty if return_dataframe: return pd.DataFrame() return { - "headers": _get_neurons_part_here_headers(), + "headers": _get_standard_query_headers() if not include_source else _get_neurons_part_here_headers(), "rows": [], "count": 0 } # Apply limit if specified (before SOLR query to save processing) if limit != -1 and limit > 0: - neuron_class_ids = neuron_class_ids[:limit] + class_ids = class_ids[:limit] - total_count = len(neuron_class_ids) + total_count = len(class_ids) - # Step 2: Query SOLR directly for just the anat_query field - # For Class terms (neuron classes), the field is 'anat_query' not 'anat_image_query' - # This matches the original VFBquery pattern and contains all result row metadata - # This is much faster than loading full term_info for each neuron + # Step 2: Query SOLR for each class to get detailed information rows = [] - for neuron_id in neuron_class_ids: + for class_id in class_ids: try: - # Query SOLR with fl=anat_query to get only the result table data - # This is the same field used in the original VFBquery implementation + # Query SOLR with specified field results = vfb_solr.search( - q=f'id:{neuron_id}', - fl='anat_query', + q=f'id:{class_id}', + fl=solr_field, rows=1 ) - if results.hits > 0 and results.docs and 'anat_query' in results.docs[0]: - # Parse the anat_query JSON string - anat_query_str = results.docs[0]['anat_query'][0] - anat_data = json.loads(anat_query_str) + if results.hits > 0 and results.docs and solr_field in results.docs[0]: + # Parse the SOLR field JSON string + field_data_str = results.docs[0][solr_field][0] + field_data = json.loads(field_data_str) # Extract core term information - term_core = anat_data.get('term', {}).get('core', {}) - neuron_short_form = term_core.get('short_form', neuron_id) + term_core = field_data.get('term', {}).get('core', {}) + class_short_form = term_core.get('short_form', class_id) - # Extract label (prefer symbol over label, matching Neo4j behavior) + # Extract label (prefer symbol over label) label_text = term_core.get('label', 'Unknown') if term_core.get('symbol') and len(term_core.get('symbol', '')) > 0: label_text = term_core.get('symbol') - # Decode URL-encoded strings from SOLR - from urllib.parse import unquote label_text = unquote(label_text) # Extract tags from unique_facets @@ -1654,9 +2010,8 @@ def get_neurons_with_part_in(short_form: str, return_dataframe=True, limit: int # Extract thumbnail from anatomy_channel_image if available thumbnail = '' - anatomy_images = anat_data.get('anatomy_channel_image', []) + anatomy_images = field_data.get('anatomy_channel_image', []) if anatomy_images and len(anatomy_images) > 0: - # Get the first anatomy channel image (example instance) first_img = anatomy_images[0] channel_image = first_img.get('channel_image', {}) image_info = channel_image.get('image', {}) @@ -1666,51 +2021,51 @@ def get_neurons_with_part_in(short_form: str, return_dataframe=True, limit: int # Convert to HTTPS and use non-transparent version thumbnail_url = thumbnail_url.replace('http://', 'https://').replace('thumbnailT.png', 'thumbnail.png') - # Format thumbnail markdown with template info + # Format thumbnail markdown template_anatomy = image_info.get('template_anatomy', {}) if template_anatomy: template_label = template_anatomy.get('symbol') or template_anatomy.get('label', '') template_label = unquote(template_label) - # Get the anatomy info for alt text anatomy_info = first_img.get('anatomy', {}) anatomy_label = anatomy_info.get('symbol') or anatomy_info.get('label', label_text) anatomy_label = unquote(anatomy_label) alt_text = f"{anatomy_label} aligned to {template_label}" - thumbnail = f"[![{alt_text}]({thumbnail_url} '{alt_text}')]({neuron_short_form})" - - # Extract source information from xrefs if available - source = '' - source_id = '' - xrefs = anat_data.get('xrefs', []) - if xrefs and len(xrefs) > 0: - # Get the first data source xref - for xref in xrefs: - if xref.get('is_data_source', False): - site_info = xref.get('site', {}) - site_label = site_info.get('symbol') or site_info.get('label', '') - site_short_form = site_info.get('short_form', '') - if site_label and site_short_form: - source = f"[{site_label}]({site_short_form})" - - accession = xref.get('accession', '') - link_base = xref.get('link_base', '') - if accession and link_base: - source_id = f"[{accession}]({link_base}{accession})" - break + thumbnail = f"[![{alt_text}]({thumbnail_url} '{alt_text}')]({class_short_form})" - # Build row matching expected format + # Build row row = { - 'id': neuron_short_form, - 'label': f"[{label_text}]({neuron_short_form})", + 'id': class_short_form, + 'label': f"[{label_text}]({class_short_form})", 'tags': tags, - 'source': source, - 'source_id': source_id, 'thumbnail': thumbnail } + + # Optionally add source information + if include_source: + source = '' + source_id = '' + xrefs = field_data.get('xrefs', []) + if xrefs and len(xrefs) > 0: + for xref in xrefs: + if xref.get('is_data_source', False): + site_info = xref.get('site', {}) + site_label = site_info.get('symbol') or site_info.get('label', '') + site_short_form = site_info.get('short_form', '') + if site_label and site_short_form: + source = f"[{site_label}]({site_short_form})" + + accession = xref.get('accession', '') + link_base = xref.get('link_base', '') + if accession and link_base: + source_id = f"[{accession}]({link_base}{accession})" + break + row['source'] = source + row['source_id'] = source_id + rows.append(row) except Exception as e: - print(f"Error fetching SOLR data for {neuron_id}: {e}") + print(f"Error fetching SOLR data for {class_id}: {e}") continue # Convert to DataFrame if requested @@ -1721,39 +2076,26 @@ def get_neurons_with_part_in(short_form: str, return_dataframe=True, limit: int df = encode_markdown_links(df, columns_to_encode) return df - # Convert to expected format with proper headers - formatted_results = { - "headers": _get_neurons_part_here_headers(), + # Return formatted dict + return { + "headers": _get_standard_query_headers() if not include_source else _get_neurons_part_here_headers(), "rows": rows, "count": total_count } - return formatted_results - except Exception as e: - print(f"Error in get_neurons_with_part_in: {e}") + print(f"Error in Owlery query: {e}") import traceback traceback.print_exc() - # Return empty results with proper structure + # Return empty results if return_dataframe: return pd.DataFrame() return { - "headers": _get_neurons_part_here_headers(), + "headers": _get_standard_query_headers() if not include_source else _get_neurons_part_here_headers(), "rows": [], "count": 0 } -def _get_neurons_part_here_headers(): - """Return standard headers for get_neurons_with_part_in results""" - return { - "id": {"title": "Add", "type": "selection_id", "order": -1}, - "label": {"title": "Name", "type": "markdown", "order": 0, "sort": {0: "Asc"}}, - "tags": {"title": "Tags", "type": "tags", "order": 2}, - "source": {"title": "Data Source", "type": "metadata", "order": 3}, - "source_id": {"title": "Data Source ID", "type": "metadata", "order": 4}, - "thumbnail": {"title": "Thumbnail", "type": "markdown", "order": 9} - } - def fill_query_results(term_info): for query in term_info['Queries']: From bcef10c3eedaa1ca48f01bf00c4b549062075402 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 4 Nov 2025 22:27:27 +0000 Subject: [PATCH 09/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index 55fcb84..dbc615d 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-04 21:17:18 UTC -**Git Commit:** 15fe2548b7ee727a66a3d5e67da770f705f629e3 +**Test Date:** 2025-11-04 22:27:27 UTC +**Git Commit:** c0d9456ee29b8baf3048d5d79d7bf714f5ec8093 **Branch:** dev -**Workflow Run:** 19083014772 +**Workflow Run:** 19084599886 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 115.7751 seconds -- **VFB_00101567 Query Time**: 0.8872 seconds -- **Total Query Time**: 116.6623 seconds +- **FBbt_00003748 Query Time**: 127.5634 seconds +- **VFB_00101567 Query Time**: 0.9622 seconds +- **Total Query Time**: 128.5257 seconds āš ļø **Result**: Some performance thresholds exceeded or test failed --- -*Last updated: 2025-11-04 21:17:18 UTC* +*Last updated: 2025-11-04 22:27:27 UTC* From f6e1315a3cb7be0ee9bb431bb8dd5310eb754d7d Mon Sep 17 00:00:00 2001 From: Rob Court Date: Wed, 5 Nov 2025 10:22:20 +0000 Subject: [PATCH 10/70] Update Owlery queries to use IRI syntax and modify test cases for SubclassesOf query --- src/test/test_new_owlery_queries.py | 10 +- src/vfbquery/vfb_queries.py | 222 ++++++++++++++-------------- 2 files changed, 120 insertions(+), 112 deletions(-) diff --git a/src/test/test_new_owlery_queries.py b/src/test/test_new_owlery_queries.py index 522abaf..bd6ef9b 100644 --- a/src/test/test_new_owlery_queries.py +++ b/src/test/test_new_owlery_queries.py @@ -239,12 +239,12 @@ class TestSubclassesOf(unittest.TestCase): """Tests for SubclassesOf query""" def setUp(self): - self.medulla_id = 'FBbt_00003748' # medulla - any Class + self.wedge_pn_id = 'FBbt_00048516' # wedge projection neuron (>45 subclasses) def test_subclasses_of_returns_results(self): - """Test that SubclassesOf query returns results for medulla""" + """Test that SubclassesOf query returns results for wedge projection neuron""" result = get_subclasses_of( - self.medulla_id, + self.wedge_pn_id, return_dataframe=False ) @@ -256,7 +256,7 @@ def test_subclasses_of_returns_results(self): def test_subclasses_of_has_expected_columns(self): """Test that result has expected column structure""" result = get_subclasses_of( - self.medulla_id, + self.wedge_pn_id, return_dataframe=False ) @@ -268,7 +268,7 @@ def test_subclasses_of_has_expected_columns(self): def test_subclasses_of_in_term_info(self): """Test that SubclassesOf appears in term_info queries""" - term_info = get_term_info(self.medulla_id, preview=True) + term_info = get_term_info(self.wedge_pn_id, preview=True) self.assertIn('Queries', term_info) # Note: Capital 'Q' query_labels = [q['label'] for q in term_info['Queries']] diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 7655cad..e1f049f 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -1782,7 +1782,7 @@ def get_neurons_with_part_in(short_form: str, return_dataframe=True, limit: int This implements the NeuronsPartHere query from the VFB XMI specification. Query chain (from XMI): Owlery (Index 1) ā†’ Process ā†’ SOLR (Index 3) - OWL query (from XMI): object= and some <$ID> + OWL query (from XMI): and some <$ID> Where: FBbt_00005106 = neuron, RO_0002131 = overlaps :param short_form: short form of the anatomical region (Class) @@ -1790,9 +1790,9 @@ def get_neurons_with_part_in(short_form: str, return_dataframe=True, limit: int :param limit: maximum number of results to return (default -1, returns all results) :return: Neuron classes with parts in the specified region """ - owl_query = f" and some <{short_form}>" + owl_query = f" and some " return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, - solr_field='anat_query', include_source=True) + solr_field='anat_query', include_source=True, query_by_label=False) @with_solr_cache('neurons_synaptic') @@ -1802,7 +1802,7 @@ def get_neurons_with_synapses_in(short_form: str, return_dataframe=True, limit: This implements the NeuronsSynaptic query from the VFB XMI specification. Query chain (from XMI): Owlery ā†’ Process ā†’ SOLR - OWL query (from XMI): object= and some <$ID> + OWL query (from XMI): object= and some Where: FBbt_00005106 = neuron, RO_0002130 = has synaptic terminals in Matching criteria: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_neuropil_domain @@ -1811,8 +1811,8 @@ def get_neurons_with_synapses_in(short_form: str, return_dataframe=True, limit: :param limit: maximum number of results to return (default -1, returns all results) :return: Neuron classes with synaptic terminals in the specified region """ - owl_query = f" and some <{short_form}>" - return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query') + owl_query = f" and some " + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query', query_by_label=False) @with_solr_cache('neurons_presynaptic') @@ -1822,7 +1822,7 @@ def get_neurons_with_presynaptic_terminals_in(short_form: str, return_dataframe= This implements the NeuronsPresynapticHere query from the VFB XMI specification. Query chain (from XMI): Owlery ā†’ Process ā†’ SOLR - OWL query (from XMI): object= and some <$ID> + OWL query (from XMI): object= and some Where: FBbt_00005106 = neuron, RO_0002113 = has presynaptic terminal in Matching criteria: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_neuropil_domain @@ -1831,8 +1831,8 @@ def get_neurons_with_presynaptic_terminals_in(short_form: str, return_dataframe= :param limit: maximum number of results to return (default -1, returns all results) :return: Neuron classes with presynaptic terminals in the specified region """ - owl_query = f" and some <{short_form}>" - return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query') + owl_query = f" and some " + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query', query_by_label=False) @with_solr_cache('neurons_postsynaptic') @@ -1842,7 +1842,7 @@ def get_neurons_with_postsynaptic_terminals_in(short_form: str, return_dataframe This implements the NeuronsPostsynapticHere query from the VFB XMI specification. Query chain (from XMI): Owlery ā†’ Process ā†’ SOLR - OWL query (from XMI): object= and some <$ID> + OWL query (from XMI): object= and some Where: FBbt_00005106 = neuron, RO_0002110 = has postsynaptic terminal in Matching criteria: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_neuropil_domain @@ -1851,8 +1851,8 @@ def get_neurons_with_postsynaptic_terminals_in(short_form: str, return_dataframe :param limit: maximum number of results to return (default -1, returns all results) :return: Neuron classes with postsynaptic terminals in the specified region """ - owl_query = f" and some <{short_form}>" - return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query') + owl_query = f" and some " + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query', query_by_label=False) @with_solr_cache('components_of') @@ -1862,7 +1862,7 @@ def get_components_of(short_form: str, return_dataframe=True, limit: int = -1): This implements the ComponentsOf query from the VFB XMI specification. Query chain (from XMI): Owlery Part of ā†’ Process ā†’ SOLR - OWL query (from XMI): object= some <$ID> + OWL query (from XMI): object= some Where: BFO_0000050 = part of Matching criteria: Class + Clone @@ -1871,8 +1871,8 @@ def get_components_of(short_form: str, return_dataframe=True, limit: int = -1): :param limit: maximum number of results to return (default -1, returns all results) :return: Components of the specified class """ - owl_query = f" some <{short_form}>" - return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query') + owl_query = f" some " + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query', query_by_label=False) @with_solr_cache('parts_of') @@ -1882,7 +1882,7 @@ def get_parts_of(short_form: str, return_dataframe=True, limit: int = -1): This implements the PartsOf query from the VFB XMI specification. Query chain (from XMI): Owlery Part of ā†’ Process ā†’ SOLR - OWL query (from XMI): object= some <$ID> + OWL query (from XMI): object= some Where: BFO_0000050 = part of Matching criteria: Class (any) @@ -1891,8 +1891,8 @@ def get_parts_of(short_form: str, return_dataframe=True, limit: int = -1): :param limit: maximum number of results to return (default -1, returns all results) :return: Parts of the specified class """ - owl_query = f" some <{short_form}>" - return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query') + owl_query = f" some " + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query', query_by_label=False) @with_solr_cache('subclasses_of') @@ -1938,7 +1938,7 @@ def _get_standard_query_headers(): def _owlery_query_to_results(owl_query_string: str, short_form: str, return_dataframe: bool = True, limit: int = -1, solr_field: str = 'anat_query', - include_source: bool = False): + include_source: bool = False, query_by_label: bool = True): """ Shared helper function for Owlery-based queries. @@ -1947,19 +1947,20 @@ def _owlery_query_to_results(owl_query_string: str, short_form: str, return_data 2. Fetch details from SOLR for each class 3. Format results as DataFrame or dict - :param owl_query_string: OWL query in VFB label syntax (e.g., "'Neuron' that 'overlaps' some 'FBbt_00003748'") + :param owl_query_string: OWL query string (format depends on query_by_label parameter) :param short_form: The anatomical region or entity short form :param return_dataframe: Returns pandas DataFrame if True, otherwise returns formatted dict :param limit: Maximum number of results to return (default -1 for all) :param solr_field: SOLR field to query (default 'anat_query' for Class, 'anat_image_query' for Individuals) :param include_source: Whether to include source and source_id columns + :param query_by_label: If True, use label syntax with quotes. If False, use IRI syntax with angle brackets. :return: Query results """ try: # Step 1: Query Owlery for classes matching the OWL pattern class_ids = vc.vfb.oc.get_subclasses( query=owl_query_string, - query_by_label=True, + query_by_label=query_by_label, verbose=False ) @@ -1973,100 +1974,107 @@ def _owlery_query_to_results(owl_query_string: str, short_form: str, return_data "count": 0 } + total_count = len(class_ids) + # Apply limit if specified (before SOLR query to save processing) if limit != -1 and limit > 0: class_ids = class_ids[:limit] - total_count = len(class_ids) - - # Step 2: Query SOLR for each class to get detailed information + # Step 2: Query SOLR for ALL classes in a single batch query + # Use the {!terms f=id} syntax from XMI to fetch all results efficiently rows = [] - for class_id in class_ids: - try: - # Query SOLR with specified field - results = vfb_solr.search( - q=f'id:{class_id}', - fl=solr_field, - rows=1 - ) - - if results.hits > 0 and results.docs and solr_field in results.docs[0]: - # Parse the SOLR field JSON string - field_data_str = results.docs[0][solr_field][0] - field_data = json.loads(field_data_str) - - # Extract core term information - term_core = field_data.get('term', {}).get('core', {}) - class_short_form = term_core.get('short_form', class_id) - - # Extract label (prefer symbol over label) - label_text = term_core.get('label', 'Unknown') - if term_core.get('symbol') and len(term_core.get('symbol', '')) > 0: - label_text = term_core.get('symbol') - label_text = unquote(label_text) + try: + # Build filter query with all class IDs + id_list = ','.join(class_ids) + results = vfb_solr.search( + q='id:*', + fq=f'{{!terms f=id}}{id_list}', + fl=solr_field, + rows=len(class_ids) + ) + + # Process all results + for doc in results.docs: + if solr_field not in doc: + continue - # Extract tags from unique_facets - tags = '|'.join(term_core.get('unique_facets', [])) + # Parse the SOLR field JSON string + field_data_str = doc[solr_field][0] + field_data = json.loads(field_data_str) + + # Extract core term information + term_core = field_data.get('term', {}).get('core', {}) + class_short_form = term_core.get('short_form', '') + + # Extract label (prefer symbol over label) + label_text = term_core.get('label', 'Unknown') + if term_core.get('symbol') and len(term_core.get('symbol', '')) > 0: + label_text = term_core.get('symbol') + label_text = unquote(label_text) + + # Extract tags from unique_facets + tags = '|'.join(term_core.get('unique_facets', [])) + + # Extract thumbnail from anatomy_channel_image if available + thumbnail = '' + anatomy_images = field_data.get('anatomy_channel_image', []) + if anatomy_images and len(anatomy_images) > 0: + first_img = anatomy_images[0] + channel_image = first_img.get('channel_image', {}) + image_info = channel_image.get('image', {}) + thumbnail_url = image_info.get('image_thumbnail', '') - # Extract thumbnail from anatomy_channel_image if available - thumbnail = '' - anatomy_images = field_data.get('anatomy_channel_image', []) - if anatomy_images and len(anatomy_images) > 0: - first_img = anatomy_images[0] - channel_image = first_img.get('channel_image', {}) - image_info = channel_image.get('image', {}) - thumbnail_url = image_info.get('image_thumbnail', '') + if thumbnail_url: + # Convert to HTTPS and use non-transparent version + thumbnail_url = thumbnail_url.replace('http://', 'https://').replace('thumbnailT.png', 'thumbnail.png') - if thumbnail_url: - # Convert to HTTPS and use non-transparent version - thumbnail_url = thumbnail_url.replace('http://', 'https://').replace('thumbnailT.png', 'thumbnail.png') - - # Format thumbnail markdown - template_anatomy = image_info.get('template_anatomy', {}) - if template_anatomy: - template_label = template_anatomy.get('symbol') or template_anatomy.get('label', '') - template_label = unquote(template_label) - anatomy_info = first_img.get('anatomy', {}) - anatomy_label = anatomy_info.get('symbol') or anatomy_info.get('label', label_text) - anatomy_label = unquote(anatomy_label) - alt_text = f"{anatomy_label} aligned to {template_label}" - thumbnail = f"[![{alt_text}]({thumbnail_url} '{alt_text}')]({class_short_form})" - - # Build row - row = { - 'id': class_short_form, - 'label': f"[{label_text}]({class_short_form})", - 'tags': tags, - 'thumbnail': thumbnail - } - - # Optionally add source information - if include_source: - source = '' - source_id = '' - xrefs = field_data.get('xrefs', []) - if xrefs and len(xrefs) > 0: - for xref in xrefs: - if xref.get('is_data_source', False): - site_info = xref.get('site', {}) - site_label = site_info.get('symbol') or site_info.get('label', '') - site_short_form = site_info.get('short_form', '') - if site_label and site_short_form: - source = f"[{site_label}]({site_short_form})" - - accession = xref.get('accession', '') - link_base = xref.get('link_base', '') - if accession and link_base: - source_id = f"[{accession}]({link_base}{accession})" - break - row['source'] = source - row['source_id'] = source_id - - rows.append(row) - - except Exception as e: - print(f"Error fetching SOLR data for {class_id}: {e}") - continue + # Format thumbnail markdown + template_anatomy = image_info.get('template_anatomy', {}) + if template_anatomy: + template_label = template_anatomy.get('symbol') or template_anatomy.get('label', '') + template_label = unquote(template_label) + anatomy_info = first_img.get('anatomy', {}) + anatomy_label = anatomy_info.get('symbol') or anatomy_info.get('label', label_text) + anatomy_label = unquote(anatomy_label) + alt_text = f"{anatomy_label} aligned to {template_label}" + thumbnail = f"[![{alt_text}]({thumbnail_url} '{alt_text}')]({class_short_form})" + + # Build row + row = { + 'id': class_short_form, + 'label': f"[{label_text}]({class_short_form})", + 'tags': tags, + 'thumbnail': thumbnail + } + + # Optionally add source information + if include_source: + source = '' + source_id = '' + xrefs = field_data.get('xrefs', []) + if xrefs and len(xrefs) > 0: + for xref in xrefs: + if xref.get('is_data_source', False): + site_info = xref.get('site', {}) + site_label = site_info.get('symbol') or site_info.get('label', '') + site_short_form = site_info.get('short_form', '') + if site_label and site_short_form: + source = f"[{site_label}]({site_short_form})" + + accession = xref.get('accession', '') + link_base = xref.get('link_base', '') + if accession and link_base: + source_id = f"[{accession}]({link_base}{accession})" + break + row['source'] = source + row['source_id'] = source_id + + rows.append(row) + + except Exception as e: + print(f"Error fetching SOLR data: {e}") + import traceback + traceback.print_exc() # Convert to DataFrame if requested if return_dataframe: From 1f9df76499f5d975034b90ff588928df46fc79a1 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 5 Nov 2025 10:45:06 +0000 Subject: [PATCH 11/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index dbc615d..aef9b41 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-04 22:27:27 UTC -**Git Commit:** c0d9456ee29b8baf3048d5d79d7bf714f5ec8093 +**Test Date:** 2025-11-05 10:45:05 UTC +**Git Commit:** 86bb48e6d1b2680ec0d74d9e686f795bbb6d4fb5 **Branch:** dev -**Workflow Run:** 19084599886 +**Workflow Run:** 19098809304 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 127.5634 seconds -- **VFB_00101567 Query Time**: 0.9622 seconds -- **Total Query Time**: 128.5257 seconds +- **FBbt_00003748 Query Time**: 1284.1489 seconds +- **VFB_00101567 Query Time**: 0.7166 seconds +- **Total Query Time**: 1284.8656 seconds āš ļø **Result**: Some performance thresholds exceeded or test failed --- -*Last updated: 2025-11-04 22:27:27 UTC* +*Last updated: 2025-11-05 10:45:06 UTC* From ceb3a6872a0a06f6ce80b166a04564f538370fc7 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Wed, 5 Nov 2025 11:16:30 +0000 Subject: [PATCH 12/70] Enhance get_term_info function to support preview parameter for faster queries --- src/vfbquery/vfb_queries.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index e1f049f..a8a5f8c 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -11,6 +11,8 @@ import numpy as np from urllib.parse import unquote from .solr_result_cache import with_solr_cache +import time +import requests # Custom JSON encoder to handle NumPy and pandas types class NumpyEncoder(json.JSONEncoder): @@ -1076,6 +1078,7 @@ def get_term_info(short_form: str, preview: bool = False): Results are cached in SOLR for 3 months to improve performance. :param short_form: short form of the term + :param preview: if False, skips executing query previews (much faster) :return: term info """ parsed_object = None @@ -1085,8 +1088,8 @@ def get_term_info(short_form: str, preview: bool = False): # Check if any results were returned parsed_object = term_info_parse_object(results, short_form) if parsed_object: - # Only try to fill query results if there are queries to fill - if parsed_object.get('Queries') and len(parsed_object['Queries']) > 0: + # Only try to fill query results if preview is enabled and there are queries to fill + if preview and parsed_object.get('Queries') and len(parsed_object['Queries']) > 0: try: term_info = fill_query_results(parsed_object) if term_info: @@ -1110,7 +1113,7 @@ def get_term_info(short_form: str, preview: bool = False): query['count'] = 0 return parsed_object else: - # No queries to fill, return parsed object directly + # No queries to fill (preview=False) or no queries defined, return parsed object directly return parsed_object else: print(f"No valid term info found for ID '{short_form}'") From 49185c8282f2a3254e8c80733be5edc409472906 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 5 Nov 2025 11:17:37 +0000 Subject: [PATCH 13/70] Update performance test results [skip ci] --- performance.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/performance.md b/performance.md index aef9b41..5e65027 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-05 10:45:05 UTC -**Git Commit:** 86bb48e6d1b2680ec0d74d9e686f795bbb6d4fb5 +**Test Date:** 2025-11-05 11:17:37 UTC +**Git Commit:** 9ddce50fdda3ada88a9eb1a55ef1dff6f3093c29 **Branch:** dev -**Workflow Run:** 19098809304 +**Workflow Run:** 19100200762 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 1284.1489 seconds -- **VFB_00101567 Query Time**: 0.7166 seconds -- **Total Query Time**: 1284.8656 seconds +- **FBbt_00003748 Query Time**: 1.4088 seconds +- **VFB_00101567 Query Time**: 0.7542 seconds +- **Total Query Time**: 2.1631 seconds -āš ļø **Result**: Some performance thresholds exceeded or test failed +šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-05 10:45:06 UTC* +*Last updated: 2025-11-05 11:17:37 UTC* From efcfd9bbb840239b0e77803aa1df337953f3f6aa Mon Sep 17 00:00:00 2001 From: Rob Court Date: Wed, 5 Nov 2025 14:42:21 +0000 Subject: [PATCH 14/70] Enhance caching mechanism for term_info queries by including preview parameter in cache key --- src/vfbquery/solr_result_cache.py | 15 +++++++++++---- src/vfbquery/vfb_queries.py | 4 ++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/vfbquery/solr_result_cache.py b/src/vfbquery/solr_result_cache.py index 086da9e..bb5586a 100644 --- a/src/vfbquery/solr_result_cache.py +++ b/src/vfbquery/solr_result_cache.py @@ -585,16 +585,23 @@ def wrapper(*args, **kwargs): logger.warning(f"No term_id found for caching {query_type}") return func(*args, **kwargs) + # Include preview parameter in cache key for term_info queries + # This ensures preview=True and preview=False have separate cache entries + cache_term_id = term_id + if query_type == 'term_info': + preview = kwargs.get('preview', True) # Default is True + cache_term_id = f"{term_id}_preview_{preview}" + cache = get_solr_cache() # Clear cache if force_refresh is True if force_refresh: logger.info(f"Force refresh requested for {query_type}({term_id})") - cache.clear_cache_entry(query_type, term_id) + cache.clear_cache_entry(query_type, cache_term_id) # Try cache first (will be empty if force_refresh was True) if not force_refresh: - cached_result = cache.get_cached_result(query_type, term_id, **kwargs) + cached_result = cache.get_cached_result(query_type, cache_term_id, **kwargs) if cached_result is not None: # Validate that cached result has essential fields for term_info if query_type == 'term_info': @@ -653,7 +660,7 @@ def wrapper(*args, **kwargs): if (result and isinstance(result, dict) and result.get('Id') and result.get('Name')): try: - cache.cache_result(query_type, term_id, result, **kwargs) + cache.cache_result(query_type, cache_term_id, result, **kwargs) logger.debug(f"Cached complete result for {term_id}") except Exception as e: logger.debug(f"Failed to cache result: {e}") @@ -661,7 +668,7 @@ def wrapper(*args, **kwargs): logger.warning(f"Not caching incomplete result for {term_id}") else: try: - cache.cache_result(query_type, term_id, result, **kwargs) + cache.cache_result(query_type, cache_term_id, result, **kwargs) except Exception as e: logger.debug(f"Failed to cache result: {e}") diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index a8a5f8c..9834b92 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -1072,13 +1072,13 @@ def serialize_solr_output(results): return json_string @with_solr_cache('term_info') -def get_term_info(short_form: str, preview: bool = False): +def get_term_info(short_form: str, preview: bool = True): """ Retrieves the term info for the given term short form. Results are cached in SOLR for 3 months to improve performance. :param short_form: short form of the term - :param preview: if False, skips executing query previews (much faster) + :param preview: if True, executes query previews to populate preview_results (default: True) :return: term info """ parsed_object = None From 57a62e95e86b3edb09f29cb608913f46caa980a4 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 5 Nov 2025 15:02:02 +0000 Subject: [PATCH 15/70] Update performance test results [skip ci] --- performance.md | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/performance.md b/performance.md index 5e65027..90179da 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-05 11:17:37 UTC -**Git Commit:** 9ddce50fdda3ada88a9eb1a55ef1dff6f3093c29 +**Test Date:** 2025-11-05 15:02:02 UTC +**Git Commit:** efcfd9bbb840239b0e77803aa1df337953f3f6aa **Branch:** dev -**Workflow Run:** 19100200762 +**Workflow Run:** 19105729538 ## Test Overview @@ -23,13 +23,7 @@ This performance test measures the execution time of VFB term info queries for s ## Summary -āœ… **Test Status**: Performance test completed - -- **FBbt_00003748 Query Time**: 1.4088 seconds -- **VFB_00101567 Query Time**: 0.7542 seconds -- **Total Query Time**: 2.1631 seconds - -šŸŽ‰ **Result**: All performance thresholds met! +āŒ **Test Status**: Performance test failed to run properly --- -*Last updated: 2025-11-05 11:17:37 UTC* +*Last updated: 2025-11-05 15:02:02 UTC* From 996e297433381169865be59430458268a7675055 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Wed, 5 Nov 2025 16:17:54 +0000 Subject: [PATCH 16/70] Add conditional validation for cached results based on preview parameter --- src/vfbquery/solr_result_cache.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/vfbquery/solr_result_cache.py b/src/vfbquery/solr_result_cache.py index bb5586a..22c6a2c 100644 --- a/src/vfbquery/solr_result_cache.py +++ b/src/vfbquery/solr_result_cache.py @@ -608,8 +608,9 @@ def wrapper(*args, **kwargs): is_valid = (cached_result and isinstance(cached_result, dict) and cached_result.get('Id') and cached_result.get('Name')) - # Additional validation for query results - if is_valid and 'Queries' in cached_result: + # Additional validation for query results - only when preview=True + preview = kwargs.get('preview', True) # Default is True + if is_valid and preview and 'Queries' in cached_result: logger.debug(f"Validating {len(cached_result['Queries'])} queries for {term_id}") for i, query in enumerate(cached_result['Queries']): count = query.get('count', 0) From 19e9770575cd7cc9b82cffb354f195fc039f1237 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 5 Nov 2025 16:35:20 +0000 Subject: [PATCH 17/70] Update performance test results [skip ci] --- performance.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/performance.md b/performance.md index 90179da..4c4aeaa 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-05 15:02:02 UTC -**Git Commit:** efcfd9bbb840239b0e77803aa1df337953f3f6aa +**Test Date:** 2025-11-05 16:35:20 UTC +**Git Commit:** 4906e444d024c5948766f3f62d06eab8ea3f5416 **Branch:** dev -**Workflow Run:** 19105729538 +**Workflow Run:** 19108639133 ## Test Overview @@ -23,7 +23,13 @@ This performance test measures the execution time of VFB term info queries for s ## Summary -āŒ **Test Status**: Performance test failed to run properly +āœ… **Test Status**: Performance test completed + +- **FBbt_00003748 Query Time**: 976.3858 seconds +- **VFB_00101567 Query Time**: 1.1636 seconds +- **Total Query Time**: 977.5494 seconds + +āš ļø **Result**: Some performance thresholds exceeded or test failed --- -*Last updated: 2025-11-05 15:02:02 UTC* +*Last updated: 2025-11-05 16:35:20 UTC* From ed15a7d24bd4def56b7aec5058194cfe12ca17e5 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Wed, 5 Nov 2025 17:59:07 +0000 Subject: [PATCH 18/70] Refactor caching logic to include query type in cache document ID and enhance validation for term_info results based on preview parameter --- src/vfbquery/solr_result_cache.py | 48 ++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/src/vfbquery/solr_result_cache.py b/src/vfbquery/solr_result_cache.py index 22c6a2c..1c7a929 100644 --- a/src/vfbquery/solr_result_cache.py +++ b/src/vfbquery/solr_result_cache.py @@ -95,11 +95,12 @@ def get_cached_result(self, query_type: str, term_id: str, **params) -> Optional Cached result or None if not found/expired """ try: - # Query for cache document with prefixed ID - cache_doc_id = f"vfb_query_{term_id}" + # Query for cache document with prefixed ID including query type + # This ensures different query types for the same term have separate cache entries + cache_doc_id = f"vfb_query_{query_type}_{term_id}" response = requests.get(f"{self.cache_url}/select", params={ - "q": f"id:{cache_doc_id} AND query_type:{query_type}", + "q": f"id:{cache_doc_id}", "fl": "cache_data", "wt": "json" }, timeout=5) # Short timeout for cache lookups @@ -194,8 +195,9 @@ def cache_result(self, query_type: str, term_id: str, result: Any, **params) -> if not cached_data: return False # Result too large or other issue - # Create cache document with prefixed ID - cache_doc_id = f"vfb_query_{term_id}" + # Create cache document with prefixed ID including query type + # This ensures different query types for the same term have separate cache entries + cache_doc_id = f"vfb_query_{query_type}_{term_id}" cache_doc = { "id": cache_doc_id, @@ -252,7 +254,8 @@ def clear_cache_entry(self, query_type: str, term_id: str) -> bool: True if successfully cleared, False otherwise """ try: - cache_doc_id = f"vfb_query_{term_id}" + # Include query_type in cache document ID to match storage format + cache_doc_id = f"vfb_query_{query_type}_{term_id}" response = requests.post( f"{self.cache_url}/update", data=f'{cache_doc_id}', @@ -299,10 +302,11 @@ def get_cache_age(self, query_type: str, term_id: str, **params) -> Optional[Dic Dictionary with cache age info or None if not cached """ try: - cache_doc_id = f"vfb_query_{term_id}" + # Include query_type in cache document ID to match storage format + cache_doc_id = f"vfb_query_{query_type}_{term_id}" response = requests.get(f"{self.cache_url}/select", params={ - "q": f"id:{cache_doc_id} AND query_type:{query_type}", + "q": f"id:{cache_doc_id}", "fl": "cache_data,hit_count,last_accessed", "wt": "json" }, timeout=5) @@ -658,8 +662,32 @@ def wrapper(*args, **kwargs): if result_is_valid: # Validate result before caching for term_info if query_type == 'term_info': - if (result and isinstance(result, dict) and - result.get('Id') and result.get('Name')): + # Basic validation: must have Id and Name + is_complete = (result and isinstance(result, dict) and + result.get('Id') and result.get('Name')) + + # Additional validation when preview=True: queries must have valid results + if is_complete: + preview = kwargs.get('preview', True) + if preview and 'Queries' in result and result['Queries']: + # Check that all queries have valid counts and preview_results + for query in result['Queries']: + count = query.get('count', -1) + preview_results = query.get('preview_results') + + # Don't cache if query has invalid count (0 or -1) + if count <= 0: + is_complete = False + logger.warning(f"Not caching result for {term_id}: query has invalid count {count}") + break + + # Don't cache if preview_results is missing or malformed + if not isinstance(preview_results, dict) or not preview_results.get('headers'): + is_complete = False + logger.warning(f"Not caching result for {term_id}: query has invalid preview_results") + break + + if is_complete: try: cache.cache_result(query_type, cache_term_id, result, **kwargs) logger.debug(f"Cached complete result for {term_id}") From ab6baa95b4e2310e273216c5db3bf645d089a57f Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 5 Nov 2025 18:12:02 +0000 Subject: [PATCH 19/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index 4c4aeaa..cafc462 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-05 16:35:20 UTC -**Git Commit:** 4906e444d024c5948766f3f62d06eab8ea3f5416 +**Test Date:** 2025-11-05 18:12:02 UTC +**Git Commit:** c9e72df77249c5903676d716d2ac1a64ffb6e0f8 **Branch:** dev -**Workflow Run:** 19108639133 +**Workflow Run:** 19111477542 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 976.3858 seconds -- **VFB_00101567 Query Time**: 1.1636 seconds -- **Total Query Time**: 977.5494 seconds +- **FBbt_00003748 Query Time**: 695.3888 seconds +- **VFB_00101567 Query Time**: 1.2484 seconds +- **Total Query Time**: 696.6372 seconds āš ļø **Result**: Some performance thresholds exceeded or test failed --- -*Last updated: 2025-11-05 16:35:20 UTC* +*Last updated: 2025-11-05 18:12:02 UTC* From 7150f8fd5ee702b6b10acdd4c829287464a2700c Mon Sep 17 00:00:00 2001 From: Rob Court Date: Wed, 5 Nov 2025 18:56:27 +0000 Subject: [PATCH 20/70] Add debug logging for Owlery query execution and error handling --- src/vfbquery/vfb_queries.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 9834b92..8e2cd2e 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -1961,6 +1961,16 @@ def _owlery_query_to_results(owl_query_string: str, short_form: str, return_data """ try: # Step 1: Query Owlery for classes matching the OWL pattern + print(f"DEBUG: Executing Owlery query: {owl_query_string}") + print(f"DEBUG: Query parameters - short_form: {short_form}, query_by_label: {query_by_label}") + + try: + # Try to get the Owlery endpoint for debugging + owlery_endpoint = getattr(vc.vfb.oc, 'owlery', 'unknown') + print(f"DEBUG: Owlery endpoint: {owlery_endpoint}") + except Exception: + pass # Ignore if we can't get the endpoint + class_ids = vc.vfb.oc.get_subclasses( query=owl_query_string, query_by_label=query_by_label, @@ -2096,6 +2106,8 @@ def _owlery_query_to_results(owl_query_string: str, short_form: str, return_data except Exception as e: print(f"Error in Owlery query: {e}") + print(f"Failed query string: {owl_query_string}") + print(f"Failed query parameters - short_form: {short_form}, query_by_label: {query_by_label}, solr_field: {solr_field}") import traceback traceback.print_exc() # Return empty results From c860d37d408d5581a1963ae168f03257ba7bb6ce Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 5 Nov 2025 18:57:39 +0000 Subject: [PATCH 21/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index cafc462..1172d87 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-05 18:12:02 UTC -**Git Commit:** c9e72df77249c5903676d716d2ac1a64ffb6e0f8 +**Test Date:** 2025-11-05 18:57:39 UTC +**Git Commit:** 172ba5a77223809ec28101b1718f59ba2ae665b2 **Branch:** dev -**Workflow Run:** 19111477542 +**Workflow Run:** 19112990567 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 695.3888 seconds -- **VFB_00101567 Query Time**: 1.2484 seconds -- **Total Query Time**: 696.6372 seconds +- **FBbt_00003748 Query Time**: 6.6323 seconds +- **VFB_00101567 Query Time**: 0.6441 seconds +- **Total Query Time**: 7.2764 seconds āš ļø **Result**: Some performance thresholds exceeded or test failed --- -*Last updated: 2025-11-05 18:12:02 UTC* +*Last updated: 2025-11-05 18:57:39 UTC* From c6708c2eb6ffd794dd26c9d78683e8beb9808ff9 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Wed, 5 Nov 2025 20:15:14 +0000 Subject: [PATCH 22/70] Enhance error handling in Owlery queries and update count logic in caching system --- src/vfbquery/solr_result_cache.py | 18 ++++++++++-------- src/vfbquery/vfb_queries.py | 27 +++++++++++++++------------ 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/vfbquery/solr_result_cache.py b/src/vfbquery/solr_result_cache.py index 1c7a929..c18a956 100644 --- a/src/vfbquery/solr_result_cache.py +++ b/src/vfbquery/solr_result_cache.py @@ -617,16 +617,17 @@ def wrapper(*args, **kwargs): if is_valid and preview and 'Queries' in cached_result: logger.debug(f"Validating {len(cached_result['Queries'])} queries for {term_id}") for i, query in enumerate(cached_result['Queries']): - count = query.get('count', 0) + count = query.get('count', -1) # Default to -1 if missing preview_results = query.get('preview_results') headers = preview_results.get('headers', []) if isinstance(preview_results, dict) else [] logger.debug(f"Query {i}: count={count}, preview_results_type={type(preview_results)}, headers={headers}") - # Check if query has unrealistic count (0 or -1) which indicates failed execution - if count <= 0: + # Check if query has error count (-1) which indicates failed execution + # Note: count of 0 is valid - it means "no matches found" + if count < 0: is_valid = False - logger.debug(f"Cached result has invalid query count {count} for {term_id}") + logger.debug(f"Cached result has error query count {count} for {term_id}") break # Check if preview_results is missing or has empty headers when it should have data if not isinstance(preview_results, dict) or not headers: @@ -672,13 +673,14 @@ def wrapper(*args, **kwargs): if preview and 'Queries' in result and result['Queries']: # Check that all queries have valid counts and preview_results for query in result['Queries']: - count = query.get('count', -1) + count = query.get('count', -1) # Default to -1 if missing preview_results = query.get('preview_results') - # Don't cache if query has invalid count (0 or -1) - if count <= 0: + # Don't cache if query has error count (-1 indicates failure) + # Note: count of 0 is valid - it means "no matches found" + if count < 0: is_complete = False - logger.warning(f"Not caching result for {term_id}: query has invalid count {count}") + logger.warning(f"Not caching result for {term_id}: query has error count {count}") break # Don't cache if preview_results is missing or malformed diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 8e2cd2e..9311a85 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -1961,15 +1961,20 @@ def _owlery_query_to_results(owl_query_string: str, short_form: str, return_data """ try: # Step 1: Query Owlery for classes matching the OWL pattern - print(f"DEBUG: Executing Owlery query: {owl_query_string}") - print(f"DEBUG: Query parameters - short_form: {short_form}, query_by_label: {query_by_label}") - + # Construct the full Owlery URL for debugging + owlery_base = "https://owl.virtualflybrain.org/kbs/vfb" # Default try: - # Try to get the Owlery endpoint for debugging - owlery_endpoint = getattr(vc.vfb.oc, 'owlery', 'unknown') - print(f"DEBUG: Owlery endpoint: {owlery_endpoint}") + if hasattr(vc.vfb, 'oc') and hasattr(vc.vfb.oc, 'owlery_endpoint'): + owlery_base = vc.vfb.oc.owlery_endpoint.rstrip('/') except Exception: - pass # Ignore if we can't get the endpoint + pass + + # Construct the actual API call URL (what vfb_connect calls) + from urllib.parse import quote + query_encoded = quote(owl_query_string, safe='') + owlery_url = f"{owlery_base}/subclasses?object={query_encoded}" + + print(f"DEBUG Owlery: {owlery_url}") class_ids = vc.vfb.oc.get_subclasses( query=owl_query_string, @@ -2105,18 +2110,16 @@ def _owlery_query_to_results(owl_query_string: str, short_form: str, return_data } except Exception as e: - print(f"Error in Owlery query: {e}") - print(f"Failed query string: {owl_query_string}") - print(f"Failed query parameters - short_form: {short_form}, query_by_label: {query_by_label}, solr_field: {solr_field}") + print(f"ERROR Owlery query failed: {e}") import traceback traceback.print_exc() - # Return empty results + # Return error indication with count=-1 if return_dataframe: return pd.DataFrame() return { "headers": _get_standard_query_headers() if not include_source else _get_neurons_part_here_headers(), "rows": [], - "count": 0 + "count": -1 # -1 indicates query error/failure } From 9b103eb805f41231afdd1a375b18d5d7ac931c2b Mon Sep 17 00:00:00 2001 From: Rob Court Date: Wed, 5 Nov 2025 20:23:36 +0000 Subject: [PATCH 23/70] Improve error handling in Owlery queries by adding debug URL construction for failed requests --- src/vfbquery/vfb_queries.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 9311a85..6dc4f60 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -1961,21 +1961,6 @@ def _owlery_query_to_results(owl_query_string: str, short_form: str, return_data """ try: # Step 1: Query Owlery for classes matching the OWL pattern - # Construct the full Owlery URL for debugging - owlery_base = "https://owl.virtualflybrain.org/kbs/vfb" # Default - try: - if hasattr(vc.vfb, 'oc') and hasattr(vc.vfb.oc, 'owlery_endpoint'): - owlery_base = vc.vfb.oc.owlery_endpoint.rstrip('/') - except Exception: - pass - - # Construct the actual API call URL (what vfb_connect calls) - from urllib.parse import quote - query_encoded = quote(owl_query_string, safe='') - owlery_url = f"{owlery_base}/subclasses?object={query_encoded}" - - print(f"DEBUG Owlery: {owlery_url}") - class_ids = vc.vfb.oc.get_subclasses( query=owl_query_string, query_by_label=query_by_label, @@ -2110,7 +2095,20 @@ def _owlery_query_to_results(owl_query_string: str, short_form: str, return_data } except Exception as e: - print(f"ERROR Owlery query failed: {e}") + # Construct the Owlery URL for debugging failed queries + owlery_base = "https://owl.virtualflybrain.org/kbs/vfb" # Default + try: + if hasattr(vc.vfb, 'oc') and hasattr(vc.vfb.oc, 'owlery_endpoint'): + owlery_base = vc.vfb.oc.owlery_endpoint.rstrip('/') + except Exception: + pass + + from urllib.parse import quote + query_encoded = quote(owl_query_string, safe='') + owlery_url = f"{owlery_base}/subclasses?object={query_encoded}" + + print(f"ERROR: Owlery query failed: {e}") + print(f" Test URL: {owlery_url}") import traceback traceback.print_exc() # Return error indication with count=-1 From bc14393187367d1c7e40c1eef06b7ae198f11f74 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 5 Nov 2025 20:24:46 +0000 Subject: [PATCH 24/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index 1172d87..cd07bc5 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-05 18:57:39 UTC -**Git Commit:** 172ba5a77223809ec28101b1718f59ba2ae665b2 +**Test Date:** 2025-11-05 20:24:46 UTC +**Git Commit:** 9b103eb805f41231afdd1a375b18d5d7ac931c2b **Branch:** dev -**Workflow Run:** 19112990567 +**Workflow Run:** 19115242485 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 6.6323 seconds -- **VFB_00101567 Query Time**: 0.6441 seconds -- **Total Query Time**: 7.2764 seconds +- **FBbt_00003748 Query Time**: 7.8917 seconds +- **VFB_00101567 Query Time**: 0.7861 seconds +- **Total Query Time**: 8.6778 seconds āš ļø **Result**: Some performance thresholds exceeded or test failed --- -*Last updated: 2025-11-05 18:57:39 UTC* +*Last updated: 2025-11-05 20:24:46 UTC* From 09583f26551ebd1b77a88ef931346f3b7c64653e Mon Sep 17 00:00:00 2001 From: Rob Court Date: Wed, 5 Nov 2025 21:08:01 +0000 Subject: [PATCH 25/70] Enhance caching logic in SolrResultCache and implement SimpleVFBConnect client to replace VFBConnect dependency --- src/vfbquery/owlery_client.py | 288 ++++++++++++++++++++++++++++++ src/vfbquery/solr_result_cache.py | 8 + src/vfbquery/vfb_queries.py | 16 +- 3 files changed, 305 insertions(+), 7 deletions(-) create mode 100644 src/vfbquery/owlery_client.py diff --git a/src/vfbquery/owlery_client.py b/src/vfbquery/owlery_client.py new file mode 100644 index 0000000..f168b44 --- /dev/null +++ b/src/vfbquery/owlery_client.py @@ -0,0 +1,288 @@ +""" +Simple Owlery REST API client to replace VFBConnect dependency. + +This module provides direct HTTP access to the Owlery OWL reasoning service, +eliminating the need for vfb_connect which has problematic GUI dependencies. +""" + +import requests +import json +import pandas as pd +import re +from urllib.parse import quote +from typing import List, Optional, Dict, Any, Union + + +def short_form_to_iri(short_form: str) -> str: + """ + Convert a short form (e.g., 'FBbt_00003748') to full IRI. + + :param short_form: Short form like 'FBbt_00003748' + :return: Full IRI like 'http://purl.obolibrary.org/obo/FBbt_00003748' + """ + # OBO library IRIs use underscores in the ID + return f"http://purl.obolibrary.org/obo/{short_form}" + + +def gen_short_form(iri: str) -> str: + """ + Generate short_form from an IRI string (VFBConnect compatible). + Splits by '/' or '#' and takes the last part. + + :param iri: An IRI string + :return: short_form + """ + return re.split('/|#', iri)[-1] + + +class OwleryClient: + """ + Simple client for Owlery OWL reasoning service. + + Provides minimal interface matching VFBConnect's OWLeryConnect functionality + for subclass queries needed by VFBquery. + """ + + def __init__(self, owlery_endpoint: str = "http://owl.virtualflybrain.org/kbs/vfb"): + """ + Initialize Owlery client. + + :param owlery_endpoint: Base URL for Owlery service (default: VFB public instance) + """ + self.owlery_endpoint = owlery_endpoint.rstrip('/') + + def get_subclasses(self, query: str, query_by_label: bool = True, + verbose: bool = False, prefixes: bool = False, direct: bool = False) -> List[str]: + """ + Query Owlery for subclasses matching an OWL class expression. + + This replicates the VFBConnect OWLeryConnect.get_subclasses() method. + Based on: https://github.com/VirtualFlyBrain/VFB_connect/blob/master/src/vfb_connect/owl/owlery_query_tools.py + + :param query: OWL class expression query string (with short forms like '') + :param query_by_label: If True, query uses label syntax (quotes). + If False, uses IRI syntax (angle brackets). + :param verbose: If True, print debug information + :param prefixes: If True, return full IRIs. If False, return short forms. + :param direct: Return direct subclasses only. Default False. + :return: List of class IDs (short forms like 'FBbt_00003748') + """ + try: + # Convert short forms in query to full IRIs + # Pattern: -> + # Match angle brackets with content that looks like a short form (alphanumeric + underscore) + import re + def convert_short_form_to_iri(match): + short_form = match.group(1) # Extract content between < > + # Only convert if it looks like a short form (contains underscore, no slashes) + if '_' in short_form and '/' not in short_form: + return f"<{short_form_to_iri(short_form)}>" + else: + # Already an IRI or other syntax, leave as-is + return match.group(0) + + # Replace all patterns with + iri_query = re.sub(r'<([^>]+)>', convert_short_form_to_iri, query) + + if verbose: + print(f"Original query: {query}") + print(f"IRI query: {iri_query}") + + # Build Owlery subclasses endpoint URL + # Based on VFBConnect's query() method + params = { + 'object': iri_query, + 'prefixes': json.dumps({ + "FBbt": "http://purl.obolibrary.org/obo/FBbt_", + "RO": "http://purl.obolibrary.org/obo/RO_", + "BFO": "http://purl.obolibrary.org/obo/BFO_" + }) + } + if direct: + params['direct'] = 'False' # Note: Owlery expects string 'False', not boolean + + # Make HTTP GET request with longer timeout for complex queries + response = requests.get( + f"{self.owlery_endpoint}/subclasses", + params=params, + timeout=120 + ) + + if verbose: + print(f"Owlery query: {response.url}") + + response.raise_for_status() + + # Parse JSON response + # Owlery returns: {"superClassOf": ["IRI1", "IRI2", ...]} + # Based on VFBConnect: return_type='superClassOf' for subclasses + data = response.json() + + if verbose: + print(f"Response keys: {data.keys() if isinstance(data, dict) else 'not a dict'}") + + # Extract IRIs from response using VFBConnect's key + iris = [] + if isinstance(data, dict) and 'superClassOf' in data: + iris = data['superClassOf'] + elif isinstance(data, list): + # Fallback: simple list response + iris = data + else: + if verbose: + print(f"Unexpected Owlery response format: {type(data)}") + print(f"Response: {data}") + return [] + + if not isinstance(iris, list): + if verbose: + print(f"Warning: No results! This is likely due to a query error") + print(f"Query: {query}") + return [] + + # Convert IRIs to short forms using gen_short_form logic from VFBConnect + # gen_short_form splits by '/' or '#' and takes the last part + import re + def gen_short_form(iri): + """Generate short_form from an IRI string (VFBConnect compatible)""" + return re.split('/|#', iri)[-1] + + short_forms = list(map(gen_short_form, iris)) + + if verbose: + print(f"Found {len(short_forms)} subclasses") + + return short_forms + + except requests.RequestException as e: + print(f"ERROR: Owlery request failed: {e}") + raise + except Exception as e: + print(f"ERROR: Unexpected error in Owlery query: {e}") + raise + + +class MockNeo4jClient: + """ + Mock Neo4j client that raises informative errors. + + Neo4j queries require full vfb_connect installation which has + GUI dependencies. This mock provides clear error messages. + """ + + def commit_list(self, queries): + """ + Mock Neo4j commit_list that raises NotImplementedError. + + :param queries: List of Cypher queries + :raises NotImplementedError: Always - Neo4j requires full vfb_connect + """ + raise NotImplementedError( + "Neo4j queries require full vfb_connect installation. " + "In development environment without GUI libraries, only Owlery-based " + "queries are available (e.g., get_neurons_with_part_in, get_parts_of, etc.). " + "Neo4j-based queries (e.g., get_instances, get_similar_neurons) are not available." + ) + + +class SimpleVFBConnect: + """ + Minimal replacement for VFBConnect that works in headless environments. + + Provides: + - Owlery client (vc.vfb.oc) for OWL reasoning queries + - Mock Neo4j client (vc.nc) that raises informative errors + - SOLR term info fetcher (vc.get_TermInfo) for term metadata + + This eliminates the need for vfb_connect which requires GUI libraries + (vispy, Quartz.framework on macOS) that aren't available in all dev environments. + """ + + def __init__(self, solr_url: str = "https://solr.virtualflybrain.org/solr/vfb_json"): + """ + Initialize simple VFB connection with Owlery and SOLR access. + + :param solr_url: Base URL for SOLR server (default: VFB public instance) + """ + self._vfb = None + self._nc = None + self.solr_url = solr_url + + @property + def vfb(self): + """Get VFB object with Owlery client.""" + if self._vfb is None: + # Create simple object with oc (Owlery client) property + class VFBObject: + def __init__(self): + self.oc = OwleryClient() + self._vfb = VFBObject() + return self._vfb + + @property + def nc(self): + """Get Neo4j client (mock that raises errors).""" + if self._nc is None: + self._nc = MockNeo4jClient() + return self._nc + + def get_TermInfo(self, short_forms: List[str], + return_dataframe: bool = False, + summary: bool = False) -> Union[List[Dict[str, Any]], pd.DataFrame]: + """ + Fetch term info from SOLR directly. + + This replicates VFBConnect's get_TermInfo method using direct SOLR queries. + + :param short_forms: List of term IDs to fetch (e.g., ['FBbt_00003748']) + :param return_dataframe: If True, return as pandas DataFrame + :param summary: If True, return summarized version (currently ignored) + :return: List of term info dictionaries or DataFrame + """ + results = [] + + for short_form in short_forms: + try: + url = f"{self.solr_url}/select" + params = { + "indent": "true", + "fl": "term_info", + "q.op": "OR", + "q": f"id:{short_form}" + } + + response = requests.get(url, params=params, timeout=30) + response.raise_for_status() + + data = response.json() + docs = data.get("response", {}).get("docs", []) + + if not docs: + print(f"WARNING: No results found for {short_form}") + continue + + if "term_info" not in docs[0] or not docs[0]["term_info"]: + print(f"WARNING: No term_info found for {short_form}") + continue + + # Extract and parse the term_info string which is itself JSON + term_info_str = docs[0]["term_info"][0] + term_info_obj = json.loads(term_info_str) + results.append(term_info_obj) + + except requests.RequestException as e: + print(f"ERROR: Error fetching data from SOLR: {e}") + except json.JSONDecodeError as e: + print(f"ERROR: Error decoding JSON for {short_form}: {e}") + except Exception as e: + print(f"ERROR: Unexpected error for {short_form}: {e}") + + # Convert to DataFrame if requested + if return_dataframe and results: + try: + return pd.json_normalize(results) + except Exception as e: + print(f"ERROR: Error converting to DataFrame: {e}") + return results + + return results diff --git a/src/vfbquery/solr_result_cache.py b/src/vfbquery/solr_result_cache.py index c18a956..2024205 100644 --- a/src/vfbquery/solr_result_cache.py +++ b/src/vfbquery/solr_result_cache.py @@ -596,6 +596,14 @@ def wrapper(*args, **kwargs): preview = kwargs.get('preview', True) # Default is True cache_term_id = f"{term_id}_preview_{preview}" + # Include return_dataframe parameter in cache key for queries that support it + # This ensures DataFrame and dict formats are cached separately + if query_type in ['instances', 'neurons_part_here', 'neurons_synaptic', + 'neurons_presynaptic', 'neurons_postsynaptic', + 'components_of', 'parts_of', 'subclasses_of']: + return_dataframe = kwargs.get('return_dataframe', True) # Default is True + cache_term_id = f"{cache_term_id}_df_{return_dataframe}" + cache = get_solr_cache() # Clear cache if force_refresh is True diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 6dc4f60..20bfbc9 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -1,7 +1,7 @@ import pysolr from .term_info_queries import deserialize_term_info -# Replace VfbConnect import with our new SolrTermInfoFetcher -from .solr_fetcher import SolrTermInfoFetcher +# Replace VfbConnect import with our new SimpleVFBConnect +from .owlery_client import SimpleVFBConnect # Keep dict_cursor if it's used elsewhere - lazy import to avoid GUI issues from marshmallow import Schema, fields, post_load from typing import List, Tuple, Dict, Any, Union @@ -59,8 +59,8 @@ def get_dict_cursor(): # Connect to the VFB SOLR server vfb_solr = pysolr.Solr('http://solr.virtualflybrain.org/solr/vfb_json/', always_commit=False, timeout=990) -# Replace VfbConnect with SolrTermInfoFetcher -vc = SolrTermInfoFetcher() +# Replace VfbConnect with SimpleVFBConnect +vc = SimpleVFBConnect() def initialize_vfb_connect(): """ @@ -2096,7 +2096,7 @@ def _owlery_query_to_results(owl_query_string: str, short_form: str, return_data except Exception as e: # Construct the Owlery URL for debugging failed queries - owlery_base = "https://owl.virtualflybrain.org/kbs/vfb" # Default + owlery_base = "http://owl.virtualflybrain.org/kbs/vfb" # Default try: if hasattr(vc.vfb, 'oc') and hasattr(vc.vfb.oc, 'owlery_endpoint'): owlery_base = vc.vfb.oc.owlery_endpoint.rstrip('/') @@ -2107,8 +2107,10 @@ def _owlery_query_to_results(owl_query_string: str, short_form: str, return_data query_encoded = quote(owl_query_string, safe='') owlery_url = f"{owlery_base}/subclasses?object={query_encoded}" - print(f"ERROR: Owlery query failed: {e}") - print(f" Test URL: {owlery_url}") + # Always use stderr for error messages to ensure they are visible + import sys + print(f"ERROR: Owlery query failed: {e}", file=sys.stderr) + print(f" Test URL: {owlery_url}", file=sys.stderr) import traceback traceback.print_exc() # Return error indication with count=-1 From 44f2f33b9e36fcbdeecd10fc35b1acd7507b9bf5 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 5 Nov 2025 21:11:26 +0000 Subject: [PATCH 26/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index cd07bc5..df97a06 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-05 20:24:46 UTC -**Git Commit:** 9b103eb805f41231afdd1a375b18d5d7ac931c2b +**Test Date:** 2025-11-05 21:11:26 UTC +**Git Commit:** ff9724b203389266ea6d469748cd3bdf893cc933 **Branch:** dev -**Workflow Run:** 19115242485 +**Workflow Run:** 19116353503 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 7.8917 seconds -- **VFB_00101567 Query Time**: 0.7861 seconds -- **Total Query Time**: 8.6778 seconds +- **FBbt_00003748 Query Time**: 133.8428 seconds +- **VFB_00101567 Query Time**: 0.7933 seconds +- **Total Query Time**: 134.6361 seconds āš ļø **Result**: Some performance thresholds exceeded or test failed --- -*Last updated: 2025-11-05 20:24:46 UTC* +*Last updated: 2025-11-05 21:11:26 UTC* From 7b9f504242bb3f25ff1427d68a7a9201941bf6e1 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Wed, 5 Nov 2025 21:21:41 +0000 Subject: [PATCH 27/70] Enhance fill_query_results to include count in preview results --- src/vfbquery/vfb_queries.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 20bfbc9..87253cf 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -2181,14 +2181,17 @@ def fill_query_results(term_info): else: print(f"Unsupported result format for filtering columns in {query['function']}") - query['preview_results'] = {'headers': filtered_headers, 'rows': filtered_result} # Handle count extraction based on result type if isinstance(result, dict) and 'count' in result: - query['count'] = result['count'] + result_count = result['count'] elif isinstance(result, pd.DataFrame): - query['count'] = len(result) + result_count = len(result) else: - query['count'] = 0 + result_count = 0 + + # Store preview results with count included + query['preview_results'] = {'headers': filtered_headers, 'rows': filtered_result, 'count': result_count} + query['count'] = result_count # print(f"Filtered result: {filtered_result}") else: print(f"Function {query['function']} not found") From a95c6218a3065a3e01cc775abdb664d1d84f48e2 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 5 Nov 2025 21:23:03 +0000 Subject: [PATCH 28/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index df97a06..1cd94eb 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-05 21:11:26 UTC -**Git Commit:** ff9724b203389266ea6d469748cd3bdf893cc933 +**Test Date:** 2025-11-05 21:23:03 UTC +**Git Commit:** dad9ec57560a3016077c7f2255a45e955e13c9ab **Branch:** dev -**Workflow Run:** 19116353503 +**Workflow Run:** 19116708313 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 133.8428 seconds -- **VFB_00101567 Query Time**: 0.7933 seconds -- **Total Query Time**: 134.6361 seconds +- **FBbt_00003748 Query Time**: 9.9958 seconds +- **VFB_00101567 Query Time**: 0.9550 seconds +- **Total Query Time**: 10.9508 seconds āš ļø **Result**: Some performance thresholds exceeded or test failed --- -*Last updated: 2025-11-05 21:11:26 UTC* +*Last updated: 2025-11-05 21:23:03 UTC* From 47ec9a8e9adc66892ca3b9a05a8338dcf919709d Mon Sep 17 00:00:00 2001 From: Rob Court Date: Wed, 5 Nov 2025 21:44:55 +0000 Subject: [PATCH 29/70] Enhance caching logic in SolrResultCache to prevent caching of error results and improve query validation in with_solr_cache function; update owl_query formatting in get_subclasses_of function for IRI compliance. --- src/vfbquery/solr_result_cache.py | 42 +++++++++++++++++++------------ src/vfbquery/vfb_queries.py | 3 ++- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/vfbquery/solr_result_cache.py b/src/vfbquery/solr_result_cache.py index 2024205..69c7a63 100644 --- a/src/vfbquery/solr_result_cache.py +++ b/src/vfbquery/solr_result_cache.py @@ -662,7 +662,12 @@ def wrapper(*args, **kwargs): if hasattr(result, 'empty'): # DataFrame result_is_valid = not result.empty elif isinstance(result, dict): - result_is_valid = bool(result) + # For dict results, check if it's not an error result (count != -1) + # Error results should not be cached + if 'count' in result: + result_is_valid = result.get('count', -1) >= 0 # Don't cache errors (count=-1) + else: + result_is_valid = bool(result) # For dicts without count field elif isinstance(result, (list, str)): result_is_valid = len(result) > 0 else: @@ -675,27 +680,32 @@ def wrapper(*args, **kwargs): is_complete = (result and isinstance(result, dict) and result.get('Id') and result.get('Name')) - # Additional validation when preview=True: queries must have valid results + # Additional validation when preview=True: check if queries have results + # We allow caching even if some queries failed (count=-1) as long as the core term_info is valid + # This is because some query functions may not be implemented yet or may legitimately fail if is_complete: preview = kwargs.get('preview', True) if preview and 'Queries' in result and result['Queries']: - # Check that all queries have valid counts and preview_results + # Count how many queries have valid results vs errors + valid_queries = 0 + failed_queries = 0 + for query in result['Queries']: - count = query.get('count', -1) # Default to -1 if missing + count = query.get('count', -1) preview_results = query.get('preview_results') - # Don't cache if query has error count (-1 indicates failure) - # Note: count of 0 is valid - it means "no matches found" - if count < 0: - is_complete = False - logger.warning(f"Not caching result for {term_id}: query has error count {count}") - break - - # Don't cache if preview_results is missing or malformed - if not isinstance(preview_results, dict) or not preview_results.get('headers'): - is_complete = False - logger.warning(f"Not caching result for {term_id}: query has invalid preview_results") - break + # Count queries with valid results (count >= 0) + if count >= 0 and isinstance(preview_results, dict): + valid_queries += 1 + else: + failed_queries += 1 + + # Only reject if ALL queries failed - at least one must succeed + if valid_queries == 0 and failed_queries > 0: + is_complete = False + logger.warning(f"Not caching result for {term_id}: all {failed_queries} queries failed") + elif failed_queries > 0: + logger.debug(f"Caching result for {term_id} with {valid_queries} valid queries ({failed_queries} failed)") if is_complete: try: diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 87253cf..f06987f 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -1914,7 +1914,8 @@ def get_subclasses_of(short_form: str, return_dataframe=True, limit: int = -1): :return: Subclasses of the specified class """ # For subclasses, we query the class itself (Owlery subclasses endpoint handles this) - owl_query = f"'{short_form}'" + # Use angle brackets for IRI conversion, not quotes + owl_query = f"<{short_form}>" return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query') def _get_neurons_part_here_headers(): From c3003e024ab7ad72efc48fe2a280b9cf0b4526b0 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 5 Nov 2025 21:46:12 +0000 Subject: [PATCH 30/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index 1cd94eb..21cc12a 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-05 21:23:03 UTC -**Git Commit:** dad9ec57560a3016077c7f2255a45e955e13c9ab +**Test Date:** 2025-11-05 21:46:12 UTC +**Git Commit:** 5b184e4d0ea3cf8e592a9eb73001d4f8b21e900b **Branch:** dev -**Workflow Run:** 19116708313 +**Workflow Run:** 19117280972 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 9.9958 seconds -- **VFB_00101567 Query Time**: 0.9550 seconds -- **Total Query Time**: 10.9508 seconds +- **FBbt_00003748 Query Time**: 11.4389 seconds +- **VFB_00101567 Query Time**: 1.1245 seconds +- **Total Query Time**: 12.5634 seconds āš ļø **Result**: Some performance thresholds exceeded or test failed --- -*Last updated: 2025-11-05 21:23:03 UTC* +*Last updated: 2025-11-05 21:46:12 UTC* From 8cf0e535065095d8171dcf6842740a007b5c8397 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Wed, 5 Nov 2025 22:16:31 +0000 Subject: [PATCH 31/70] Enhance SolrResultCache to validate cached results and reject errors; update owl_query formatting in get_subclasses_of for improved query handling. --- src/vfbquery/solr_result_cache.py | 8 ++++++++ src/vfbquery/vfb_queries.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/vfbquery/solr_result_cache.py b/src/vfbquery/solr_result_cache.py index 69c7a63..761424a 100644 --- a/src/vfbquery/solr_result_cache.py +++ b/src/vfbquery/solr_result_cache.py @@ -162,6 +162,14 @@ def get_cached_result(self, query_type: str, term_id: str, **params) -> Optional logger.warning(f"Failed to parse cached result for {term_id}") return None + # IMPORTANT: Validate cached result - reject error results (count=-1) + # This ensures old cached errors get retried when the service is working again + if isinstance(result, dict) and 'count' in result: + if result.get('count', -1) < 0: + logger.warning(f"Rejecting cached error result for {query_type}({term_id}): count={result.get('count')}") + self._clear_expired_cache_document(cache_doc_id) + return None + logger.info(f"Cache hit for {query_type}({term_id})") return result diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index f06987f..b433d52 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -1916,7 +1916,7 @@ def get_subclasses_of(short_form: str, return_dataframe=True, limit: int = -1): # For subclasses, we query the class itself (Owlery subclasses endpoint handles this) # Use angle brackets for IRI conversion, not quotes owl_query = f"<{short_form}>" - return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query') + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query', query_by_label=False) def _get_neurons_part_here_headers(): """Return standard headers for get_neurons_with_part_in results""" From b1a0237844737b53c2882cfa579d87a6265af2f5 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 5 Nov 2025 22:17:51 +0000 Subject: [PATCH 32/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index 21cc12a..394a6a4 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-05 21:46:12 UTC -**Git Commit:** 5b184e4d0ea3cf8e592a9eb73001d4f8b21e900b +**Test Date:** 2025-11-05 22:17:51 UTC +**Git Commit:** 3d02d8176c60733d28b5d4d1dd79c58de98d9ee0 **Branch:** dev -**Workflow Run:** 19117280972 +**Workflow Run:** 19118039042 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 11.4389 seconds -- **VFB_00101567 Query Time**: 1.1245 seconds -- **Total Query Time**: 12.5634 seconds +- **FBbt_00003748 Query Time**: 9.1223 seconds +- **VFB_00101567 Query Time**: 0.8121 seconds +- **Total Query Time**: 9.9344 seconds āš ļø **Result**: Some performance thresholds exceeded or test failed --- -*Last updated: 2025-11-05 21:46:12 UTC* +*Last updated: 2025-11-05 22:17:51 UTC* From 9d17e1064e23706f5cba291deed06ea62decf65b Mon Sep 17 00:00:00 2001 From: Rob Court Date: Wed, 5 Nov 2025 22:33:11 +0000 Subject: [PATCH 33/70] Enhance with_solr_cache function to track and handle error results; clear cache entry for transient failures to prevent caching of erroneous data. --- src/vfbquery/solr_result_cache.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/vfbquery/solr_result_cache.py b/src/vfbquery/solr_result_cache.py index 761424a..132f1bd 100644 --- a/src/vfbquery/solr_result_cache.py +++ b/src/vfbquery/solr_result_cache.py @@ -666,6 +666,8 @@ def wrapper(*args, **kwargs): # Cache the result asynchronously to avoid blocking # Handle DataFrame, dict, and other result types properly result_is_valid = False + result_is_error = False # Track if result is an error that should clear cache + if result is not None: if hasattr(result, 'empty'): # DataFrame result_is_valid = not result.empty @@ -673,7 +675,9 @@ def wrapper(*args, **kwargs): # For dict results, check if it's not an error result (count != -1) # Error results should not be cached if 'count' in result: - result_is_valid = result.get('count', -1) >= 0 # Don't cache errors (count=-1) + count_value = result.get('count', -1) + result_is_valid = count_value >= 0 # Don't cache errors (count=-1) + result_is_error = count_value < 0 # Mark as error if count is negative else: result_is_valid = bool(result) # For dicts without count field elif isinstance(result, (list, str)): @@ -681,6 +685,15 @@ def wrapper(*args, **kwargs): else: result_is_valid = True + # If result is an error, actively clear any existing cache entry + # This ensures that transient failures don't get stuck in cache + if result_is_error: + logger.warning(f"Query returned error result for {query_type}({term_id}), clearing cache entry") + try: + cache.clear_cache_entry(query_type, cache_term_id) + except Exception as e: + logger.debug(f"Failed to clear cache entry: {e}") + if result_is_valid: # Validate result before caching for term_info if query_type == 'term_info': From bbda43145c35d124cdd7777e4715ee2928870eae Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 5 Nov 2025 22:34:22 +0000 Subject: [PATCH 34/70] Update performance test results [skip ci] --- performance.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/performance.md b/performance.md index 394a6a4..ac788e3 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-05 22:17:51 UTC -**Git Commit:** 3d02d8176c60733d28b5d4d1dd79c58de98d9ee0 +**Test Date:** 2025-11-05 22:34:21 UTC +**Git Commit:** 08c79abc46e864b26a991b387ae08f1ce5b9db04 **Branch:** dev -**Workflow Run:** 19118039042 +**Workflow Run:** 19118413401 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 9.1223 seconds -- **VFB_00101567 Query Time**: 0.8121 seconds -- **Total Query Time**: 9.9344 seconds +- **FBbt_00003748 Query Time**: 1.7221 seconds +- **VFB_00101567 Query Time**: 1.7979 seconds +- **Total Query Time**: 3.5200 seconds -āš ļø **Result**: Some performance thresholds exceeded or test failed +šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-05 22:17:51 UTC* +*Last updated: 2025-11-05 22:34:21 UTC* From c394908d1543eea1779d61dfb382480a156f194e Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 09:20:12 +0000 Subject: [PATCH 35/70] Add minimal fallback implementation for get_templates when Neo4j is unavailable --- src/vfbquery/vfb_queries.py | 58 +++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index b433d52..8a3299b 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -1433,6 +1433,49 @@ def _get_instances_headers(): return formatted_results +def _get_templates_minimal(limit: int = -1, return_dataframe: bool = False): + """ + Minimal fallback implementation for get_templates when Neo4j is unavailable. + Returns hardcoded list of core templates with basic information. + """ + # Core templates with their basic information + templates_data = [ + {"id": "VFB_00101567", "name": "JRC2018Unisex", "tags": "VFB|VFB_vol|has_image", "order": 1}, + {"id": "VFB_00200000", "name": "JRC_FlyEM_Hemibrain", "tags": "VFB|VFB_vol|has_image", "order": 2}, + {"id": "VFB_00017894", "name": "Adult Brain", "tags": "VFB|VFB_painted|has_image", "order": 3}, + {"id": "VFB_00101384", "name": "JFRC2", "tags": "VFB|VFB_vol|has_image", "order": 4}, + {"id": "VFB_00050000", "name": "JFRC2010", "tags": "VFB|VFB_vol|has_image", "order": 5}, + {"id": "VFB_00049000", "name": "Ito2014", "tags": "VFB|VFB_painted|has_image", "order": 6}, + {"id": "VFB_00100000", "name": "FCWB", "tags": "VFB|VFB_vol|has_image", "order": 7}, + {"id": "VFB_00030786", "name": "Adult VNS", "tags": "VFB|VFB_painted|has_image", "order": 8}, + {"id": "VFB_00110000", "name": "L3 CNS", "tags": "VFB|VFB_vol|has_image", "order": 9}, + {"id": "VFB_00120000", "name": "L1 CNS", "tags": "VFB|VFB_vol|has_image", "order": 10}, + ] + + # Apply limit if specified + if limit > 0: + templates_data = templates_data[:limit] + + count = len(templates_data) + + if return_dataframe: + df = pd.DataFrame(templates_data) + return df + + # Format as dict with headers and rows + formatted_results = { + "headers": { + "id": {"title": "Add", "type": "selection_id", "order": -1}, + "order": {"title": "Order", "type": "numeric", "order": 1, "sort": {0: "Asc"}}, + "name": {"title": "Name", "type": "markdown", "order": 1, "sort": {1: "Asc"}}, + "tags": {"title": "Tags", "type": "tags", "order": 2}, + }, + "rows": templates_data, + "count": count + } + + return formatted_results + def get_templates(limit: int = -1, return_dataframe: bool = False): """Get list of templates @@ -1442,12 +1485,17 @@ def get_templates(limit: int = -1, return_dataframe: bool = False): :rtype: pandas.DataFrame or list of dicts """ - count_query = """MATCH (t:Template)<-[:depicts]-(tc:Template)-[r:in_register_with]->(tc:Template) - RETURN COUNT(DISTINCT t) AS total_count""" + try: + count_query = """MATCH (t:Template)<-[:depicts]-(tc:Template)-[r:in_register_with]->(tc:Template) + RETURN COUNT(DISTINCT t) AS total_count""" - count_results = vc.nc.commit_list([count_query]) - count_df = pd.DataFrame.from_records(get_dict_cursor()(count_results)) - total_count = count_df['total_count'][0] if not count_df.empty else 0 + count_results = vc.nc.commit_list([count_query]) + count_df = pd.DataFrame.from_records(get_dict_cursor()(count_results)) + total_count = count_df['total_count'][0] if not count_df.empty else 0 + except Exception as e: + # Fallback to minimal template list when Neo4j is unavailable + print(f"Neo4j unavailable ({e}), using minimal template list fallback") + return _get_templates_minimal(limit, return_dataframe) # Define the main Cypher query query = f""" From de41ff17505fa40f82289d88990f25bff797371d Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 09:21:25 +0000 Subject: [PATCH 36/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index ac788e3..f472337 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-05 22:34:21 UTC -**Git Commit:** 08c79abc46e864b26a991b387ae08f1ce5b9db04 +**Test Date:** 2025-11-06 09:21:25 UTC +**Git Commit:** d5c4d54428a8095073bc5f80ddf36b1c69f3c264 **Branch:** dev -**Workflow Run:** 19118413401 +**Workflow Run:** 19130788290 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 1.7221 seconds -- **VFB_00101567 Query Time**: 1.7979 seconds -- **Total Query Time**: 3.5200 seconds +- **FBbt_00003748 Query Time**: 1.8085 seconds +- **VFB_00101567 Query Time**: 1.1379 seconds +- **Total Query Time**: 2.9464 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-05 22:34:21 UTC* +*Last updated: 2025-11-06 09:21:25 UTC* From a7b570873e2ad470c5a566e428b81cda85516e2b Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 11:08:03 +0000 Subject: [PATCH 37/70] Refactor Neo4j client handling: implement fallback to mock client when Neo4j is unavailable; update error messages for clarity. Add lightweight Neo4j REST client module. --- src/vfbquery/neo4j_client.py | 120 ++++++++++++++++++++++++++++++++++ src/vfbquery/owlery_client.py | 51 +++++++++------ src/vfbquery/vfb_queries.py | 5 +- 3 files changed, 155 insertions(+), 21 deletions(-) create mode 100644 src/vfbquery/neo4j_client.py diff --git a/src/vfbquery/neo4j_client.py b/src/vfbquery/neo4j_client.py new file mode 100644 index 0000000..985eef6 --- /dev/null +++ b/src/vfbquery/neo4j_client.py @@ -0,0 +1,120 @@ +""" +Lightweight Neo4j REST client. + +This module provides a minimal Neo4j client extracted from vfb_connect +to avoid loading heavy GUI dependencies (navis, vispy, matplotlib, etc.) +that come with the full vfb_connect package. + +Based on vfb_connect.neo.neo4j_tools.Neo4jConnect +""" + +import requests +import json +import time + + +def dict_cursor(results): + """ + Takes JSON results from a neo4j query and turns them into a list of dicts. + + :param results: neo4j query results + :return: list of dicts + """ + dc = [] + for n in results: + # Add conditional to skip any failures + if n: + for d in n['data']: + dc.append(dict(zip(n['columns'], d['row']))) + return dc + + +class Neo4jConnect: + """ + Thin layer over Neo4j REST API to handle connections and queries. + + :param endpoint: Neo4j REST endpoint (default: VFB production server) + :param usr: username for authentication + :param pwd: password for authentication + """ + + def __init__(self, + endpoint: str = "http://pdb.virtualflybrain.org", + usr: str = "neo4j", + pwd: str = "vfb"): + self.base_uri = endpoint + self.usr = usr + self.pwd = pwd + self.commit = "/db/neo4j/tx/commit" + self.headers = {'Content-type': 'application/json'} + + # Test connection and fall back to v3 API if needed + if not self.test_connection(): + print("Falling back to Neo4j v3 connection") + self.commit = "/db/data/transaction/commit" + self.headers = {} + if not self.test_connection(): + raise Exception("Failed to connect to Neo4j.") + + def commit_list(self, statements, return_graphs=False): + """ + Commit a list of Cypher statements to Neo4j via REST API. + + :param statements: A list of Cypher statements + :param return_graphs: If True, returns graphs under 'graph' key + :return: List of results or False if errors encountered + """ + cstatements = [] + if return_graphs: + for s in statements: + cstatements.append({'statement': s, "resultDataContents": ["row", "graph"]}) + else: + for s in statements: + cstatements.append({'statement': s}) + + payload = {'statements': cstatements} + + try: + response = requests.post( + url=f"{self.base_uri}{self.commit}", + auth=(self.usr, self.pwd), + data=json.dumps(payload), + headers=self.headers + ) + except requests.exceptions.RequestException as e: + print(f"\033[31mConnection Error:\033[0m {e}") + print("Retrying in 10 seconds...") + time.sleep(10) + return self.commit_list(statements) + + if self.rest_return_check(response): + return response.json()['results'] + else: + return False + + def rest_return_check(self, response): + """ + Check status response and report errors. + + :param response: requests.Response object + :return: True if OK and no errors, False otherwise + """ + if response.status_code != 200: + print(f"\033[31mConnection Error:\033[0m {response.status_code} ({response.reason})") + return False + else: + j = response.json() + if j['errors']: + for e in j['errors']: + print(f"\033[31mQuery Error:\033[0m {e}") + return False + else: + return True + + def test_connection(self): + """Test neo4j endpoint connection""" + statements = ["MATCH (n) RETURN n LIMIT 1"] + if self.commit_list(statements): + return True + else: + return False diff --git a/src/vfbquery/owlery_client.py b/src/vfbquery/owlery_client.py index f168b44..d84dce2 100644 --- a/src/vfbquery/owlery_client.py +++ b/src/vfbquery/owlery_client.py @@ -164,24 +164,13 @@ def gen_short_form(iri): class MockNeo4jClient: """ - Mock Neo4j client that raises informative errors. - - Neo4j queries require full vfb_connect installation which has - GUI dependencies. This mock provides clear error messages. + Mock Neo4j client that raises NotImplementedError for all queries. + Used when Neo4j is not available or connection fails. """ - - def commit_list(self, queries): - """ - Mock Neo4j commit_list that raises NotImplementedError. - - :param queries: List of Cypher queries - :raises NotImplementedError: Always - Neo4j requires full vfb_connect - """ + def commit_list(self, statements): raise NotImplementedError( - "Neo4j queries require full vfb_connect installation. " - "In development environment without GUI libraries, only Owlery-based " - "queries are available (e.g., get_neurons_with_part_in, get_parts_of, etc.). " - "Neo4j-based queries (e.g., get_instances, get_similar_neurons) are not available." + "Neo4j queries are not available. " + "Either Neo4j server is unavailable or connection failed." ) @@ -191,7 +180,7 @@ class SimpleVFBConnect: Provides: - Owlery client (vc.vfb.oc) for OWL reasoning queries - - Mock Neo4j client (vc.nc) that raises informative errors + - Neo4j client (vc.nc) - tries real Neo4j first, falls back to mock - SOLR term info fetcher (vc.get_TermInfo) for term metadata This eliminates the need for vfb_connect which requires GUI libraries @@ -201,11 +190,13 @@ class SimpleVFBConnect: def __init__(self, solr_url: str = "https://solr.virtualflybrain.org/solr/vfb_json"): """ Initialize simple VFB connection with Owlery and SOLR access. + Attempts to use real Neo4j if available, falls back to mock otherwise. :param solr_url: Base URL for SOLR server (default: VFB public instance) """ self._vfb = None self._nc = None + self._nc_available = None # Cache whether Neo4j is available self.solr_url = solr_url @property @@ -221,9 +212,31 @@ def __init__(self): @property def nc(self): - """Get Neo4j client (mock that raises errors).""" + """ + Get Neo4j client - tries real Neo4j first, falls back to mock. + + Attempts to connect to Neo4j using our lightweight client. + If unavailable (server down, network issues), returns mock client. + """ if self._nc is None: - self._nc = MockNeo4jClient() + # Try to connect to real Neo4j + if self._nc_available is None: + try: + from .neo4j_client import Neo4jConnect + # Try to initialize - this will fail if Neo4j server unreachable + self._nc = Neo4jConnect() + self._nc_available = True + print("āœ… Neo4j connection established") + except Exception as e: + # Fall back to mock client + self._nc = MockNeo4jClient() + self._nc_available = False + print(f"ā„¹ļø Neo4j unavailable ({type(e).__name__}), using Owlery-only mode") + elif self._nc_available: + from .neo4j_client import Neo4jConnect + self._nc = Neo4jConnect() + else: + self._nc = MockNeo4jClient() return self._nc def get_TermInfo(self, short_forms: List[str], diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 8a3299b..a13abc0 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -51,10 +51,10 @@ def safe_to_dict(df, sort_by_id=True): def get_dict_cursor(): """Lazy import dict_cursor to avoid import issues during testing""" try: - from vfb_connect.cross_server_tools import dict_cursor + from .neo4j_client import dict_cursor return dict_cursor except ImportError as e: - raise ImportError(f"vfb_connect is required but could not be imported: {e}") + raise ImportError(f"Could not import dict_cursor: {e}") # Connect to the VFB SOLR server vfb_solr = pysolr.Solr('http://solr.virtualflybrain.org/solr/vfb_json/', always_commit=False, timeout=990) @@ -1476,6 +1476,7 @@ def _get_templates_minimal(limit: int = -1, return_dataframe: bool = False): return formatted_results +@with_solr_cache('templates') def get_templates(limit: int = -1, return_dataframe: bool = False): """Get list of templates From 3889502f05ace8d5379d36fba6c53d328c44851d Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 11:09:11 +0000 Subject: [PATCH 38/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index f472337..58b3158 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 09:21:25 UTC -**Git Commit:** d5c4d54428a8095073bc5f80ddf36b1c69f3c264 +**Test Date:** 2025-11-06 11:09:11 UTC +**Git Commit:** 82e638d16d8acc30bcdfbe6b3f14cf1266c4969b **Branch:** dev -**Workflow Run:** 19130788290 +**Workflow Run:** 19133644242 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 1.8085 seconds -- **VFB_00101567 Query Time**: 1.1379 seconds -- **Total Query Time**: 2.9464 seconds +- **FBbt_00003748 Query Time**: 1.5410 seconds +- **VFB_00101567 Query Time**: 1.2278 seconds +- **Total Query Time**: 2.7688 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-06 09:21:25 UTC* +*Last updated: 2025-11-06 11:09:11 UTC* From eae6162ae5fa79e6bd866ea2763dec02e0399107 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 11:54:09 +0000 Subject: [PATCH 39/70] Enhance _get_templates_minimal and get_templates functions: include additional fields (thumbnail, dataset, license) in templates data structure for consistency with full get_templates() output. --- src/vfbquery/vfb_queries.py | 38 ++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index a13abc0..6ac69ea 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -1439,17 +1439,18 @@ def _get_templates_minimal(limit: int = -1, return_dataframe: bool = False): Returns hardcoded list of core templates with basic information. """ # Core templates with their basic information + # Include all columns to match full get_templates() structure templates_data = [ - {"id": "VFB_00101567", "name": "JRC2018Unisex", "tags": "VFB|VFB_vol|has_image", "order": 1}, - {"id": "VFB_00200000", "name": "JRC_FlyEM_Hemibrain", "tags": "VFB|VFB_vol|has_image", "order": 2}, - {"id": "VFB_00017894", "name": "Adult Brain", "tags": "VFB|VFB_painted|has_image", "order": 3}, - {"id": "VFB_00101384", "name": "JFRC2", "tags": "VFB|VFB_vol|has_image", "order": 4}, - {"id": "VFB_00050000", "name": "JFRC2010", "tags": "VFB|VFB_vol|has_image", "order": 5}, - {"id": "VFB_00049000", "name": "Ito2014", "tags": "VFB|VFB_painted|has_image", "order": 6}, - {"id": "VFB_00100000", "name": "FCWB", "tags": "VFB|VFB_vol|has_image", "order": 7}, - {"id": "VFB_00030786", "name": "Adult VNS", "tags": "VFB|VFB_painted|has_image", "order": 8}, - {"id": "VFB_00110000", "name": "L3 CNS", "tags": "VFB|VFB_vol|has_image", "order": 9}, - {"id": "VFB_00120000", "name": "L1 CNS", "tags": "VFB|VFB_vol|has_image", "order": 10}, + {"id": "VFB_00101567", "name": "JRC2018Unisex", "tags": "VFB|VFB_vol|has_image", "order": 1, "thumbnail": "", "dataset": "", "license": ""}, + {"id": "VFB_00200000", "name": "JRC_FlyEM_Hemibrain", "tags": "VFB|VFB_vol|has_image", "order": 2, "thumbnail": "", "dataset": "", "license": ""}, + {"id": "VFB_00017894", "name": "Adult Brain", "tags": "VFB|VFB_painted|has_image", "order": 3, "thumbnail": "", "dataset": "", "license": ""}, + {"id": "VFB_00101384", "name": "JFRC2", "tags": "VFB|VFB_vol|has_image", "order": 4, "thumbnail": "", "dataset": "", "license": ""}, + {"id": "VFB_00050000", "name": "JFRC2010", "tags": "VFB|VFB_vol|has_image", "order": 5, "thumbnail": "", "dataset": "", "license": ""}, + {"id": "VFB_00049000", "name": "Ito2014", "tags": "VFB|VFB_painted|has_image", "order": 6, "thumbnail": "", "dataset": "", "license": ""}, + {"id": "VFB_00100000", "name": "FCWB", "tags": "VFB|VFB_vol|has_image", "order": 7, "thumbnail": "", "dataset": "", "license": ""}, + {"id": "VFB_00030786", "name": "Adult VNS", "tags": "VFB|VFB_painted|has_image", "order": 8, "thumbnail": "", "dataset": "", "license": ""}, + {"id": "VFB_00110000", "name": "L3 CNS", "tags": "VFB|VFB_vol|has_image", "order": 9, "thumbnail": "", "dataset": "", "license": ""}, + {"id": "VFB_00120000", "name": "L1 CNS", "tags": "VFB|VFB_vol|has_image", "order": 10, "thumbnail": "", "dataset": "", "license": ""}, ] # Apply limit if specified @@ -1462,13 +1463,16 @@ def _get_templates_minimal(limit: int = -1, return_dataframe: bool = False): df = pd.DataFrame(templates_data) return df - # Format as dict with headers and rows + # Format as dict with headers and rows (match full get_templates structure) formatted_results = { "headers": { "id": {"title": "Add", "type": "selection_id", "order": -1}, "order": {"title": "Order", "type": "numeric", "order": 1, "sort": {0: "Asc"}}, "name": {"title": "Name", "type": "markdown", "order": 1, "sort": {1: "Asc"}}, "tags": {"title": "Tags", "type": "tags", "order": 2}, + "thumbnail": {"title": "Thumbnail", "type": "markdown", "order": 9}, + "dataset": {"title": "Dataset", "type": "metadata", "order": 3}, + "license": {"title": "License", "type": "metadata", "order": 4} }, "rows": templates_data, "count": count @@ -1500,15 +1504,15 @@ def get_templates(limit: int = -1, return_dataframe: bool = False): # Define the main Cypher query query = f""" - MATCH (t:Template)-[:INSTANCEOF]->(p:Class), - (t)<-[:depicts]-(tc:Template)-[r:in_register_with]->(tc:Template), - (t)-[:has_source]->(ds:DataSet)-[:has_license]->(lic:License) + MATCH (t:Template)-[:INSTANCEOF]->(p:Class) + OPTIONAL MATCH (t)<-[:depicts]-(tc:Template)-[r:in_register_with]->(tc:Template) + OPTIONAL MATCH (t)-[:has_source]->(ds:DataSet)-[:has_license|license]->(lic:License) RETURN t.short_form as id, apoc.text.format("[%s](%s)",[COALESCE(t.symbol[0],t.label),t.short_form]) AS name, apoc.text.join(t.uniqueFacets, '|') AS tags, - apoc.text.format("[%s](%s)",[COALESCE(ds.symbol[0],ds.label),ds.short_form]) AS dataset, - REPLACE(apoc.text.format("[%s](%s)",[COALESCE(lic.symbol[0],lic.label),lic.short_form]), '[null](null)', '') AS license, - REPLACE(apoc.text.format("[![%s](%s '%s')](%s)",[COALESCE(t.symbol[0],t.label), REPLACE(COALESCE(r.thumbnail[0],""),"thumbnailT.png","thumbnail.png"), COALESCE(t.symbol[0],t.label), t.short_form]), "[![null]( 'null')](null)", "") as thumbnail, + COALESCE(apoc.text.format("[%s](%s)",[COALESCE(ds.symbol[0],ds.label),ds.short_form]), '') AS dataset, + COALESCE(REPLACE(apoc.text.format("[%s](%s)",[COALESCE(lic.symbol[0],lic.label),lic.short_form]), '[null](null)', ''), '') AS license, + COALESCE(REPLACE(apoc.text.format("[![%s](%s '%s')](%s)",[COALESCE(t.symbol[0],t.label), REPLACE(COALESCE(r.thumbnail[0],""),"thumbnailT.png","thumbnail.png"), COALESCE(t.symbol[0],t.label), t.short_form]), "[![null]( 'null')](null)", ""), "") as thumbnail, 99 as order ORDER BY id Desc """ From fe6454f1a6221d954cfd2d5d4ee0405862757986 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 11:55:14 +0000 Subject: [PATCH 40/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index 58b3158..fd163b9 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 11:09:11 UTC -**Git Commit:** 82e638d16d8acc30bcdfbe6b3f14cf1266c4969b +**Test Date:** 2025-11-06 11:55:14 UTC +**Git Commit:** 879901a3b10f687dc24d4a93bafe15c78a76ad5f **Branch:** dev -**Workflow Run:** 19133644242 +**Workflow Run:** 19134785578 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 1.5410 seconds -- **VFB_00101567 Query Time**: 1.2278 seconds -- **Total Query Time**: 2.7688 seconds +- **FBbt_00003748 Query Time**: 1.0246 seconds +- **VFB_00101567 Query Time**: 0.8317 seconds +- **Total Query Time**: 1.8562 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-06 11:09:11 UTC* +*Last updated: 2025-11-06 11:55:14 UTC* From 0f4d47240fe989b97940c0587a4e13355ad93401 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 15:50:40 +0000 Subject: [PATCH 41/70] Refactor get_templates query: adjust Cypher pattern and improve license and thumbnail handling --- src/vfbquery/vfb_queries.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 6ac69ea..e522a64 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -1503,10 +1503,11 @@ def get_templates(limit: int = -1, return_dataframe: bool = False): return _get_templates_minimal(limit, return_dataframe) # Define the main Cypher query + # Match full pattern to exclude template channel nodes query = f""" - MATCH (t:Template)-[:INSTANCEOF]->(p:Class) - OPTIONAL MATCH (t)<-[:depicts]-(tc:Template)-[r:in_register_with]->(tc:Template) - OPTIONAL MATCH (t)-[:has_source]->(ds:DataSet)-[:has_license|license]->(lic:License) + MATCH (p:Class)<-[:INSTANCEOF]-(t:Template)<-[:depicts]-(tc:Template)-[r:in_register_with]->(tc) + OPTIONAL MATCH (t)-[:has_source]->(ds:DataSet) + OPTIONAL MATCH (ds)-[:has_license|license]->(lic:License) RETURN t.short_form as id, apoc.text.format("[%s](%s)",[COALESCE(t.symbol[0],t.label),t.short_form]) AS name, apoc.text.join(t.uniqueFacets, '|') AS tags, @@ -1514,7 +1515,7 @@ def get_templates(limit: int = -1, return_dataframe: bool = False): COALESCE(REPLACE(apoc.text.format("[%s](%s)",[COALESCE(lic.symbol[0],lic.label),lic.short_form]), '[null](null)', ''), '') AS license, COALESCE(REPLACE(apoc.text.format("[![%s](%s '%s')](%s)",[COALESCE(t.symbol[0],t.label), REPLACE(COALESCE(r.thumbnail[0],""),"thumbnailT.png","thumbnail.png"), COALESCE(t.symbol[0],t.label), t.short_form]), "[![null]( 'null')](null)", ""), "") as thumbnail, 99 as order - ORDER BY id Desc + ORDER BY id DESC """ if limit != -1: From 323cd7de9789eca9bcacc8667aa95048a98d0753 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 15:51:42 +0000 Subject: [PATCH 42/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index fd163b9..005f541 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 11:55:14 UTC -**Git Commit:** 879901a3b10f687dc24d4a93bafe15c78a76ad5f +**Test Date:** 2025-11-06 15:51:42 UTC +**Git Commit:** 0f4d47240fe989b97940c0587a4e13355ad93401 **Branch:** dev -**Workflow Run:** 19134785578 +**Workflow Run:** 19141485505 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 1.0246 seconds -- **VFB_00101567 Query Time**: 0.8317 seconds -- **Total Query Time**: 1.8562 seconds +- **FBbt_00003748 Query Time**: 0.8206 seconds +- **VFB_00101567 Query Time**: 0.9078 seconds +- **Total Query Time**: 1.7284 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-06 11:55:14 UTC* +*Last updated: 2025-11-06 15:51:42 UTC* From 8c3a3dcd4d1fd9ae5cbace860914ec8a0c5dac67 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 16:44:18 +0000 Subject: [PATCH 43/70] Refactor get_templates function: streamline formatting of results for improved readability --- src/vfbquery/vfb_queries.py | 51 +++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index e522a64..6532c4a 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -1547,31 +1547,32 @@ def get_templates(limit: int = -1, return_dataframe: bool = False): # Format the results formatted_results = { "headers": { - "id": {"title": "Add", "type": "selection_id", "order": -1}, - "order": {"title": "Order", "type": "numeric", "order": 1, "sort": {0: "Asc"}}, - "name": {"title": "Name", "type": "markdown", "order": 1, "sort": {1: "Asc"}}, - "tags": {"title": "Tags", "type": "tags", "order": 2}, - "thumbnail": {"title": "Thumbnail", "type": "markdown", "order": 9}, - "dataset": {"title": "Dataset", "type": "metadata", "order": 3}, - "license": {"title": "License", "type": "metadata", "order": 4} - }, - "rows": [ - { - key: row[key] - for key in [ - "id", - "order", - "name", - "tags", - "thumbnail", - "dataset", - "license" - ] - } - for row in safe_to_dict(df) - ], - "count": total_count - } + "id": {"title": "Add", "type": "selection_id", "order": -1}, + "order": {"title": "Order", "type": "numeric", "order": 1, "sort": {0: "Asc"}}, + "name": {"title": "Name", "type": "markdown", "order": 1, "sort": {1: "Asc"}}, + "tags": {"title": "Tags", "type": "tags", "order": 2}, + "thumbnail": {"title": "Thumbnail", "type": "markdown", "order": 9}, + "dataset": {"title": "Dataset", "type": "metadata", "order": 3}, + "license": {"title": "License", "type": "metadata", "order": 4} + }, + "rows": [ + { + key: row[key] + for key in [ + "id", + "order", + "name", + "tags", + "thumbnail", + "dataset", + "license" + ] + } + for row in safe_to_dict(df) + ], + "count": total_count + } + return formatted_results def get_related_anatomy(template_short_form: str, limit: int = -1, return_dataframe: bool = False): From 8f62360caa247616629d233bac58d752edb4c075 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 16:45:35 +0000 Subject: [PATCH 44/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index 005f541..0e0ecbf 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 15:51:42 UTC -**Git Commit:** 0f4d47240fe989b97940c0587a4e13355ad93401 +**Test Date:** 2025-11-06 16:45:35 UTC +**Git Commit:** 2cf6a0ec66dda74f91636d1b4e47e7ad6959f8ae **Branch:** dev -**Workflow Run:** 19141485505 +**Workflow Run:** 19143050774 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 0.8206 seconds -- **VFB_00101567 Query Time**: 0.9078 seconds -- **Total Query Time**: 1.7284 seconds +- **FBbt_00003748 Query Time**: 1.5730 seconds +- **VFB_00101567 Query Time**: 1.2510 seconds +- **Total Query Time**: 2.8239 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-06 15:51:42 UTC* +*Last updated: 2025-11-06 16:45:35 UTC* From 1e8d225b42e62148da8aebbbe513338b54078a91 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 17:02:06 +0000 Subject: [PATCH 45/70] Update README.md: enhance thumbnail URLs and add new neuron queries for medulla --- README.md | 611 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 604 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c5ba6ae..46c67b3 100644 --- a/README.md +++ b/README.md @@ -97,30 +97,627 @@ vfb.get_term_info('FBbt_00003748') "id": "VFB_00102107", "label": "[ME on JRC2018Unisex adult brain](VFB_00102107)", "tags": "Nervous_system|Adult|Visual_system|Synaptic_neuropil_domain", - "thumbnail": "[![ME on JRC2018Unisex adult brain aligned to JRC2018U](http://www.virtualflybrain.org/data/VFB/i/0010/2107/VFB_00101567/thumbnail.png 'ME on JRC2018Unisex adult brain aligned to JRC2018U')](VFB_00101567,VFB_00102107)" + "thumbnail": "[![ME on JRC2018Unisex adult brain aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/0010/2107/VFB_00101567/thumbnail.png 'ME on JRC2018Unisex adult brain aligned to JRC2018U')](VFB_00101567,VFB_00102107)" }, { "id": "VFB_00101385", "label": "[ME(R) on JRC_FlyEM_Hemibrain](VFB_00101385)", "tags": "Nervous_system|Adult|Visual_system|Synaptic_neuropil_domain", - "thumbnail": "[![ME(R) on JRC_FlyEM_Hemibrain aligned to JRCFIB2018Fum](http://www.virtualflybrain.org/data/VFB/i/0010/1385/VFB_00101384/thumbnail.png 'ME(R) on JRC_FlyEM_Hemibrain aligned to JRCFIB2018Fum')](VFB_00101384,VFB_00101385)" + "thumbnail": "[![ME(R) on JRC_FlyEM_Hemibrain aligned to JRCFIB2018Fum](https://www.virtualflybrain.org/data/VFB/i/0010/1385/VFB_00101384/thumbnail.png 'ME(R) on JRC_FlyEM_Hemibrain aligned to JRCFIB2018Fum')](VFB_00101384,VFB_00101385)" }, { "id": "VFB_00030810", "label": "[medulla on adult brain template Ito2014](VFB_00030810)", - "tags": "Nervous_system|Visual_system|Adult|Synaptic_neuropil_domain", - "thumbnail": "[![medulla on adult brain template Ito2014 aligned to adult brain template Ito2014](http://www.virtualflybrain.org/data/VFB/i/0003/0810/VFB_00030786/thumbnail.png 'medulla on adult brain template Ito2014 aligned to adult brain template Ito2014')](VFB_00030786,VFB_00030810)" + "tags": "Nervous_system|Adult|Visual_system|Synaptic_neuropil_domain", + "thumbnail": "[![medulla on adult brain template Ito2014 aligned to adult brain template Ito2014](https://www.virtualflybrain.org/data/VFB/i/0003/0810/VFB_00030786/thumbnail.png 'medulla on adult brain template Ito2014 aligned to adult brain template Ito2014')](VFB_00030786,VFB_00030810)" }, { "id": "VFB_00030624", "label": "[medulla on adult brain template JFRC2](VFB_00030624)", - "tags": "Nervous_system|Visual_system|Adult|Synaptic_neuropil_domain", - "thumbnail": "[![medulla on adult brain template JFRC2 aligned to JFRC2](http://www.virtualflybrain.org/data/VFB/i/0003/0624/VFB_00017894/thumbnail.png 'medulla on adult brain template JFRC2 aligned to JFRC2')](VFB_00017894,VFB_00030624)" + "tags": "Nervous_system|Adult|Visual_system|Synaptic_neuropil_domain", + "thumbnail": "[![medulla on adult brain template JFRC2 aligned to JFRC2](https://www.virtualflybrain.org/data/VFB/i/0003/0624/VFB_00017894/thumbnail.png 'medulla on adult brain template JFRC2 aligned to JFRC2')](VFB_00017894,VFB_00030624)" } - ] + ], + "count": 4 }, "output_format": "table", "count": 4 + }, + { + "query": "NeuronsPartHere", + "label": "Neurons with some part in medulla", + "function": "get_neurons_with_part_in", + "takes": { + "short_form": { + "$and": [ + "Class", + "Anatomy" + ] + }, + "default": { + "short_form": "FBbt_00003748" + } + }, + "preview": 10, + "preview_columns": [ + "id", + "label", + "tags", + "thumbnail" + ], + "preview_results": { + "headers": { + "id": { + "title": "Add", + "type": "selection_id", + "order": -1 + }, + "label": { + "title": "Name", + "type": "markdown", + "order": 0, + "sort": { + "0": "Asc" + } + }, + "tags": { + "title": "Tags", + "type": "tags", + "order": 2 + }, + "thumbnail": { + "title": "Thumbnail", + "type": "markdown", + "order": 9 + } + }, + "rows": [ + { + "id": "FBbt_00053385", + "label": "[medulla intrinsic neuron](FBbt_00053385)", + "tags": "Adult|Nervous_system|Neuron|Visual_system", + "thumbnail": "[![ME.8543 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw06/0696/VFB_00101567/thumbnail.png 'ME.8543 aligned to JRC2018U')](FBbt_00053385)" + }, + { + "id": "FBbt_00110033", + "label": "[medulla intrinsic neuron vGlutMinew1a](FBbt_00110033)", + "tags": "Adult|Glutamatergic|Nervous_system|Visual_system", + "thumbnail": "" + }, + { + "id": "FBbt_00110142", + "label": "[OA-AL2i2](FBbt_00110142)", + "tags": "Adult|Nervous_system|Octopaminergic", + "thumbnail": "[![SPS.ME.7 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw04/2336/VFB_00101567/thumbnail.png 'SPS.ME.7 aligned to JRC2018U')](FBbt_00110142)" + }, + { + "id": "FBbt_00110143", + "label": "[OA-AL2i3](FBbt_00110143)", + "tags": "Adult|Nervous_system|Octopaminergic|Visual_system", + "thumbnail": "[![ME.970 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw03/6562/VFB_00101567/thumbnail.png 'ME.970 aligned to JRC2018U')](FBbt_00110143)" + }, + { + "id": "FBbt_00110144", + "label": "[OA-AL2i4](FBbt_00110144)", + "tags": "Adult|Nervous_system|Octopaminergic", + "thumbnail": "[![OA-AL2i4_R (JRC_OpticLobe:10677) aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/0010/450b/VFB_00101567/thumbnail.png 'OA-AL2i4_R (JRC_OpticLobe:10677) aligned to JRC2018U')](FBbt_00110144)" + } + ], + "count": 472 + }, + "output_format": "table", + "count": 472 + }, + { + "query": "NeuronsSynaptic", + "label": "Neurons with synaptic terminals in medulla", + "function": "get_neurons_with_synapses_in", + "takes": { + "short_form": { + "$and": [ + "Class", + "Anatomy" + ] + }, + "default": { + "short_form": "FBbt_00003748" + } + }, + "preview": 10, + "preview_columns": [ + "id", + "label", + "tags", + "thumbnail" + ], + "preview_results": { + "headers": { + "id": { + "title": "Add", + "type": "selection_id", + "order": -1 + }, + "label": { + "title": "Name", + "type": "markdown", + "order": 0, + "sort": { + "0": "Asc" + } + }, + "tags": { + "title": "Tags", + "type": "tags", + "order": 2 + }, + "thumbnail": { + "title": "Thumbnail", + "type": "markdown", + "order": 9 + } + }, + "rows": [ + { + "id": "FBbt_00053287", + "label": "[Cm](FBbt_00053287)", + "tags": "Adult|Nervous_system|Neuron|Visual_system", + "thumbnail": "[![ME.8457 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw05/7997/VFB_00101567/thumbnail.png 'ME.8457 aligned to JRC2018U')](FBbt_00053287)" + }, + { + "id": "FBbt_00053385", + "label": "[medulla intrinsic neuron](FBbt_00053385)", + "tags": "Adult|Nervous_system|Neuron|Visual_system", + "thumbnail": "[![ME.8543 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw06/0696/VFB_00101567/thumbnail.png 'ME.8543 aligned to JRC2018U')](FBbt_00053385)" + }, + { + "id": "FBbt_00110033", + "label": "[medulla intrinsic neuron vGlutMinew1a](FBbt_00110033)", + "tags": "Adult|Glutamatergic|Nervous_system|Visual_system", + "thumbnail": "" + }, + { + "id": "FBbt_00110142", + "label": "[OA-AL2i2](FBbt_00110142)", + "tags": "Adult|Nervous_system|Octopaminergic", + "thumbnail": "[![SPS.ME.7 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw04/2336/VFB_00101567/thumbnail.png 'SPS.ME.7 aligned to JRC2018U')](FBbt_00110142)" + }, + { + "id": "FBbt_00110143", + "label": "[OA-AL2i3](FBbt_00110143)", + "tags": "Adult|Nervous_system|Octopaminergic|Visual_system", + "thumbnail": "[![ME.970 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw03/6562/VFB_00101567/thumbnail.png 'ME.970 aligned to JRC2018U')](FBbt_00110143)" + }, + { + "id": "FBbt_00110144", + "label": "[OA-AL2i4](FBbt_00110144)", + "tags": "Adult|Nervous_system|Octopaminergic", + "thumbnail": "[![OA-AL2i4_R (JRC_OpticLobe:10677) aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/0010/450b/VFB_00101567/thumbnail.png 'OA-AL2i4_R (JRC_OpticLobe:10677) aligned to JRC2018U')](FBbt_00110144)" + }, + { + "id": "FBbt_00110168", + "label": "[OA-ASM1](FBbt_00110168)", + "tags": "Adult|Nervous_system|Octopaminergic|Visual_system", + "thumbnail": "[![OA-ASM1_R aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/jrch/k105/VFB_00101567/thumbnail.png 'OA-ASM1_R aligned to JRC2018U')](FBbt_00110168)" + }, + { + "id": "FBbt_20004313", + "label": "[MeVPMe13](FBbt_20004313)", + "tags": "Adult|Cholinergic|Nervous_system|Visual_system", + "thumbnail": "[![ME.27 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw03/5802/VFB_00101567/thumbnail.png 'ME.27 aligned to JRC2018U')](FBbt_20004313)" + }, + { + "id": "FBbt_20005420", + "label": "[MeVP3](FBbt_20005420)", + "tags": "Adult|Cholinergic|Nervous_system|Visual_system", + "thumbnail": "[![ME.4801 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw01/4271/VFB_00101567/thumbnail.png 'ME.4801 aligned to JRC2018U')](FBbt_20005420)" + }, + { + "id": "FBbt_20007601", + "label": "[MeVPLp2](FBbt_20007601)", + "tags": "Adult|Nervous_system|Neuron|Visual_system", + "thumbnail": "[![MeVPLp2_L (JRC_OpticLobe:11641) aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/0010/40wo/VFB_00101567/thumbnail.png 'MeVPLp2_L (JRC_OpticLobe:11641) aligned to JRC2018U')](FBbt_20007601)" + } + ], + "count": 465 + }, + "output_format": "table", + "count": 465 + }, + { + "query": "NeuronsPresynapticHere", + "label": "Neurons with presynaptic terminals in medulla", + "function": "get_neurons_with_presynaptic_terminals_in", + "takes": { + "short_form": { + "$and": [ + "Class", + "Anatomy" + ] + }, + "default": { + "short_form": "FBbt_00003748" + } + }, + "preview": 10, + "preview_columns": [ + "id", + "label", + "tags", + "thumbnail" + ], + "preview_results": { + "headers": { + "id": { + "title": "Add", + "type": "selection_id", + "order": -1 + }, + "label": { + "title": "Name", + "type": "markdown", + "order": 0, + "sort": { + "0": "Asc" + } + }, + "tags": { + "title": "Tags", + "type": "tags", + "order": 2 + }, + "thumbnail": { + "title": "Thumbnail", + "type": "markdown", + "order": 9 + } + }, + "rows": [ + { + "id": "FBbt_00110144", + "label": "[OA-AL2i4](FBbt_00110144)", + "tags": "Adult|Nervous_system|Octopaminergic", + "thumbnail": "[![OA-AL2i4_R (JRC_OpticLobe:10677) aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/0010/450b/VFB_00101567/thumbnail.png 'OA-AL2i4_R (JRC_OpticLobe:10677) aligned to JRC2018U')](FBbt_00110144)" + }, + { + "id": "FBbt_02000003", + "label": "[yR8](FBbt_02000003)", + "tags": "Adult|Cholinergic|Histaminergic|Nervous_system|Sensory_neuron|Visual_system", + "thumbnail": "[![R8y_R (JRC_OpticLobe:203836) aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/0010/48b9/VFB_00101567/thumbnail.png 'R8y_R (JRC_OpticLobe:203836) aligned to JRC2018U')](FBbt_02000003)" + }, + { + "id": "FBbt_02000004", + "label": "[pR8](FBbt_02000004)", + "tags": "Adult|Cholinergic|Histaminergic|Nervous_system|Sensory_neuron|Visual_system", + "thumbnail": "[![R8p_R (JRC_OpticLobe:238050) aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/0010/484j/VFB_00101567/thumbnail.png 'R8p_R (JRC_OpticLobe:238050) aligned to JRC2018U')](FBbt_02000004)" + }, + { + "id": "FBbt_02000005", + "label": "[yR7](FBbt_02000005)", + "tags": "Adult|Histaminergic|Nervous_system|Sensory_neuron|Visual_system", + "thumbnail": "[![R7y_R (JRC_OpticLobe:168619) aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/0010/47sf/VFB_00101567/thumbnail.png 'R7y_R (JRC_OpticLobe:168619) aligned to JRC2018U')](FBbt_02000005)" + }, + { + "id": "FBbt_02000006", + "label": "[pR7](FBbt_02000006)", + "tags": "Adult|Histaminergic|Nervous_system|Sensory_neuron|Visual_system", + "thumbnail": "[![R7p_R (JRC_OpticLobe:142122) aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/0010/47l7/VFB_00101567/thumbnail.png 'R7p_R (JRC_OpticLobe:142122) aligned to JRC2018U')](FBbt_02000006)" + }, + { + "id": "FBbt_20007253", + "label": "[CB3838](FBbt_20007253)", + "tags": "Adult|GABAergic|Nervous_system|Visual_system", + "thumbnail": "[![ME.38 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw06/2030/VFB_00101567/thumbnail.png 'ME.38 aligned to JRC2018U')](FBbt_20007253)" + }, + { + "id": "FBbt_20007256", + "label": "[Cm31a](FBbt_20007256)", + "tags": "Adult|GABAergic|Nervous_system|Visual_system", + "thumbnail": "[![ME.5 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw06/2043/VFB_00101567/thumbnail.png 'ME.5 aligned to JRC2018U')](FBbt_20007256)" + }, + { + "id": "FBbt_20007257", + "label": "[Mi19](FBbt_20007257)", + "tags": "Adult|Nervous_system|Neuron|Visual_system", + "thumbnail": "[![ME.5256 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw06/1990/VFB_00101567/thumbnail.png 'ME.5256 aligned to JRC2018U')](FBbt_20007257)" + }, + { + "id": "FBbt_20007258", + "label": "[Cm35](FBbt_20007258)", + "tags": "Adult|GABAergic|Nervous_system|Visual_system", + "thumbnail": "[![ME.18 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw06/2034/VFB_00101567/thumbnail.png 'ME.18 aligned to JRC2018U')](FBbt_20007258)" + }, + { + "id": "FBbt_20007259", + "label": "[Cm32](FBbt_20007259)", + "tags": "Adult|GABAergic|Nervous_system|Visual_system", + "thumbnail": "[![ME.278 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw06/1913/VFB_00101567/thumbnail.png 'ME.278 aligned to JRC2018U')](FBbt_20007259)" + } + ], + "count": 253 + }, + "output_format": "table", + "count": 253 + }, + { + "query": "NeuronsPostsynapticHere", + "label": "Neurons with postsynaptic terminals in medulla", + "function": "get_neurons_with_postsynaptic_terminals_in", + "takes": { + "short_form": { + "$and": [ + "Class", + "Anatomy" + ] + }, + "default": { + "short_form": "FBbt_00003748" + } + }, + "preview": 10, + "preview_columns": [ + "id", + "label", + "tags", + "thumbnail" + ], + "preview_results": { + "headers": { + "id": { + "title": "Add", + "type": "selection_id", + "order": -1 + }, + "label": { + "title": "Name", + "type": "markdown", + "order": 0, + "sort": { + "0": "Asc" + } + }, + "tags": { + "title": "Tags", + "type": "tags", + "order": 2 + }, + "thumbnail": { + "title": "Thumbnail", + "type": "markdown", + "order": 9 + } + }, + "rows": [ + { + "id": "FBbt_02000005", + "label": "[yR7](FBbt_02000005)", + "tags": "Adult|Histaminergic|Nervous_system|Sensory_neuron|Visual_system", + "thumbnail": "[![R7y_R (JRC_OpticLobe:168619) aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/0010/47sf/VFB_00101567/thumbnail.png 'R7y_R (JRC_OpticLobe:168619) aligned to JRC2018U')](FBbt_02000005)" + }, + { + "id": "FBbt_02000006", + "label": "[pR7](FBbt_02000006)", + "tags": "Adult|Histaminergic|Nervous_system|Sensory_neuron|Visual_system", + "thumbnail": "[![R7p_R (JRC_OpticLobe:142122) aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/0010/47l7/VFB_00101567/thumbnail.png 'R7p_R (JRC_OpticLobe:142122) aligned to JRC2018U')](FBbt_02000006)" + }, + { + "id": "FBbt_20007251", + "label": "[MeLo2](FBbt_20007251)", + "tags": "Adult|Cholinergic|Nervous_system|Visual_system", + "thumbnail": "[![MeLo2_R (JRC_OpticLobe:60182) aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/0010/3ztx/VFB_00101567/thumbnail.png 'MeLo2_R (JRC_OpticLobe:60182) aligned to JRC2018U')](FBbt_20007251)" + }, + { + "id": "FBbt_20007253", + "label": "[CB3838](FBbt_20007253)", + "tags": "Adult|GABAergic|Nervous_system|Visual_system", + "thumbnail": "[![ME.38 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw06/2030/VFB_00101567/thumbnail.png 'ME.38 aligned to JRC2018U')](FBbt_20007253)" + }, + { + "id": "FBbt_20007256", + "label": "[Cm31a](FBbt_20007256)", + "tags": "Adult|GABAergic|Nervous_system|Visual_system", + "thumbnail": "[![ME.5 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw06/2043/VFB_00101567/thumbnail.png 'ME.5 aligned to JRC2018U')](FBbt_20007256)" + }, + { + "id": "FBbt_20007257", + "label": "[Mi19](FBbt_20007257)", + "tags": "Adult|Nervous_system|Neuron|Visual_system", + "thumbnail": "[![ME.5256 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw06/1990/VFB_00101567/thumbnail.png 'ME.5256 aligned to JRC2018U')](FBbt_20007257)" + }, + { + "id": "FBbt_20007258", + "label": "[Cm35](FBbt_20007258)", + "tags": "Adult|GABAergic|Nervous_system|Visual_system", + "thumbnail": "[![ME.18 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw06/2034/VFB_00101567/thumbnail.png 'ME.18 aligned to JRC2018U')](FBbt_20007258)" + }, + { + "id": "FBbt_20007259", + "label": "[Cm32](FBbt_20007259)", + "tags": "Adult|GABAergic|Nervous_system|Visual_system", + "thumbnail": "[![ME.278 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw06/1913/VFB_00101567/thumbnail.png 'ME.278 aligned to JRC2018U')](FBbt_20007259)" + }, + { + "id": "FBbt_20007264", + "label": "[CB3849](FBbt_20007264)", + "tags": "Adult|Nervous_system|Neuron|Visual_system", + "thumbnail": "[![NO_CONS.90 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw09/5500/VFB_00101567/thumbnail.png 'NO_CONS.90 aligned to JRC2018U')](FBbt_20007264)" + }, + { + "id": "FBbt_20007265", + "label": "[Pm7](FBbt_20007265)", + "tags": "Adult|GABAergic|Nervous_system|Visual_system", + "thumbnail": "[![ME.768 aligned to JRC2018U](https://www.virtualflybrain.org/data/VFB/i/fw06/1336/VFB_00101567/thumbnail.png 'ME.768 aligned to JRC2018U')](FBbt_20007265)" + } + ], + "count": 331 + }, + "output_format": "table", + "count": 331 + }, + { + "query": "PartsOf", + "label": "Parts of medulla", + "function": "get_parts_of", + "takes": { + "short_form": { + "$and": [ + "Class" + ] + }, + "default": { + "short_form": "FBbt_00003748" + } + }, + "preview": 10, + "preview_columns": [ + "id", + "label", + "tags", + "thumbnail" + ], + "preview_results": { + "headers": { + "id": { + "title": "Add", + "type": "selection_id", + "order": -1 + }, + "label": { + "title": "Name", + "type": "markdown", + "order": 0, + "sort": { + "0": "Asc" + } + }, + "tags": { + "title": "Tags", + "type": "tags", + "order": 2 + }, + "thumbnail": { + "title": "Thumbnail", + "type": "markdown", + "order": 9 + } + }, + "rows": [ + { + "id": "FBbt_00003760", + "label": "[medulla layer M10](FBbt_00003760)", + "tags": "Adult|Nervous_system|Synaptic_neuropil_subdomain|Visual_system", + "thumbnail": "" + }, + { + "id": "FBbt_00045001", + "label": "[plexiform medulla](FBbt_00045001)", + "tags": "Adult|Nervous_system|Synaptic_neuropil_subdomain|Visual_system", + "thumbnail": "" + }, + { + "id": "FBbt_00045002", + "label": "[medulla dorsal rim area](FBbt_00045002)", + "tags": "Adult|Nervous_system|Synaptic_neuropil_subdomain|Visual_system", + "thumbnail": "" + }, + { + "id": "FBbt_00048343", + "label": "[medulla layer](FBbt_00048343)", + "tags": "Adult|Nervous_system|Synaptic_neuropil_subdomain|Visual_system", + "thumbnail": "" + }, + { + "id": "FBbt_00100528", + "label": "[distal medulla glial cell](FBbt_00100528)", + "tags": "Adult|Glial_cell|Nervous_system|Visual_system", + "thumbnail": "" + }, + { + "id": "FBbt_00111270", + "label": "[medulla sublayer M6B](FBbt_00111270)", + "tags": "Adult|Nervous_system|Synaptic_neuropil_subdomain|Visual_system", + "thumbnail": "" + }, + { + "id": "FBbt_00049496", + "label": "[adult medulla astrocyte-like glial cell](FBbt_00049496)", + "tags": "Adult|Glial_cell|Nervous_system|Visual_system", + "thumbnail": "" + }, + { + "id": "FBbt_00049497", + "label": "[adult medulla ensheathing glial cell](FBbt_00049497)", + "tags": "Adult|Glial_cell|Nervous_system|Visual_system", + "thumbnail": "" + }, + { + "id": "FBbt_00049500", + "label": "[adult serpentine medulla ensheathing glial cell](FBbt_00049500)", + "tags": "Adult|Glial_cell|Nervous_system|Visual_system", + "thumbnail": "" + }, + { + "id": "FBbt_00003749", + "label": "[outer medulla](FBbt_00003749)", + "tags": "Adult|Nervous_system|Synaptic_neuropil_subdomain|Visual_system", + "thumbnail": "" + } + ], + "count": 28 + }, + "output_format": "table", + "count": 28 + }, + { + "query": "SubclassesOf", + "label": "Subclasses of medulla", + "function": "get_subclasses_of", + "takes": { + "short_form": { + "$and": [ + "Class" + ] + }, + "default": { + "short_form": "FBbt_00003748" + } + }, + "preview": 10, + "preview_columns": [ + "id", + "label", + "tags", + "thumbnail" + ], + "preview_results": { + "headers": { + "id": { + "title": "Add", + "type": "selection_id", + "order": -1 + }, + "label": { + "title": "Name", + "type": "markdown", + "order": 0, + "sort": { + "0": "Asc" + } + }, + "tags": { + "title": "Tags", + "type": "tags", + "order": 2 + }, + "thumbnail": { + "title": "Thumbnail", + "type": "markdown", + "order": 9 + } + }, + "count": 0 + }, + "output_format": "table", + "count": 0 } ], "IsIndividual": False, From 6156f50ff212bc6aa804dd0fe29703d1f4874c26 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 17:03:19 +0000 Subject: [PATCH 46/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index 0e0ecbf..73541c2 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 16:45:35 UTC -**Git Commit:** 2cf6a0ec66dda74f91636d1b4e47e7ad6959f8ae +**Test Date:** 2025-11-06 17:03:19 UTC +**Git Commit:** fab46e98c14c836e7b952a6b50f91b0ef3ad084b **Branch:** dev -**Workflow Run:** 19143050774 +**Workflow Run:** 19143547620 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 1.5730 seconds -- **VFB_00101567 Query Time**: 1.2510 seconds -- **Total Query Time**: 2.8239 seconds +- **FBbt_00003748 Query Time**: 1.1708 seconds +- **VFB_00101567 Query Time**: 1.0747 seconds +- **Total Query Time**: 2.2455 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-06 16:45:35 UTC* +*Last updated: 2025-11-06 17:03:19 UTC* From f0cdd3667b38314ae253eeac8438d65b083378d2 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 17:08:07 +0000 Subject: [PATCH 47/70] Enhance get_templates function: aggregate datasets and licenses for improved query results --- src/vfbquery/vfb_queries.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 6532c4a..21d1993 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -1504,15 +1504,17 @@ def get_templates(limit: int = -1, return_dataframe: bool = False): # Define the main Cypher query # Match full pattern to exclude template channel nodes + # Use COLLECT to aggregate multiple datasets/licenses into single row per template query = f""" MATCH (p:Class)<-[:INSTANCEOF]-(t:Template)<-[:depicts]-(tc:Template)-[r:in_register_with]->(tc) OPTIONAL MATCH (t)-[:has_source]->(ds:DataSet) OPTIONAL MATCH (ds)-[:has_license|license]->(lic:License) - RETURN t.short_form as id, + WITH t, r, COLLECT(DISTINCT ds) as datasets, COLLECT(DISTINCT lic) as licenses + RETURN DISTINCT t.short_form as id, apoc.text.format("[%s](%s)",[COALESCE(t.symbol[0],t.label),t.short_form]) AS name, apoc.text.join(t.uniqueFacets, '|') AS tags, - COALESCE(apoc.text.format("[%s](%s)",[COALESCE(ds.symbol[0],ds.label),ds.short_form]), '') AS dataset, - COALESCE(REPLACE(apoc.text.format("[%s](%s)",[COALESCE(lic.symbol[0],lic.label),lic.short_form]), '[null](null)', ''), '') AS license, + apoc.text.join([ds IN datasets | apoc.text.format("[%s](%s)",[COALESCE(ds.symbol[0],ds.label),ds.short_form])], ', ') AS dataset, + apoc.text.join([lic IN licenses | REPLACE(apoc.text.format("[%s](%s)",[COALESCE(lic.symbol[0],lic.label),lic.short_form]), '[null](null)', '')], ', ') AS license, COALESCE(REPLACE(apoc.text.format("[![%s](%s '%s')](%s)",[COALESCE(t.symbol[0],t.label), REPLACE(COALESCE(r.thumbnail[0],""),"thumbnailT.png","thumbnail.png"), COALESCE(t.symbol[0],t.label), t.short_form]), "[![null]( 'null')](null)", ""), "") as thumbnail, 99 as order ORDER BY id DESC From 35f877621e8627c244032b07e2a196f2ccc7875b Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 17:09:19 +0000 Subject: [PATCH 48/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index 73541c2..b0cc0f2 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 17:03:19 UTC -**Git Commit:** fab46e98c14c836e7b952a6b50f91b0ef3ad084b +**Test Date:** 2025-11-06 17:09:19 UTC +**Git Commit:** a6f08172fa07f1aa8399cf99d9e6e9e54f8d76f6 **Branch:** dev -**Workflow Run:** 19143547620 +**Workflow Run:** 19143720777 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 1.1708 seconds -- **VFB_00101567 Query Time**: 1.0747 seconds -- **Total Query Time**: 2.2455 seconds +- **FBbt_00003748 Query Time**: 1.4304 seconds +- **VFB_00101567 Query Time**: 1.3393 seconds +- **Total Query Time**: 2.7697 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-06 17:03:19 UTC* +*Last updated: 2025-11-06 17:09:19 UTC* From ebe9e851e0238f3a383ebbabf7fb034e9807922b Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 17:18:37 +0000 Subject: [PATCH 49/70] Refactor fill_query_results: store count at query level instead of in preview_results --- src/vfbquery/vfb_queries.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 21d1993..4d773bc 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -2247,8 +2247,8 @@ def fill_query_results(term_info): else: result_count = 0 - # Store preview results with count included - query['preview_results'] = {'headers': filtered_headers, 'rows': filtered_result, 'count': result_count} + # Store preview results (count is stored at query level, not in preview_results) + query['preview_results'] = {'headers': filtered_headers, 'rows': filtered_result} query['count'] = result_count # print(f"Filtered result: {filtered_result}") else: From a79c002a2c3975f400c72ad36a8a7fe5faace065 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 17:19:42 +0000 Subject: [PATCH 50/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index b0cc0f2..27390aa 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 17:09:19 UTC -**Git Commit:** a6f08172fa07f1aa8399cf99d9e6e9e54f8d76f6 +**Test Date:** 2025-11-06 17:19:42 UTC +**Git Commit:** c6daeee9db1fa8aa13c2b2e185f0a0d077bed164 **Branch:** dev -**Workflow Run:** 19143720777 +**Workflow Run:** 19144019225 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 1.4304 seconds -- **VFB_00101567 Query Time**: 1.3393 seconds -- **Total Query Time**: 2.7697 seconds +- **FBbt_00003748 Query Time**: 0.9748 seconds +- **VFB_00101567 Query Time**: 0.9678 seconds +- **Total Query Time**: 1.9426 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-06 17:09:19 UTC* +*Last updated: 2025-11-06 17:19:42 UTC* From f0aa380dbc8b336e9562b69cfc99fc601d4afc14 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 17:55:47 +0000 Subject: [PATCH 51/70] Refactor get_instances function: update Cypher query pattern for improved clarity --- src/vfbquery/vfb_queries.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 4d773bc..890e6d9 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -1162,9 +1162,10 @@ def get_instances(short_form: str, return_dataframe=True, limit: int = -1): total_count = count_df['total_count'][0] if not count_df.empty else 0 # Define the main Cypher query + # Pattern: Individual ā† depicts ā† TemplateChannel ā†’ in_register_with ā†’ TemplateChannelTemplate ā†’ depicts ā†’ ActualTemplate query = f""" MATCH (i:Individual:has_image)-[:INSTANCEOF]->(p:Class {{ short_form: '{short_form}' }}), - (i)<-[:depicts]-(:Individual)-[r:in_register_with]->(:Template)-[:depicts]->(templ:Template), + (i)<-[:depicts]-(tc:Individual)-[r:in_register_with]->(tct:Template)-[:depicts]->(templ:Template), (i)-[:has_source]->(ds:DataSet) OPTIONAL MATCH (i)-[rx:database_cross_reference]->(site:Site) OPTIONAL MATCH (ds)-[:license|licence]->(lic:License) From 66ce8c5ae66f99c018c553e5e25a3f1021aa1d1e Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 17:56:48 +0000 Subject: [PATCH 52/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index 27390aa..79a6720 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 17:19:42 UTC -**Git Commit:** c6daeee9db1fa8aa13c2b2e185f0a0d077bed164 +**Test Date:** 2025-11-06 17:56:48 UTC +**Git Commit:** 16022a78d578e3c683445ad2b077bea90e27ec57 **Branch:** dev -**Workflow Run:** 19144019225 +**Workflow Run:** 19144999220 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 0.9748 seconds -- **VFB_00101567 Query Time**: 0.9678 seconds -- **Total Query Time**: 1.9426 seconds +- **FBbt_00003748 Query Time**: 0.9845 seconds +- **VFB_00101567 Query Time**: 0.9237 seconds +- **Total Query Time**: 1.9083 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-06 17:19:42 UTC* +*Last updated: 2025-11-06 17:56:48 UTC* From a0800ad3798eae93ac8c37f5e715450f0f329ef1 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 18:47:16 +0000 Subject: [PATCH 53/70] Enhance encode_markdown_links function to handle multiple comma-separated markdown links --- src/vfbquery/vfb_queries.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 890e6d9..70c6342 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -327,9 +327,12 @@ def encode_markdown_links(df, columns): """ Encodes brackets in the labels within markdown links, leaving the link syntax intact. Does NOT encode alt text in linked images ([![...](...)(...)] format). + Handles multiple comma-separated markdown links in a single string. :param df: DataFrame containing the query results. :param columns: List of column names to apply encoding to. """ + import re + def encode_label(label): if not isinstance(label, str): return label @@ -340,17 +343,21 @@ def encode_label(label): if label.startswith("[!["): return label - # Process regular markdown links - elif label.startswith("[") and "](" in label: - parts = label.split("](") - if len(parts) < 2: - return label + # Process regular markdown links - handle multiple links separated by commas + # Pattern matches [label](url) format + elif "[" in label and "](" in label: + # Use regex to find all markdown links and encode each one separately + # Pattern: \[([^\]]+)\]\(([^\)]+)\) + # Matches: [anything except ]](anything except )) + def encode_single_link(match): + label_part = match.group(1) # The label part (between [ and ]) + url_part = match.group(2) # The URL part (between ( and )) + # Encode brackets in the label part only + label_part_encoded = encode_brackets(label_part) + return f"[{label_part_encoded}]({url_part})" - label_part = parts[0][1:] # Remove the leading '[' - # Encode brackets in the label part - label_part_encoded = encode_brackets(label_part) - # Reconstruct the markdown link with the encoded label - encoded_label = f"[{label_part_encoded}]({parts[1]}" + # Replace all markdown links with their encoded versions + encoded_label = re.sub(r'\[([^\]]+)\]\(([^\)]+)\)', encode_single_link, label) return encoded_label except Exception as e: From b8565c41015ddf71e4add8296d6011fc151e0714 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 18:48:24 +0000 Subject: [PATCH 54/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index 79a6720..dc6e829 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 17:56:48 UTC -**Git Commit:** 16022a78d578e3c683445ad2b077bea90e27ec57 +**Test Date:** 2025-11-06 18:48:24 UTC +**Git Commit:** 16ca842f38a510efd4f3954e37ca9cf421c0b1bb **Branch:** dev -**Workflow Run:** 19144999220 +**Workflow Run:** 19146351463 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 0.9845 seconds -- **VFB_00101567 Query Time**: 0.9237 seconds -- **Total Query Time**: 1.9083 seconds +- **FBbt_00003748 Query Time**: 1.0513 seconds +- **VFB_00101567 Query Time**: 1.0490 seconds +- **Total Query Time**: 2.1003 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-06 17:56:48 UTC* +*Last updated: 2025-11-06 18:48:24 UTC* From 03b88591d5a797c6015a22a71339aee14b85f673 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 19:54:39 +0000 Subject: [PATCH 55/70] Refactor template entries in README.md to update dataset and license information --- README.md | 87 +++++++++---------------------------------------------- 1 file changed, 14 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index 46c67b3..7529da3 100644 --- a/README.md +++ b/README.md @@ -1783,53 +1783,12 @@ vfb.get_templates(return_dataframe=False) "license": { "title": "License", "type": "metadata", - "order": 4 - } - }, - "rows": [ - { - "id": "VFB_00101567", - "order": 1, - "name": "[JRC2018U](VFB_00101567)", - "tags": "Nervous_system|Adult", - "thumbnail": "[![JRC2018U](http://www.virtualflybrain.org/data/VFB/i/0010/1567/VFB_00101567/thumbnail.png 'JRC2018U')](VFB_00101567)", - "dataset": "[JRC 2018 templates & ROIs](JRC2018)", - "license": "[CC-BY-NC-SA](VFBlicense_CC_BY_NC_SA_4_0)" - }, - { - "id": "VFB_00200000", - "order": 2, - "name": "[JRCVNC2018U](VFB_00200000)", + "id": "VFB_00100000", + "order": 7, + "name": "[COURT2018VNS](VFB_00100000)", "tags": "Nervous_system|Adult|Ganglion", - "thumbnail": "[![JRCVNC2018U](http://www.virtualflybrain.org/data/VFB/i/0020/0000/VFB_00200000/thumbnail.png 'JRCVNC2018U')](VFB_00200000)", - "dataset": "[JRC 2018 templates & ROIs](JRC2018)", - "license": "[CC-BY-NC-SA](VFBlicense_CC_BY_NC_SA_4_0)" - }, - { - "id": "VFB_00017894", - "order": 3, - "name": "[JFRC2](VFB_00017894)", - "tags": "Nervous_system|Adult", - "thumbnail": "[![JFRC2](http://www.virtualflybrain.org/data/VFB/i/0001/7894/VFB_00017894/thumbnail.png 'JFRC2')](VFB_00017894)", - "dataset": "[FlyLight - GMR GAL4 collection (Jenett2012)](Jenett2012)", - "license": "[CC-BY-NC-SA](VFBlicense_CC_BY_NC_SA_4_0)" - }, - { - "id": "VFB_00101384", - "order": 4, - "name": "[JRCFIB2018Fum](VFB_00101384)", - "tags": "Nervous_system|Adult", - "thumbnail": "[![JRCFIB2018Fum](http://www.virtualflybrain.org/data/VFB/i/0010/1384/VFB_00101384/thumbnail.png 'JRCFIB2018Fum')](VFB_00101384)", - "dataset": "[JRC_FlyEM_Hemibrain painted domains](Xu2020roi)", - "license": "[CC_BY](VFBlicense_CC_BY_4_0)" - }, - { - "id": "VFB_00050000", - "order": 5, - "name": "[L1 larval CNS ssTEM - Cardona/Janelia](VFB_00050000)", - "tags": "Nervous_system|Larva", - "thumbnail": "[![L1 larval CNS ssTEM - Cardona/Janelia](http://www.virtualflybrain.org/data/VFB/i/0005/0000/VFB_00050000/thumbnail.png 'L1 larval CNS ssTEM - Cardona/Janelia')](VFB_00050000)", - "dataset": "[Neurons involved in larval fast escape response - EM (Ohyama2016)](Ohyama2015)", + "thumbnail": "[![COURT2018VNS](http://www.virtualflybrain.org/data/VFB/i/0010/0000/VFB_00100000/thumbnail.png 'COURT2018VNS')](VFB_00100000)", + "dataset": "[Adult VNS neuropils (Court2017)](Court2017)", "license": "[CC_BY_SA](VFBlicense_CC_BY_SA_4_0)" }, { @@ -1838,8 +1797,8 @@ vfb.get_templates(return_dataframe=False) "name": "[L1 larval CNS ssTEM - Cardona/Janelia](VFB_00050000)", "tags": "Nervous_system|Larva", "thumbnail": "[![L1 larval CNS ssTEM - Cardona/Janelia](http://www.virtualflybrain.org/data/VFB/i/0005/0000/VFB_00050000/thumbnail.png 'L1 larval CNS ssTEM - Cardona/Janelia')](VFB_00050000)", - "dataset": "[larval hugin neurons - EM (Schlegel2016)](Schlegel2016)", - "license": "[CC_BY](VFBlicense_CC_BY_4_0)" + "dataset": "[larval hugin neurons - EM (Schlegel2016)](Schlegel2016), [Neurons involved in larval fast escape response - EM (Ohyama2016)](Ohyama2015)", + "license": "[CC_BY](VFBlicense_CC_BY_4_0), [CC_BY_SA](VFBlicense_CC_BY_SA_4_0)" }, { "id": "VFB_00049000", @@ -1850,15 +1809,6 @@ vfb.get_templates(return_dataframe=False) "dataset": "[L3 Larval CNS Template (Truman2016)](Truman2016)", "license": "[CC_BY_SA](VFBlicense_CC_BY_SA_4_0)" }, - { - "id": "VFB_00100000", - "order": 7, - "name": "[COURT2018VNS](VFB_00100000)", - "tags": "Nervous_system|Adult|Ganglion", - "thumbnail": "[![COURT2018VNS](http://www.virtualflybrain.org/data/VFB/i/0010/0000/VFB_00100000/thumbnail.png 'COURT2018VNS')](VFB_00100000)", - "dataset": "[Adult VNS neuropils (Court2017)](Court2017)", - "license": "[CC_BY_SA](VFBlicense_CC_BY_SA_4_0)" - }, { "id": "VFB_00030786", "order": 8, @@ -1869,22 +1819,13 @@ vfb.get_templates(return_dataframe=False) "license": "[CC_BY_SA](VFBlicense_CC_BY_SA_4_0)" }, { - "id": "VFB_00110000", - "order": 9, - "name": "[Adult Head (McKellar2020)](VFB_00110000)", - "tags": "Adult|Anatomy", - "thumbnail": "[![Adult Head (McKellar2020)](http://www.virtualflybrain.org/data/VFB/i/0011/0000/VFB_00110000/thumbnail.png 'Adult Head (McKellar2020)')](VFB_00110000)", - "dataset": "[GAL4 lines from McKellar et al., 2020](McKellar2020)", - "license": "[CC_BY_SA](VFBlicense_CC_BY_SA_4_0)" - }, - { - "id": "VFB_00120000", - "order": 10, - "name": "[Adult T1 Leg (Kuan2020)](VFB_00120000)", - "tags": "Adult|Anatomy", - "thumbnail": "[![Adult T1 Leg (Kuan2020)](http://www.virtualflybrain.org/data/VFB/i/0012/0000/VFB_00120000/thumbnail.png 'Adult T1 Leg (Kuan2020)')](VFB_00120000)", - "dataset": "[Millimeter-scale imaging of a Drosophila leg at single-neuron resolution](Kuan2020)", - "license": "[CC_BY](VFBlicense_CC_BY_4_0)" + "id": "VFB_00017894", + "order": 3, + "name": "[JFRC2](VFB_00017894)", + "tags": "Nervous_system|Adult", + "thumbnail": "[![JFRC2](http://www.virtualflybrain.org/data/VFB/i/0001/7894/VFB_00017894/thumbnail.png 'JFRC2')](VFB_00017894)", + "dataset": "[FlyLight - GMR GAL4 collection (Jenett2012)](Jenett2012)", + "license": "[CC-BY-NC-SA](VFBlicense_CC_BY_NC_SA_4_0)" } ], "count": 10 From 0c031b689c5c4b170d19bd10b8d1300442abd898 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 19:55:34 +0000 Subject: [PATCH 56/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index dc6e829..697de8a 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 18:48:24 UTC -**Git Commit:** 16ca842f38a510efd4f3954e37ca9cf421c0b1bb +**Test Date:** 2025-11-06 19:55:34 UTC +**Git Commit:** 03b88591d5a797c6015a22a71339aee14b85f673 **Branch:** dev -**Workflow Run:** 19146351463 +**Workflow Run:** 19148077265 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 1.0513 seconds -- **VFB_00101567 Query Time**: 1.0490 seconds -- **Total Query Time**: 2.1003 seconds +- **FBbt_00003748 Query Time**: 1.2774 seconds +- **VFB_00101567 Query Time**: 0.7816 seconds +- **Total Query Time**: 2.0590 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-06 18:48:24 UTC* +*Last updated: 2025-11-06 19:55:34 UTC* From f4491d7fd736428fd0685e89f04d9c2d62e951c5 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 20:12:10 +0000 Subject: [PATCH 57/70] Fix missing newline at end of README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7529da3..fee9ca3 100644 --- a/README.md +++ b/README.md @@ -1830,4 +1830,4 @@ vfb.get_templates(return_dataframe=False) ], "count": 10 } -``` \ No newline at end of file +``` From bdd55478ad83378279c6e79a9a55b56a68c3bbce Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 20:13:15 +0000 Subject: [PATCH 58/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index 697de8a..fca5522 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 19:55:34 UTC -**Git Commit:** 03b88591d5a797c6015a22a71339aee14b85f673 +**Test Date:** 2025-11-06 20:13:15 UTC +**Git Commit:** 30dec1dce5d10331eff337b937272e092e93c963 **Branch:** dev -**Workflow Run:** 19148077265 +**Workflow Run:** 19148521621 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 1.2774 seconds -- **VFB_00101567 Query Time**: 0.7816 seconds -- **Total Query Time**: 2.0590 seconds +- **FBbt_00003748 Query Time**: 1.1181 seconds +- **VFB_00101567 Query Time**: 1.0476 seconds +- **Total Query Time**: 2.1657 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-06 19:55:34 UTC* +*Last updated: 2025-11-06 20:13:15 UTC* From 32a0caa22846c748d87ec0c1edc991c6017d9901 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 20:49:56 +0000 Subject: [PATCH 59/70] Update order field in template metadata and correct entry order in README.md --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fee9ca3..b3e09a1 100644 --- a/README.md +++ b/README.md @@ -1783,8 +1783,13 @@ vfb.get_templates(return_dataframe=False) "license": { "title": "License", "type": "metadata", + "order": 7 + } + }, + "rows": [ + { "id": "VFB_00100000", - "order": 7, + "order": 4, "name": "[COURT2018VNS](VFB_00100000)", "tags": "Nervous_system|Adult|Ganglion", "thumbnail": "[![COURT2018VNS](http://www.virtualflybrain.org/data/VFB/i/0010/0000/VFB_00100000/thumbnail.png 'COURT2018VNS')](VFB_00100000)", From d7fa77a50f650033004e3da5d1750e1ccdb4732a Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 20:50:57 +0000 Subject: [PATCH 60/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index fca5522..f87358f 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 20:13:15 UTC -**Git Commit:** 30dec1dce5d10331eff337b937272e092e93c963 +**Test Date:** 2025-11-06 20:50:57 UTC +**Git Commit:** 017cb43b70bf175705632be0edb2a2fffb8c78b2 **Branch:** dev -**Workflow Run:** 19148521621 +**Workflow Run:** 19149456336 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 1.1181 seconds -- **VFB_00101567 Query Time**: 1.0476 seconds -- **Total Query Time**: 2.1657 seconds +- **FBbt_00003748 Query Time**: 0.8290 seconds +- **VFB_00101567 Query Time**: 0.8786 seconds +- **Total Query Time**: 1.7076 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-06 20:13:15 UTC* +*Last updated: 2025-11-06 20:50:57 UTC* From 4457ed418d289c5f732c3bc404497da2843c4f8b Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 20:58:53 +0000 Subject: [PATCH 61/70] Update template metadata with new entries and correct order values --- README.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b3e09a1..140180e 100644 --- a/README.md +++ b/README.md @@ -1783,13 +1783,58 @@ vfb.get_templates(return_dataframe=False) "license": { "title": "License", "type": "metadata", - "order": 7 + "order": 4 } }, "rows": [ { - "id": "VFB_00100000", + "id": "VFB_00200000", + "order": 2, + "name": "[JRCVNC2018U](VFB_00200000)", + "tags": "Nervous_system|Adult|Ganglion", + "thumbnail": "[![JRCVNC2018U](http://www.virtualflybrain.org/data/VFB/i/0020/0000/VFB_00200000/thumbnail.png 'JRCVNC2018U')](VFB_00200000)", + "dataset": "[JRC 2018 templates & ROIs](JRC2018)", + "license": "[CC-BY-NC-SA](VFBlicense_CC_BY_NC_SA_4_0)" + }, + { + "id": "VFB_00120000", + "order": 10, + "name": "[Adult T1 Leg (Kuan2020)](VFB_00120000)", + "tags": "Adult|Anatomy", + "thumbnail": "[![Adult T1 Leg (Kuan2020)](http://www.virtualflybrain.org/data/VFB/i/0012/0000/VFB_00120000/thumbnail.png 'Adult T1 Leg (Kuan2020)')](VFB_00120000)", + "dataset": "[Millimeter-scale imaging of a Drosophila leg at single-neuron resolution](Kuan2020)", + "license": "[CC_BY](VFBlicense_CC_BY_4_0)" + }, + { + "id": "VFB_00110000", + "order": 9, + "name": "[Adult Head (McKellar2020)](VFB_00110000)", + "tags": "Adult|Anatomy", + "thumbnail": "[![Adult Head (McKellar2020)](http://www.virtualflybrain.org/data/VFB/i/0011/0000/VFB_00110000/thumbnail.png 'Adult Head (McKellar2020)')](VFB_00110000)", + "dataset": "[GAL4 lines from McKellar et al., 2020](McKellar2020)", + "license": "[CC_BY_SA](VFBlicense_CC_BY_SA_4_0)" + }, + { + "id": "VFB_00101567", + "order": 1, + "name": "[JRC2018U](VFB_00101567)", + "tags": "Nervous_system|Adult", + "thumbnail": "[![JRC2018U](http://www.virtualflybrain.org/data/VFB/i/0010/1567/VFB_00101567/thumbnail.png 'JRC2018U')](VFB_00101567)", + "dataset": "[JRC 2018 templates & ROIs](JRC2018)", + "license": "[CC-BY-NC-SA](VFBlicense_CC_BY_NC_SA_4_0)" + }, + { + "id": "VFB_00101384", "order": 4, + "name": "[JRCFIB2018Fum](VFB_00101384)", + "tags": "Nervous_system|Adult", + "thumbnail": "[![JRCFIB2018Fum](http://www.virtualflybrain.org/data/VFB/i/0010/1384/VFB_00101384/thumbnail.png 'JRCFIB2018Fum')](VFB_00101384)", + "dataset": "[JRC_FlyEM_Hemibrain painted domains](Xu2020roi)", + "license": "[CC_BY](VFBlicense_CC_BY_4_0)" + }, + { + "id": "VFB_00100000", + "order": 7, "name": "[COURT2018VNS](VFB_00100000)", "tags": "Nervous_system|Adult|Ganglion", "thumbnail": "[![COURT2018VNS](http://www.virtualflybrain.org/data/VFB/i/0010/0000/VFB_00100000/thumbnail.png 'COURT2018VNS')](VFB_00100000)", From 7723ce038b584992d8ba9cd2dd9989d74ca72594 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 20:59:55 +0000 Subject: [PATCH 62/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index f87358f..8928e6f 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 20:50:57 UTC -**Git Commit:** 017cb43b70bf175705632be0edb2a2fffb8c78b2 +**Test Date:** 2025-11-06 20:59:55 UTC +**Git Commit:** e852ffbaefd5410ca183f8c261b8174d18b4f4ba **Branch:** dev -**Workflow Run:** 19149456336 +**Workflow Run:** 19149670397 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 0.8290 seconds -- **VFB_00101567 Query Time**: 0.8786 seconds -- **Total Query Time**: 1.7076 seconds +- **FBbt_00003748 Query Time**: 0.9410 seconds +- **VFB_00101567 Query Time**: 0.7934 seconds +- **Total Query Time**: 1.7343 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-06 20:50:57 UTC* +*Last updated: 2025-11-06 20:59:55 UTC* From 3d3a8dd64fb62540c2b019cbd82a8d28ca52d355 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 21:28:12 +0000 Subject: [PATCH 63/70] Add test suites for NeuronClassesFasciculatingHere, LineageClonesIn, and TractsNervesInnervatingHere queries --- src/test/test_lineage_clones_in.py | 190 ++++++++++++++++++ src/test/test_neuron_classes_fasciculating.py | 187 +++++++++++++++++ src/test/test_tracts_nerves_innervating.py | 188 +++++++++++++++++ src/vfbquery/vfb_queries.py | 160 +++++++++++++++ 4 files changed, 725 insertions(+) create mode 100644 src/test/test_lineage_clones_in.py create mode 100644 src/test/test_neuron_classes_fasciculating.py create mode 100644 src/test/test_tracts_nerves_innervating.py diff --git a/src/test/test_lineage_clones_in.py b/src/test/test_lineage_clones_in.py new file mode 100644 index 0000000..95d86bd --- /dev/null +++ b/src/test/test_lineage_clones_in.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +""" +Test suite for LineageClonesIn query. + +Tests the query that finds lineage clones that overlap with a synaptic neuropil. +This implements the LineageClonesIn query from the VFB XMI specification. + +Test cases: +1. Query execution with known neuropil +2. Schema generation and validation +3. Term info integration +4. Preview results validation +5. Cache functionality +""" + +import unittest +import sys +import os + +# Add the src directory to the path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +from vfbquery.vfb_queries import ( + get_lineage_clones_in, + LineageClonesIn_to_schema, + get_term_info +) + + +class LineageClonesInTest(unittest.TestCase): + """Test suite for LineageClonesIn query""" + + def setUp(self): + """Set up test fixtures""" + # Example synaptic neuropil: adult antennal lobe (FBbt_00007401) + self.test_neuropil = "FBbt_00007401" # antennal lobe + + def test_query_execution(self): + """Test that the query executes successfully""" + print(f"\n=== Testing LineageClonesIn query execution ===") + + # Execute the query + result = get_lineage_clones_in(self.test_neuropil, return_dataframe=False, limit=5) + + # Validate result structure + self.assertIsNotNone(result, "Query should return a result") + self.assertIsInstance(result, dict, "Result should be a dictionary") + + # Check for expected keys + if result: + print(f"Query returned {len(result.get('data', []))} results") + + # Validate data structure + if 'data' in result and len(result['data']) > 0: + first_result = result['data'][0] + self.assertIn('id', first_result, "Result should contain 'id' field") + self.assertIn('label', first_result, "Result should contain 'label' field") + print(f"First result: {first_result.get('label', 'N/A')} ({first_result.get('id', 'N/A')})") + else: + print("No results found (this is OK if no clones overlap this neuropil)") + + def test_schema_generation(self): + """Test schema function generates correct structure""" + print(f"\n=== Testing LineageClonesIn schema generation ===") + + test_name = "Test Neuropil" + test_takes = {"short_form": self.test_neuropil} + + schema = LineageClonesIn_to_schema(test_name, test_takes) + + # Validate schema structure + self.assertIsNotNone(schema, "Schema should not be None") + self.assertEqual(schema.query, "LineageClonesIn", "Query name should match") + self.assertEqual(schema.label, f"Lineage clones found in {test_name}", "Label should be formatted correctly") + self.assertEqual(schema.function, "get_lineage_clones_in", "Function name should match") + self.assertEqual(schema.preview, 10, "Preview should be 10") + + # Check preview columns + expected_columns = ["id", "label", "tags", "thumbnail"] + self.assertEqual(schema.preview_columns, expected_columns, f"Preview columns should be {expected_columns}") + + print(f"Schema generated successfully: {schema.label}") + + def test_term_info_integration(self): + """Test that query appears in term info for appropriate terms""" + print(f"\n=== Testing term info integration ===") + + # Get term info for a synaptic neuropil + term_info = get_term_info(self.test_neuropil, preview=False) + + self.assertIsNotNone(term_info, "Term info should not be None") + self.assertIn("Queries", term_info, "Term info should contain Queries") + + # Check if our query is present + queries = term_info.get("Queries", []) + query_names = [q.get('query') for q in queries] + + print(f"Available queries for {self.test_neuropil}: {query_names}") + + # For synaptic neuropils, this query should be available + if "Synaptic_neuropil" in term_info.get("SuperTypes", []) or \ + "Synaptic_neuropil_domain" in term_info.get("SuperTypes", []): + self.assertIn("LineageClonesIn", query_names, + "LineageClonesIn should be available for Synaptic_neuropil") + print("āœ“ Query correctly appears for Synaptic_neuropil type") + else: + print(f"Warning: {self.test_neuropil} does not have Synaptic_neuropil type") + print(f"SuperTypes: {term_info.get('SuperTypes', [])}") + + def test_preview_results(self): + """Test that preview results are properly formatted""" + print(f"\n=== Testing preview results ===") + + # Get term info with preview enabled + term_info = get_term_info(self.test_neuropil, preview=True) + + self.assertIsNotNone(term_info, "Term info should not be None") + + # Find our query in the results + queries = term_info.get("Queries", []) + clones_query = None + for q in queries: + if q.get('query') == "LineageClonesIn": + clones_query = q + break + + if clones_query: + print(f"Found LineageClonesIn query") + + # Check if preview_results exist + if clones_query.get('preview_results'): + preview = clones_query['preview_results'] + data_key = 'data' if 'data' in preview else 'rows' + print(f"Preview contains {len(preview.get(data_key, []))} results") + + # Validate preview structure + self.assertIn(data_key, preview, f"Preview should contain '{data_key}' key") + self.assertIn('headers', preview, "Preview should contain 'headers' key") + + # Check first result if available + if preview.get(data_key) and len(preview[data_key]) > 0: + first_result = preview[data_key][0] + print(f"First preview result: {first_result.get('label', 'N/A')}") + + # Validate required fields + self.assertIn('id', first_result, "Preview result should have 'id'") + self.assertIn('label', first_result, "Preview result should have 'label'") + else: + print("No preview results available (this is OK if no clones overlap this neuropil)") + else: + print("LineageClonesIn query not found in term info") + + def test_with_different_neuropils(self): + """Test with multiple synaptic neuropil types""" + print(f"\n=== Testing with different neuropils ===") + + test_neuropils = [ + ("FBbt_00007401", "antennal lobe"), + ("FBbt_00003982", "medulla"), + ("FBbt_00003679", "mushroom body"), + ] + + for neuropil_id, neuropil_name in test_neuropils: + print(f"\nTesting {neuropil_name} ({neuropil_id})...") + + try: + result = get_lineage_clones_in(neuropil_id, return_dataframe=False, limit=3) + + if result and 'data' in result: + print(f" āœ“ Query successful, found {len(result['data'])} results") + else: + print(f" āœ“ Query successful, no results found") + + except Exception as e: + print(f" āœ— Query failed: {str(e)}") + # Don't fail the test, just log the error + # raise + + +def run_tests(): + """Run the test suite""" + suite = unittest.TestLoader().loadTestsFromTestCase(LineageClonesInTest) + runner = unittest.TextTestRunner(verbosity=2) + result = runner.run(suite) + return result.wasSuccessful() + + +if __name__ == '__main__': + success = run_tests() + sys.exit(0 if success else 1) diff --git a/src/test/test_neuron_classes_fasciculating.py b/src/test/test_neuron_classes_fasciculating.py new file mode 100644 index 0000000..f06ba23 --- /dev/null +++ b/src/test/test_neuron_classes_fasciculating.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +""" +Test suite for NeuronClassesFasciculatingHere query. + +Tests the query that finds neuron classes that fasciculate with (run along) tracts or nerves. +This implements the NeuronClassesFasciculatingHere query from the VFB XMI specification. + +Test cases: +1. Query execution with known tract +2. Schema generation and validation +3. Term info integration +4. Preview results validation +5. Cache functionality +""" + +import unittest +import sys +import os + +# Add the src directory to the path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +from vfbquery.vfb_queries import ( + get_neuron_classes_fasciculating_here, + NeuronClassesFasciculatingHere_to_schema, + get_term_info +) + + +class NeuronClassesFasciculatingTest(unittest.TestCase): + """Test suite for NeuronClassesFasciculatingHere query""" + + def setUp(self): + """Set up test fixtures""" + # Example tract/nerve: broad root (FBbt_00003987) - a neuron projection bundle + self.test_tract = "FBbt_00003987" # broad root + + def test_query_execution(self): + """Test that the query executes successfully""" + print(f"\n=== Testing NeuronClassesFasciculatingHere query execution ===") + + # Execute the query + result = get_neuron_classes_fasciculating_here(self.test_tract, return_dataframe=False, limit=5) + + # Validate result structure + self.assertIsNotNone(result, "Query should return a result") + self.assertIsInstance(result, dict, "Result should be a dictionary") + + # Check for expected keys + if result: + print(f"Query returned {len(result.get('data', []))} results") + + # Validate data structure + if 'data' in result and len(result['data']) > 0: + first_result = result['data'][0] + self.assertIn('id', first_result, "Result should contain 'id' field") + self.assertIn('label', first_result, "Result should contain 'label' field") + print(f"First result: {first_result.get('label', 'N/A')} ({first_result.get('id', 'N/A')})") + + def test_schema_generation(self): + """Test schema function generates correct structure""" + print(f"\n=== Testing NeuronClassesFasciculatingHere schema generation ===") + + test_name = "Test Tract" + test_takes = {"short_form": self.test_tract} + + schema = NeuronClassesFasciculatingHere_to_schema(test_name, test_takes) + + # Validate schema structure + self.assertIsNotNone(schema, "Schema should not be None") + self.assertEqual(schema.query, "NeuronClassesFasciculatingHere", "Query name should match") + self.assertEqual(schema.label, f"Neurons fasciculating in {test_name}", "Label should be formatted correctly") + self.assertEqual(schema.function, "get_neuron_classes_fasciculating_here", "Function name should match") + self.assertEqual(schema.preview, 10, "Preview should be 10") + + # Check preview columns + expected_columns = ["id", "label", "tags", "thumbnail"] + self.assertEqual(schema.preview_columns, expected_columns, f"Preview columns should be {expected_columns}") + + print(f"Schema generated successfully: {schema.label}") + + def test_term_info_integration(self): + """Test that query appears in term info for appropriate terms""" + print(f"\n=== Testing term info integration ===") + + # Get term info for a tract/nerve + term_info = get_term_info(self.test_tract, preview=False) + + self.assertIsNotNone(term_info, "Term info should not be None") + self.assertIn("Queries", term_info, "Term info should contain Queries") + + # Check if our query is present + queries = term_info.get("Queries", []) + query_names = [q.get('query') for q in queries] + + print(f"Available queries for {self.test_tract}: {query_names}") + + # For tracts/nerves (Neuron_projection_bundle), this query should be available + if "Neuron_projection_bundle" in term_info.get("SuperTypes", []): + self.assertIn("NeuronClassesFasciculatingHere", query_names, + "NeuronClassesFasciculatingHere should be available for Neuron_projection_bundle") + print("āœ“ Query correctly appears for Neuron_projection_bundle type") + else: + print(f"Warning: {self.test_tract} does not have Neuron_projection_bundle type") + print(f"SuperTypes: {term_info.get('SuperTypes', [])}") + + def test_preview_results(self): + """Test that preview results are properly formatted""" + print(f"\n=== Testing preview results ===") + + # Get term info with preview enabled + term_info = get_term_info(self.test_tract, preview=True) + + self.assertIsNotNone(term_info, "Term info should not be None") + + # Find our query in the results + queries = term_info.get("Queries", []) + fasciculating_query = None + for q in queries: + if q.get('query') == "NeuronClassesFasciculatingHere": + fasciculating_query = q + break + + if fasciculating_query: + print(f"Found NeuronClassesFasciculatingHere query") + + # Check if preview_results exist + if fasciculating_query.get('preview_results'): + preview = fasciculating_query['preview_results'] + data_key = 'data' if 'data' in preview else 'rows' + print(f"Preview contains {len(preview.get(data_key, []))} results") + + # Validate preview structure + self.assertIn(data_key, preview, f"Preview should contain '{data_key}' key") + self.assertIn('headers', preview, "Preview should contain 'headers' key") + + # Check first result if available + if preview.get(data_key) and len(preview[data_key]) > 0: + first_result = preview[data_key][0] + print(f"First preview result: {first_result.get('label', 'N/A')}") + + # Validate required fields + self.assertIn('id', first_result, "Preview result should have 'id'") + self.assertIn('label', first_result, "Preview result should have 'label'") + else: + print("No preview results available (this is OK if no matching neurons exist)") + else: + print("NeuronClassesFasciculatingHere query not found in term info") + + def test_with_different_tracts(self): + """Test with multiple tract/nerve types""" + print(f"\n=== Testing with different tracts/nerves ===") + + test_tracts = [ + ("FBbt_00003987", "broad root"), + ("FBbt_00007354", "adult antenno-subesophageal tract"), + ("FBbt_00003985", "adult medial antennal lobe tract"), + ] + + for tract_id, tract_name in test_tracts: + print(f"\nTesting {tract_name} ({tract_id})...") + + try: + result = get_neuron_classes_fasciculating_here(tract_id, return_dataframe=False, limit=3) + + if result and 'data' in result: + print(f" āœ“ Query successful, found {len(result['data'])} results") + else: + print(f" āœ“ Query successful, no results found") + + except Exception as e: + print(f" āœ— Query failed: {str(e)}") + # Don't fail the test, just log the error + # raise + + +def run_tests(): + """Run the test suite""" + suite = unittest.TestLoader().loadTestsFromTestCase(NeuronClassesFasciculatingTest) + runner = unittest.TextTestRunner(verbosity=2) + result = runner.run(suite) + return result.wasSuccessful() + + +if __name__ == '__main__': + success = run_tests() + sys.exit(0 if success else 1) diff --git a/src/test/test_tracts_nerves_innervating.py b/src/test/test_tracts_nerves_innervating.py new file mode 100644 index 0000000..8e2b38f --- /dev/null +++ b/src/test/test_tracts_nerves_innervating.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +""" +Test suite for TractsNervesInnervatingHere query. + +Tests the query that finds tracts and nerves that innervate a synaptic neuropil. +This implements the TractsNervesInnervatingHere query from the VFB XMI specification. + +Test cases: +1. Query execution with known neuropil +2. Schema generation and validation +3. Term info integration +4. Preview results validation +5. Cache functionality +""" + +import unittest +import sys +import os + +# Add the src directory to the path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +from vfbquery.vfb_queries import ( + get_tracts_nerves_innervating_here, + TractsNervesInnervatingHere_to_schema, + get_term_info +) + + +class TractsNervesInnervatingTest(unittest.TestCase): + """Test suite for TractsNervesInnervatingHere query""" + + def setUp(self): + """Set up test fixtures""" + # Example synaptic neuropil: adult antennal lobe (FBbt_00007401) + self.test_neuropil = "FBbt_00007401" # antennal lobe + + def test_query_execution(self): + """Test that the query executes successfully""" + print(f"\n=== Testing TractsNervesInnervatingHere query execution ===") + + # Execute the query + result = get_tracts_nerves_innervating_here(self.test_neuropil, return_dataframe=False, limit=5) + + # Validate result structure + self.assertIsNotNone(result, "Query should return a result") + self.assertIsInstance(result, dict, "Result should be a dictionary") + + # Check for expected keys + if result: + print(f"Query returned {len(result.get('data', []))} results") + + # Validate data structure + if 'data' in result and len(result['data']) > 0: + first_result = result['data'][0] + self.assertIn('id', first_result, "Result should contain 'id' field") + self.assertIn('label', first_result, "Result should contain 'label' field") + print(f"First result: {first_result.get('label', 'N/A')} ({first_result.get('id', 'N/A')})") + + def test_schema_generation(self): + """Test schema function generates correct structure""" + print(f"\n=== Testing TractsNervesInnervatingHere schema generation ===") + + test_name = "Test Neuropil" + test_takes = {"short_form": self.test_neuropil} + + schema = TractsNervesInnervatingHere_to_schema(test_name, test_takes) + + # Validate schema structure + self.assertIsNotNone(schema, "Schema should not be None") + self.assertEqual(schema.query, "TractsNervesInnervatingHere", "Query name should match") + self.assertEqual(schema.label, f"Tracts/nerves innervating {test_name}", "Label should be formatted correctly") + self.assertEqual(schema.function, "get_tracts_nerves_innervating_here", "Function name should match") + self.assertEqual(schema.preview, 10, "Preview should be 10") + + # Check preview columns + expected_columns = ["id", "label", "tags", "thumbnail"] + self.assertEqual(schema.preview_columns, expected_columns, f"Preview columns should be {expected_columns}") + + print(f"Schema generated successfully: {schema.label}") + + def test_term_info_integration(self): + """Test that query appears in term info for appropriate terms""" + print(f"\n=== Testing term info integration ===") + + # Get term info for a synaptic neuropil + term_info = get_term_info(self.test_neuropil, preview=False) + + self.assertIsNotNone(term_info, "Term info should not be None") + self.assertIn("Queries", term_info, "Term info should contain Queries") + + # Check if our query is present + queries = term_info.get("Queries", []) + query_names = [q.get('query') for q in queries] + + print(f"Available queries for {self.test_neuropil}: {query_names}") + + # For synaptic neuropils, this query should be available + if "Synaptic_neuropil" in term_info.get("SuperTypes", []) or \ + "Synaptic_neuropil_domain" in term_info.get("SuperTypes", []): + self.assertIn("TractsNervesInnervatingHere", query_names, + "TractsNervesInnervatingHere should be available for Synaptic_neuropil") + print("āœ“ Query correctly appears for Synaptic_neuropil type") + else: + print(f"Warning: {self.test_neuropil} does not have Synaptic_neuropil type") + print(f"SuperTypes: {term_info.get('SuperTypes', [])}") + + def test_preview_results(self): + """Test that preview results are properly formatted""" + print(f"\n=== Testing preview results ===") + + # Get term info with preview enabled + term_info = get_term_info(self.test_neuropil, preview=True) + + self.assertIsNotNone(term_info, "Term info should not be None") + + # Find our query in the results + queries = term_info.get("Queries", []) + innervating_query = None + for q in queries: + if q.get('query') == "TractsNervesInnervatingHere": + innervating_query = q + break + + if innervating_query: + print(f"Found TractsNervesInnervatingHere query") + + # Check if preview_results exist + if innervating_query.get('preview_results'): + preview = innervating_query['preview_results'] + data_key = 'data' if 'data' in preview else 'rows' + print(f"Preview contains {len(preview.get(data_key, []))} results") + + # Validate preview structure + self.assertIn(data_key, preview, f"Preview should contain '{data_key}' key") + self.assertIn('headers', preview, "Preview should contain 'headers' key") + + # Check first result if available + if preview.get(data_key) and len(preview[data_key]) > 0: + first_result = preview[data_key][0] + print(f"First preview result: {first_result.get('label', 'N/A')}") + + # Validate required fields + self.assertIn('id', first_result, "Preview result should have 'id'") + self.assertIn('label', first_result, "Preview result should have 'label'") + else: + print("No preview results available (this is OK if no innervating tracts exist)") + else: + print("TractsNervesInnervatingHere query not found in term info") + + def test_with_different_neuropils(self): + """Test with multiple synaptic neuropil types""" + print(f"\n=== Testing with different neuropils ===") + + test_neuropils = [ + ("FBbt_00007401", "antennal lobe"), + ("FBbt_00003982", "medulla"), + ("FBbt_00003679", "mushroom body"), + ] + + for neuropil_id, neuropil_name in test_neuropils: + print(f"\nTesting {neuropil_name} ({neuropil_id})...") + + try: + result = get_tracts_nerves_innervating_here(neuropil_id, return_dataframe=False, limit=3) + + if result and 'data' in result: + print(f" āœ“ Query successful, found {len(result['data'])} results") + else: + print(f" āœ“ Query successful, no results found") + + except Exception as e: + print(f" āœ— Query failed: {str(e)}") + # Don't fail the test, just log the error + # raise + + +def run_tests(): + """Run the test suite""" + suite = unittest.TestLoader().loadTestsFromTestCase(TractsNervesInnervatingTest) + runner = unittest.TextTestRunner(verbosity=2) + result = runner.run(suite) + return result.wasSuccessful() + + +if __name__ == '__main__': + success = run_tests() + sys.exit(0 if success else 1) diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 70c6342..47158c3 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -724,6 +724,30 @@ def term_info_parse_object(results, short_form): q = SubclassesOf_to_schema(termInfo["Name"], {"short_form": vfbTerm.term.core.short_form}) queries.append(q) + # NeuronClassesFasciculatingHere query - for tracts/nerves + # Matches XMI criteria: Class + Tract_or_nerve (VFB uses Neuron_projection_bundle type) + if contains_all_tags(termInfo["SuperTypes"], ["Class"]) and "Neuron_projection_bundle" in termInfo["SuperTypes"]: + q = NeuronClassesFasciculatingHere_to_schema(termInfo["Name"], {"short_form": vfbTerm.term.core.short_form}) + queries.append(q) + + # TractsNervesInnervatingHere query - for synaptic neuropils + # Matches XMI criteria: Class + (Synaptic_neuropil OR Synaptic_neuropil_domain) + if contains_all_tags(termInfo["SuperTypes"], ["Class"]) and ( + "Synaptic_neuropil" in termInfo["SuperTypes"] or + "Synaptic_neuropil_domain" in termInfo["SuperTypes"] + ): + q = TractsNervesInnervatingHere_to_schema(termInfo["Name"], {"short_form": vfbTerm.term.core.short_form}) + queries.append(q) + + # LineageClonesIn query - for synaptic neuropils + # Matches XMI criteria: Class + (Synaptic_neuropil OR Synaptic_neuropil_domain) + if contains_all_tags(termInfo["SuperTypes"], ["Class"]) and ( + "Synaptic_neuropil" in termInfo["SuperTypes"] or + "Synaptic_neuropil_domain" in termInfo["SuperTypes"] + ): + q = LineageClonesIn_to_schema(termInfo["Name"], {"short_form": vfbTerm.term.core.short_form}) + queries.append(q) + # Add Publications to the termInfo object if vfbTerm.pubs and len(vfbTerm.pubs) > 0: publications = [] @@ -1064,6 +1088,81 @@ def SubclassesOf_to_schema(name, take_default): return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) + +def NeuronClassesFasciculatingHere_to_schema(name, take_default): + """ + Schema for NeuronClassesFasciculatingHere query. + Finds neuron classes that fasciculate with (run along) a tract or nerve. + + Matching criteria from XMI: + - Class + Tract_or_nerve (VFB uses Neuron_projection_bundle type) + + Query chain: Owlery subclass query ā†’ process ā†’ SOLR + OWL query: 'Neuron' that 'fasciculates with' some '{short_form}' + """ + query = "NeuronClassesFasciculatingHere" + label = f"Neurons fasciculating in {name}" + function = "get_neuron_classes_fasciculating_here" + takes = { + "short_form": {"$and": ["Class", "Neuron_projection_bundle"]}, + "default": take_default, + } + preview = 10 + preview_columns = ["id", "label", "tags", "thumbnail"] + + return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) + + +def TractsNervesInnervatingHere_to_schema(name, take_default): + """ + Schema for TractsNervesInnervatingHere query. + Finds tracts and nerves that innervate a synaptic neuropil. + + Matching criteria from XMI: + - Class + Synaptic_neuropil + - Class + Synaptic_neuropil_domain + + Query chain: Owlery subclass query ā†’ process ā†’ SOLR + OWL query: 'Tract_or_nerve' that 'innervates' some '{short_form}' + """ + query = "TractsNervesInnervatingHere" + label = f"Tracts/nerves innervating {name}" + function = "get_tracts_nerves_innervating_here" + takes = { + "short_form": {"$and": ["Class", "Synaptic_neuropil"]}, + "default": take_default, + } + preview = 10 + preview_columns = ["id", "label", "tags", "thumbnail"] + + return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) + + +def LineageClonesIn_to_schema(name, take_default): + """ + Schema for LineageClonesIn query. + Finds lineage clones that overlap with a synaptic neuropil or domain. + + Matching criteria from XMI: + - Class + Synaptic_neuropil + - Class + Synaptic_neuropil_domain + + Query chain: Owlery subclass query ā†’ process ā†’ SOLR + OWL query: 'Clone' that 'overlaps' some '{short_form}' + """ + query = "LineageClonesIn" + label = f"Lineage clones found in {name}" + function = "get_lineage_clones_in" + takes = { + "short_form": {"$and": ["Class", "Synaptic_neuropil"]}, + "default": take_default, + } + preview = 10 + preview_columns = ["id", "label", "tags", "thumbnail"] + + return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) + + def serialize_solr_output(results): # Create a copy of the document and remove Solr-specific fields doc = dict(results.docs[0]) @@ -1983,6 +2082,67 @@ def get_subclasses_of(short_form: str, return_dataframe=True, limit: int = -1): owl_query = f"<{short_form}>" return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query', query_by_label=False) + +@with_solr_cache('neuron_classes_fasciculating_here') +def get_neuron_classes_fasciculating_here(short_form: str, return_dataframe=True, limit: int = -1): + """ + Retrieves neuron classes that fasciculate with (run along) the specified tract or nerve. + + This implements the NeuronClassesFasciculatingHere query from the VFB XMI specification. + Query chain (from XMI): Owlery ā†’ Process ā†’ SOLR + OWL query (from XMI): object= and some + Where: FBbt_00005106 = neuron, RO_0002101 = fasciculates with + Matching criteria: Class + Tract_or_nerve + + :param short_form: short form of the tract or nerve (Class) + :param return_dataframe: Returns pandas dataframe if true, otherwise returns formatted dict + :param limit: maximum number of results to return (default -1, returns all results) + :return: Neuron classes that fasciculate with the specified tract or nerve + """ + owl_query = f" and some " + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query', query_by_label=False) + + +@with_solr_cache('tracts_nerves_innervating_here') +def get_tracts_nerves_innervating_here(short_form: str, return_dataframe=True, limit: int = -1): + """ + Retrieves tracts and nerves that innervate the specified synaptic neuropil. + + This implements the TractsNervesInnervatingHere query from the VFB XMI specification. + Query chain (from XMI): Owlery ā†’ Process ā†’ SOLR + OWL query (from XMI): object= and some + Where: FBbt_00005099 = tract or nerve, RO_0002134 = innervates + Matching criteria: Class + Synaptic_neuropil, Class + Synaptic_neuropil_domain + + :param short_form: short form of the synaptic neuropil (Class) + :param return_dataframe: Returns pandas dataframe if true, otherwise returns formatted dict + :param limit: maximum number of results to return (default -1, returns all results) + :return: Tracts and nerves that innervate the specified neuropil + """ + owl_query = f" and some " + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query', query_by_label=False) + + +@with_solr_cache('lineage_clones_in') +def get_lineage_clones_in(short_form: str, return_dataframe=True, limit: int = -1): + """ + Retrieves lineage clones that overlap with the specified synaptic neuropil. + + This implements the LineageClonesIn query from the VFB XMI specification. + Query chain (from XMI): Owlery ā†’ Process ā†’ SOLR + OWL query (from XMI): object= and some + Where: FBbt_00007683 = clone, RO_0002131 = overlaps + Matching criteria: Class + Synaptic_neuropil, Class + Synaptic_neuropil_domain + + :param short_form: short form of the synaptic neuropil (Class) + :param return_dataframe: Returns pandas dataframe if true, otherwise returns formatted dict + :param limit: maximum number of results to return (default -1, returns all results) + :return: Lineage clones that overlap with the specified neuropil + """ + owl_query = f" and some " + return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query', query_by_label=False) + + def _get_neurons_part_here_headers(): """Return standard headers for get_neurons_with_part_in results""" return { From a27bdaece4fd838e1ec969ab4cef9d038b624ee5 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 21:31:03 +0000 Subject: [PATCH 64/70] Update performance test results [skip ci] --- performance.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/performance.md b/performance.md index 8928e6f..4422ae1 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 20:59:55 UTC -**Git Commit:** e852ffbaefd5410ca183f8c261b8174d18b4f4ba +**Test Date:** 2025-11-06 21:31:03 UTC +**Git Commit:** bcf8be83f7715103c33cee69009a7c8da1ea5164 **Branch:** dev -**Workflow Run:** 19149670397 +**Workflow Run:** 19150456365 ## Test Overview @@ -25,11 +25,11 @@ This performance test measures the execution time of VFB term info queries for s āœ… **Test Status**: Performance test completed -- **FBbt_00003748 Query Time**: 0.9410 seconds -- **VFB_00101567 Query Time**: 0.7934 seconds -- **Total Query Time**: 1.7343 seconds +- **FBbt_00003748 Query Time**: 1.1114 seconds +- **VFB_00101567 Query Time**: 0.8394 seconds +- **Total Query Time**: 1.9508 seconds šŸŽ‰ **Result**: All performance thresholds met! --- -*Last updated: 2025-11-06 20:59:55 UTC* +*Last updated: 2025-11-06 21:31:03 UTC* From 160079a0c99df42c8141cfceeb7ae6410f9525ca Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 21:40:51 +0000 Subject: [PATCH 65/70] Refactor performance tests to include new queries and improve reporting --- .github/workflows/performance-test.yml | 111 +++++++--- src/test/test_query_performance.py | 284 +++++++++++++++++++++++++ 2 files changed, 362 insertions(+), 33 deletions(-) create mode 100644 src/test/test_query_performance.py diff --git a/.github/workflows/performance-test.yml b/.github/workflows/performance-test.yml index 411706f..900924a 100644 --- a/.github/workflows/performance-test.yml +++ b/.github/workflows/performance-test.yml @@ -31,10 +31,11 @@ jobs: - name: Run Performance Test run: | - export PYTHONPATH=$PYTHONPATH:$PWD/ - echo "Running performance test for term info queries..." - python -m unittest -v src.test.term_info_queries_test.TermInfoQueriesTest.test_term_info_performance 2>&1 | tee performance_test_output.log - continue-on-error: true # Continue even if performance thresholds are exceeded + python -m unittest src.test.test_query_performance -v 2>&1 | tee performance_test_output.log + + - name: Run Legacy Performance Test + run: | + python -m unittest -v src.test.term_info_queries_test.TermInfoQueriesTest.test_term_info_performance 2>&1 | tee -a performance_test_output.log - name: Create Performance Report if: always() # Always run this step, even if the test fails @@ -46,63 +47,107 @@ jobs: **Test Date:** $(date -u '+%Y-%m-%d %H:%M:%S UTC') **Git Commit:** ${{ github.sha }} **Branch:** ${{ github.ref_name }} - **Workflow Run:** ${{ github.run_id }} + **Workflow Run:** [${{ github.run_id }}](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) ## Test Overview - This performance test measures the execution time of VFB term info queries for specific terms: + This performance test measures the execution time of all implemented VFB queries including: + + ### Core Queries + - **Term Info Queries**: Basic term information retrieval + - **Neuron Part Queries**: Neurons with parts overlapping regions + - **Synaptic Terminal Queries**: Pre/post synaptic terminals + - **Anatomical Hierarchy**: Components, parts, subclasses + - **Instance Queries**: Available images and instances - - **FBbt_00003748**: mushroom body (anatomical class) - - **VFB_00101567**: individual anatomy data + ### New Queries (2025) + - **NeuronClassesFasciculatingHere**: Neurons fasciculating with tracts + - **TractsNervesInnervatingHere**: Tracts/nerves innervating neuropils + - **LineageClonesIn**: Lineage clones in neuropils ## Performance Thresholds - - Maximum single query time: 2 seconds - - Maximum total time for both queries: 4 seconds + - **Fast queries**: < 1 second (SOLR lookups) + - **Medium queries**: < 3 seconds (Owlery + SOLR) + - **Slow queries**: < 10 seconds (Neo4j + complex processing) ## Test Results - ``` + \`\`\` $(cat performance_test_output.log) - ``` + \`\`\` ## Summary EOF - # Extract timing information from the test output - if grep -q "Performance Test Results:" performance_test_output.log; then - echo "āœ… **Test Status**: Performance test completed" >> performance.md + # Check overall test status + if grep -q "OK" performance_test_output.log || grep -q "Ran.*test" performance_test_output.log; then + echo "āœ… **Test Status**: Performance tests completed" >> performance.md echo "" >> performance.md - # Extract timing data - if grep -q "FBbt_00003748 query took:" performance_test_output.log; then - TIMING1=$(grep "FBbt_00003748 query took:" performance_test_output.log | sed 's/.*took: \([0-9.]*\) seconds.*/\1/') - echo "- **FBbt_00003748 Query Time**: ${TIMING1} seconds" >> performance.md - fi + # Count successes and failures + TOTAL_TESTS=$(grep -c "^test_" performance_test_output.log || echo "0") + FAILED_TESTS=$(grep -c "FAIL:" performance_test_output.log || echo "0") + ERROR_TESTS=$(grep -c "ERROR:" performance_test_output.log || echo "0") + PASSED_TESTS=$((TOTAL_TESTS - FAILED_TESTS - ERROR_TESTS)) - if grep -q "VFB_00101567 query took:" performance_test_output.log; then - TIMING2=$(grep "VFB_00101567 query took:" performance_test_output.log | sed 's/.*took: \([0-9.]*\) seconds.*/\1/') - echo "- **VFB_00101567 Query Time**: ${TIMING2} seconds" >> performance.md - fi + echo "### Test Statistics" >> performance.md + echo "" >> performance.md + echo "- **Total Tests**: ${TOTAL_TESTS}" >> performance.md + echo "- **Passed**: ${PASSED_TESTS} āœ…" >> performance.md + echo "- **Failed**: ${FAILED_TESTS} āŒ" >> performance.md + echo "- **Errors**: ${ERROR_TESTS} āš ļø" >> performance.md + echo "" >> performance.md + + # Extract timing information for key queries + echo "### Query Performance Details" >> performance.md + echo "" >> performance.md - if grep -q "Total time for both queries:" performance_test_output.log; then - TOTAL_TIME=$(grep "Total time for both queries:" performance_test_output.log | sed 's/.*queries: \([0-9.]*\) seconds.*/\1/') - echo "- **Total Query Time**: ${TOTAL_TIME} seconds" >> performance.md + # Extract all timing lines + if grep -q "seconds" performance_test_output.log; then + echo "| Query | Duration | Status |" >> performance.md + echo "|-------|----------|--------|" >> performance.md + + # Parse timing information + grep -E "^(get_term_info|NeuronsPartHere|NeuronsSynaptic|NeuronsPresynapticHere|NeuronsPostsynapticHere|ComponentsOf|PartsOf|SubclassesOf|NeuronClassesFasciculatingHere|TractsNervesInnervatingHere|LineageClonesIn|ListAllAvailableImages):" performance_test_output.log | while read line; do + QUERY=$(echo "$line" | sed 's/:.*//') + DURATION=$(echo "$line" | sed 's/.*: \([0-9.]*\)s.*/\1/') + if echo "$line" | grep -q "āœ…"; then + STATUS="āœ… Pass" + else + STATUS="āŒ Fail" + fi + echo "| $QUERY | ${DURATION}s | $STATUS |" >> performance.md + done fi - # Check if test passed or failed - if grep -q "OK" performance_test_output.log; then - echo "" >> performance.md + echo "" >> performance.md + + # Overall result + if [ "$FAILED_TESTS" -eq "0" ] && [ "$ERROR_TESTS" -eq "0" ]; then echo "šŸŽ‰ **Result**: All performance thresholds met!" >> performance.md - elif grep -q "FAILED" performance_test_output.log; then + else + echo "āš ļø **Result**: Some performance thresholds exceeded or tests failed" >> performance.md echo "" >> performance.md - echo "āš ļø **Result**: Some performance thresholds exceeded or test failed" >> performance.md + echo "Please review the failed tests above. Common causes:" >> performance.md + echo "- Network latency to VFB services" >> performance.md + echo "- SOLR/Neo4j/Owlery server load" >> performance.md + echo "- First-time cache population (expected to be slower)" >> performance.md fi else - echo "āŒ **Test Status**: Performance test failed to run properly" >> performance.md + echo "āŒ **Test Status**: Performance tests failed to run properly" >> performance.md + echo "" >> performance.md + echo "Please check the test output above for errors." >> performance.md fi + echo "" >> performance.md + echo "---" >> performance.md + echo "" >> performance.md + echo "## Historical Performance" >> performance.md + echo "" >> performance.md + echo "Track performance trends across commits:" >> performance.md + echo "- [GitHub Actions History](https://github.com/${{ github.repository }}/actions/workflows/performance-test.yml)" >> performance.md echo "" >> performance.md echo "---" >> performance.md echo "*Last updated: $(date -u '+%Y-%m-%d %H:%M:%S UTC')*" >> performance.md diff --git a/src/test/test_query_performance.py b/src/test/test_query_performance.py new file mode 100644 index 0000000..3cc7b91 --- /dev/null +++ b/src/test/test_query_performance.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +""" +Comprehensive performance test for all VFB queries. + +Tests the execution time of all implemented queries to ensure they meet performance thresholds. +Results are formatted for GitHub Actions reporting. +""" + +import unittest +import time +import sys +import os + +# Add the src directory to the path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +from vfbquery.vfb_queries import ( + get_term_info, + get_neurons_with_part_in, + get_neurons_with_synapses_in, + get_neurons_with_presynaptic_terminals_in, + get_neurons_with_postsynaptic_terminals_in, + get_components_of, + get_parts_of, + get_subclasses_of, + get_neuron_classes_fasciculating_here, + get_tracts_nerves_innervating_here, + get_lineage_clones_in, + get_instances, + get_similar_neurons, + get_individual_neuron_inputs +) + + +class QueryPerformanceTest(unittest.TestCase): + """Comprehensive performance tests for all VFB queries""" + + # Performance thresholds (in seconds) + THRESHOLD_FAST = 1.0 # Fast queries (simple SOLR lookups) + THRESHOLD_MEDIUM = 3.0 # Medium queries (Owlery + SOLR) + THRESHOLD_SLOW = 10.0 # Slow queries (Neo4j + complex processing) + + def setUp(self): + """Set up test data""" + self.test_terms = { + 'mushroom_body': 'FBbt_00003748', # Class - mushroom body + 'antennal_lobe': 'FBbt_00007401', # Synaptic neuropil + 'medulla': 'FBbt_00003982', # Visual system + 'broad_root': 'FBbt_00003987', # Neuron projection bundle (tract) + 'individual_neuron': 'VFB_00101567', # Individual anatomy + 'neuron_with_nblast': 'VFB_00017894', # Neuron with NBLAST data + 'clone': 'FBbt_00050024', # Clone + } + + self.results = [] + + def _time_query(self, query_name, query_func, *args, **kwargs): + """Helper to time a query execution""" + start_time = time.time() + try: + result = query_func(*args, **kwargs) + duration = time.time() - start_time + success = result is not None + error = None + except Exception as e: + duration = time.time() - start_time + success = False + result = None + error = str(e) + + self.results.append({ + 'name': query_name, + 'duration': duration, + 'success': success, + 'error': error + }) + + return result, duration, success + + def test_01_term_info_queries(self): + """Test term info query performance""" + print("\n" + "="*80) + print("TERM INFO QUERIES") + print("="*80) + + # Test basic term info retrieval + result, duration, success = self._time_query( + "get_term_info (mushroom body)", + get_term_info, + self.test_terms['mushroom_body'], + preview=True + ) + print(f"get_term_info (mushroom body): {duration:.4f}s {'āœ…' if success else 'āŒ'}") + self.assertLess(duration, self.THRESHOLD_MEDIUM, "term_info query exceeded threshold") + + result, duration, success = self._time_query( + "get_term_info (individual)", + get_term_info, + self.test_terms['individual_neuron'], + preview=True + ) + print(f"get_term_info (individual): {duration:.4f}s {'āœ…' if success else 'āŒ'}") + self.assertLess(duration, self.THRESHOLD_MEDIUM, "term_info query exceeded threshold") + + def test_02_neuron_part_queries(self): + """Test neuron part overlap queries""" + print("\n" + "="*80) + print("NEURON PART OVERLAP QUERIES") + print("="*80) + + result, duration, success = self._time_query( + "NeuronsPartHere (antennal lobe)", + get_neurons_with_part_in, + self.test_terms['antennal_lobe'], + return_dataframe=False, + limit=10 + ) + print(f"NeuronsPartHere: {duration:.4f}s {'āœ…' if success else 'āŒ'}") + self.assertLess(duration, self.THRESHOLD_SLOW, "NeuronsPartHere exceeded threshold") + + def test_03_synaptic_queries(self): + """Test synaptic terminal queries""" + print("\n" + "="*80) + print("SYNAPTIC TERMINAL QUERIES") + print("="*80) + + test_term = self.test_terms['antennal_lobe'] + + result, duration, success = self._time_query( + "NeuronsSynaptic", + get_neurons_with_synapses_in, + test_term, + return_dataframe=False, + limit=10 + ) + print(f"NeuronsSynaptic: {duration:.4f}s {'āœ…' if success else 'āŒ'}") + self.assertLess(duration, self.THRESHOLD_SLOW, "NeuronsSynaptic exceeded threshold") + + result, duration, success = self._time_query( + "NeuronsPresynapticHere", + get_neurons_with_presynaptic_terminals_in, + test_term, + return_dataframe=False, + limit=10 + ) + print(f"NeuronsPresynapticHere: {duration:.4f}s {'āœ…' if success else 'āŒ'}") + self.assertLess(duration, self.THRESHOLD_SLOW, "NeuronsPresynapticHere exceeded threshold") + + result, duration, success = self._time_query( + "NeuronsPostsynapticHere", + get_neurons_with_postsynaptic_terminals_in, + test_term, + return_dataframe=False, + limit=10 + ) + print(f"NeuronsPostsynapticHere: {duration:.4f}s {'āœ…' if success else 'āŒ'}") + self.assertLess(duration, self.THRESHOLD_SLOW, "NeuronsPostsynapticHere exceeded threshold") + + def test_04_anatomy_hierarchy_queries(self): + """Test anatomical hierarchy queries""" + print("\n" + "="*80) + print("ANATOMICAL HIERARCHY QUERIES") + print("="*80) + + test_term = self.test_terms['mushroom_body'] + + result, duration, success = self._time_query( + "ComponentsOf", + get_components_of, + test_term, + return_dataframe=False, + limit=10 + ) + print(f"ComponentsOf: {duration:.4f}s {'āœ…' if success else 'āŒ'}") + self.assertLess(duration, self.THRESHOLD_SLOW, "ComponentsOf exceeded threshold") + + result, duration, success = self._time_query( + "PartsOf", + get_parts_of, + test_term, + return_dataframe=False, + limit=10 + ) + print(f"PartsOf: {duration:.4f}s {'āœ…' if success else 'āŒ'}") + self.assertLess(duration, self.THRESHOLD_SLOW, "PartsOf exceeded threshold") + + result, duration, success = self._time_query( + "SubclassesOf", + get_subclasses_of, + test_term, + return_dataframe=False, + limit=10 + ) + print(f"SubclassesOf: {duration:.4f}s {'āœ…' if success else 'āŒ'}") + self.assertLess(duration, self.THRESHOLD_SLOW, "SubclassesOf exceeded threshold") + + def test_05_new_queries(self): + """Test newly implemented queries""" + print("\n" + "="*80) + print("NEW QUERIES (2025)") + print("="*80) + + # NeuronClassesFasciculatingHere + result, duration, success = self._time_query( + "NeuronClassesFasciculatingHere", + get_neuron_classes_fasciculating_here, + self.test_terms['broad_root'], + return_dataframe=False, + limit=10 + ) + print(f"NeuronClassesFasciculatingHere: {duration:.4f}s {'āœ…' if success else 'āŒ'}") + self.assertLess(duration, self.THRESHOLD_SLOW, "NeuronClassesFasciculatingHere exceeded threshold") + + # TractsNervesInnervatingHere + result, duration, success = self._time_query( + "TractsNervesInnervatingHere", + get_tracts_nerves_innervating_here, + self.test_terms['antennal_lobe'], + return_dataframe=False, + limit=10 + ) + print(f"TractsNervesInnervatingHere: {duration:.4f}s {'āœ…' if success else 'āŒ'}") + self.assertLess(duration, self.THRESHOLD_SLOW, "TractsNervesInnervatingHere exceeded threshold") + + # LineageClonesIn + result, duration, success = self._time_query( + "LineageClonesIn", + get_lineage_clones_in, + self.test_terms['antennal_lobe'], + return_dataframe=False, + limit=10 + ) + print(f"LineageClonesIn: {duration:.4f}s {'āœ…' if success else 'āŒ'}") + self.assertLess(duration, self.THRESHOLD_SLOW, "LineageClonesIn exceeded threshold") + + def test_06_instance_queries(self): + """Test instance retrieval queries""" + print("\n" + "="*80) + print("INSTANCE QUERIES") + print("="*80) + + result, duration, success = self._time_query( + "ListAllAvailableImages", + get_instances, + self.test_terms['medulla'], + return_dataframe=False, + limit=5 + ) + print(f"ListAllAvailableImages: {duration:.4f}s {'āœ…' if success else 'āŒ'}") + self.assertLess(duration, self.THRESHOLD_SLOW, "ListAllAvailableImages exceeded threshold") + + def tearDown(self): + """Generate performance summary""" + pass + + @classmethod + def tearDownClass(cls): + """Generate final performance report""" + print("\n" + "="*80) + print("PERFORMANCE TEST SUMMARY") + print("="*80) + + # This will be populated by the test instance + # For now, just print a summary message + print("All performance tests completed!") + print("="*80) + + +def run_tests(): + """Run the performance test suite""" + # Create test suite + loader = unittest.TestLoader() + suite = loader.loadTestsFromTestCase(QueryPerformanceTest) + + # Run tests with detailed output + runner = unittest.TextTestRunner(verbosity=2) + result = runner.run(suite) + + return result.wasSuccessful() + + +if __name__ == '__main__': + success = run_tests() + sys.exit(0 if success else 1) From 9f15e30ed3e13a4a3e4bc4b9b1ab99fc659e9ebd Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 21:42:09 +0000 Subject: [PATCH 66/70] Update performance test results [skip ci] --- performance.md | 140 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 126 insertions(+), 14 deletions(-) diff --git a/performance.md b/performance.md index 4422ae1..753848d 100644 --- a/performance.md +++ b/performance.md @@ -1,35 +1,147 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 21:31:03 UTC -**Git Commit:** bcf8be83f7715103c33cee69009a7c8da1ea5164 +**Test Date:** 2025-11-06 21:42:09 UTC +**Git Commit:** 160079a0c99df42c8141cfceeb7ae6410f9525ca **Branch:** dev -**Workflow Run:** 19150456365 +**Workflow Run:** [19150715535](https://github.com/VirtualFlyBrain/VFBquery/actions/runs/19150715535) ## Test Overview -This performance test measures the execution time of VFB term info queries for specific terms: +This performance test measures the execution time of all implemented VFB queries including: -- **FBbt_00003748**: mushroom body (anatomical class) -- **VFB_00101567**: individual anatomy data +### Core Queries +- **Term Info Queries**: Basic term information retrieval +- **Neuron Part Queries**: Neurons with parts overlapping regions +- **Synaptic Terminal Queries**: Pre/post synaptic terminals +- **Anatomical Hierarchy**: Components, parts, subclasses +- **Instance Queries**: Available images and instances + +### New Queries (2025) +- **NeuronClassesFasciculatingHere**: Neurons fasciculating with tracts +- **TractsNervesInnervatingHere**: Tracts/nerves innervating neuropils +- **LineageClonesIn**: Lineage clones in neuropils ## Performance Thresholds -- Maximum single query time: 2 seconds -- Maximum total time for both queries: 4 seconds +- **Fast queries**: < 1 second (SOLR lookups) +- **Medium queries**: < 3 seconds (Owlery + SOLR) +- **Slow queries**: < 10 seconds (Neo4j + complex processing) ## Test Results +``` +test_01_term_info_queries (src.test.test_query_performance.QueryPerformanceTest) +Test term info query performance ... FAIL +test_02_neuron_part_queries (src.test.test_query_performance.QueryPerformanceTest) +Test neuron part overlap queries ... ok +test_03_synaptic_queries (src.test.test_query_performance.QueryPerformanceTest) +Test synaptic terminal queries ... ok +test_04_anatomy_hierarchy_queries (src.test.test_query_performance.QueryPerformanceTest) +Test anatomical hierarchy queries ... ok +test_05_new_queries (src.test.test_query_performance.QueryPerformanceTest) +Test newly implemented queries ... ok +test_06_instance_queries (src.test.test_query_performance.QueryPerformanceTest) +Test instance retrieval queries ... ok + +====================================================================== +FAIL: test_01_term_info_queries (src.test.test_query_performance.QueryPerformanceTest) +Test term info query performance +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/home/runner/work/VFBquery/VFBquery/src/test/test_query_performance.py", line 94, in test_01_term_info_queries + self.assertLess(duration, self.THRESHOLD_MEDIUM, "term_info query exceeded threshold") +AssertionError: 11.490196466445923 not less than 3.0 : term_info query exceeded threshold + +---------------------------------------------------------------------- +Ran 6 tests in 21.573s + +FAILED (failures=1) +VFBquery caching enabled: TTL=2160h (90 days), Memory=2048MB +VFBquery functions patched with caching support +VFBquery: Caching enabled by default (3-month TTL, 2GB memory) + Disable with: export VFBQUERY_CACHE_ENABLED=false + +================================================================================ +TERM INFO QUERIES +================================================================================ +DEBUG: Cache lookup for FBbt_00003748: MISS +āœ… Neo4j connection established +get_term_info (mushroom body): 11.4902s āœ… + +================================================================================ +NEURON PART OVERLAP QUERIES +================================================================================ +NeuronsPartHere: 0.7864s āœ… + +================================================================================ +SYNAPTIC TERMINAL QUERIES +================================================================================ +NeuronsSynaptic: 0.6386s āœ… +NeuronsPresynapticHere: 0.5441s āœ… +NeuronsPostsynapticHere: 0.6352s āœ… + +================================================================================ +ANATOMICAL HIERARCHY QUERIES +================================================================================ +ComponentsOf: 2.2629s āœ… +PartsOf: 0.8743s āœ… +SubclassesOf: 0.8658s āœ… +================================================================================ +NEW QUERIES (2025) +================================================================================ +NeuronClassesFasciculatingHere: 0.5527s āœ… +TractsNervesInnervatingHere: 0.5502s āœ… +LineageClonesIn: 0.5484s āœ… + +================================================================================ +INSTANCE QUERIES +================================================================================ +ListAllAvailableImages: 1.8236s āœ… + +================================================================================ +PERFORMANCE TEST SUMMARY +================================================================================ +All performance tests completed! +================================================================================ +test_term_info_performance (src.test.term_info_queries_test.TermInfoQueriesTest) +Performance test for specific term info queries. ... ok + +---------------------------------------------------------------------- +Ran 1 test in 1.117s + +OK +VFBquery caching enabled: TTL=2160h (90 days), Memory=2048MB +VFBquery functions patched with caching support +VFBquery: Caching enabled by default (3-month TTL, 2GB memory) + Disable with: export VFBQUERY_CACHE_ENABLED=false +VFBquery caching enabled: TTL=2160h (90 days), Memory=2048MB +VFBquery functions patched with caching support +VFBquery: Caching enabled by default (3-month TTL, 2GB memory) + Disable with: export VFBQUERY_CACHE_ENABLED=false + +================================================== +Performance Test Results: +================================================== +FBbt_00003748 query took: 0.5602 seconds +VFB_00101567 query took: 0.5561 seconds +Total time for both queries: 1.1163 seconds +Performance Level: šŸŸ¢ Excellent (< 1.5 seconds) +================================================== +Performance test completed successfully! +``` ## Summary -āœ… **Test Status**: Performance test completed +āœ… **Test Status**: Performance tests completed + + +--- -- **FBbt_00003748 Query Time**: 1.1114 seconds -- **VFB_00101567 Query Time**: 0.8394 seconds -- **Total Query Time**: 1.9508 seconds +## Historical Performance -šŸŽ‰ **Result**: All performance thresholds met! +Track performance trends across commits: +- [GitHub Actions History](https://github.com/VirtualFlyBrain/VFBquery/actions/workflows/performance-test.yml) --- -*Last updated: 2025-11-06 21:31:03 UTC* +*Last updated: 2025-11-06 21:42:09 UTC* From 5dfeb639109ac06e49316b69b9679d4080560e8e Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 22:52:43 +0000 Subject: [PATCH 67/70] Implement ImagesNeurons query and associated tests; update Owlery client for instance retrieval --- VFB_QUERIES_REFERENCE.md | 51 +++++-- src/test/test_images_neurons.py | 152 +++++++++++++++++++++ src/test/test_query_performance.py | 13 +- src/vfbquery/owlery_client.py | 103 ++++++++++++++ src/vfbquery/vfb_queries.py | 212 +++++++++++++++++++++++++++++ 5 files changed, 518 insertions(+), 13 deletions(-) create mode 100644 src/test/test_images_neurons.py diff --git a/VFB_QUERIES_REFERENCE.md b/VFB_QUERIES_REFERENCE.md index ddd4094..c0486f2 100644 --- a/VFB_QUERIES_REFERENCE.md +++ b/VFB_QUERIES_REFERENCE.md @@ -285,16 +285,22 @@ Applies to: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_n - **Query Chain**: Multi-step Owlery and Neo4j queries - **Status**: āŒ **NOT IMPLEMENTED** -#### 9. **NeuronClassesFasciculatingHere** āŒ +#### 9. **NeuronClassesFasciculatingHere** āœ… - **ID**: `NeuronClassesFasciculatingHere` / `AberNeuronClassesFasciculatingHere` - **Name**: "Neuron classes fasciculating here" - **Description**: "Neurons fasciculating in $NAME" -- **Matching Criteria**: Class + Tract_or_nerve +- **Matching Criteria**: Class + Neuron_projection_bundle (note: XMI specifies Tract_or_nerve, but VFB SOLR uses Neuron_projection_bundle) - **Query Chain**: Owlery ā†’ Process ā†’ SOLR - **OWL Query**: `object= and some <$ID>` -- **Status**: āŒ **NOT IMPLEMENTED** - -#### 10. **ImagesNeurons** āŒ +- **Status**: āœ… **FULLY IMPLEMENTED** (November 2025) +- **Implementation**: + - Schema: `NeuronClassesFasciculatingHere_to_schema()` + - Execution: `get_neuron_classes_fasciculating_here(term_id)` + - Tests: `src/test/test_neuron_classes_fasciculating.py` + - Preview: neuron_label, neuron_id + - Test term: FBbt_00003987 (broad root) + +#### 10. **ImagesNeurons** āœ… - **ID**: `ImagesNeurons` - **Name**: "Images of neurons with some part here" - **Description**: "Images of neurons with some part in $NAME" @@ -303,7 +309,16 @@ Applies to: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_n - Class + Synaptic_neuropil_domain - **Query Chain**: Owlery instances ā†’ Process ā†’ SOLR - **OWL Query**: `object= and some <$ID>` (instances, not classes) -- **Status**: āŒ **NOT IMPLEMENTED** +- **Status**: āœ… **FULLY IMPLEMENTED** (November 2025) +- **Implementation**: + - Schema: `ImagesNeurons_to_schema()` āœ… + - Execution: `get_images_neurons(term_id)` āœ… + - Helper: `_owlery_instances_query_to_results()` āœ… + - Tests: `src/test/test_images_neurons.py` āœ… + - Preview: id, label, tags, thumbnail + - Test term: FBbt_00007401 (antennal lobe) ā†’ Returns 9,657 neuron images + - Note: Returns individual neuron images (instances) not neuron classes + - Query successfully retrieves VFB instance IDs from Owlery and enriches with SOLR anat_image_query data #### 12. **PaintedDomains** āŒ - **ID**: `PaintedDomains` / `domainsForTempId` @@ -321,18 +336,24 @@ Applies to: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_n - **Query Chain**: Neo4j ā†’ Process ā†’ SOLR - **Status**: āŒ **NOT IMPLEMENTED** -#### 16. **TractsNervesInnervatingHere** āŒ +#### 16. **TractsNervesInnervatingHere** āœ… - **ID**: `TractsNervesInnervatingHere` / `innervatesX` - **Name**: "Tracts/nerves innervating synaptic neuropil" - **Description**: "Tracts/nerves innervating $NAME" -- **Matching Criteria**: +- **Matching Criteria**: - Class + Synaptic_neuropil - Class + Synaptic_neuropil_domain - **Query Chain**: Owlery ā†’ Process ā†’ SOLR - **OWL Query**: `object= and some <$ID>` -- **Status**: āŒ **NOT IMPLEMENTED** - -#### 17. **LineageClonesIn** āŒ +- **Status**: āœ… **FULLY IMPLEMENTED** (November 2025) +- **Implementation**: + - Schema: `TractsNervesInnervatingHere_to_schema()` + - Execution: `get_tracts_nerves_innervating_here(term_id)` + - Tests: `src/test/test_tracts_nerves_innervating.py` + - Preview: tract_label, tract_id + - Test term: FBbt_00007401 (antennal lobe) + +#### 17. **LineageClonesIn** āœ… - **ID**: `LineageClonesIn` / `lineageClones` - **Name**: "Lineage clones found here" - **Description**: "Lineage clones found in $NAME" @@ -341,7 +362,13 @@ Applies to: Class + Synaptic_neuropil, Class + Visual_system, Class + Synaptic_n - Class + Synaptic_neuropil_domain - **Query Chain**: Owlery ā†’ Process ā†’ SOLR - **OWL Query**: `object= and some <$ID>` -- **Status**: āŒ **NOT IMPLEMENTED** +- **Status**: āœ… **FULLY IMPLEMENTED** (November 2025) +- **Implementation**: + - Schema: `LineageClonesIn_to_schema()` + - Execution: `get_lineage_clones_in(term_id)` + - Tests: `src/test/test_lineage_clones_in.py` + - Preview: clone_label, clone_id + - Test term: FBbt_00007401 (antennal lobe) #### 18. **AllAlignedImages** āŒ - **ID**: `AllAlignedImages` / `imagesForTempQuery` diff --git a/src/test/test_images_neurons.py b/src/test/test_images_neurons.py new file mode 100644 index 0000000..9cf77cd --- /dev/null +++ b/src/test/test_images_neurons.py @@ -0,0 +1,152 @@ +""" +Unit tests for the ImagesNeurons query. + +This tests the ImagesNeurons query which retrieves individual neuron images +(instances) with parts in a synaptic neuropil or domain. + +Test term: FBbt_00007401 (antennal lobe) - a synaptic neuropil +""" + +import unittest +import sys +import os + +# Add src directory to path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) + +from vfbquery.vfb_queries import ( + get_images_neurons, + ImagesNeurons_to_schema, + get_term_info +) + + +class ImagesNeuronsTest(unittest.TestCase): + """Test cases for ImagesNeurons query""" + + def setUp(self): + """Set up test fixtures""" + self.test_term = 'FBbt_00007401' # antennal lobe - synaptic neuropil with individual images + + def test_get_images_neurons_execution(self): + """Test that get_images_neurons executes and returns results""" + result = get_images_neurons(self.test_term, return_dataframe=True, limit=3) + + # Should return a DataFrame + self.assertIsNotNone(result, "Result should not be None") + + # Check result type - handle both DataFrame and dict (from cache) + import pandas as pd + if isinstance(result, pd.DataFrame): + # DataFrame result + if len(result) > 0: + print(f"\nāœ“ Found {len(result)} individual neuron images for {self.test_term}") + + # Verify DataFrame has expected columns + self.assertIn('id', result.columns, "Result should have 'id' column") + self.assertIn('label', result.columns, "Result should have 'label' column") + + # Print first few results for verification + print("\nSample results:") + for idx, row in result.head(3).iterrows(): + print(f" - {row.get('label', 'N/A')} ({row.get('id', 'N/A')})") + else: + print(f"\nāš  No individual neuron images found for {self.test_term} (this may be expected)") + elif isinstance(result, dict): + # Dict result (from cache) + count = result.get('count', 0) + rows = result.get('rows', []) + print(f"\nāœ“ Found {count} total individual neuron images for {self.test_term} (showing {len(rows)})") + if rows: + print("\nSample results:") + for row in rows[:3]: + print(f" - {row.get('label', 'N/A')} ({row.get('id', 'N/A')})") + else: + self.fail(f"Unexpected result type: {type(result)}") + + def test_images_neurons_schema(self): + """Test that ImagesNeurons_to_schema generates correct schema""" + name = "antennal lobe" + take_default = {"short_form": self.test_term} + + schema = ImagesNeurons_to_schema(name, take_default) + + # Verify schema structure + self.assertEqual(schema.query, "ImagesNeurons") + self.assertEqual(schema.label, f"Images of neurons with some part in {name}") + self.assertEqual(schema.function, "get_images_neurons") + self.assertEqual(schema.preview, 10) + self.assertIn("id", schema.preview_columns) + self.assertIn("label", schema.preview_columns) + + print(f"\nāœ“ Schema generated correctly") + print(f" Query: {schema.query}") + print(f" Label: {schema.label}") + print(f" Function: {schema.function}") + + def test_term_info_integration(self): + """Test that ImagesNeurons query appears in term_info for synaptic neuropils""" + term_info = get_term_info(self.test_term, preview=True) + + # Should have queries + self.assertIn('Queries', term_info, "term_info should have 'Queries' key") + + # Look for ImagesNeurons query + query_names = [q['query'] for q in term_info['Queries']] + print(f"\nāœ“ Queries available for {self.test_term}: {query_names}") + + if 'ImagesNeurons' in query_names: + images_query = next(q for q in term_info['Queries'] if q['query'] == 'ImagesNeurons') + print(f"āœ“ ImagesNeurons query found: {images_query['label']}") + + # Verify preview results if available + if 'preview_results' in images_query: + preview = images_query['preview_results'] + # Handle both 'data' and 'rows' keys + data_key = 'data' if 'data' in preview else 'rows' + if data_key in preview and len(preview[data_key]) > 0: + print(f" Preview has {len(preview[data_key])} individual neuron images") + print(f" Sample: {preview[data_key][0]}") + else: + print(f"āš  ImagesNeurons query not found in term_info") + print(f" Available queries: {query_names}") + print(f" SuperTypes: {term_info.get('SuperTypes', [])}") + + def test_images_neurons_preview(self): + """Test preview results format""" + result = get_images_neurons(self.test_term, return_dataframe=False, limit=5) + + # Should be a dict with specific structure + self.assertIsInstance(result, dict, "Result should be a dictionary") + self.assertIn('rows', result, "Result should have 'rows' key") + self.assertIn('headers', result, "Result should have 'headers' key") + self.assertIn('count', result, "Result should have 'count' key") + + if result['count'] > 0: + print(f"\nāœ“ Preview format validated") + print(f" Total count: {result['count']}") + print(f" Returned rows: {len(result['rows'])}") + print(f" Headers: {list(result['headers'].keys())}") + else: + print(f"\nāš  No results in preview (this may be expected)") + + def test_multiple_terms(self): + """Test query with multiple synaptic neuropil terms""" + test_terms = [ + ('FBbt_00007401', 'antennal lobe'), + ('FBbt_00003982', 'medulla'), # another synaptic neuropil + ] + + print("\nāœ“ Testing ImagesNeurons with multiple terms:") + for term_id, term_name in test_terms: + try: + result = get_images_neurons(term_id, return_dataframe=True, limit=10) + count = len(result) if result is not None else 0 + print(f" - {term_name} ({term_id}): {count} individual neuron images") + except Exception as e: + print(f" - {term_name} ({term_id}): Error - {e}") + + +if __name__ == '__main__': + # Run tests with verbose output + unittest.main(verbosity=2) diff --git a/src/test/test_query_performance.py b/src/test/test_query_performance.py index 3cc7b91..73986e9 100644 --- a/src/test/test_query_performance.py +++ b/src/test/test_query_performance.py @@ -26,9 +26,9 @@ get_neuron_classes_fasciculating_here, get_tracts_nerves_innervating_here, get_lineage_clones_in, + get_images_neurons, get_instances, get_similar_neurons, - get_individual_neuron_inputs ) @@ -232,6 +232,17 @@ def test_05_new_queries(self): ) print(f"LineageClonesIn: {duration:.4f}s {'āœ…' if success else 'āŒ'}") self.assertLess(duration, self.THRESHOLD_SLOW, "LineageClonesIn exceeded threshold") + + # ImagesNeurons + result, duration, success = self._time_query( + "ImagesNeurons", + get_images_neurons, + self.test_terms['antennal_lobe'], + return_dataframe=False, + limit=10 + ) + print(f"ImagesNeurons: {duration:.4f}s {'āœ…' if success else 'āŒ'}") + self.assertLess(duration, self.THRESHOLD_SLOW, "ImagesNeurons exceeded threshold") def test_06_instance_queries(self): """Test instance retrieval queries""" diff --git a/src/vfbquery/owlery_client.py b/src/vfbquery/owlery_client.py index d84dce2..add8263 100644 --- a/src/vfbquery/owlery_client.py +++ b/src/vfbquery/owlery_client.py @@ -160,6 +160,109 @@ def gen_short_form(iri): except Exception as e: print(f"ERROR: Unexpected error in Owlery query: {e}") raise + + def get_instances(self, query: str, query_by_label: bool = True, + verbose: bool = False, prefixes: bool = False, direct: bool = False) -> List[str]: + """ + Query Owlery for instances matching an OWL class expression. + + Similar to get_subclasses but returns individuals/instances instead of classes. + Used for queries like ImagesNeurons that need individual images rather than classes. + + :param query: OWL class expression query string (with short forms like '') + :param query_by_label: If True, query uses label syntax (quotes). + If False, uses IRI syntax (angle brackets). + :param verbose: If True, print debug information + :param prefixes: If True, return full IRIs. If False, return short forms. + :param direct: Return direct instances only. Default False. + :return: List of instance IDs (short forms like 'VFB_00101567') + """ + try: + # Convert short forms in query to full IRIs + import re + def convert_short_form_to_iri(match): + short_form = match.group(1) + if '_' in short_form and '/' not in short_form: + return f"<{short_form_to_iri(short_form)}>" + else: + return match.group(0) + + iri_query = re.sub(r'<([^>]+)>', convert_short_form_to_iri, query) + + if verbose: + print(f"Original query: {query}") + print(f"IRI query: {iri_query}") + + # Build Owlery instances endpoint URL + params = { + 'object': iri_query, + 'prefixes': json.dumps({ + "FBbt": "http://purl.obolibrary.org/obo/FBbt_", + "RO": "http://purl.obolibrary.org/obo/RO_", + "BFO": "http://purl.obolibrary.org/obo/BFO_", + "VFB": "http://virtualflybrain.org/reports/VFB_" + }) + } + if direct: + params['direct'] = 'False' + + # Make HTTP GET request to instances endpoint + response = requests.get( + f"{self.owlery_endpoint}/instances", + params=params, + timeout=120 + ) + + if verbose: + print(f"Owlery instances query: {response.url}") + + response.raise_for_status() + + # Parse JSON response + # KEY DIFFERENCE: Owlery returns {"hasInstance": ["IRI1", "IRI2", ...]} for instances + # whereas subclasses returns {"superClassOf": [...]} + data = response.json() + + if verbose: + print(f"Response keys: {data.keys() if isinstance(data, dict) else 'not a dict'}") + + # Extract IRIs from response using correct key + iris = [] + if isinstance(data, dict) and 'hasInstance' in data: + iris = data['hasInstance'] + elif isinstance(data, list): + iris = data + else: + if verbose: + print(f"Unexpected Owlery response format: {type(data)}") + print(f"Response: {data}") + return [] + + if not isinstance(iris, list): + if verbose: + print(f"Warning: No results! This is likely due to a query error") + print(f"Query: {query}") + return [] + + # Convert IRIs to short forms + def gen_short_form(iri): + return re.split('/|#', iri)[-1] + + short_forms = list(map(gen_short_form, iris)) + + if verbose: + print(f"Found {len(short_forms)} instances") + if short_forms: + print(f"Sample instances: {short_forms[:5]}") + + return short_forms + + except requests.RequestException as e: + print(f"ERROR: Owlery instances request failed: {e}") + raise + except Exception as e: + print(f"ERROR: Unexpected error in Owlery instances query: {e}") + raise class MockNeo4jClient: diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 47158c3..808a9dc 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -748,6 +748,16 @@ def term_info_parse_object(results, short_form): q = LineageClonesIn_to_schema(termInfo["Name"], {"short_form": vfbTerm.term.core.short_form}) queries.append(q) + # ImagesNeurons query - for synaptic neuropils + # Matches XMI criteria: Class + (Synaptic_neuropil OR Synaptic_neuropil_domain) + # Returns individual neuron images (instances) rather than neuron classes + if contains_all_tags(termInfo["SuperTypes"], ["Class"]) and ( + "Synaptic_neuropil" in termInfo["SuperTypes"] or + "Synaptic_neuropil_domain" in termInfo["SuperTypes"] + ): + q = ImagesNeurons_to_schema(termInfo["Name"], {"short_form": vfbTerm.term.core.short_form}) + queries.append(q) + # Add Publications to the termInfo object if vfbTerm.pubs and len(vfbTerm.pubs) > 0: publications = [] @@ -1163,6 +1173,31 @@ def LineageClonesIn_to_schema(name, take_default): return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) +def ImagesNeurons_to_schema(name, take_default): + """ + Schema for ImagesNeurons query. + Finds individual neuron images with parts in a synaptic neuropil or domain. + + Matching criteria from XMI: + - Class + Synaptic_neuropil + - Class + Synaptic_neuropil_domain + + Query chain: Owlery instances query ā†’ process ā†’ SOLR + OWL query: 'Neuron' that 'overlaps' some '{short_form}' (returns instances, not classes) + """ + query = "ImagesNeurons" + label = f"Images of neurons with some part in {name}" + function = "get_images_neurons" + takes = { + "short_form": {"$and": ["Class", "Synaptic_neuropil"]}, + "default": take_default, + } + preview = 10 + preview_columns = ["id", "label", "tags", "thumbnail"] + + return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) + + def serialize_solr_output(results): # Create a copy of the document and remove Solr-specific fields doc = dict(results.docs[0]) @@ -2143,6 +2178,28 @@ def get_lineage_clones_in(short_form: str, return_dataframe=True, limit: int = - return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_query', query_by_label=False) +@with_solr_cache('images_neurons') +def get_images_neurons(short_form: str, return_dataframe=True, limit: int = -1): + """ + Retrieves individual neuron images with parts in the specified synaptic neuropil. + + This implements the ImagesNeurons query from the VFB XMI specification. + Query chain (from XMI): Owlery instances ā†’ Process ā†’ SOLR + OWL query (from XMI): object= and some <$ID> (instances) + Where: FBbt_00005106 = neuron, RO_0002131 = overlaps + Matching criteria: Class + Synaptic_neuropil, Class + Synaptic_neuropil_domain + + Note: This query returns INSTANCES (individual neuron images) not classes. + + :param short_form: short form of the synaptic neuropil (Class) + :param return_dataframe: Returns pandas dataframe if true, otherwise returns formatted dict + :param limit: maximum number of results to return (default -1, returns all results) + :return: Individual neuron images with parts in the specified neuropil + """ + owl_query = f" and some " + return _owlery_instances_query_to_results(owl_query, short_form, return_dataframe, limit, solr_field='anat_image_query') + + def _get_neurons_part_here_headers(): """Return standard headers for get_neurons_with_part_in results""" return { @@ -2349,6 +2406,161 @@ def _owlery_query_to_results(owl_query_string: str, short_form: str, return_data } +def _owlery_instances_query_to_results(owl_query_string: str, short_form: str, return_dataframe: bool = True, + limit: int = -1, solr_field: str = 'anat_image_query'): + """ + Shared helper function for Owlery-based instance queries (e.g., ImagesNeurons). + + Similar to _owlery_query_to_results but queries for instances (individuals) instead of classes. + This implements the common pattern: + 1. Query Owlery for instance IDs matching an OWL pattern + 2. Fetch details from SOLR for each instance + 3. Format results as DataFrame or dict + + :param owl_query_string: OWL query string with IRI syntax (angle brackets) + :param short_form: The anatomical region or entity short form + :param return_dataframe: Returns pandas DataFrame if True, otherwise returns formatted dict + :param limit: Maximum number of results to return (default -1 for all) + :param solr_field: SOLR field to query (default 'anat_image_query' for Individual images) + :return: Query results + """ + try: + # Step 1: Query Owlery for instances matching the OWL pattern + instance_ids = vc.vfb.oc.get_instances( + query=owl_query_string, + query_by_label=False, # Use IRI syntax + verbose=False + ) + + if not instance_ids: + # No results found - return empty + if return_dataframe: + return pd.DataFrame() + return { + "headers": _get_standard_query_headers(), + "rows": [], + "count": 0 + } + + total_count = len(instance_ids) + + # Apply limit if specified (before SOLR query to save processing) + if limit != -1 and limit > 0: + instance_ids = instance_ids[:limit] + + # Step 2: Query SOLR for ALL instances in a single batch query + rows = [] + try: + # Build filter query with all instance IDs + id_list = ','.join(instance_ids) + results = vfb_solr.search( + q='id:*', + fq=f'{{!terms f=id}}{id_list}', + fl=solr_field, + rows=len(instance_ids) + ) + + # Process all results + for doc in results.docs: + if solr_field not in doc: + continue + + # Parse the SOLR field JSON string + try: + field_data = json.loads(doc[solr_field][0]) + except (json.JSONDecodeError, IndexError) as e: + print(f"Error parsing {solr_field} JSON: {e}") + continue + + # Extract from term.core structure (VFBConnect pattern) + term_core = field_data.get('term', {}).get('core', {}) + instance_short_form = term_core.get('short_form', '') + label_text = term_core.get('label', '') + + # Build tags list from unique_facets + tags = term_core.get('unique_facets', []) + + # Get thumbnail from channel_image array (VFBConnect pattern) + thumbnail = '' + channel_images = field_data.get('channel_image', []) + if channel_images and len(channel_images) > 0: + first_channel = channel_images[0] + image_info = first_channel.get('image', {}) + thumbnail_url = image_info.get('image_thumbnail', '') + + if thumbnail_url: + # Convert to HTTPS (thumbnails are already non-transparent) + thumbnail_url = thumbnail_url.replace('http://', 'https://') + + # Format thumbnail markdown with template info + template_anatomy = image_info.get('template_anatomy', {}) + if template_anatomy: + template_label = template_anatomy.get('symbol') or template_anatomy.get('label', '') + template_label = unquote(template_label) + anatomy_label = term_core.get('symbol') or label_text + anatomy_label = unquote(anatomy_label) + alt_text = f"{anatomy_label} aligned to {template_label}" + template_short_form = template_anatomy.get('short_form', '') + thumbnail = f"[![{alt_text}]({thumbnail_url} '{alt_text}')]({template_short_form},{instance_short_form})" + + # Build row + row = { + 'id': instance_short_form, + 'label': f"[{label_text}]({instance_short_form})", + 'tags': tags, + 'thumbnail': thumbnail + } + + rows.append(row) + + except Exception as e: + print(f"Error fetching SOLR data for instances: {e}") + import traceback + traceback.print_exc() + + # Convert to DataFrame if requested + if return_dataframe: + df = pd.DataFrame(rows) + # Apply markdown encoding + columns_to_encode = ['label', 'thumbnail'] + df = encode_markdown_links(df, columns_to_encode) + return df + + # Return formatted dict + return { + "headers": _get_standard_query_headers(), + "rows": rows, + "count": total_count + } + + except Exception as e: + # Construct the Owlery URL for debugging failed queries + owlery_base = "http://owl.virtualflybrain.org/kbs/vfb" + try: + if hasattr(vc.vfb, 'oc') and hasattr(vc.vfb.oc, 'owlery_endpoint'): + owlery_base = vc.vfb.oc.owlery_endpoint.rstrip('/') + except Exception: + pass + + from urllib.parse import quote + query_encoded = quote(owl_query_string, safe='') + owlery_url = f"{owlery_base}/instances?object={query_encoded}" + + import sys + print(f"ERROR: Owlery instances query failed: {e}", file=sys.stderr) + print(f" Test URL: {owlery_url}", file=sys.stderr) + import traceback + traceback.print_exc() + # Return error indication with count=-1 + if return_dataframe: + return pd.DataFrame() + return { + "headers": _get_standard_query_headers(), + "rows": [], + "count": -1 + } + + def fill_query_results(term_info): for query in term_info['Queries']: # print(f"Query Keys:{query.keys()}") From a955327ae9149fac10b91f29b271481d5a3f519e Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 22:55:24 +0000 Subject: [PATCH 68/70] Update performance test results [skip ci] --- performance.md | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/performance.md b/performance.md index 753848d..b682987 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 21:42:09 UTC -**Git Commit:** 160079a0c99df42c8141cfceeb7ae6410f9525ca +**Test Date:** 2025-11-06 22:55:24 UTC +**Git Commit:** e9345a34a70d715cbba306e5de8f07971bf2f06b **Branch:** dev -**Workflow Run:** [19150715535](https://github.com/VirtualFlyBrain/VFBquery/actions/runs/19150715535) +**Workflow Run:** [19152298747](https://github.com/VirtualFlyBrain/VFBquery/actions/runs/19152298747) ## Test Overview @@ -50,10 +50,10 @@ Test term info query performance Traceback (most recent call last): File "/home/runner/work/VFBquery/VFBquery/src/test/test_query_performance.py", line 94, in test_01_term_info_queries self.assertLess(duration, self.THRESHOLD_MEDIUM, "term_info query exceeded threshold") -AssertionError: 11.490196466445923 not less than 3.0 : term_info query exceeded threshold +AssertionError: 91.05893087387085 not less than 3.0 : term_info query exceeded threshold ---------------------------------------------------------------------- -Ran 6 tests in 21.573s +Ran 6 tests in 99.000s FAILED (failures=1) VFBquery caching enabled: TTL=2160h (90 days), Memory=2048MB @@ -65,39 +65,39 @@ VFBquery: Caching enabled by default (3-month TTL, 2GB memory) TERM INFO QUERIES ================================================================================ DEBUG: Cache lookup for FBbt_00003748: MISS -āœ… Neo4j connection established -get_term_info (mushroom body): 11.4902s āœ… +get_term_info (mushroom body): 91.0589s āœ… ================================================================================ NEURON PART OVERLAP QUERIES ================================================================================ -NeuronsPartHere: 0.7864s āœ… +NeuronsPartHere: 0.7412s āœ… ================================================================================ SYNAPTIC TERMINAL QUERIES ================================================================================ -NeuronsSynaptic: 0.6386s āœ… -NeuronsPresynapticHere: 0.5441s āœ… -NeuronsPostsynapticHere: 0.6352s āœ… +NeuronsSynaptic: 0.7658s āœ… +NeuronsPresynapticHere: 0.5882s āœ… +NeuronsPostsynapticHere: 0.6004s āœ… ================================================================================ ANATOMICAL HIERARCHY QUERIES ================================================================================ -ComponentsOf: 2.2629s āœ… -PartsOf: 0.8743s āœ… -SubclassesOf: 0.8658s āœ… +ComponentsOf: 0.6777s āœ… +PartsOf: 0.5956s āœ… +SubclassesOf: 0.6793s āœ… ================================================================================ NEW QUERIES (2025) ================================================================================ -NeuronClassesFasciculatingHere: 0.5527s āœ… -TractsNervesInnervatingHere: 0.5502s āœ… -LineageClonesIn: 0.5484s āœ… +NeuronClassesFasciculatingHere: 0.7705s āœ… +TractsNervesInnervatingHere: 0.5785s āœ… +LineageClonesIn: 0.5851s āœ… +ImagesNeurons: 0.5834s āœ… ================================================================================ INSTANCE QUERIES ================================================================================ -ListAllAvailableImages: 1.8236s āœ… +ListAllAvailableImages: 0.7739s āœ… ================================================================================ PERFORMANCE TEST SUMMARY @@ -108,7 +108,7 @@ test_term_info_performance (src.test.term_info_queries_test.TermInfoQueriesTest) Performance test for specific term info queries. ... ok ---------------------------------------------------------------------- -Ran 1 test in 1.117s +Ran 1 test in 1.189s OK VFBquery caching enabled: TTL=2160h (90 days), Memory=2048MB @@ -123,9 +123,9 @@ VFBquery: Caching enabled by default (3-month TTL, 2GB memory) ================================================== Performance Test Results: ================================================== -FBbt_00003748 query took: 0.5602 seconds -VFB_00101567 query took: 0.5561 seconds -Total time for both queries: 1.1163 seconds +FBbt_00003748 query took: 0.5922 seconds +VFB_00101567 query took: 0.5970 seconds +Total time for both queries: 1.1891 seconds Performance Level: šŸŸ¢ Excellent (< 1.5 seconds) ================================================== Performance test completed successfully! @@ -144,4 +144,4 @@ Track performance trends across commits: - [GitHub Actions History](https://github.com/VirtualFlyBrain/VFBquery/actions/workflows/performance-test.yml) --- -*Last updated: 2025-11-06 21:42:09 UTC* +*Last updated: 2025-11-06 22:55:24 UTC* From 252031708b2d8ad1156bd35047fe0f7139f86256 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Thu, 6 Nov 2025 23:08:04 +0000 Subject: [PATCH 69/70] Reduce preview results from 10 to 5 in neuron-related schema functions --- src/vfbquery/vfb_queries.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index 808a9dc..59b9a51 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -943,7 +943,7 @@ def NeuronsPartHere_to_schema(name, take_default): "short_form": {"$and": ["Class", "Anatomy"]}, "default": take_default, } - preview = 10 # Show 10 preview results with example images + preview = 5 # Show 5 preview results with example images preview_columns = ["id", "label", "tags", "thumbnail"] return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) @@ -969,7 +969,7 @@ def NeuronsSynaptic_to_schema(name, take_default): "short_form": {"$and": ["Class", "Anatomy"]}, "default": take_default, } - preview = 10 + preview = 5 preview_columns = ["id", "label", "tags", "thumbnail"] return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) @@ -995,7 +995,7 @@ def NeuronsPresynapticHere_to_schema(name, take_default): "short_form": {"$and": ["Class", "Anatomy"]}, "default": take_default, } - preview = 10 + preview = 5 preview_columns = ["id", "label", "tags", "thumbnail"] return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) @@ -1021,7 +1021,7 @@ def NeuronsPostsynapticHere_to_schema(name, take_default): "short_form": {"$and": ["Class", "Anatomy"]}, "default": take_default, } - preview = 10 + preview = 5 preview_columns = ["id", "label", "tags", "thumbnail"] return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) @@ -1045,7 +1045,7 @@ def ComponentsOf_to_schema(name, take_default): "short_form": {"$and": ["Class", "Anatomy"]}, "default": take_default, } - preview = 10 + preview = 5 preview_columns = ["id", "label", "tags", "thumbnail"] return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) @@ -1069,7 +1069,7 @@ def PartsOf_to_schema(name, take_default): "short_form": {"$and": ["Class"]}, "default": take_default, } - preview = 10 + preview = 5 preview_columns = ["id", "label", "tags", "thumbnail"] return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) @@ -1093,7 +1093,7 @@ def SubclassesOf_to_schema(name, take_default): "short_form": {"$and": ["Class"]}, "default": take_default, } - preview = 10 + preview = 5 preview_columns = ["id", "label", "tags", "thumbnail"] return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) @@ -1117,7 +1117,7 @@ def NeuronClassesFasciculatingHere_to_schema(name, take_default): "short_form": {"$and": ["Class", "Neuron_projection_bundle"]}, "default": take_default, } - preview = 10 + preview = 5 preview_columns = ["id", "label", "tags", "thumbnail"] return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) @@ -1142,7 +1142,7 @@ def TractsNervesInnervatingHere_to_schema(name, take_default): "short_form": {"$and": ["Class", "Synaptic_neuropil"]}, "default": take_default, } - preview = 10 + preview = 5 preview_columns = ["id", "label", "tags", "thumbnail"] return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) @@ -1167,7 +1167,7 @@ def LineageClonesIn_to_schema(name, take_default): "short_form": {"$and": ["Class", "Synaptic_neuropil"]}, "default": take_default, } - preview = 10 + preview = 5 preview_columns = ["id", "label", "tags", "thumbnail"] return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) @@ -1192,7 +1192,7 @@ def ImagesNeurons_to_schema(name, take_default): "short_form": {"$and": ["Class", "Synaptic_neuropil"]}, "default": take_default, } - preview = 10 + preview = 5 preview_columns = ["id", "label", "tags", "thumbnail"] return Query(query=query, label=label, function=function, takes=takes, preview=preview, preview_columns=preview_columns) From 91a0f36e493a991e3777e974797ecd51c0f543d8 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 6 Nov 2025 23:09:28 +0000 Subject: [PATCH 70/70] Update performance test results [skip ci] --- performance.md | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/performance.md b/performance.md index b682987..6fbb5b4 100644 --- a/performance.md +++ b/performance.md @@ -1,9 +1,9 @@ # VFBquery Performance Test Results -**Test Date:** 2025-11-06 22:55:24 UTC -**Git Commit:** e9345a34a70d715cbba306e5de8f07971bf2f06b +**Test Date:** 2025-11-06 23:09:28 UTC +**Git Commit:** bd346650d667a1f6980990ab476db9233479e89f **Branch:** dev -**Workflow Run:** [19152298747](https://github.com/VirtualFlyBrain/VFBquery/actions/runs/19152298747) +**Workflow Run:** [19152612234](https://github.com/VirtualFlyBrain/VFBquery/actions/runs/19152612234) ## Test Overview @@ -50,10 +50,10 @@ Test term info query performance Traceback (most recent call last): File "/home/runner/work/VFBquery/VFBquery/src/test/test_query_performance.py", line 94, in test_01_term_info_queries self.assertLess(duration, self.THRESHOLD_MEDIUM, "term_info query exceeded threshold") -AssertionError: 91.05893087387085 not less than 3.0 : term_info query exceeded threshold +AssertionError: 8.598119258880615 not less than 3.0 : term_info query exceeded threshold ---------------------------------------------------------------------- -Ran 6 tests in 99.000s +Ran 6 tests in 16.573s FAILED (failures=1) VFBquery caching enabled: TTL=2160h (90 days), Memory=2048MB @@ -65,39 +65,39 @@ VFBquery: Caching enabled by default (3-month TTL, 2GB memory) TERM INFO QUERIES ================================================================================ DEBUG: Cache lookup for FBbt_00003748: MISS -get_term_info (mushroom body): 91.0589s āœ… +get_term_info (mushroom body): 8.5981s āœ… ================================================================================ NEURON PART OVERLAP QUERIES ================================================================================ -NeuronsPartHere: 0.7412s āœ… +NeuronsPartHere: 0.6646s āœ… ================================================================================ SYNAPTIC TERMINAL QUERIES ================================================================================ -NeuronsSynaptic: 0.7658s āœ… -NeuronsPresynapticHere: 0.5882s āœ… -NeuronsPostsynapticHere: 0.6004s āœ… +NeuronsSynaptic: 0.6417s āœ… +NeuronsPresynapticHere: 0.6458s āœ… +NeuronsPostsynapticHere: 0.6492s āœ… ================================================================================ ANATOMICAL HIERARCHY QUERIES ================================================================================ -ComponentsOf: 0.6777s āœ… -PartsOf: 0.5956s āœ… -SubclassesOf: 0.6793s āœ… +ComponentsOf: 0.6447s āœ… +PartsOf: 0.6637s āœ… +SubclassesOf: 0.8425s āœ… ================================================================================ NEW QUERIES (2025) ================================================================================ -NeuronClassesFasciculatingHere: 0.7705s āœ… -TractsNervesInnervatingHere: 0.5785s āœ… -LineageClonesIn: 0.5851s āœ… -ImagesNeurons: 0.5834s āœ… +NeuronClassesFasciculatingHere: 0.6454s āœ… +TractsNervesInnervatingHere: 0.6351s āœ… +LineageClonesIn: 0.6446s āœ… +ImagesNeurons: 0.6604s āœ… ================================================================================ INSTANCE QUERIES ================================================================================ -ListAllAvailableImages: 0.7739s āœ… +ListAllAvailableImages: 0.6358s āœ… ================================================================================ PERFORMANCE TEST SUMMARY @@ -108,7 +108,7 @@ test_term_info_performance (src.test.term_info_queries_test.TermInfoQueriesTest) Performance test for specific term info queries. ... ok ---------------------------------------------------------------------- -Ran 1 test in 1.189s +Ran 1 test in 1.306s OK VFBquery caching enabled: TTL=2160h (90 days), Memory=2048MB @@ -123,9 +123,9 @@ VFBquery: Caching enabled by default (3-month TTL, 2GB memory) ================================================== Performance Test Results: ================================================== -FBbt_00003748 query took: 0.5922 seconds -VFB_00101567 query took: 0.5970 seconds -Total time for both queries: 1.1891 seconds +FBbt_00003748 query took: 0.6560 seconds +VFB_00101567 query took: 0.6502 seconds +Total time for both queries: 1.3062 seconds Performance Level: šŸŸ¢ Excellent (< 1.5 seconds) ================================================== Performance test completed successfully! @@ -144,4 +144,4 @@ Track performance trends across commits: - [GitHub Actions History](https://github.com/VirtualFlyBrain/VFBquery/actions/workflows/performance-test.yml) --- -*Last updated: 2025-11-06 22:55:24 UTC* +*Last updated: 2025-11-06 23:09:28 UTC*