CHORE: Add LCOV_EXCL_LINE markers to LOG() statements for coverage analysis

.coveragerc

@@ -4,6 +4,9 @@ omit =
     setup.py
     bcp_options.py
     tests/*
+    add_lcov_exclusions.py
+    fix_multiline_log_exclusions.py
+    run_coverage_docker.ps1
 
 [report]
 # Exclude lines that don't need coverage (logging, defensive code, etc.)

.gitignore

@@ -48,6 +48,16 @@ build/
 
 # wheel files
 *.whl
+
+# Coverage reports and artifacts
+.coverage
+coverage.json
+coverage*.xml
+htmlcov/
+unified-coverage/
+*.profraw
+*.profdata
+*.info
 *.tar.gz
 *.zip
 

add_lcov_exclusions.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""
+Script to add LCOV_EXCL_LINE markers to LOG() statements in C++ code.
+This excludes diagnostic logging from code coverage analysis.
+"""
+
+import re
+from pathlib import Path
+
+def process_file(filepath):
+    """Add LCOV_EXCL_LINE to LOG() statements in a C++ file (including multi-line statements)."""
+    with open(filepath, 'r', encoding='utf-8') as f:
+        content = f.read()
+
+    lines = content.split('\n')
+    modified = False
+    new_lines = []
+    in_log_statement = False
+    paren_depth = 0
+
+    for i, line in enumerate(lines):
+        # Skip comment lines that mention LOG() but aren't actual LOG calls
+        if line.strip().startswith('//'):
+            new_lines.append(line)
+            continue
+
+        # Check if we're starting a new LOG statement
+        if 'LOG(' in line and not in_log_statement and 'LCOV_EXCL' not in line:
+            in_log_statement = True
+            # Count parentheses to track when LOG statement ends
+            paren_depth = line.count('(') - line.count(')')
+
+        # Process line if it's part of a LOG statement
+        if in_log_statement and 'LCOV_EXCL' not in line:
+            stripped = line.rstrip()
+            new_line = stripped + '  // LCOV_EXCL_LINE'
+            new_lines.append(new_line)
+            modified = True
+
+            # Show first line only for clarity
+            if 'LOG(' in line:
+                print(f"  Modified: {filepath.name}:{len(new_lines)} - {stripped[:60]}...")
+
+            # Update parenthesis depth
+            paren_depth += line.count('(') - line.count(')')
+
+            # Check if LOG statement ends on this line
+            if paren_depth <= 0 or ');' in line:
+                in_log_statement = False
+                paren_depth = 0
+        else:
+            new_lines.append(line)
+
+    if modified:
+        with open(filepath, 'w', encoding='utf-8') as f:
+            f.write('\n'.join(new_lines))
+        return True
+    return False
+
+def main():
+    """Process all C++ files in mssql_python/pybind directory."""
+    base_path = Path(__file__).parent / 'mssql_python' / 'pybind'
+
+    # Find all .cpp and .h files
+    cpp_files = list(base_path.rglob('*.cpp')) + list(base_path.rglob('*.h'))
+
+    print(f"Found {len(cpp_files)} C++ files to process")
+    print("=" * 70)
+
+    modified_count = 0
+    for filepath in sorted(cpp_files):
+        if process_file(filepath):
+            modified_count += 1
+
+    print("=" * 70)
+    print(f"\nCompleted: {modified_count} files modified")
+
+if __name__ == '__main__':
+    main()

fix_multiline_log_exclusions.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""
+Script to add LCOV_EXCL_LINE markers to ALL lines of LOG() statements (including continuations).
+"""
+
+import re
+from pathlib import Path
+
+def process_file(filepath):
+    """Add LCOV_EXCL_LINE to all lines of LOG() statements, including multi-line continuations."""
+    with open(filepath, 'r', encoding='utf-8') as f:
+        lines = f.readlines()
+
+    new_lines = []
+    modified = False
+    i = 0
+
+    while i < len(lines):
+        line = lines[i].rstrip('\n')
+
+        # Skip comment-only lines
+        if line.strip().startswith('//') and 'LOG(' not in line:
+            new_lines.append(line)
+            i += 1
+            continue
+
+        # Check if this line starts a LOG statement
+        if 'LOG(' in line:
+            # Collect all lines of this LOG statement
+            log_lines = [line]
+            j = i + 1
+
+            # Track parenthesis depth to find where LOG statement ends
+            depth = line.count('(') - line.count(')')
+
+            while j < len(lines) and depth > 0:
+                next_line = lines[j].rstrip('\n')
+                log_lines.append(next_line)
+                depth += next_line.count('(') - next_line.count(')')
+                j += 1
+
+            # Add LCOV_EXCL_LINE to all lines that don't already have it
+            for log_line in log_lines:
+                if 'LCOV_EXCL' not in log_line:
+                    new_lines.append(log_line.rstrip() + '  // LCOV_EXCL_LINE')
+                    modified = True
+                else:
+                    new_lines.append(log_line)
+
+            if modified and 'LOG(' in log_lines[0]:
+                print(f"  Modified: {filepath.name} lines {i+1}-{j}: {log_lines[0][:60]}...")
+
+            i = j
+        else:
+            new_lines.append(line)
+            i += 1
+
+    if modified:
+        with open(filepath, 'w', encoding='utf-8', newline='\n') as f:
+            f.write('\n'.join(new_lines))
+        return True
+    return False
+
+def main():
+    """Process all C++ files in mssql_python/pybind directory."""
+    base_path = Path(__file__).parent / 'mssql_python' / 'pybind'
+
+    # Find all .cpp and .h files
+    cpp_files = list(base_path.rglob('*.cpp')) + list(base_path.rglob('*.h'))
+
+    print(f"Found {len(cpp_files)} C++ files to process")
+    print("=" * 70)
+
+    modified_count = 0
+    for filepath in sorted(cpp_files):
+        if process_file(filepath):
+            modified_count += 1
+
+    print("=" * 70)
+    print(f"\nCompleted: {modified_count} files modified")
+
+if __name__ == '__main__':
+    main()

generate_codecov.sh

@@ -81,20 +81,108 @@ llvm-cov export "$PYBIND_SO" \
   --skip-functions \
   -format=lcov > cpp-coverage.info
 
-# Note: LCOV exclusion markers (LCOV_EXCL_LINE) should be added to source code
-# to exclude LOG() statements from coverage. However, for automated exclusion
-# of all LOG lines without modifying source code, we can use geninfo's --omit-lines
-# feature during the merge step (see below).
+# Note: LCOV exclusion markers (LCOV_EXCL_LINE) are processed below
 
 echo "==================================="
 echo "[STEP 4] Merging Python + C++ coverage"
 echo "==================================="
 
 # Merge LCOV reports (ignore inconsistencies in Python LCOV export)
 echo "[ACTION] Merging Python and C++ coverage"
-lcov -a python-coverage.info -a cpp-coverage.info -o total.info \
+lcov -a python-coverage.info -a cpp-coverage.info -o total-unfiltered.info \
   --ignore-errors inconsistent,corrupt
 
+# Filter out LOG() statements with LCOV_EXCL_LINE markers and recalculate statistics
+echo "[ACTION] Filtering LCOV_EXCL_LINE markers from coverage"
+python3 - <<'PYTHON_FILTER'
+import os
+import re
+
+with open('total-unfiltered.info', 'r') as f:
+    content = f.read()
+
+# Process file section by section
+sections = content.split('SF:')
+output_sections = []
+
+if sections[0].strip():
+    output_sections.append(sections[0])  # Keep any header
+
+total_excluded = 0
+
+for section in sections[1:]:
+    if not section.strip():
+        continue
+
+    lines = section.split('\n')
+    source_file = lines[0].strip()
+
+    # Read source file to find LCOV_EXCL_LINE markers
+    excluded_lines = set()
+    if os.path.exists(source_file):
+        try:
+            with open(source_file, 'r', encoding='utf-8', errors='ignore') as src:
+                for line_num, src_line in enumerate(src, 1):
+                    if 'LCOV_EXCL_LINE' in src_line:
+                        excluded_lines.add(line_num)
+        except:
+            pass
+
+    # Process section lines - filter DA entries and recalculate LH/LF
+    new_section = [f'SF:{source_file}']
+    da_entries = []
+    other_lines = []
+    lines_hit = 0
+    lines_found = 0
+
+    for line in lines[1:]:
+        line = line.rstrip()
+        if not line:
+            continue
+
+        if line.startswith('DA:'):
+            # DA:line_number,hit_count
+            match = re.match(r'DA:(\d+),(\d+)', line)
+            if match:
+                line_num = int(match.group(1))
+                hit_count = int(match.group(2))
+
+                if line_num not in excluded_lines:
+                    da_entries.append(line)
+                    lines_found += 1
+                    if hit_count > 0:
+                        lines_hit += 1
+                else:
+                    total_excluded += 1
+        elif line.startswith('LH:') or line.startswith('LF:'):
+            # Skip old counters - we'll add new ones
+            continue
+        else:
+            other_lines.append(line)
+
+    # Rebuild section with correct order: SF, DA entries, other, LH/LF, end_of_record
+    new_section.extend(da_entries)
+
+    # Add other lines (FN, FNDA, etc.) but keep end_of_record for last
+    for line in other_lines:
+        if line != 'end_of_record':
+            new_section.append(line)
+
+    # Add recalculated counters
+    new_section.append(f'LH:{lines_hit}')
+    new_section.append(f'LF:{lines_found}')
+    new_section.append('end_of_record')
+
+    output_sections.append('\n'.join(new_section))
+
+# Write filtered coverage with recalculated statistics
+with open('total.info', 'w') as f:
+    f.write('\n'.join(output_sections))
+
+print(f"[INFO] Filtered {total_excluded} lines marked with LCOV_EXCL_LINE")
+print(f"[INFO] Recalculated coverage statistics")
+PYTHON_FILTER
+
 # Normalize paths so everything starts from mssql_python/
 echo "[ACTION] Normalizing paths in LCOV report"
 sed -i "s|$(pwd)/||g" total.info