Merge Dev Into Staging

.dockerignore

@@ -7,5 +7,13 @@
 .pre-commit-config.yaml
 .readthedocs.yml
 .travis.yml
-venv
 .git
+
+# ignore local python environments
+venv
+.venv
+
+# prevent large backup files from being copied into the image
+/backups
+*.sql
+*.gz

.envs/.local/.django

@@ -33,7 +33,17 @@ SINEQUA_CONFIGS_REPO_WEBAPP_PR_BRANCH='dummy_branch'
 # Slack Webhook
 # ------------------------------------------------------------------------------
 SLACK_WEBHOOK_URL=''
-LRM_USER=''
-LRM_PASSWORD=''
+
+#Server Credentials
+#--------------------------------------------------------------------------------
+LRM_DEV_USER=''
+LRM_DEV_PASSWORD=''
 XLI_USER=''
 XLI_PASSWORD=''
+LRM_QA_USER=''
+LRM_QA_PASSWORD=''
+
+#Server Tokens
+#--------------------------------------------------------------------------------
+LRM_DEV_TOKEN=''
+XLI_TOKEN=''

.github/workflows/run_full_test_suite.yml

@@ -0,0 +1,44 @@
+name: Django Test Suite on PR
+
+on:
+  pull_request:
+    branches:
+      - dev
+    paths-ignore:
+      - '**/*.md'
+
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+
+    services:
+      docker:
+        image: docker:24.0.5
+        options: --privileged
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Check out merged code
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Compose
+        run: |
+          sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+
+      - name: Build the Docker environment
+        run: docker-compose -f local.yml build
+
+      - name: Run test suite
+        env:
+          DJANGO_ENV: test
+        run: docker-compose -f local.yml run --rm django bash ./init.sh
+
+      - name: Generate Coverage Report
+        env:
+          DJANGO_ENV: test
+        run: docker-compose -f local.yml run --rm django bash -c "coverage report"
+
+      - name: Cleanup
+        run: docker-compose -f local.yml down --volumes

.gitignore

@@ -292,8 +292,7 @@ config_generation/config.py
 # Model's inference files
 Document_Classifier_inference/model.pt
 
-# Database backup
-backup.json
-
-# Prod backup
-prod_backup-20240423.json
+# Ignore Database Backup files
+/backups
+*.sql
+*.gz

.pre-commit-config.yaml

@@ -3,48 +3,77 @@ default_stages: [commit]
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.6.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
+      - id: check-merge-conflict
+      - id: debug-statements
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.1
+    rev: v3.17.0
     hooks:
       - id: pyupgrade
         args: [--py310-plus]
 
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 24.8.0
     hooks:
       - id: black
 
   - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
 
   - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
+    rev: 7.1.1
     hooks:
       - id: flake8
         args: ["--config=setup.cfg"]
         additional_dependencies: [flake8-isort]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.4.0
+    rev: v1.11.2
     hooks:
       - id: mypy
         args: ["--strict"]
-        # ignoring everything for now
-        exclude: .
-        additional_dependencies: [django-stubs, celery,  django-environ, django-extensions, django-crispy-forms,
-        crispy-bootstrap5, django-allauth, django-celery-beat, djangorestframework, djangorestframework-datatables,
-        django-debug-toolbar, psycopg2-binary, python-slugify, xmltodict, PyGithub, boto3, scrapy, types-requests]
+        exclude: "."
+        additional_dependencies:
+          - django-stubs
+          - celery
+          - django-environ
+          - django-extensions
+          - django-crispy-forms
+          - crispy-bootstrap5
+          - django-allauth
+          - django-celery-beat
+          - djangorestframework
+          - djangorestframework-datatables
+          - django-debug-toolbar
+          - psycopg2-binary
+          - python-slugify
+          - xmltodict
+          - PyGithub
+          - boto3
+          - scrapy
+          - types-requests
+
+  - repo: https://github.com/PyCQA/bandit
+    rev: '1.7.0'
+    hooks:
+      - id: bandit
+        args: ['-r', '--configfile=bandit-config.yml']
+
+  - repo: https://github.com/zricethezav/gitleaks
+    rev: 'v8.0.4'
+    hooks:
+      - id: gitleaks
+        args: ['--config=gitleaks-config.toml']
+
 
 
-# sets up .pre-commit-ci.yaml to ensure pre-commit dependencies stay up to date
 ci:
   autoupdate_schedule: weekly
   skip: []

CHANGELOG.md

@@ -0,0 +1,67 @@
+## Overview
+These are not the release notes, which can be found https://github.com/NASA-IMPACT/COSMOS/releases. Instead, this is a changelog that developers use to log key changes to the codebase with each pull request.
+
+## What to Include
+For each PR made, an entry should be added to this changelog. It should contain
+- a brief description of the deliverable of the feature or bugfix
+- exact listing of key changes such as:
+  - API endpoint modified
+  - frontend components added
+  - model updates
+  - deployment changes needed on the servers
+  - etc.
+
+## Changelog
+- 2889-serialize-the-tdamm-tags
+  - Description: Have TDAMM serialzed in a specific way and exposed via the Curated URLs API to be consumed into SDE Test/Prod
+  - Changes:
+    - Changed `get_tdamm_tag` method in the `CuratedURLAPISerializer` to process the TDAMM tags and pass them to the API endpoint
+
+- 960-notifications-add-a-dropdown-with-options-on-the-feedback-form
+  - Description: Generate an API endpoint and publish all the dropdown options necessary as a list for LRM to consume it.
+  - Changes:
+    - Created a new model `FeedbackFormDropdown`
+    - Added the migration file
+    - Added the `dropdown_option` field to the `Feedback` model
+    - Updated the slack notification structure by adding the dropdown option text
+    - Created a new serializer called `FeedbackFormDropdownSerializer`
+    - Added a new API endpoint `feedback-form-dropdown-options-api/` where the list is going to be accesible
+    - Added a list view called `FeedbackFormDropdownListView`
+    - Added tests
+
+- 1217-add-data-validation-to-the-feedback-form-api-to-restrict-html-content
+  - Description: The feedback form API does not currently have any form of data validation on the backend which makes it easy for the user with the endpoint to send in data with html tags. We need to have a validation scheme on the backend to protect this from happening.
+  - Changes:
+    - Defined a class `HTMLFreeCharField` which inherits `serializers.CharField`
+    - Used regex to catch any HTML content comming in as an input to form fields
+    - Called this class within the serializer for necessary fields
+
+- 3227-bugfix-title-patterns-selecting-multi-url-pattern-does-nothing
+  - Description: When selecting options from the match pattern type filter, the system does not filter the results as expected. Instead of displaying only the chosen variety of patterns, it continues to show all patterns.
+  - Changes:
+    - In `title_patterns_table` definition, corrected the column reference
+    - Made `match_pattern_type` searchable
+    - Corrected the column references and made code consistent on all the other tables, i.e., `exclude_patterns_table`, `include_patterns_table`, `division_patterns_table` and `document_type_patterns_table`
+
+- 1001-tests-for-critical-functionalities
+  - Description: Critical functionalities have been identified and listed, and critical areas lacking tests listed
+  - Changes:
+    - Integrated coverage.py as an indicative tool in the workflow for automated coverage reports on PRs, with separate display from test results.
+    - Introduced docs/architecture-decisions/testing_strategy.md, which includes the coverage report, lists critical areas, and specifically identifies those critical areas that are untested or under-tested.
+
+- 1192-finalize-the-infrastructure-for-frontend-testing
+  - Description: Set up comprehensive frontend testing infrastructure using Selenium WebDriver with Chrome, establishing a foundation for automated UI testing.
+  - Changes:
+    - Added Selenium testing dependency to `requirements/local.txt`
+    - Updated Dockerfile to support Chrome and ChromeDriver
+    - Created BaseTestCase and AuthenticationMixin for reusable test components
+    - Implemented core authentication test suite
+
+- 1195-implement-unit-test-for-forms-on-the-frontend
+  - Description: Implemented comprehensive frontend test suite covering authentication, collection management, search functionality, and pattern application forms.
+  - Changes:
+    - Added tests for authentication flows
+    - Implemented collection display and data table tests
+    - Added universal search functionality tests
+    - Created search pane filter tests
+    - Added pattern application form tests with validation checks

CODE_STANDARDS.md

@@ -0,0 +1,63 @@
+# Coding Standards and Conventions for COSMOS
+
+## Overview
+To maintain high-quality code and ensure consistency across the entire COSMOS project, we have established coding standards and conventions. This document outlines the key standards and practices that all contributors are expected to follow. Adhering to these guidelines helps us to achieve a codebase that appears as if it were written by a single entity, regardless of the number of contributors.
+
+## Coding Standards
+
+### Formatting Standards
+- **Line Length**: Maximum of 120 characters per line to ensure readability across various environments.
+- **Code Formatting**: Utilize tools like Black for Python code to ensure consistent formatting across the entire codebase.
+- **Import Ordering**: Follow a consistent import order:
+  - Standard library imports.
+  - Third-party imports.
+  - Application-specific imports.
+
+### Naming Conventions
+- **Variables and Functions**: Use `snake_case`.
+- **Classes and Exceptions**: Use `CamelCase`.
+- **Constants**: Use `UPPER_CASE`.
+
+### Commenting
+- Inline comments should be used sparingly and only when necessary to explain "why" something is done, not "what" is done.
+- All public methods, classes, and modules should include docstrings that follow the [Google style guide](https://google.github.io/styleguide/pyguide.html).
+
+### Error Handling
+- Explicit is better than implicit. Raise exceptions rather than returning None or any error codes.
+- Use custom exceptions over generic exceptions when possible to make error handling more predictive.
+
+## Tool Configurations and Pre-commit Hooks
+
+To automate and enforce these standards, the following tools are configured with pre-commit hooks in our development process:
+
+### Pre-commit Hooks Setup
+
+To ensure that these tools are run automatically on every commit, contributors must set up pre-commit hooks locally. Run the following commands to install and configure pre-commit hooks:
+
+```bash
+pip install pre-commit
+pre-commit install
+pre-commit run --all-files
+```
+
+The following pre-commit hooks are configured:
+
+- trailing-whitespace, end-of-file-fixer, check-yaml, check-merge-conflict, debug-statements: Checks for common formatting issues.
+- pyupgrade: Automatically upgrades syntax for newer versions of the language.
+- black: Formats Python code to ensure consistent styling.
+- isort: Sorts imports alphabetically and automatically separated into sections.
+- flake8: Lints code to catch styling errors and potential bugs.
+- mypy: Checks type annotations to catch potential bugs.
+- bandit: Scans code for common security issues.
+- gitleaks: Prevents secrets from being committed to the repository.
+- hadolint: Lints Dockerfiles to ensure best practices and common conventions are followed.
+
+## Continuous Integration (CI)
+When a commit is pushed to a branch that is part of a Pull Request, our Continuous Integration (CI) pipeline automatically runs specified tools to check code quality, style, security and other standards. If these checks fail, the PR cannot be merged until all issues are resolved.
+
+## Quality Standards Enforcement
+- PRs must pass all checks from the configured pre-commit hooks and CI pipeline to be eligible for merging.
+- Code reviews additionally focus on logical errors and code quality beyond what automated tools can detect.
+
+## Conclusion
+By adhering to these standards and utilizing the tools set up, we maintain the high quality and consistency of our codebase, making it easier for developers to collaborate effectively.

CONTRIBUTING.md

@@ -0,0 +1,69 @@
+Thank you for your interest in contributing to COSMOS! We welcome contributions and appreciate your help in making this project better. Please follow the guidelines below to ensure a smooth contribution process.
+
+## Pull Requests
+
+### Prerequisites
+
+- **GitHub CLI (`gh`)**: Make sure you have the GitHub CLI installed. If not, you can install it from [GitHub CLI installation page](https://cli.github.com/).
+
+### 1. **Create an Issue on the Repo**
+
+1. **Navigate to Your Repository**:
+
+    ```bash
+    $ cd path/to/your/repository
+    ```
+
+2. **Create an Issue**:
+Use the `gh issue create` command to create a new issue.
+
+    ```bash
+    $ gh issue create --title "Issue Title" --body "Description of the issue"
+    ```
+
+    After running this command, you’ll get an issue number in the output. Note this number as it will be used to create a branch.
+
+
+### 2. **Create a Branch for the Issue**
+
+1. **Create a Branch**:
+Use the `gh` CLI to create a branch associated with the issue. The `gh` CLI can automatically create a branch for you based on the issue number. In this case, the `<issue_number>` is 989.
+
+    ```bash
+    $ gh issue develop -c 989
+    github.com/NASA-IMPACT/COSMOS/tree/989-make-coding-syntax-consistent
+    From https://github.com/NASA-IMPACT/COSMOS
+     * [new branch]      989-make-coding-syntax-consistent -> origin/989-make-coding-syntax-consistent
+
+    ```
+
+    This command creates a new branch named `<issue_number>-issue` and switches to it. This branch will be used to work on the issue.
+
+2. **Make Your Changes and Push:**
+Edit files, add code, or make any changes needed to address the issue. Commit your changes and push the branch to the remote repository.
+
+    ```bash
+    git add .
+    git commit -m "Fixes issue #<issue_number>"
+    git push origin <issue_number>-issue
+    ```
+
+
+### 3. **Create a Pull Request**
+
+1. **Create the Pull Request**:
+After pushing the branch, create a pull request using the `gh pr create` command:
+
+    ```bash
+    gh pr create --base dev --head <issue_number>-issue --title "Title of the Pull Request" --body "Description of the changes"
+    ```
+
+    - **`-base`**: The base branch you want to merge your changes into (`dev` in our case)
+    - **`-head`**: The branch that contains your changes (e.g., `<issue_number>-issue`).
+    - **`-title`**: The title of the pull request.
+    - **`-body`**: The description or body of the pull request.
+
+    This command will create a pull request from your branch into the base branch specified.
+
+2. **Review and Merge**:
+Once the pull request is created, we will review it on GitHub and merge it if everything looks good. If any changes are required, we might ask you to make adjustments before the merge.