From 4af78231ed9410db8edb25ca3c52d0ee0d7a749c Mon Sep 17 00:00:00 2001 From: Greeshma Rajendran Date: Mon, 11 May 2026 16:35:12 +0530 Subject: [PATCH 1/6] docs(api): added dph documentation Signed-off-by: Greeshma Rajendran --- .secrets.baseline | 62 +++++++++++++++++++++++++- docs/api/dph_services/core.rst | 23 ---------- docs/api/dph_services/index.rst | 73 ++----------------------------- docs/api/odcs_generator/index.rst | 5 --- src/wxdi/dph_services/dph_v1.py | 4 +- 5 files changed, 66 insertions(+), 101 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index e1737e6..6a81058 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2026-04-09T04:39:13Z", + "generated_at": "2026-05-11T09:05:52Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -301,6 +301,66 @@ "verified_result": null } ], + "docs/chapters/05_dph_services/overview.rst": [ + { + "hashed_secret": "45d676e7c6ab44cf4b8fa366ef2d8fccd3e6d6e6", + "is_secret": false, + "is_verified": false, + "line_number": 123, + "type": "Secret Keyword", + "verified_result": null + } + ], + "docs/chapters/05_dph_services/usage_guide.rst": [ + { + "hashed_secret": "11fa7c37d697f30e6aee828b4426a10f83ab2380", + "is_secret": false, + "is_verified": false, + "line_number": 63, + "type": "Secret Keyword", + "verified_result": null + } + ], + "docs/chapters/06_odcs_generator/collibra_integration.rst": [ + { + "hashed_secret": "564e340cd48437d2dfe876ee154cc99dc4d0d137", + "is_secret": false, + "is_verified": false, + "line_number": 97, + "type": "Secret Keyword", + "verified_result": null + } + ], + "docs/chapters/06_odcs_generator/examples.rst": [ + { + "hashed_secret": "564e340cd48437d2dfe876ee154cc99dc4d0d137", + "is_secret": false, + "is_verified": false, + "line_number": 61, + "type": "Secret Keyword", + "verified_result": null + } + ], + "docs/chapters/06_odcs_generator/index.rst": [ + { + "hashed_secret": "564e340cd48437d2dfe876ee154cc99dc4d0d137", + "is_secret": false, + "is_verified": false, + "line_number": 93, + "type": "Secret Keyword", + "verified_result": null + } + ], + "docs/chapters/06_odcs_generator/informatica_integration.rst": [ + { + "hashed_secret": "564e340cd48437d2dfe876ee154cc99dc4d0d137", + "is_secret": false, + "is_verified": false, + "line_number": 77, + "type": "Secret Keyword", + "verified_result": null + } + ], "examples/auth_provider_usage.py": [ { "hashed_secret": "df5cc5832dc34a455c18662ac84587ea19cf2435", diff --git a/docs/api/dph_services/core.rst b/docs/api/dph_services/core.rst index 2009b39..aa92c32 100644 --- a/docs/api/dph_services/core.rst +++ b/docs/api/dph_services/core.rst @@ -31,27 +31,4 @@ DphV1 Service :show-inheritance: :inherited-members: -Common Models -------------- - -.. autoclass:: wxdi.dph_services.common.DataProduct - :members: - :undoc-members: - :show-inheritance: - -.. autoclass:: wxdi.dph_services.common.DataProductDraft - :members: - :undoc-members: - :show-inheritance: - -.. autoclass:: wxdi.dph_services.common.ContractTerms - :members: - :undoc-members: - :show-inheritance: - -.. autoclass:: wxdi.dph_services.common.Domain - :members: - :undoc-members: - :show-inheritance: - .. Made with Bob \ No newline at end of file diff --git a/docs/api/dph_services/index.rst b/docs/api/dph_services/index.rst index d9adaa6..c42ff65 100644 --- a/docs/api/dph_services/index.rst +++ b/docs/api/dph_services/index.rst @@ -28,6 +28,9 @@ API reference for the Data Product Hub Services module. Main Service Class ------------------ +The DphV1 class provides access to all Data Product Hub Services operations. +For detailed API reference including all methods, see :ref:`api_dph_services_core`. + .. currentmodule:: wxdi.dph_services .. autoclass:: DphV1 @@ -35,74 +38,6 @@ Main Service Class :undoc-members: :show-inheritance: :special-members: __init__ - -Container Operations --------------------- - -.. automethod:: DphV1.initialize -.. automethod:: DphV1.get_initialize_status -.. automethod:: DphV1.get_service_id_credentials -.. automethod:: DphV1.manage_api_keys - -Data Product Operations ------------------------ - -.. automethod:: DphV1.create_data_product -.. automethod:: DphV1.list_data_products -.. automethod:: DphV1.list_data_products_with_pager -.. automethod:: DphV1.get_data_product -.. automethod:: DphV1.update_data_product -.. automethod:: DphV1.delete_data_product - -Draft Operations ----------------- - -.. automethod:: DphV1.create_data_product_draft -.. automethod:: DphV1.list_data_product_drafts -.. automethod:: DphV1.get_data_product_draft -.. automethod:: DphV1.update_data_product_draft -.. automethod:: DphV1.delete_data_product_draft -.. automethod:: DphV1.publish_data_product_draft - -Release Operations ------------------- - -.. automethod:: DphV1.list_data_product_releases -.. automethod:: DphV1.get_data_product_release -.. automethod:: DphV1.update_data_product_release -.. automethod:: DphV1.retire_data_product_release - -Contract Terms Operations -------------------------- - -.. automethod:: DphV1.create_draft_contract_terms_document -.. automethod:: DphV1.get_data_product_draft_contract_terms -.. automethod:: DphV1.update_draft_contract_terms_document -.. automethod:: DphV1.delete_draft_contract_terms_document - -Domain Operations ------------------ - -.. automethod:: DphV1.list_data_product_domains -.. automethod:: DphV1.create_data_product_domain -.. automethod:: DphV1.create_data_product_subdomain -.. automethod:: DphV1.get_domain -.. automethod:: DphV1.update_data_product_domain -.. automethod:: DphV1.delete_domain - -Asset Visualization Operations -------------------------------- - -.. automethod:: DphV1.create_data_asset_visualization -.. automethod:: DphV1.reinitiate_data_asset_visualization - -Contract Template Operations ----------------------------- - -.. automethod:: DphV1.create_contract_template -.. automethod:: DphV1.list_data_product_contract_template -.. automethod:: DphV1.get_contract_template -.. automethod:: DphV1.update_data_product_contract_template -.. automethod:: DphV1.delete_data_product_contract_template + :no-index: .. Made with Bob \ No newline at end of file diff --git a/docs/api/odcs_generator/index.rst b/docs/api/odcs_generator/index.rst index d7e3203..2b45be6 100644 --- a/docs/api/odcs_generator/index.rst +++ b/docs/api/odcs_generator/index.rst @@ -45,9 +45,4 @@ Informatica Integration :undoc-members: :show-inheritance: -.. autoclass:: ODCSGenerator - :members: - :undoc-members: - :show-inheritance: - .. Made with Bob diff --git a/src/wxdi/dph_services/dph_v1.py b/src/wxdi/dph_services/dph_v1.py index a05cec9..d7908ff 100644 --- a/src/wxdi/dph_services/dph_v1.py +++ b/src/wxdi/dph_services/dph_v1.py @@ -1392,9 +1392,7 @@ def replace_data_product_draft_contract_terms( contract. :param List[ContractTemplateSLA] sla: (optional) Service Level Agreement details. - :param List[ContractTemplateSupportAndCommunication] - support_and_communication: (optional) Support and communication details for - the contract. + :param List[ContractTemplateSupportAndCommunication] support_and_communication: (optional) Support and communication details for the contract. :param List[ContractTemplateCustomProperty] custom_properties: (optional) Custom properties that are not part of the standard contract. :param ContractTest contract_test: (optional) Contains the contract test From 4e46af80d3dc0065ae385ca44c70c954a85cb1c3 Mon Sep 17 00:00:00 2001 From: Greeshma Rajendran Date: Mon, 11 May 2026 20:11:58 +0530 Subject: [PATCH 2/6] docs(api): added dph documentation Signed-off-by: Greeshma Rajendran --- docs/api/data_product_recommender/index.rst | 29 +------ docs/api/dph_services/core.rst | 9 ++- docs/api/dph_services/index.rst | 11 ++- docs/api/odcs_generator/index.rst | 21 +++-- src/wxdi/__init__.py | 81 +------------------ src/wxdi/data_product_recommender/__init__.py | 18 +++++ 6 files changed, 51 insertions(+), 118 deletions(-) diff --git a/docs/api/data_product_recommender/index.rst b/docs/api/data_product_recommender/index.rst index 713e0f5..1f853a9 100644 --- a/docs/api/data_product_recommender/index.rst +++ b/docs/api/data_product_recommender/index.rst @@ -18,14 +18,12 @@ Data Product Recommender Reference =================================== -Class reference for the Data Product Recommender module. +The data product recommender module provides query-log analysis and platform-specific parsers. Core Classes ------------ -.. currentmodule:: wxdi.data_product_recommender.recommender - -.. autoclass:: DataProductRecommender +.. automodule:: wxdi.data_product_recommender.recommender :members: :undoc-members: :show-inheritance: @@ -33,24 +31,7 @@ Core Classes Platform Parsers ---------------- -.. currentmodule:: wxdi.data_product_recommender.platforms - -.. autoclass:: SnowflakeQueryParser - :members: - :undoc-members: - :show-inheritance: - -.. autoclass:: DatabricksQueryParser - :members: - :undoc-members: - :show-inheritance: - -.. autoclass:: BigQueryQueryParser - :members: - :undoc-members: - :show-inheritance: - -.. autoclass:: WatsonxDataQueryParser +.. automodule:: wxdi.data_product_recommender.platforms :members: :undoc-members: :show-inheritance: @@ -58,9 +39,7 @@ Platform Parsers Base Classes ------------ -.. currentmodule:: wxdi.data_product_recommender.base - -.. autoclass:: QueryLogParser +.. automodule:: wxdi.data_product_recommender.base :members: :undoc-members: :show-inheritance: diff --git a/docs/api/dph_services/core.rst b/docs/api/dph_services/core.rst index aa92c32..2c8c645 100644 --- a/docs/api/dph_services/core.rst +++ b/docs/api/dph_services/core.rst @@ -18,14 +18,17 @@ Core Classes ============ -Main service class and data models for Data Product Hub Services. +The core DPH services API provides the main service client and its available operations. + +.. currentmodule:: wxdi.dph_services.dph_v1 DphV1 Service ------------- -.. currentmodule:: wxdi.dph_services +DphV1 +~~~~~ -.. autoclass:: DphV1 +.. autoclass:: wxdi.dph_services.dph_v1.DphV1 :members: :undoc-members: :show-inheritance: diff --git a/docs/api/dph_services/index.rst b/docs/api/dph_services/index.rst index c42ff65..791166a 100644 --- a/docs/api/dph_services/index.rst +++ b/docs/api/dph_services/index.rst @@ -18,22 +18,25 @@ DPH Services API ================ -API reference for the Data Product Hub Services module. +The DPH services module provides the main client for Data Product Hub Services operations. .. toctree:: :maxdepth: 2 core +.. currentmodule:: wxdi.dph_services.dph_v1 + Main Service Class ------------------ +DphV1 +~~~~~ + The DphV1 class provides access to all Data Product Hub Services operations. For detailed API reference including all methods, see :ref:`api_dph_services_core`. -.. currentmodule:: wxdi.dph_services - -.. autoclass:: DphV1 +.. autoclass:: wxdi.dph_services.dph_v1.DphV1 :members: :undoc-members: :show-inheritance: diff --git a/docs/api/odcs_generator/index.rst b/docs/api/odcs_generator/index.rst index 2b45be6..c56af78 100644 --- a/docs/api/odcs_generator/index.rst +++ b/docs/api/odcs_generator/index.rst @@ -18,29 +18,38 @@ ODCS Generator Reference ======================== -Class reference for the ODCS Generator module. +The ODCS generator module provides integrations for generating ODCS YAML from supported catalog platforms. + +.. currentmodule:: wxdi.odcs_generator.generate_odcs_from_collibra Collibra Integration -------------------- -.. currentmodule:: wxdi.odcs_generator.generate_odcs_from_collibra +CollibraClient +~~~~~~~~~~~~~~ -.. autoclass:: CollibraClient +.. autoclass:: wxdi.odcs_generator.generate_odcs_from_collibra.CollibraClient :members: :undoc-members: :show-inheritance: -.. autoclass:: ODCSGenerator +ODCSGenerator +~~~~~~~~~~~~~ + +.. autoclass:: wxdi.odcs_generator.generate_odcs_from_collibra.ODCSGenerator :members: :undoc-members: :show-inheritance: +.. currentmodule:: wxdi.odcs_generator.generate_odcs_from_informatica + Informatica Integration ----------------------- -.. currentmodule:: wxdi.odcs_generator.generate_odcs_from_informatica +InformaticaClient +~~~~~~~~~~~~~~~~~ -.. autoclass:: InformaticaClient +.. autoclass:: wxdi.odcs_generator.generate_odcs_from_informatica.InformaticaClient :members: :undoc-members: :show-inheritance: diff --git a/src/wxdi/__init__.py b/src/wxdi/__init__.py index 63261f8..c120913 100644 --- a/src/wxdi/__init__.py +++ b/src/wxdi/__init__.py @@ -20,89 +20,10 @@ data product hub services, ODCS generation, and data product recommendations. """ -# Re-export commonly used modules for convenience -from wxdi.dq_validator import ( - # Metadata - DataType, - ColumnMetadata, - AssetMetadata, - # Base classes - BaseCheck, - ValidationError, - # Core classes - ValidationResult, - ValidationResultConsolidated, - ValidationRule, - Validator, - # Checks - LengthCheck, - ValidValuesCheck, - ComparisonCheck, - ComparisonOperator, - CaseCheck, - ColumnCaseEnum, - CompletenessCheck, - RangeCheck, - RegexCheck, - DataTypeCheck, - FormatCheck, - FormatConstraintType, - # DateTime Formats - DateTimeFormats, - # Data Quality Dimensions - DataQualityDimension, -) - -from wxdi.common.auth import ( - AuthConfig, - EnvironmentType, - GovCloudAuthenticator, - AuthProvider, -) - from .version import __version__ __all__ = [ - # Metadata - "DataType", - "ColumnMetadata", - "AssetMetadata", - # Base classes - "BaseCheck", - "ValidationError", - # Core classes - "ValidationResult", - "ValidationResultConsolidated", - "ValidationRule", - "Validator", - # Checks - "LengthCheck", - "ValidValuesCheck", - "ComparisonCheck", - "ComparisonOperator", - "CaseCheck", - "ColumnCaseEnum", - "CompletenessCheck", - "RangeCheck", - "RegexCheck", - "DataTypeCheck", - "FormatCheck", - "FormatConstraintType", - # Authentication - "AuthConfig", - "EnvironmentType", - "GovCloudAuthenticator", - "AuthProvider", - # DateTime Formats - "DateTimeFormats", - # Data Quality Dimensions - "DataQualityDimension", + "__version__", ] -# Note: dph_services, odcs_generator, and data_product_recommender are available as submodules -# Import them explicitly: -# from wxdi.dph_services import DphV1 -# from wxdi.odcs_generator import CollibraClient, ODCSGenerator, InformaticaClient -# from wxdi.data_product_recommender import DataProductRecommender - # Made with Bob diff --git a/src/wxdi/data_product_recommender/__init__.py b/src/wxdi/data_product_recommender/__init__.py index d226660..1320b61 100644 --- a/src/wxdi/data_product_recommender/__init__.py +++ b/src/wxdi/data_product_recommender/__init__.py @@ -21,4 +21,22 @@ based on usage patterns, user diversity, and table relationships. """ +from .recommender import DataProductRecommender +from .platforms import ( + SnowflakeQueryParser, + DatabricksQueryParser, + BigQueryQueryParser, + WatsonxDataQueryParser, +) +from .base import QueryLogParser + +__all__ = [ + "DataProductRecommender", + "SnowflakeQueryParser", + "DatabricksQueryParser", + "BigQueryQueryParser", + "WatsonxDataQueryParser", + "QueryLogParser", +] + __version__ = "0.1.0" \ No newline at end of file From 2d7666abb9e6224f0f47d419e92a9f919ec48524 Mon Sep 17 00:00:00 2001 From: Greeshma Rajendran Date: Tue, 12 May 2026 11:14:57 +0530 Subject: [PATCH 3/6] docs(api): reverting few changes Signed-off-by: Greeshma Rajendran --- src/wxdi/__init__.py | 74 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/src/wxdi/__init__.py b/src/wxdi/__init__.py index c120913..073f41e 100644 --- a/src/wxdi/__init__.py +++ b/src/wxdi/__init__.py @@ -20,9 +20,83 @@ data product hub services, ODCS generation, and data product recommendations. """ +# Re-export commonly used modules for convenience +from wxdi.dq_validator import ( + # Metadata + DataType, + ColumnMetadata, + AssetMetadata, + # Base classes + BaseCheck, + ValidationError, + # Core classes + ValidationResult, + ValidationResultConsolidated, + ValidationRule, + Validator, + # Checks + LengthCheck, + ValidValuesCheck, + ComparisonCheck, + ComparisonOperator, + CaseCheck, + ColumnCaseEnum, + CompletenessCheck, + RangeCheck, + RegexCheck, + DataTypeCheck, + FormatCheck, + FormatConstraintType, + # DateTime Formats + DateTimeFormats, + # Data Quality Dimensions + DataQualityDimension, +) + +from wxdi.common.auth import ( + AuthConfig, + EnvironmentType, + GovCloudAuthenticator, + AuthProvider, +) + from .version import __version__ __all__ = [ + # Metadata + "DataType", + "ColumnMetadata", + "AssetMetadata", + # Base classes + "BaseCheck", + "ValidationError", + # Core classes + "ValidationResult", + "ValidationResultConsolidated", + "ValidationRule", + "Validator", + # Checks + "LengthCheck", + "ValidValuesCheck", + "ComparisonCheck", + "ComparisonOperator", + "CaseCheck", + "ColumnCaseEnum", + "CompletenessCheck", + "RangeCheck", + "RegexCheck", + "DataTypeCheck", + "FormatCheck", + "FormatConstraintType", + # Authentication + "AuthConfig", + "EnvironmentType", + "GovCloudAuthenticator", + "AuthProvider", + # DateTime Formats + "DateTimeFormats", + # Data Quality Dimensions + "DataQualityDimension", "__version__", ] From 1fd8d4282bc27770bcb6b2e9d76250ae6837bf39 Mon Sep 17 00:00:00 2001 From: Greeshma Rajendran Date: Tue, 12 May 2026 11:43:47 +0530 Subject: [PATCH 4/6] docs(api): reverting few changes Signed-off-by: Greeshma Rajendran --- src/wxdi/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/wxdi/__init__.py b/src/wxdi/__init__.py index 073f41e..63261f8 100644 --- a/src/wxdi/__init__.py +++ b/src/wxdi/__init__.py @@ -97,7 +97,12 @@ "DateTimeFormats", # Data Quality Dimensions "DataQualityDimension", - "__version__", ] +# Note: dph_services, odcs_generator, and data_product_recommender are available as submodules +# Import them explicitly: +# from wxdi.dph_services import DphV1 +# from wxdi.odcs_generator import CollibraClient, ODCSGenerator, InformaticaClient +# from wxdi.data_product_recommender import DataProductRecommender + # Made with Bob From e4b2e8b532aceb219ceb83f77d5ef1bbb8d82484 Mon Sep 17 00:00:00 2001 From: Greeshma Rajendran Date: Tue, 12 May 2026 11:51:50 +0530 Subject: [PATCH 5/6] docs(api): reverting few changes Signed-off-by: Greeshma Rajendran --- src/wxdi/__init__.py | 75 +------------------------------------------- 1 file changed, 1 insertion(+), 74 deletions(-) diff --git a/src/wxdi/__init__.py b/src/wxdi/__init__.py index 63261f8..8393d35 100644 --- a/src/wxdi/__init__.py +++ b/src/wxdi/__init__.py @@ -20,83 +20,10 @@ data product hub services, ODCS generation, and data product recommendations. """ -# Re-export commonly used modules for convenience -from wxdi.dq_validator import ( - # Metadata - DataType, - ColumnMetadata, - AssetMetadata, - # Base classes - BaseCheck, - ValidationError, - # Core classes - ValidationResult, - ValidationResultConsolidated, - ValidationRule, - Validator, - # Checks - LengthCheck, - ValidValuesCheck, - ComparisonCheck, - ComparisonOperator, - CaseCheck, - ColumnCaseEnum, - CompletenessCheck, - RangeCheck, - RegexCheck, - DataTypeCheck, - FormatCheck, - FormatConstraintType, - # DateTime Formats - DateTimeFormats, - # Data Quality Dimensions - DataQualityDimension, -) - -from wxdi.common.auth import ( - AuthConfig, - EnvironmentType, - GovCloudAuthenticator, - AuthProvider, -) - from .version import __version__ __all__ = [ - # Metadata - "DataType", - "ColumnMetadata", - "AssetMetadata", - # Base classes - "BaseCheck", - "ValidationError", - # Core classes - "ValidationResult", - "ValidationResultConsolidated", - "ValidationRule", - "Validator", - # Checks - "LengthCheck", - "ValidValuesCheck", - "ComparisonCheck", - "ComparisonOperator", - "CaseCheck", - "ColumnCaseEnum", - "CompletenessCheck", - "RangeCheck", - "RegexCheck", - "DataTypeCheck", - "FormatCheck", - "FormatConstraintType", - # Authentication - "AuthConfig", - "EnvironmentType", - "GovCloudAuthenticator", - "AuthProvider", - # DateTime Formats - "DateTimeFormats", - # Data Quality Dimensions - "DataQualityDimension", + "__version__", ] # Note: dph_services, odcs_generator, and data_product_recommender are available as submodules From 259abd0299790cd5a6ef86c8458bb0afb0a01dfa Mon Sep 17 00:00:00 2001 From: Greeshma Rajendran Date: Tue, 12 May 2026 15:18:54 +0530 Subject: [PATCH 6/6] docs(api): reverting few changes Signed-off-by: Greeshma Rajendran --- src/wxdi/__init__.py | 75 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/src/wxdi/__init__.py b/src/wxdi/__init__.py index 8393d35..63261f8 100644 --- a/src/wxdi/__init__.py +++ b/src/wxdi/__init__.py @@ -20,10 +20,83 @@ data product hub services, ODCS generation, and data product recommendations. """ +# Re-export commonly used modules for convenience +from wxdi.dq_validator import ( + # Metadata + DataType, + ColumnMetadata, + AssetMetadata, + # Base classes + BaseCheck, + ValidationError, + # Core classes + ValidationResult, + ValidationResultConsolidated, + ValidationRule, + Validator, + # Checks + LengthCheck, + ValidValuesCheck, + ComparisonCheck, + ComparisonOperator, + CaseCheck, + ColumnCaseEnum, + CompletenessCheck, + RangeCheck, + RegexCheck, + DataTypeCheck, + FormatCheck, + FormatConstraintType, + # DateTime Formats + DateTimeFormats, + # Data Quality Dimensions + DataQualityDimension, +) + +from wxdi.common.auth import ( + AuthConfig, + EnvironmentType, + GovCloudAuthenticator, + AuthProvider, +) + from .version import __version__ __all__ = [ - "__version__", + # Metadata + "DataType", + "ColumnMetadata", + "AssetMetadata", + # Base classes + "BaseCheck", + "ValidationError", + # Core classes + "ValidationResult", + "ValidationResultConsolidated", + "ValidationRule", + "Validator", + # Checks + "LengthCheck", + "ValidValuesCheck", + "ComparisonCheck", + "ComparisonOperator", + "CaseCheck", + "ColumnCaseEnum", + "CompletenessCheck", + "RangeCheck", + "RegexCheck", + "DataTypeCheck", + "FormatCheck", + "FormatConstraintType", + # Authentication + "AuthConfig", + "EnvironmentType", + "GovCloudAuthenticator", + "AuthProvider", + # DateTime Formats + "DateTimeFormats", + # Data Quality Dimensions + "DataQualityDimension", ] # Note: dph_services, odcs_generator, and data_product_recommender are available as submodules