Skip to content

Commit 6d1ca69

Browse files
siddhant1siddhant1
authored
Merge branch 'main' into sid/query-filtering
2 parents a9f410b + 4acf0a2 commit 6d1ca69

File tree

125 files changed

+22826
-164
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

125 files changed

+22826
-164
lines changed

‎.github/copilot-instructions.md‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -429,10 +429,23 @@ mvn verify # Run integration tests
429429
```bash
430430
cd openmetadata-ui/src/main/resources/ui
431431
yarn lint:fix # Fix ESLint issues
432-
yarn pretty # Format with Prettier
432+
yarn pretty # Format with Prettier
433433
yarn license-header-fix # Add license headers
434+
yarn pre-commit # Run precommit checks (lint-staged): license headers, i18n sync, organize imports, ESLint, and Prettier
434435
```
435436

437+
**IMPORTANT: Precommit Hook Standards**
438+
- The project uses `lint-staged` with `husky` for precommit checks
439+
- When making UI changes, ALWAYS run `yarn pre-commit` before committing
440+
- Precommit automatically runs:
441+
1. License header insertion (`yarn license-header-fix`)
442+
2. i18n localization sync (`yarn i18n`)
443+
3. Import organization (`organize-imports-cli`)
444+
4. ESLint with auto-fix (`./lint-staged-eslint.sh`)
445+
5. Prettier formatting (`prettier --write`)
446+
- These checks run on staged files only (via lint-staged)
447+
- CI will reject commits that don't pass these checks
448+
436449
### Python
437450
```bash
438451
make py_format # Format with black, isort, pycln
@@ -511,18 +524,23 @@ ALWAYS run these validation steps:
511524
# Java formatting
512525
mvn spotless:apply
513526

514-
# Frontend linting
515-
cd openmetadata-ui/src/main/resources/ui && yarn lint:fix
527+
# Frontend precommit checks (PREFERRED - runs all formatting and linting)
528+
cd openmetadata-ui/src/main/resources/ui && yarn pre-commit
529+
530+
# OR run individual frontend checks
531+
cd openmetadata-ui/src/main/resources/ui && yarn lint:fix && yarn pretty
516532

517-
# Python formatting
533+
# Python formatting
518534
make py_format
519535

520536
# Run tests relevant to your changes
521537
mvn test # For Java changes
522-
yarn test # For UI changes
538+
yarn test # For UI changes
523539
make unit_ingestion_dev_env # For Python changes
524540
```
525541

542+
**Note**: The project uses Git hooks (husky + lint-staged) that automatically run precommit checks on staged files. The `yarn pre-commit` command manually runs the same checks.
543+
526544
### CI Build Expectations
527545
- **Maven Build**: 45-60 minutes
528546
- **Playwright E2E Tests**: 30-45 minutes

‎CLAUDE.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ OpenMetadata is a unified metadata platform for data discovery, data observabili
1010

1111
- **Backend**: Java 21 + Dropwizard REST API framework, multi-module Maven project
1212
- **Frontend**: React + TypeScript + Ant Design, built with Webpack and Yarn
13-
- **Ingestion**: Python 3.9+ with Pydantic 2.x, 75+ data source connectors
13+
- **Ingestion**: Python 3.10-3.12 with Pydantic 2.x, 75+ data source connectors
1414
- **Database**: MySQL (default) or PostgreSQL with Flyway migrations
1515
- **Search**: Elasticsearch 7.17+ or OpenSearch 2.6+ for metadata discovery
1616
- **Infrastructure**: Apache Airflow for workflow orchestration

‎bootstrap/sql/migrations/native/1.11.0/mysql/schemaChanges.sql‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,3 +137,96 @@ CREATE INDEX idx_entity_relationship_bidirectional
137137
ON entity_relationship(fromId, toId, relation, deleted);
138138

139139

140+
-- Create ai_application_entity table
141+
CREATE TABLE IF NOT EXISTS ai_application_entity (
142+
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
143+
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
144+
fqnHash VARCHAR(768) NOT NULL,
145+
json JSON NOT NULL,
146+
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
147+
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
148+
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.impersonatedBy') VIRTUAL,
149+
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
150+
PRIMARY KEY (id),
151+
UNIQUE KEY unique_name (fqnHash),
152+
INDEX name_index (name),
153+
INDEX deleted_index (deleted)
154+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='AI Application entities';
155+
156+
-- Create llm_model_entity table
157+
CREATE TABLE IF NOT EXISTS llm_model_entity (
158+
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
159+
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
160+
fqnHash VARCHAR(768) NOT NULL,
161+
json JSON NOT NULL,
162+
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
163+
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
164+
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.impersonatedBy') VIRTUAL,
165+
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
166+
PRIMARY KEY (id),
167+
UNIQUE KEY unique_name (fqnHash),
168+
INDEX name_index (name),
169+
INDEX deleted_index (deleted)
170+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='LLM Model entities';
171+
172+
-- Create prompt_template_entity table
173+
CREATE TABLE IF NOT EXISTS prompt_template_entity (
174+
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
175+
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
176+
fqnHash VARCHAR(768) NOT NULL,
177+
json JSON NOT NULL,
178+
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
179+
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
180+
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.impersonatedBy') VIRTUAL,
181+
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
182+
PRIMARY KEY (id),
183+
UNIQUE KEY unique_name (fqnHash),
184+
INDEX name_index (name),
185+
INDEX deleted_index (deleted)
186+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Prompt Template entities';
187+
188+
-- Create agent_execution_entity table
189+
CREATE TABLE IF NOT EXISTS agent_execution_entity (
190+
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
191+
agentId VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.agentId') STORED NOT NULL,
192+
json JSON NOT NULL,
193+
timestamp BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.timestamp') NOT NULL,
194+
PRIMARY KEY (id),
195+
INDEX agent_index (agentId),
196+
INDEX timestamp_index (timestamp)
197+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='AI Agent Execution logs';
198+
199+
-- Create ai_governance_policy_entity table
200+
CREATE TABLE IF NOT EXISTS ai_governance_policy_entity (
201+
id VARCHAR(36) GENERATED ALWAYS AS (json ->> '$.id') STORED NOT NULL,
202+
name VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.name') NOT NULL,
203+
fqnHash VARCHAR(768) NOT NULL,
204+
json JSON NOT NULL,
205+
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json ->> '$.updatedAt') NOT NULL,
206+
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.updatedBy') NOT NULL,
207+
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.impersonatedBy') VIRTUAL,
208+
deleted BOOLEAN GENERATED ALWAYS AS (JSON_EXTRACT(json, '$.deleted')),
209+
PRIMARY KEY (id),
210+
UNIQUE KEY unique_name (fqnHash),
211+
INDEX name_index (name),
212+
INDEX deleted_index (deleted)
213+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='AI Governance Policy entities';
214+
215+
-- Create llm_service_entity table
216+
CREATE TABLE IF NOT EXISTS llm_service_entity (
217+
id VARCHAR(36) GENERATED ALWAYS AS (json_unquote(json_extract(`json`, '$.id'))) STORED NOT NULL,
218+
name VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(`json`, '$.name'))) VIRTUAL NOT NULL,
219+
serviceType VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(`json`, '$.serviceType'))) VIRTUAL NOT NULL,
220+
json JSON NOT NULL,
221+
updatedAt BIGINT UNSIGNED GENERATED ALWAYS AS (json_unquote(json_extract(`json`, '$.updatedAt'))) VIRTUAL NOT NULL,
222+
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json_unquote(json_extract(`json`, '$.updatedBy'))) VIRTUAL NOT NULL,
223+
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.impersonatedBy') VIRTUAL,
224+
deleted TINYINT(1) GENERATED ALWAYS AS (json_extract(`json`, '$.deleted')) VIRTUAL,
225+
nameHash VARCHAR(256) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,
226+
PRIMARY KEY (id),
227+
UNIQUE KEY nameHash (nameHash),
228+
INDEX name_index (name),
229+
INDEX service_type_index (serviceType),
230+
INDEX deleted_index (deleted)
231+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='LLM Service entities';
232+

‎bootstrap/sql/migrations/native/1.11.0/postgres/schemaChanges.sql‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,3 +166,116 @@ CREATE INDEX IF NOT EXISTS idx_entity_relationship_bidirectional
166166
ON entity_relationship(fromId, toId, relation)
167167
WHERE deleted = FALSE;
168168

169+
-- AI Application and LLM Model entities for AI/LLM governance
170+
-- Version 1.11.0
171+
172+
-- Create ai_application_entity table
173+
CREATE TABLE IF NOT EXISTS ai_application_entity (
174+
id VARCHAR(36) GENERATED ALWAYS AS (json->>'id') STORED NOT NULL,
175+
name VARCHAR(256) GENERATED ALWAYS AS (json->>'name') STORED NOT NULL,
176+
fqnHash VARCHAR(768) NOT NULL,
177+
json JSONB NOT NULL,
178+
updatedAt BIGINT GENERATED ALWAYS AS ((json->>'updatedAt')::bigint) STORED NOT NULL,
179+
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'updatedBy') STORED NOT NULL,
180+
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'impersonatedBy') STORED,
181+
deleted BOOLEAN GENERATED ALWAYS AS ((json->>'deleted')::boolean) STORED,
182+
PRIMARY KEY (id),
183+
UNIQUE (fqnHash)
184+
);
185+
186+
CREATE INDEX IF NOT EXISTS ai_application_name_index ON ai_application_entity(name);
187+
CREATE INDEX IF NOT EXISTS ai_application_deleted_index ON ai_application_entity(deleted);
188+
189+
COMMENT ON TABLE ai_application_entity IS 'AI Application entities';
190+
191+
-- Create llm_model_entity table
192+
CREATE TABLE IF NOT EXISTS llm_model_entity (
193+
id VARCHAR(36) GENERATED ALWAYS AS (json->>'id') STORED NOT NULL,
194+
name VARCHAR(256) GENERATED ALWAYS AS (json->>'name') STORED NOT NULL,
195+
fqnHash VARCHAR(768) NOT NULL,
196+
json JSONB NOT NULL,
197+
updatedAt BIGINT GENERATED ALWAYS AS ((json->>'updatedAt')::bigint) STORED NOT NULL,
198+
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'updatedBy') STORED NOT NULL,
199+
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'impersonatedBy') STORED,
200+
deleted BOOLEAN GENERATED ALWAYS AS ((json->>'deleted')::boolean) STORED,
201+
PRIMARY KEY (id),
202+
UNIQUE (fqnHash)
203+
);
204+
205+
CREATE INDEX IF NOT EXISTS llm_model_name_index ON llm_model_entity(name);
206+
CREATE INDEX IF NOT EXISTS llm_model_deleted_index ON llm_model_entity(deleted);
207+
208+
COMMENT ON TABLE llm_model_entity IS 'LLM Model entities';
209+
210+
-- Create prompt_template_entity table
211+
CREATE TABLE IF NOT EXISTS prompt_template_entity (
212+
id VARCHAR(36) GENERATED ALWAYS AS (json->>'id') STORED NOT NULL,
213+
name VARCHAR(256) GENERATED ALWAYS AS (json->>'name') STORED NOT NULL,
214+
fqnHash VARCHAR(768) NOT NULL,
215+
json JSONB NOT NULL,
216+
updatedAt BIGINT GENERATED ALWAYS AS ((json->>'updatedAt')::bigint) STORED NOT NULL,
217+
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'updatedBy') STORED NOT NULL,
218+
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'impersonatedBy') STORED,
219+
deleted BOOLEAN GENERATED ALWAYS AS ((json->>'deleted')::boolean) STORED,
220+
PRIMARY KEY (id),
221+
UNIQUE (fqnHash)
222+
);
223+
224+
CREATE INDEX IF NOT EXISTS prompt_template_name_index ON prompt_template_entity(name);
225+
CREATE INDEX IF NOT EXISTS prompt_template_deleted_index ON prompt_template_entity(deleted);
226+
227+
COMMENT ON TABLE prompt_template_entity IS 'Prompt Template entities';
228+
229+
-- Create agent_execution_entity table
230+
CREATE TABLE IF NOT EXISTS agent_execution_entity (
231+
id VARCHAR(36) GENERATED ALWAYS AS (json->>'id') STORED NOT NULL,
232+
agentId VARCHAR(36) GENERATED ALWAYS AS (json->>'agentId') STORED NOT NULL,
233+
json JSONB NOT NULL,
234+
timestamp BIGINT GENERATED ALWAYS AS ((json->>'timestamp')::bigint) STORED NOT NULL,
235+
PRIMARY KEY (id)
236+
);
237+
238+
CREATE INDEX IF NOT EXISTS agent_execution_agent_index ON agent_execution_entity(agentId);
239+
CREATE INDEX IF NOT EXISTS agent_execution_timestamp_index ON agent_execution_entity(timestamp);
240+
241+
COMMENT ON TABLE agent_execution_entity IS 'AI Agent Execution logs';
242+
243+
-- Create ai_governance_policy_entity table
244+
CREATE TABLE IF NOT EXISTS ai_governance_policy_entity (
245+
id VARCHAR(36) GENERATED ALWAYS AS (json->>'id') STORED NOT NULL,
246+
name VARCHAR(256) GENERATED ALWAYS AS (json->>'name') STORED NOT NULL,
247+
fqnHash VARCHAR(768) NOT NULL,
248+
json JSONB NOT NULL,
249+
updatedAt BIGINT GENERATED ALWAYS AS ((json->>'updatedAt')::bigint) STORED NOT NULL,
250+
updatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'updatedBy') STORED NOT NULL,
251+
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'impersonatedBy') STORED,
252+
deleted BOOLEAN GENERATED ALWAYS AS ((json->>'deleted')::boolean) STORED,
253+
PRIMARY KEY (id),
254+
UNIQUE (fqnHash)
255+
);
256+
257+
CREATE INDEX IF NOT EXISTS ai_governance_policy_name_index ON ai_governance_policy_entity(name);
258+
CREATE INDEX IF NOT EXISTS ai_governance_policy_deleted_index ON ai_governance_policy_entity(deleted);
259+
260+
COMMENT ON TABLE ai_governance_policy_entity IS 'AI Governance Policy entities';
261+
262+
-- Create llm_service_entity table
263+
CREATE TABLE IF NOT EXISTS llm_service_entity (
264+
id VARCHAR(36) GENERATED ALWAYS AS ((json ->> 'id'::text)) STORED NOT NULL,
265+
name VARCHAR(256) GENERATED ALWAYS AS ((json ->> 'name'::text)) STORED NOT NULL,
266+
serviceType VARCHAR(256) GENERATED ALWAYS AS ((json ->> 'serviceType'::text)) STORED NOT NULL,
267+
json JSONB NOT NULL,
268+
updatedAt BIGINT GENERATED ALWAYS AS (((json ->> 'updatedAt'::text))::bigint) STORED NOT NULL,
269+
updatedBy VARCHAR(256) GENERATED ALWAYS AS ((json ->> 'updatedBy'::text)) STORED NOT NULL,
270+
impersonatedBy VARCHAR(256) GENERATED ALWAYS AS (json->>'impersonatedBy') STORED,
271+
deleted BOOLEAN GENERATED ALWAYS AS (((json ->> 'deleted'::text))::boolean) STORED,
272+
nameHash VARCHAR(256) NOT NULL,
273+
PRIMARY KEY (id),
274+
UNIQUE (nameHash)
275+
);
276+
277+
CREATE INDEX IF NOT EXISTS llm_service_name_index ON llm_service_entity(name);
278+
CREATE INDEX IF NOT EXISTS llm_service_type_index ON llm_service_entity(serviceType);
279+
CREATE INDEX IF NOT EXISTS llm_service_deleted_index ON llm_service_entity(deleted);
280+
281+
COMMENT ON TABLE llm_service_entity IS 'LLM Service entities';

0 commit comments

Comments
 (0)