process_tracker_python-86 Add Source Object Attribute Lookup

✨ Source Object Attributes can now be registered
✨  Attributes can be associated to processes as sources or targets

Attributes can now be registered for processes and data sources.  This
allows for metadata injection within scripts.

Author: Alex Meadows

dbscripts/mysql_process_tracker.sql

@@ -1,5 +1,14 @@
 USE process_tracker;
 
+create table process_tracker.data_type_lkup
+(
+	data_type_id int auto_increment
+		primary key,
+	data_type varchar(75) not null,
+	constraint data_type_lkup_data_type_uindex
+		unique (data_type)
+);
+
 create table contact_lkup
 (
 	contact_id int auto_increment
@@ -439,3 +448,49 @@ create table process_contact
 		foreign key (contact_id) references contact_lkup (contact_id)
 );
 
+create table source_object_attribute
+(
+	source_object_attribute_id int auto_increment
+		primary key,
+	source_object_attribute_name varchar(250) not null,
+	source_object_id int not null,
+	attribute_path varchar(750) null,
+	data_type_id int null,
+	data_length int null,
+	data_decimal int null,
+	is_pii tinyint(1) default 0 not null,
+	default_value_string varchar(250) null,
+	default_value_number decimal null,
+	constraint source_object_attribute_udx01
+		unique (source_object_id, source_object_attribute_name),
+	constraint source_object_attribute_fk01
+		foreign key (source_object_id) references source_object_lkup (source_object_id),
+	constraint source_object_attribute_fk02
+		foreign key (data_type_id) references data_type_lkup (data_type_id)
+);
+
+create table process_source_object_attribute
+(
+	process_id int not null,
+	source_object_attribute_id int not null,
+	source_object_attribute_alias varchar(250) null,
+	source_object_attribute_expression varchar(250) null,
+	primary key (process_id, source_object_attribute_id),
+	constraint process_source_object_attribute_fk01
+		foreign key (process_id) references process (process_id),
+	constraint process_source_object_attribute_fk02
+		foreign key (source_object_attribute_id) references source_object_attribute (source_object_attribute_id)
+);
+
+create table if not exists process_target_object_attribute
+(
+	process_id int not null,
+	target_object_attribute_id int not null,
+	target_object_attribute_alias varchar(250) null,
+	target_object_attribute_expression varchar(250) null,
+	primary key (process_id, target_object_attribute_id),
+	constraint process_target_object_attribute_fk01
+		foreign key (process_id) references process (process_id),
+	constraint process_target_object_attribute_fk02
+		foreign key (target_object_attribute_id) references source_object_attribute (source_object_attribute_id)
+);

dbscripts/postgresql_process_tracker.sql

@@ -4,6 +4,21 @@ create schema process_tracker;
 
 alter schema process_tracker owner to pt_admin;
 
+create table process_tracker.data_type_lkup
+(
+	data_type_id serial not null
+		constraint data_type_lkup_pk
+			primary key,
+	data_type varchar(75) not null
+);
+
+alter table process_tracker.data_type_lkup owner to pt_admin;
+
+create unique index data_type_lkup_data_type_uindex
+	on process_tracker.data_type_lkup (data_type);
+
+
+
 create table process_tracker.contact_lkup
 (
 	contact_id serial not null
@@ -630,4 +645,62 @@ create table process_contact
 		primary key (process_id, contact_id)
 );
 
-alter table process_contact owner to pt_admin;
+alter table process_contact owner to pt_admin;
+
+create table process_tracker.source_object_attribute
+(
+	source_object_attribute_id serial not null
+		constraint source_object_attribute_pk
+			primary key,
+	source_object_attribute_name varchar(250) not null,
+	source_object_id integer
+		constraint source_object_attribute_fk01
+			references process_tracker.source_object_lkup,
+	attribute_path varchar(750),
+	data_type_id integer
+		constraint source_object_attribute_fk02
+			references process_tracker.data_type_lkup,
+	data_length integer,
+	data_decimal integer,
+	is_pii boolean default false not null,
+	default_value_string varchar(250),
+	default_value_number numeric
+);
+
+alter table process_tracker.source_object_attribute owner to pt_admin;
+
+create unique index source_object_attribute_udx01
+	on process_tracker.source_object_attribute (source_object_id, source_object_attribute_name);
+
+create table process_tracker.process_target_object_attribute
+(
+	process_id integer not null
+		constraint process_target_object_attribute_fk01
+			references process_tracker.process,
+	target_object_attribute_id integer not null
+		constraint process_target_object_attribute_fk02
+			references process_tracker.source_object_attribute,
+	target_object_attribute_alias varchar(250),
+	target_object_attribute_expression varchar(250),
+	constraint process_target_object_attribute_pk
+		primary key (process_id, target_object_attribute_id)
+);
+
+alter table process_tracker.process_target_object_attribute owner to pt_admin;
+
+create table process_tracker.process_source_object_attribute
+(
+	process_id integer not null
+		constraint process_source_object_attribute_fk01
+			references process_tracker.process,
+	source_object_attribute_id integer not null
+		constraint process_source_object_attribute_fk02
+			references process_tracker.source_object_attribute,
+	source_object_attribute_alias varchar(250),
+	source_object_attribute_expression varchar(250),
+	constraint process_source_object_attribute_pk
+		primary key (process_id, source_object_attribute_id)
+);
+
+alter table process_tracker.process_source_object_attribute owner to pt_admin;
+

process_tracker/models/process.py

@@ -282,6 +282,37 @@ def __repr__(self):
         )
 
 
+class ProcessSourceObjectAttribute(Base):
+    __tablename__ = "process_source_object_attribute"
+    __table_args__ = {"schema": "process_tracker"}
+
+    process_id = Column(
+        Integer,
+        ForeignKey("process_tracker.process.process_id"),
+        primary_key=True,
+        nullable=False,
+    )
+    source_object_attribute_id = Column(
+        Integer,
+        ForeignKey(
+            "process_tracker.source_object_attribute.source_object_attribute_id"
+        ),
+        primary_key=True,
+        nullable=False,
+    )
+    source_object_attribute_alias = Column(String(250), nullable=True)
+    source_object_attribute_expression = Column(String(250), nullable=True)
+
+    attributes = relationship("SourceObjectAttribute")
+    processes = relationship("Process")
+
+    def __repr__(self):
+        return "<ProcessSourceObjectAttribute process_id=%s, attribute_id=%s>" % (
+            self.process_id,
+            self.source_object_attribute_id,
+        )
+
+
 class ProcessTarget(Base):
     __tablename__ = "process_target"
     __table_args__ = {"schema": "process_tracker"}
@@ -337,6 +368,37 @@ def __repr__(self):
         )
 
 
+class ProcessTargetObjectAttribute(Base):
+    __tablename__ = "process_target_object_attribute"
+    __table_args__ = {"schema": "process_tracker"}
+
+    process_id = Column(
+        Integer,
+        ForeignKey("process_tracker.process.process_id"),
+        primary_key=True,
+        nullable=False,
+    )
+    target_object_attribute_id = Column(
+        Integer,
+        ForeignKey(
+            "process_tracker.source_object_attribute.source_object_attribute_id"
+        ),
+        primary_key=True,
+        nullable=False,
+    )
+    target_object_attribute_alias = Column(String(250), nullable=True)
+    target_object_attribute_expression = Column(String(250), nullable=True)
+
+    attributes = relationship("SourceObjectAttribute")
+    processes = relationship("Process")
+
+    def __repr__(self):
+        return "<ProcessTargetObjectAttribute process_id=%s, attribute_id=%s>" % (
+            self.process_id,
+            self.source_object_attribute_id,
+        )
+
+
 class ProcessDependency(Base):
 
     __tablename__ = "process_dependency"

process_tracker/models/source.py

@@ -1,12 +1,39 @@
 # SQLAlchemy Models
 # Models for Source entities
 
-from sqlalchemy import Column, ForeignKey, Integer, Sequence, String, UniqueConstraint
+from sqlalchemy import (
+    Boolean,
+    Column,
+    ForeignKey,
+    Integer,
+    Numeric,
+    Sequence,
+    String,
+    UniqueConstraint,
+)
 from sqlalchemy.orm import relationship
 
 from process_tracker.models.model_base import Base
 
 
+class DataType(Base):
+
+    __tablename__ = "data_type_lkup"
+    __table_args__ = {"schema": "process_tracker"}
+
+    data_type_id = Column(
+        Integer,
+        Sequence("data_type_lkup_data_type_id", schema="process_tracker"),
+        primary_key=True,
+        nullable=False,
+    )
+    data_type = Column(String(75), unique=True, nullable=False)
+
+    def __repr__(self):
+
+        return "<DataType id=%s, name=%s>" % (self.data_type_id, self.data_type)
+
+
 class DatasetType(Base):
 
     __tablename__ = "dataset_type_lkup"
@@ -143,6 +170,49 @@ def __repr__(self):
         )
 
 
+class SourceObjectAttribute(Base):
+
+    __tablename__ = "source_object_attribute"
+    __table_args__ = {"schema": "process_tracker"}
+
+    source_object_attribute_id = Column(
+        Integer,
+        Sequence(
+            "source_object_attribute_source_object_attribute_id_seq",
+            schema="process_tracker",
+        ),
+        primary_key=True,
+        nullable=False,
+    )
+    source_object_attribute_name = Column(String(250), nullable=False)
+    source_object_id = Column(
+        Integer,
+        ForeignKey("process_tracker.source_object_lkup.source_object_id"),
+        nullable=False,
+    )
+    attribute_path = Column(String(750), nullable=True)
+    data_type_id = Column(
+        Integer,
+        ForeignKey("process_tracker.data_type_lkup.data_type_id"),
+        nullable=True,
+    )
+    data_length = Column(Integer, nullable=True)
+    data_decimal = Column(Integer, nullable=True)
+    is_pii = Column(Boolean, nullable=False, default=False)
+    default_value_string = Column(String(250), nullable=True)
+    default_value_number = Column(Numeric, nullable=True)
+
+    UniqueConstraint(source_object_id, source_object_attribute_name)
+
+    def __repr__(self):
+
+        return "<Source Object Attribute id=%s, name=%s, source_object=%s>" % (
+            self.source_object_attribute_id,
+            self.source_object_attribute_name,
+            self.source_object_id,
+        )
+
+
 class SourceObjectDatasetType(Base):
 
     __tablename__ = "source_object_dataset_type"