Skip to content

Commit a5e6157

Browse files
OpenDataAlexOpenDataAlex
authored
Merge pull request #99 from OpenDataAlex/process_tracker_python-89
Process tracker python 89
2 parents 45b9897 + 6e5092f commit a5e6157

File tree

10 files changed

+587
-18
lines changed

10 files changed

+587
-18
lines changed

‎dbscripts/mysql_process_tracker.sql‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,14 @@
11
USE process_tracker;
22

3+
create table schedule_frequency_lkup
4+
(
5+
schedule_frequency_id int auto_increment
6+
primary key,
7+
schedule_frequency_name varchar(25) not null,
8+
constraint schedule_frequency_lkup_schedule_frequency_name_uindex
9+
unique (schedule_frequency_name)
10+
);
11+
312
create table data_type_lkup
413
(
514
data_type_id int auto_increment
@@ -210,8 +219,11 @@ create table process
210219
process_type_id int null,
211220
process_tool_id int null,
212221
last_failed_run_date_time datetime not null,
222+
schedule_frequency_id int default 0 not null,
213223
constraint process_name
214224
unique (process_name),
225+
constraint process_fk03
226+
foreign key (schedule_frequency_id) references schedule_frequency_lkup (schedule_frequency_id),
215227
constraint process_ibfk_1
216228
foreign key (process_type_id) references process_type_lkup (process_type_id),
217229
constraint process_ibfk_2
@@ -523,3 +535,37 @@ create table if not exists process_target_object_attribute
523535
constraint process_target_object_attribute_fk02
524536
foreign key (target_object_attribute_id) references source_object_attribute (source_object_attribute_id)
525537
);
538+
539+
create table filter_type_lkup
540+
(
541+
filter_type_id int auto_increment
542+
primary key,
543+
filter_type_code varchar(3) not null,
544+
filter_type_name varchar(75) not null,
545+
constraint filter_type_lkup_filter_type_code_uindex
546+
unique (filter_type_code),
547+
constraint filter_type_lkup_filter_type_name_uindex
548+
unique (filter_type_name)
549+
);
550+
551+
create table process_filter
552+
(
553+
process_filter_id int auto_increment
554+
primary key,
555+
process_id int not null,
556+
source_object_attribute_id int not null,
557+
filter_type_id int not null,
558+
filter_value_string varchar(250) null,
559+
filter_value_numeric decimal null,
560+
constraint process_filter_udx
561+
unique (process_id, source_object_attribute_id, filter_type_id),
562+
constraint process_filter_fk01
563+
foreign key (process_id) references process (process_id),
564+
constraint process_filter_fk02
565+
foreign key (source_object_attribute_id) references source_object_attribute (source_object_attribute_id),
566+
constraint process_filter_fk03
567+
foreign key (filter_type_id) references filter_type_lkup (filter_type_id)
568+
);
569+
570+
571+

‎dbscripts/mysql_process_tracker_defaults.sql‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,23 @@ INSERT INTO process_tracker.system_lkup (system_id, system_key, system_value) VA
2424

2525
INSERT INTO process_tracker.extract_compression_type_lkup (extract_compression_type_id, extract_compression_type) VALUES (1, 'zip');
2626

27-
INSERT INTO process_tracker.extract_filetype_lkup (extract_filetype_id, extract_filetype_code, extract_filetype, delimiter_char, quote_char, escape_char) VALUES (1, 'csv', 'Comma Separated Values', ',', '"', '/');
27+
INSERT INTO process_tracker.extract_filetype_lkup (extract_filetype_id, extract_filetype_code, extract_filetype, delimiter_char, quote_char, escape_char) VALUES (1, 'csv', 'Comma Separated Values', ',', '"', '/');
28+
29+
INSERT INTO process_tracker.schedule_frequency_lkup (schedule_frequency_id, schedule_frequency_name) VALUES (0, 'unscheduled');
30+
INSERT INTO process_tracker.schedule_frequency_lkup (schedule_frequency_id, schedule_frequency_name) VALUES (7, 'annually');
31+
INSERT INTO process_tracker.schedule_frequency_lkup (schedule_frequency_id, schedule_frequency_name) VALUES (3, 'daily');
32+
INSERT INTO process_tracker.schedule_frequency_lkup (schedule_frequency_id, schedule_frequency_name) VALUES (2, 'hourly');
33+
INSERT INTO process_tracker.schedule_frequency_lkup (schedule_frequency_id, schedule_frequency_name) VALUES (5, 'monthly');
34+
INSERT INTO process_tracker.schedule_frequency_lkup (schedule_frequency_id, schedule_frequency_name) VALUES (6, 'quarterly');
35+
INSERT INTO process_tracker.schedule_frequency_lkup (schedule_frequency_id, schedule_frequency_name) VALUES (4, 'weekly');
36+
37+
UPDATE process_tracker.schedule_frequency_lkup SET schedule_frequency_id = 0 WHERE schedule_frequency_name = 'unscheduled';
38+
39+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (1, 'eq', 'equal to');
40+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (2, 'lt', 'less than');
41+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (3, 'gt', 'greater than');
42+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (4, 'lte', 'less than or equal');
43+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (5, 'gte', 'greater than or equal');
44+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (6, 'not', 'not equal');
45+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (7, 'lke', 'like');
46+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (8, 'in', 'in set');

‎dbscripts/postgresql_process_tracker.sql‎

Lines changed: 71 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,19 @@ create schema process_tracker;
44

55
alter schema process_tracker owner to pt_admin;
66

7+
create table schedule_frequency_lkup
8+
(
9+
schedule_frequency_id serial not null
10+
constraint schedule_frequency_lkup_pk
11+
primary key,
12+
schedule_frequency_name varchar(25) not null
13+
);
14+
15+
alter table schedule_frequency_lkup owner to pt_admin;
16+
17+
create unique index schedule_frequency_lkup_schedule_frequency_name_uindex
18+
on schedule_frequency_lkup (schedule_frequency_name);
19+
720
create table data_type_lkup
821
(
922
data_type_id serial not null
@@ -180,38 +193,59 @@ alter table process_type_lkup owner to pt_admin;
180193
create unique index process_type_lkup_udx01
181194
on process_type_lkup (process_type_name);
182195

183-
create table process
196+
create table process_tracker.filter_type_lkup
197+
(
198+
filter_type_id serial not null
199+
constraint filter_type_lkup_pk
200+
primary key,
201+
filter_type_code varchar(3) not null,
202+
filter_type_name varchar(75) not null
203+
);
204+
205+
alter table process_tracker.filter_type_lkup owner to pt_admin;
206+
207+
create unique index filter_type_lkup_filter_type_code_uindex
208+
on process_tracker.filter_type_lkup (filter_type_code);
209+
210+
create unique index filter_type_lkup_filter_type_name_uindex
211+
on process_tracker.filter_type_lkup (filter_type_name);
212+
213+
214+
create table process_tracker.process
184215
(
185216
process_id serial not null
186217
constraint process_pk
187218
primary key,
188219
process_name varchar(250) not null,
189220
total_record_count integer default 0 not null,
190-
process_type_id integer null
221+
process_type_id integer
191222
constraint process_fk02
192-
references process_type_lkup,
193-
process_tool_id integer null
223+
references process_tracker.process_type_lkup,
224+
process_tool_id integer
194225
constraint process_fk03
195-
references tool_lkup,
196-
last_failed_run_date_time timestamp default '1900-01-01 00:00:00'::timestamp without time zone not null
226+
references process_tracker.tool_lkup,
227+
last_failed_run_date_time timestamp default '1900-01-01 00:00:00'::timestamp without time zone not null,
228+
schedule_frequency_id integer default 0 not null
229+
constraint process_fk04
230+
references process_tracker.schedule_frequency_lkup
197231
);
198232

199-
comment on table process is 'Processes being tracked';
233+
comment on table process_tracker.process is 'Processes being tracked';
200234

201-
comment on column process.process_name is 'Unique name for process.';
235+
comment on column process_tracker.process.process_name is 'Unique name for process.';
202236

203-
comment on column process.total_record_count is 'Total number of records processed over all runs of process.';
237+
comment on column process_tracker.process.total_record_count is 'Total number of records processed over all runs of process.';
204238

205-
comment on column process.process_type_id is 'The type of process being tracked.';
239+
comment on column process_tracker.process.process_type_id is 'The type of process being tracked.';
206240

207-
comment on column process.process_tool_id is 'The type of tool used to execute the process.';
241+
comment on column process_tracker.process.process_tool_id is 'The type of tool used to execute the process.';
208242

209-
comment on column process.last_failed_run_date_time is 'The last time the process failed to run.';
243+
comment on column process_tracker.process.last_failed_run_date_time is 'The last time the process failed to run.';
210244

211-
alter table process owner to pt_admin;
245+
alter table process_tracker.process owner to pt_admin;
212246

213247
create unique index process_udx01
214-
on process (process_name);
248+
on process_tracker.process (process_name);
215249

216250
create table process_dependency
217251
(
@@ -750,3 +784,26 @@ create table process_tracker.process_source_object_attribute
750784

751785
alter table process_tracker.process_source_object_attribute owner to pt_admin;
752786

787+
788+
create table process_tracker.process_filter
789+
(
790+
process_filter_id serial not null
791+
constraint process_filter_pk
792+
primary key,
793+
process_id integer not null
794+
constraint process_filter_fk01
795+
references process_tracker.process,
796+
source_object_attribute_id integer not null
797+
constraint process_filter_fk02
798+
references process_tracker.source_object_attribute,
799+
filter_type_id integer not null
800+
constraint process_filter_fk03
801+
references process_tracker.filter_type_lkup,
802+
filter_value_string varchar(250),
803+
filter_value_numeric numeric
804+
);
805+
806+
alter table process_tracker.process_filter owner to pt_admin;
807+
808+
create unique index process_filter_udx01
809+
on process_tracker.process_filter (process_id, source_object_attribute_id, filter_type_id);

‎dbscripts/postgresql_process_tracker_defaults.sql‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,21 @@ INSERT INTO process_tracker.system_lkup (system_id, system_key, system_value) VA
2525

2626
INSERT INTO process_tracker.extract_compression_type_lkup (extract_compression_type_id, extract_compression_type) VALUES (1, 'zip');
2727

28-
INSERT INTO process_tracker.extract_filetype_lkup (extract_filetype_id, extract_filetype_code, extract_filetype, delimiter_char, quote_char, escape_char) VALUES (1, 'csv', 'Comma Separated Values', ',', '"', '/');
28+
INSERT INTO process_tracker.extract_filetype_lkup (extract_filetype_id, extract_filetype_code, extract_filetype, delimiter_char, quote_char, escape_char) VALUES (1, 'csv', 'Comma Separated Values', ',', '"', '/');
29+
30+
INSERT INTO process_tracker.schedule_frequency_lkup (schedule_frequency_id, schedule_frequency_name) VALUES (0, 'unscheduled');
31+
INSERT INTO process_tracker.schedule_frequency_lkup (schedule_frequency_id, schedule_frequency_name) VALUES (1, 'hourly');
32+
INSERT INTO process_tracker.schedule_frequency_lkup (schedule_frequency_id, schedule_frequency_name) VALUES (2, 'daily');
33+
INSERT INTO process_tracker.schedule_frequency_lkup (schedule_frequency_id, schedule_frequency_name) VALUES (3, 'weekly');
34+
INSERT INTO process_tracker.schedule_frequency_lkup (schedule_frequency_id, schedule_frequency_name) VALUES (4, 'monthly');
35+
INSERT INTO process_tracker.schedule_frequency_lkup (schedule_frequency_id, schedule_frequency_name) VALUES (5, 'quarterly');
36+
INSERT INTO process_tracker.schedule_frequency_lkup (schedule_frequency_id, schedule_frequency_name) VALUES (6, 'annually');
37+
38+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (1, 'eq', 'equal to');
39+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (2, 'lt', 'less than');
40+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (3, 'gt', 'greater than');
41+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (4, 'lte', 'less than or equal');
42+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (5, 'gte', 'greater than or equal');
43+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (6, 'not', 'not equal');
44+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (7, 'lke', 'like');
45+
INSERT INTO process_tracker.filter_type_lkup (filter_type_id, filter_type_code, filter_type_name) VALUES (8, 'in', 'in set');

‎process_tracker/models/process.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
DateTime,
99
ForeignKey,
1010
Integer,
11+
Numeric,
1112
Sequence,
1213
String,
1314
UniqueConstraint,
@@ -147,6 +148,12 @@ class Process(Base):
147148
last_failed_run_date_time = Column(
148149
DateTime(timezone=True), nullable=False, default=default_date
149150
)
151+
schedule_frequency_id = Column(
152+
Integer,
153+
ForeignKey("process_tracker.schedule_frequency_lkup.schedule_frequency_id"),
154+
nullable=False,
155+
default=0,
156+
)
150157

151158
cluster_processes = relationship("ClusterProcess", passive_deletes="all")
152159
process_tracking = relationship("ProcessTracking", passive_deletes="all")
@@ -226,6 +233,48 @@ def __repr__(self):
226233
)
227234

228235

236+
class ProcessFilter(Base):
237+
238+
__tablename__ = "process_filter"
239+
__table_args__ = {"schema": "process_tracker"}
240+
241+
process_filter_id = Column(
242+
Integer,
243+
Sequence("process_filter_process_filter_id_seq", schema="process_tracker"),
244+
primary_key=True,
245+
nullable=False,
246+
)
247+
process_id = Column(
248+
Integer, ForeignKey("process_tracker.process.process_id"), nullable=False
249+
)
250+
source_object_attribute_id = Column(
251+
Integer,
252+
ForeignKey(
253+
"process_tracker.source_object_attribute.source_object_attribute_id"
254+
),
255+
nullable=False,
256+
)
257+
filter_type_id = Column(
258+
Integer,
259+
ForeignKey("process_tracker.filter_type_lkup.filter_type_id"),
260+
nullable=False,
261+
)
262+
filter_value_string = Column(String(250), nullable=True)
263+
filter_value_numeric = Column(Numeric, nullable=True)
264+
265+
attributes = relationship("SourceObjectAttribute")
266+
267+
UniqueConstraint = (process_id, source_object_attribute_id, filter_type_id)
268+
269+
def __repr__(self):
270+
271+
return "<ProcessFilter process=%s, attribute=%s, filter_type=%s>" % (
272+
self.process_id,
273+
self.source_object_attribute_id,
274+
self.filter_type_id,
275+
)
276+
277+
229278
class ProcessSource(Base):
230279

231280
__tablename__ = "process_source"

‎process_tracker/models/schedule.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# SQLAlchemy Models
2+
# Models for Schedule entities
3+
4+
5+
from sqlalchemy import (
6+
Boolean,
7+
Column,
8+
DateTime,
9+
ForeignKey,
10+
Integer,
11+
Sequence,
12+
String,
13+
UniqueConstraint,
14+
)
15+
from sqlalchemy.orm import relationship
16+
17+
from process_tracker.models.model_base import default_date, Base
18+
19+
20+
class ScheduleFrequency(Base):
21+
22+
__tablename__ = "schedule_frequency_lkup"
23+
__table_args__ = {"schema": "process_tracker"}
24+
25+
schedule_frequency_id = Column(
26+
Integer,
27+
Sequence("schedule_frequency_schedule_frequency_id", schema="process_tracker"),
28+
primary_key=True,
29+
nullable=False,
30+
)
31+
schedule_frequency_name = Column(String(25), unique=True, nullable=False)

‎process_tracker/models/source.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,28 @@ def __repr__(self):
5959
return "<DatasetType %s>" % self.dataset_type
6060

6161

62+
class FilterType(Base):
63+
64+
__tablename__ = "filter_type_lkup"
65+
__table_args__ = {"schema": "process_tracker"}
66+
67+
filter_type_id = Column(
68+
Integer,
69+
Sequence("filter_type_lkup_filter_type_id_seq", schema="process_tracker"),
70+
primary_key=True,
71+
nullable=False,
72+
)
73+
filter_type_code = Column(String(3), nullable=False, unique=True)
74+
filter_type_name = Column(String(75), nullable=False, unique=True)
75+
76+
def __repr__(self):
77+
78+
return "<FilterType code=%s, name=%s>" % (
79+
self.filter_type_code,
80+
self.filter_type_name,
81+
)
82+
83+
6284
class Source(Base):
6385

6486
__tablename__ = "source_lkup"
@@ -162,6 +184,8 @@ class SourceObject(Base):
162184
passive_deletes="all",
163185
)
164186

187+
sources = relationship("Source")
188+
165189
def __repr__(self):
166190

167191
return "<SourceObject (source_id=%s, source_object_name=%s)>" % (
@@ -204,6 +228,8 @@ class SourceObjectAttribute(Base):
204228

205229
UniqueConstraint(source_object_id, source_object_attribute_name)
206230

231+
source_objects = relationship("SourceObject")
232+
207233
def __repr__(self):
208234

209235
return "<Source Object Attribute id=%s, name=%s, source_object=%s>" % (

0 commit comments

Comments
 (0)