Skip to content

Commit 6c7c8bb

Browse files
OpenDataAlexOpenDataAlex
authored
Merge pull request #171 from OpenDataAlex/process_tracker_python-144
Process tracker python 144
2 parents ec39e28 + 6061fff commit 6c7c8bb

File tree

7 files changed

+92
-4
lines changed

7 files changed

+92
-4
lines changed

‎dbscripts/mysql_process_tracker.sql‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,18 @@
11
USE process_tracker;
22

3+
create table process_tracker.dependency_type_lkup
4+
(
5+
dependency_type_id int auto_increment
6+
primary key,
7+
dependency_type_name varchar(75) not null,
8+
created_date_time timestamp default CURRENT_TIMESTAMP not null,
9+
created_by int default 0 not null,
10+
update_date_time timestamp default CURRENT_TIMESTAMP not null on update CURRENT_TIMESTAMP,
11+
updated_by int default 0 not null,
12+
constraint dependency_type_lkup_dependency_type_name_uindex
13+
unique (dependency_type_name)
14+
);
15+
316
create table process_tracker.character_set_lkup
417
(
518
character_set_id int auto_increment
@@ -253,6 +266,7 @@ create table process_tracker.extract_dependency
253266
(
254267
parent_extract_id int not null,
255268
child_extract_id int not null,
269+
dependency_type_id int default 1 not null,
256270
created_date_time timestamp default CURRENT_TIMESTAMP not null,
257271
created_by int default 0 not null,
258272
update_date_time timestamp default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP not null,
@@ -261,7 +275,9 @@ create table process_tracker.extract_dependency
261275
constraint extract_dependency_fk01
262276
foreign key (parent_extract_id) references extract_tracking (extract_id),
263277
constraint extract_dependency_fk02
264-
foreign key (child_extract_id) references extract_tracking (extract_id)
278+
foreign key (child_extract_id) references extract_tracking (extract_id),
279+
constraint extract_dependency_fk03
280+
foreign key (dependency_type_id) references dependency_type_lkup (dependency_type_id)
265281
)
266282
comment 'Table tracking interdependencies between extract files.';
267283

@@ -373,6 +389,7 @@ create table process_dependency
373389
(
374390
parent_process_id int not null,
375391
child_process_id int not null,
392+
dependency_type_id int default 1 not null,
376393
created_date_time timestamp default CURRENT_TIMESTAMP not null,
377394
created_by int default 0 not null,
378395
update_date_time timestamp default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP not null,
@@ -381,7 +398,9 @@ create table process_dependency
381398
constraint process_dependency_ibfk_1
382399
foreign key (parent_process_id) references process (process_id),
383400
constraint process_dependency_ibfk_2
384-
foreign key (child_process_id) references process (process_id)
401+
foreign key (child_process_id) references process (process_id),
402+
constraint process_dependency_fk03
403+
foreign key (dependency_type_id) references dependency_type_lkup (dependency_type_id)
385404
);
386405

387406
create index child_process_id

‎dbscripts/mysql_process_tracker_defaults.sql‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,4 +53,9 @@ INSERT INTO process_tracker.filesize_type_lkup (filesize_type_id, filesize_type_
5353
INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (1, 'Undefined');
5454
INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (2, 'Database');
5555
INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (3, 'Internal');
56-
INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External');
56+
INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External');
57+
58+
INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (1, 'Undefined');
59+
INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (2, 'Hard');
60+
INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (3, 'Soft');
61+

‎dbscripts/postgresql_process_tracker.sql‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,21 @@ create schema process_tracker;
77

88
alter schema process_tracker owner to pt_admin;
99

10+
create table process_tracker.dependency_type_lkup
11+
(
12+
dependency_type_id serial not null
13+
constraint dependency_type_lkup_pk
14+
primary key,
15+
dependency_type_name varchar(75) not null,
16+
created_date_time timestamptz default CURRENT_TIMESTAMP not null,
17+
created_by int default 0 not null,
18+
update_date_time timestamptz default CURRENT_TIMESTAMP not null,
19+
updated_by int default 0 not null
20+
);
21+
22+
create unique index dependency_type_lkup_dependency_type_name_uindex
23+
on process_tracker.dependency_type_lkup (dependency_type_name);
24+
1025
create table dataset_type_lkup
1126
(
1227
dataset_type_id serial not null
@@ -406,6 +421,9 @@ create table process_dependency
406421
references process,
407422
constraint process_dependency_pk
408423
primary key (child_process_id, parent_process_id),
424+
dependency_type_id int default 1 not null
425+
constraint process_dependency_fk03
426+
references dependency_type_lkup,
409427
created_date_time timestamp with time zone default CURRENT_TIMESTAMP not null,
410428
created_by integer default 0 not null,
411429
update_date_time timestamp with time zone default CURRENT_TIMESTAMP not null,
@@ -1036,6 +1054,9 @@ create table extract_dependency
10361054
child_extract_id integer not null
10371055
constraint extract_dependency_fk02
10381056
references extract_tracking,
1057+
dependency_type_id int default 1 not null
1058+
constraint extract_dependency_fk03
1059+
references dependency_type_lkup,
10391060
constraint extract_dependency_pk
10401061
primary key (parent_extract_id, child_extract_id),
10411062
created_date_time timestamp with time zone default CURRENT_TIMESTAMP not null,
@@ -1136,6 +1157,8 @@ CREATE TRIGGER data_type_lkup_update_date_time_trg BEFORE UPDATE
11361157
ON process_tracker.data_type_lkup FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger();
11371158
CREATE TRIGGER dataset_type_lkup_update_date_time_trg BEFORE UPDATE
11381159
ON process_tracker.dataset_type_lkup FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger();
1160+
CREATE TRIGGER dependency_type_lkup_update_date_time_trg BEFORE UPDATE
1161+
ON process_tracker.dependency_type_lkup FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger();
11391162
CREATE TRIGGER error_tracking_update_date_time_trg BEFORE UPDATE
11401163
ON process_tracker.error_tracking FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger();
11411164
CREATE TRIGGER error_type_lkup_update_date_time_trg BEFORE UPDATE

‎dbscripts/postgresql_process_tracker_defaults.sql‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,4 +52,8 @@ INSERT INTO process_tracker.filesize_type_lkup (filesize_type_id, filesize_type_
5252
INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (1, 'Undefined');
5353
INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (2, 'Database');
5454
INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (3, 'Internal');
55-
INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External');
55+
INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External');
56+
57+
INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (1, 'Undefined');
58+
INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (2, 'Hard');
59+
INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (3, 'Soft');

‎process_tracker/models/extract.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,12 @@ class ExtractDependency(Base, BaseColumn):
211211
primary_key=True,
212212
nullable=False,
213213
)
214+
dependency_type_id = Column(
215+
Integer,
216+
ForeignKey("process_tracker.dependency_type_lkup.dependency_type_id"),
217+
nullable=False,
218+
default=1,
219+
)
214220

215221
child_extract = relationship("Extract", foreign_keys=[child_extract_id])
216222
parent_extract = relationship("Extract", foreign_keys=[parent_extract_id])

‎process_tracker/models/process.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,26 @@
1818
from process_tracker.models.model_base import default_date, Base, BaseColumn
1919

2020

21+
class DependencyType(Base, BaseColumn):
22+
23+
__tablename__ = "dependency_type_lkup"
24+
__table_args__ = {"schema": "process_tracker"}
25+
26+
dependency_type_id = Column(
27+
Integer,
28+
Sequence(
29+
"dependency_type_lkup_dependency_type_id_seq", schema="process_tracker"
30+
),
31+
primary_key=True,
32+
nullable=False,
33+
)
34+
dependency_type_name = Column(String(75), unique=True, nullable=False)
35+
36+
def __repr__(self):
37+
38+
return "<DependencyType (name=%s)>" % self.dependency_type_name
39+
40+
2141
class ErrorType(Base, BaseColumn):
2242

2343
__tablename__ = "error_type_lkup"
@@ -473,6 +493,12 @@ class ProcessDependency(Base, BaseColumn):
473493
primary_key=True,
474494
nullable=False,
475495
)
496+
dependency_type_id = Column(
497+
Integer,
498+
ForeignKey("process_tracker.dependency_type_lkup.dependency_type_id"),
499+
nullable=False,
500+
default=1,
501+
)
476502

477503
child_process = relationship("Process", foreign_keys=[child_process_id])
478504
parent_process = relationship("Process", foreign_keys=[parent_process_id])

‎process_tracker/utilities/aws_utilities.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@ def __init__(self):
1515
self.logger = logging.getLogger(__name__)
1616
self.logger.setLevel(self.log_level)
1717

18+
logging.getLogger("boto3").setLevel(logging.CRITICAL)
19+
logging.getLogger("botocore").setLevel(logging.CRITICAL)
20+
logging.getLogger("s3transfer").setLevel(logging.CRITICAL)
21+
logging.getLogger("urllib3").setLevel(logging.CRITICAL)
22+
1823
self.s3 = boto3.resource("s3")
1924

2025
self.url_match = re.compile(

0 commit comments

Comments
 (0)