Skip to content

Commit a4c7cfe

Browse files
OpenDataAlexOpenDataAlex
committed
process_tracker_python-1 Add support for other relational databases
✨ MySQL support verified Tested with MySQL. Found issue with datetime columns with MySQL where sub-seconds are not tracked by default. While SQLAlchemy does have the ability to take advantage of newer MySQL versions allowing for sub-seconds, instead of adding complexity to the data model classes the decision was made to NOT support sub-seconds at this time in MySQL. If someone else wants to add it, I'm fine, but hit circular import problem when determining data store type. Working on #1
1 parent ba3601b commit a4c7cfe

15 files changed

+388
-50
lines changed

‎.idea/inspectionProfiles/Project_Default.xml‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.travis.yml‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,23 @@ install:
66
- make
77
services:
88
- postgresql
9+
- mysql
910
script:
1011
- make test
1112
after_success:
1213
- coveralls
1314
before_script:
1415
- mkdir $HOME/.process_tracker
1516
- mv configs/${database_type}_config.ini $HOME/.process_tracker/process_tracker_config.ini
16-
- psql -f dbscripts/postgresql_account_create.sql -U postgres
17-
- psql -f dbscripts/postgresql_process_tracker.sql process_tracking -U postgres
18-
- psql -f dbscripts/postgresql_process_tracker_defaults.sql process_tracking -U postgres
17+
- sh -c "if [ '$DB' == 'postgresq' ]; then psql -f dbscripts/postgresql_account_create.sql -U postgres; fi"
18+
- sh -c "if [ '$DB' == 'postgresq' ]; then psql -f dbscripts/postgresql_process_tracker.sql process_tracking -U postgres; fi"
19+
- sh -c "if [ '$DB' == 'postgresq' ]; then psql -f dbscripts/postgresqlpostgresql_process_tracker_defaults.sql_process_tracker_defaults.sql process_tracking -U postgres; fi"
20+
- sh -c "if [ '$DB' == 'mysql' ]; then mysql -u root < dbscripts/mysql_account_create.sql; fi"
21+
- sh -c "if [ '$DB' == 'mysql' ]; then mysql -u root < dbscripts/mysql_process_tracker.sql; fi"
22+
- sh -c "if [ '$DB' == 'mysql' ]; then mysql -u root < dbscripts/mysql_process_tracker_defaults.sql; fi"
1923
env:
20-
- database_type=postgresql
24+
- DB=postgres database_type=postgresql
25+
- DB=mysql database_type=mysql
2126
deploy:
2227
- provider: releases
2328
api_key:

‎Pipfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ sqlalchemy-utils="*"
1717
python-dateutil="*"
1818
psycopg2-binary="*"
1919
google-compute-engine = "*"
20+
snowflake-sqlalchemy = "*"
21+
pymysql = "*"
2022

2123
[requires]
2224
python_version = "3.7"

‎configs/mysql_config.ini‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[DEFAULT]
2+
log_level = ERROR
3+
data_store_type = mysql
4+
data_store_username = pt_admin
5+
data_store_password = Testing1!
6+
data_store_host = localhost
7+
data_store_port = 3306
8+
data_store_name = process_tracker

‎configs/postgresql_config.ini‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@ data_store_username = pt_admin
55
data_store_password = Testing1!
66
data_store_host = localhost
77
data_store_port = 5432
8-
data_store_name = process_tracking
8+
data_store_name = process_tracker

‎dbscripts/mysql_account_create.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
drop database if exists process_tracker;
2+
create database process_tracker;
3+
create user pt_admin identified by 'Testing1!';
4+
grant all privileges on process_tracker.* to pt_admin;

‎dbscripts/mysql_process_tracker.sql‎

Lines changed: 262 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
USE process_tracker;
2+
3+
4+
create schema process_tracker collate utf8mb4_0900_ai_ci;
5+
6+
create table actor_lkup
7+
(
8+
actor_id int auto_increment
9+
primary key,
10+
actor_name varchar(250) not null,
11+
constraint actor_name
12+
unique (actor_name)
13+
);
14+
15+
create table error_type_lkup
16+
(
17+
error_type_id int auto_increment
18+
primary key,
19+
error_type_name varchar(250) not null,
20+
constraint error_type_name
21+
unique (error_type_name)
22+
);
23+
24+
create table extract_status_lkup
25+
(
26+
extract_status_id int auto_increment
27+
primary key,
28+
extract_status_name varchar(75) not null,
29+
constraint extract_status_name
30+
unique (extract_status_name)
31+
);
32+
33+
create table location_type_lkup
34+
(
35+
location_type_id int auto_increment
36+
primary key,
37+
location_type_name varchar(25) not null,
38+
constraint location_type_name
39+
unique (location_type_name)
40+
);
41+
42+
create table location_lkup
43+
(
44+
location_id int auto_increment
45+
primary key,
46+
location_name varchar(750) not null,
47+
location_path varchar(750) not null,
48+
location_type int null,
49+
constraint location_name
50+
unique (location_name),
51+
constraint location_path
52+
unique (location_path),
53+
constraint location_lkup_ibfk_1
54+
foreign key (location_type) references location_type_lkup (location_type_id)
55+
);
56+
57+
create table extract_tracking
58+
(
59+
extract_id int auto_increment
60+
primary key,
61+
extract_filename varchar(750) not null,
62+
extract_location_id int null,
63+
extract_status_id int null,
64+
extract_registration_date_time datetime not null,
65+
constraint extract_filename
66+
unique (extract_filename),
67+
constraint extract_tracking_ibfk_1
68+
foreign key (extract_location_id) references location_lkup (location_id),
69+
constraint extract_tracking_ibfk_2
70+
foreign key (extract_status_id) references extract_status_lkup (extract_status_id)
71+
);
72+
73+
create index extract_location_id
74+
on extract_tracking (extract_location_id);
75+
76+
create index extract_status_id
77+
on extract_tracking (extract_status_id);
78+
79+
create index location_type
80+
on location_lkup (location_type);
81+
82+
create table process_status_lkup
83+
(
84+
process_status_id int auto_increment
85+
primary key,
86+
process_status_name varchar(75) not null,
87+
constraint process_status_name
88+
unique (process_status_name)
89+
);
90+
91+
create table process_type_lkup
92+
(
93+
process_type_id int auto_increment
94+
primary key,
95+
process_type_name varchar(250) not null
96+
);
97+
98+
create table source_lkup
99+
(
100+
source_id int auto_increment
101+
primary key,
102+
source_name varchar(250) not null,
103+
constraint source_name
104+
unique (source_name)
105+
);
106+
107+
create table system_lkup
108+
(
109+
system_id int auto_increment
110+
primary key,
111+
system_key varchar(250) not null,
112+
system_value varchar(250) not null,
113+
constraint system_key
114+
unique (system_key)
115+
);
116+
117+
create table tool_lkup
118+
(
119+
tool_id int auto_increment
120+
primary key,
121+
tool_name varchar(250) not null,
122+
constraint tool_name
123+
unique (tool_name)
124+
);
125+
126+
create table process
127+
(
128+
process_id int auto_increment
129+
primary key,
130+
process_name varchar(250) not null,
131+
total_record_count int not null,
132+
process_type_id int null,
133+
process_tool_id int null,
134+
last_failed_run_date_time datetime not null,
135+
constraint process_name
136+
unique (process_name),
137+
constraint process_ibfk_1
138+
foreign key (process_type_id) references process_type_lkup (process_type_id),
139+
constraint process_ibfk_2
140+
foreign key (process_tool_id) references tool_lkup (tool_id)
141+
);
142+
143+
create index process_tool_id
144+
on process (process_tool_id);
145+
146+
create index process_type_id
147+
on process (process_type_id);
148+
149+
create table process_dependency
150+
(
151+
parent_process_id int not null,
152+
child_process_id int not null,
153+
primary key (parent_process_id, child_process_id),
154+
constraint process_dependency_ibfk_1
155+
foreign key (parent_process_id) references process (process_id),
156+
constraint process_dependency_ibfk_2
157+
foreign key (child_process_id) references process (process_id)
158+
);
159+
160+
create index child_process_id
161+
on process_dependency (child_process_id);
162+
163+
create table process_source
164+
(
165+
source_id int not null,
166+
process_id int not null,
167+
primary key (source_id, process_id),
168+
constraint process_source_ibfk_1
169+
foreign key (source_id) references source_lkup (source_id),
170+
constraint process_source_ibfk_2
171+
foreign key (process_id) references process (process_id)
172+
);
173+
174+
create index process_id
175+
on process_source (process_id);
176+
177+
create table process_target
178+
(
179+
target_source_id int not null,
180+
process_id int not null,
181+
primary key (target_source_id, process_id),
182+
constraint process_target_ibfk_1
183+
foreign key (target_source_id) references source_lkup (source_id),
184+
constraint process_target_ibfk_2
185+
foreign key (process_id) references process (process_id)
186+
);
187+
188+
create index process_id
189+
on process_target (process_id);
190+
191+
create table process_tracking
192+
(
193+
process_tracking_id int auto_increment
194+
primary key,
195+
process_id int null,
196+
process_status_id int null,
197+
process_run_id int not null,
198+
process_run_low_date_time datetime null,
199+
process_run_high_date_time datetime null,
200+
process_run_start_date_time datetime not null,
201+
process_run_end_date_time datetime null,
202+
process_run_record_count int not null,
203+
process_run_actor_id int null,
204+
is_latest_run tinyint(1) not null,
205+
constraint process_tracking_ibfk_1
206+
foreign key (process_id) references process (process_id),
207+
constraint process_tracking_ibfk_2
208+
foreign key (process_status_id) references process_status_lkup (process_status_id),
209+
constraint process_tracking_ibfk_3
210+
foreign key (process_run_actor_id) references actor_lkup (actor_id)
211+
);
212+
213+
create table error_tracking
214+
(
215+
error_tracking_id int auto_increment
216+
primary key,
217+
error_type_id int null,
218+
error_description varchar(750) null,
219+
error_occurrence_date_time datetime not null,
220+
process_tracking_id int null,
221+
constraint error_tracking_ibfk_1
222+
foreign key (error_type_id) references error_type_lkup (error_type_id),
223+
constraint error_tracking_ibfk_2
224+
foreign key (process_tracking_id) references process_tracking (process_tracking_id)
225+
);
226+
227+
create index error_type_id
228+
on error_tracking (error_type_id);
229+
230+
create index process_tracking_id
231+
on error_tracking (process_tracking_id);
232+
233+
create table extract_process_tracking
234+
(
235+
extract_tracking_id int not null,
236+
process_tracking_id int not null,
237+
extract_process_status_id int null,
238+
extract_process_event_date_time datetime not null,
239+
primary key (extract_tracking_id, process_tracking_id),
240+
constraint extract_process_tracking_ibfk_1
241+
foreign key (extract_tracking_id) references extract_tracking (extract_id),
242+
constraint extract_process_tracking_ibfk_2
243+
foreign key (process_tracking_id) references process_tracking (process_tracking_id),
244+
constraint extract_process_tracking_ibfk_3
245+
foreign key (extract_process_status_id) references extract_status_lkup (extract_status_id)
246+
);
247+
248+
create index extract_process_status_id
249+
on extract_process_tracking (extract_process_status_id);
250+
251+
create index process_tracking_id
252+
on extract_process_tracking (process_tracking_id);
253+
254+
create index process_id
255+
on process_tracking (process_id);
256+
257+
create index process_run_actor_id
258+
on process_tracking (process_run_actor_id);
259+
260+
create index process_status_id
261+
on process_tracking (process_status_id);
262+

‎dbscripts/mysql_process_tracker_defaults.sql‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
USE process_tracker;
2+
3+
INSERT INTO process_tracker.extract_status_lkup (extract_status_id, extract_status_name) VALUES (default, 'initializing');
4+
INSERT INTO process_tracker.extract_status_lkup (extract_status_id, extract_status_name) VALUES (default, 'ready');
5+
INSERT INTO process_tracker.extract_status_lkup (extract_status_id, extract_status_name) VALUES (default, 'loading');
6+
INSERT INTO process_tracker.extract_status_lkup (extract_status_id, extract_status_name) VALUES (default, 'loaded');
7+
INSERT INTO process_tracker.extract_status_lkup (extract_status_id, extract_status_name) VALUES (default, 'archived');
8+
INSERT INTO process_tracker.extract_status_lkup (extract_status_id, extract_status_name) VALUES (default, 'deleted');
9+
INSERT INTO process_tracker.extract_status_lkup (extract_status_id, extract_status_name) VALUES (default, 'error');
10+
11+
INSERT INTO process_tracker.process_status_lkup (process_status_id, process_status_name) VALUES (default, 'running');
12+
INSERT INTO process_tracker.process_status_lkup (process_status_id, process_status_name) VALUES (default, 'completed');
13+
INSERT INTO process_tracker.process_status_lkup (process_status_id, process_status_name) VALUES (default, 'failed');
14+
15+
INSERT INTO process_tracker.error_type_lkup (error_type_id, error_type_name) VALUES (default, 'File Error');
16+
INSERT INTO process_tracker.error_type_lkup (error_type_id, error_type_name) VALUES (default, 'Data Error');
17+
INSERT INTO process_tracker.error_type_lkup (error_type_id, error_type_name) VALUES (default, 'Process Error');
18+
19+
INSERT INTO process_tracker.process_type_lkup (process_type_id, process_type_name) VALUES (default, 'Extract');
20+
INSERT INTO process_tracker.process_type_lkup (process_type_id, process_type_name) VALUES (default, 'Load');
21+
22+
INSERT INTO process_tracker.system_lkup (system_id, system_key, system_value) VALUES (default, 'version', '0.1.0');

‎dbscripts/postgresql_account_create.sql‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
drop database if exists process_tracking;
2-
create database process_tracking;
1+
drop database if exists process_tracker;
2+
create database process_tracker;
33
create user pt_admin with password 'Testing1!';
4-
grant all privileges on database process_tracking to pt_admin;
4+
grant all privileges on database process_tracker to pt_admin;

‎dbscripts/postgresql_process_tracker.sql‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
SET search_path TO process_tracking;
1+
SET search_path TO process_tracker;
22

3-
create schema process_tracking;
3+
create schema process_tracker;
44

5-
alter schema process_tracking owner to pt_admin;
5+
alter schema process_tracker owner to pt_admin;
66

77
create table error_type_lkup
88
(

0 commit comments

Comments
 (0)