Skip to content

Commit 4a21db1

Browse files
OpenDataAlexOpenDataAlex
committed
Configuring build stack
👷 Continuing to configure/debug build stack. Continuing to configure/debug build stack.
1 parent 34b31a5 commit 4a21db1

File tree

4 files changed

+363
-1
lines changed

4 files changed

+363
-1
lines changed

‎.idea/sqldialects.xml‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.travis.yml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,14 @@ python:
44
- "3.7"
55
install:
66
- "make"
7+
services:
8+
- postgresql
79
script:
810
- make test
911
after_success:
10-
- coveralls
12+
- coveralls
13+
before_script:
14+
- psql -f dbscripts/postgresql_account_create.sql process_tracker -U postgres
15+
- psql -f dbscripts/postgresql_process_tracker.sql process_tracker -U postgres
16+
env:
17+
- process_tracking_data_store_host= process_tracking_data_store_port=5432 process_tracking_data_store_name=process_tracking process_tracking_data_store_type=postgresql process_tracking_data_store_password=Testing1! proccess_tracking_data_store_username=pt_admin

‎dbscripts/postgresql_account_create.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
create database process_tracking;
2+
create user pt_admin with password 'Testing1!';
3+
grant all privileges on database process_tracking to pt_admin;

‎dbscripts/postgresql_process_tracker.sql‎

Lines changed: 345 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,345 @@
1+
SET search_path TO process_tracking;
2+
3+
create schema process_tracking;
4+
5+
alter schema process_tracking owner to pt_admin;
6+
7+
create sequence actor_lkup_actor_id_seq;
8+
9+
alter sequence actor_lkup_actor_id_seq owner to pt_admin;
10+
11+
create table location_lkup
12+
(
13+
location_id serial not null
14+
constraint location_lkup_pk
15+
primary key,
16+
location_name varchar(750) not null,
17+
location_path varchar(750) not null
18+
);
19+
20+
comment on table location_lkup is 'Locations where files are located.';
21+
22+
alter table location_lkup owner to pt_admin;
23+
24+
create unique index location_lkup_udx01
25+
on location_lkup (location_name);
26+
27+
create unique index location_lkup_udx02
28+
on location_lkup (location_path);
29+
30+
create table error_type_lkup
31+
(
32+
error_type_id serial not null
33+
constraint error_type_lkup_pk
34+
primary key,
35+
error_type_name varchar(250) not null
36+
);
37+
38+
comment on table error_type_lkup is 'Types of errors that are being tracked.';
39+
40+
comment on column error_type_lkup.error_type_name is 'Unique error type name.';
41+
42+
alter table error_type_lkup owner to pt_admin;
43+
44+
create unique index error_type_lkup_udx01
45+
on error_type_lkup (error_type_name);
46+
47+
create table error_tracking
48+
(
49+
error_tracking_id serial not null
50+
constraint error_tracking_pk
51+
primary key,
52+
error_type_id integer not null,
53+
process_tracking_id integer not null,
54+
error_description varchar(750),
55+
error_occurrence_date_time timestamp not null
56+
);
57+
58+
comment on table error_tracking is 'Tracking of process errors';
59+
60+
comment on column error_tracking.error_type_id is 'The type of error being recorded.';
61+
62+
comment on column error_tracking.process_tracking_id is 'The specific process run that triggered the error.';
63+
64+
comment on column error_tracking.error_occurrence_date_time is 'The timestamp when the error occurred.';
65+
66+
alter table error_tracking owner to pt_admin;
67+
68+
create index error_tracking_idx01
69+
on error_tracking (process_tracking_id, error_type_id);
70+
71+
create table tool_lkup
72+
(
73+
tool_id serial not null
74+
constraint tool_lkup_pk
75+
primary key,
76+
tool_name varchar(250) not null
77+
);
78+
79+
comment on table tool_lkup is 'List of tools that are used to run processes';
80+
81+
alter table tool_lkup owner to pt_admin;
82+
83+
create unique index tool_lkup_tool_udx01
84+
on tool_lkup (tool_name);
85+
86+
create table source_lkup
87+
(
88+
source_id serial not null
89+
constraint source_lkup_pk
90+
primary key,
91+
source_name varchar(250) not null
92+
);
93+
94+
comment on table source_lkup is 'Source system where data originates.';
95+
96+
alter table source_lkup owner to pt_admin;
97+
98+
create unique index source_lkup_udx01
99+
on source_lkup (source_name);
100+
101+
create table process_status_lkup
102+
(
103+
process_status_id serial not null
104+
constraint process_status_lkup_pk
105+
primary key,
106+
process_status_name varchar(75) not null
107+
);
108+
109+
comment on table process_status_lkup is 'Process status states';
110+
111+
alter table process_status_lkup owner to pt_admin;
112+
113+
create unique index process_status_lkup_udx01
114+
on process_status_lkup (process_status_name);
115+
116+
create table process_type_lkup
117+
(
118+
process_type_id serial not null
119+
constraint process_type_lkup_pk
120+
primary key,
121+
process_type_name varchar(250) not null
122+
);
123+
124+
comment on table process_type_lkup is 'Valid process types for processes';
125+
126+
comment on column process_type_lkup.process_type_name is 'Unique process type name.';
127+
128+
alter table process_type_lkup owner to pt_admin;
129+
130+
create unique index process_type_lkup_udx01
131+
on process_type_lkup (process_type_name);
132+
133+
create table process
134+
(
135+
process_id serial not null
136+
constraint process_pk
137+
primary key,
138+
process_name varchar(250) not null,
139+
process_source_id integer not null
140+
constraint process_fk01
141+
references source_lkup,
142+
total_record_count integer default 0 not null,
143+
process_type_id integer not null
144+
constraint process_fk02
145+
references process_type_lkup,
146+
process_tool_id integer not null
147+
constraint process_fk03
148+
references tool_lkup,
149+
last_failed_run_date_time timestamp default '1900-01-01 00:00:00'::timestamp without time zone not null
150+
);
151+
152+
comment on table process is 'Processes being tracked';
153+
154+
comment on column process.process_name is 'Unique name for process.';
155+
156+
comment on column process.process_source_id is 'The source that the process is extracting from.';
157+
158+
comment on column process.total_record_count is 'Total number of records processed over all runs of process.';
159+
160+
comment on column process.process_type_id is 'The type of process being tracked.';
161+
162+
comment on column process.process_tool_id is 'The type of tool used to execute the process.';
163+
164+
comment on column process.last_failed_run_date_time is 'The last time the process failed to run.';
165+
166+
alter table process owner to pt_admin;
167+
168+
create unique index process_udx01
169+
on process (process_name);
170+
171+
create table process_dependency
172+
(
173+
parent_process_id integer not null
174+
constraint process_dependency_fk01
175+
references process,
176+
child_process_id integer not null
177+
constraint process_dependency_pk
178+
primary key
179+
constraint process_dependency_fk02
180+
references process
181+
);
182+
183+
comment on table process_dependency is 'Dependency tracking between processes.';
184+
185+
comment on column process_dependency.parent_process_id is 'The parent process.';
186+
187+
comment on column process_dependency.child_process_id is 'The child process.';
188+
189+
alter table process_dependency owner to pt_admin;
190+
191+
create table actor_lkup
192+
(
193+
actor_id serial not null
194+
constraint actor_lkup_pk
195+
primary key,
196+
actor_name varchar(250) not null
197+
);
198+
199+
comment on table actor_lkup is 'List of developers or applications that can run processes.';
200+
201+
alter table actor_lkup owner to pt_admin;
202+
203+
create unique index actor_lkup_udx01
204+
on actor_lkup (actor_name);
205+
206+
create table process_tracking
207+
(
208+
process_tracking_id serial not null
209+
constraint process_tracking_pk
210+
primary key,
211+
process_id integer not null
212+
constraint process_tracking_fk01
213+
references process,
214+
process_status_id integer not null
215+
constraint process_tracking_fk02
216+
references process_status_lkup,
217+
process_run_id integer default 0 not null,
218+
process_run_low_date_time timestamp,
219+
process_run_high_date_time timestamp,
220+
process_run_start_date_time timestamp not null,
221+
process_run_end_date_time timestamp,
222+
process_run_record_count integer default 0,
223+
process_run_actor_id integer
224+
constraint process_tracking_fk03
225+
references actor_lkup,
226+
is_latest_run boolean default false
227+
);
228+
229+
comment on table process_tracking is 'Tracking table of process runs.';
230+
231+
comment on column process_tracking.process_id is 'The process that is being run.';
232+
233+
comment on column process_tracking.process_status_id is 'The current status of the given process run.';
234+
235+
comment on column process_tracking.process_run_id is 'The unique run identifier for the process. Sequential to the unique process.';
236+
237+
comment on column process_tracking.process_run_low_date_time is 'The lowest datetime provided by the extract dataset being processed.';
238+
239+
comment on column process_tracking.process_run_high_date_time is 'The highest datetime provided by the extract dataset being processed.';
240+
241+
comment on column process_tracking.process_run_start_date_time is 'The datetime which the process run kicked off.';
242+
243+
comment on column process_tracking.process_run_end_date_time is 'The datetime which the process run ended (either in failure or success).';
244+
245+
comment on column process_tracking.process_run_record_count is 'The number of unique records processed by the run.';
246+
247+
comment on column process_tracking.process_run_actor_id is 'The actor who kicked the process run off.';
248+
249+
comment on column process_tracking.is_latest_run is 'Flag for determining if the run record is the latest for a given process.';
250+
251+
alter table process_tracking owner to pt_admin;
252+
253+
create index process_tracking_idx01
254+
on process_tracking (process_id, process_status_id);
255+
256+
create index process_tracking_idx02
257+
on process_tracking (process_run_start_date_time, process_run_end_date_time);
258+
259+
create index process_tracking_idx03
260+
on process_tracking (process_run_low_date_time, process_run_high_date_time);
261+
262+
create table extract_status_lkup
263+
(
264+
extract_status_id serial not null
265+
constraint extract_status_lkup_pk
266+
primary key,
267+
extract_status_name varchar(75) not null
268+
);
269+
270+
comment on table extract_status_lkup is 'List of valid extract processing statuses.';
271+
272+
alter table extract_status_lkup owner to pt_admin;
273+
274+
create unique index extract_status_lkup_extract_status_name_uindex
275+
on extract_status_lkup (extract_status_name);
276+
277+
create table extract_tracking
278+
(
279+
extract_id serial not null
280+
constraint extract_tracking_pk
281+
primary key,
282+
extract_source_id integer not null,
283+
extract_filename varchar(750) not null,
284+
extract_location_id integer not null
285+
constraint extract_tracking_fk01
286+
references location_lkup,
287+
extract_process_run_id integer
288+
constraint extract_tracking_fk03
289+
references process_tracking,
290+
extract_status_id integer
291+
constraint extract_tracking_fk02
292+
references extract_status_lkup,
293+
extract_registration_date_time timestamp not null
294+
);
295+
296+
comment on table extract_tracking is 'Tracking table for all extract/staging data files.';
297+
298+
comment on column extract_tracking.extract_source_id is 'Source identifier (source_lkup) for where the extract originated.';
299+
300+
comment on column extract_tracking.extract_filename is 'The unique filename for a given extract from a given source.';
301+
302+
comment on column extract_tracking.extract_location_id is 'The location where the given extract can be found.';
303+
304+
comment on column extract_tracking.extract_process_run_id is 'The process that registered or created the extract file.';
305+
306+
comment on column extract_tracking.extract_status_id is 'The status of the extract.';
307+
308+
comment on column extract_tracking.extract_registration_date_time is 'The datetime that the extract was loaded into extract tracking.';
309+
310+
alter table extract_tracking owner to pt_admin;
311+
312+
create unique index extract_tracking_udx01
313+
on extract_tracking (extract_source_id, extract_filename);
314+
315+
create table extract_process_tracking
316+
(
317+
extract_tracking_id integer not null
318+
constraint extract_process_tracking_fk01
319+
references extract_tracking,
320+
process_tracking_id integer not null
321+
constraint extract_process_tracking_fk02
322+
references process_tracking,
323+
extract_process_event_date_time timestamp with time zone not null,
324+
extract_process_status_id integer
325+
constraint extract_process_tracking_fk03
326+
references extract_status_lkup,
327+
constraint extract_process_tracking_pk
328+
primary key (process_tracking_id, extract_tracking_id)
329+
);
330+
331+
comment on table extract_process_tracking is 'Showing which processes have impacted which extracts';
332+
333+
alter table extract_process_tracking owner to pt_admin;
334+
335+
INSERT INTO process_tracking.extract_status_lkup (extract_status_id, extract_status_name) VALUES (1, 'initializing');
336+
INSERT INTO process_tracking.extract_status_lkup (extract_status_id, extract_status_name) VALUES (2, 'ready');
337+
INSERT INTO process_tracking.extract_status_lkup (extract_status_id, extract_status_name) VALUES (3, 'loading');
338+
INSERT INTO process_tracking.extract_status_lkup (extract_status_id, extract_status_name) VALUES (4, 'loaded');
339+
INSERT INTO process_tracking.extract_status_lkup (extract_status_id, extract_status_name) VALUES (5, 'archived');
340+
INSERT INTO process_tracking.extract_status_lkup (extract_status_id, extract_status_name) VALUES (6, 'deleted');
341+
INSERT INTO process_tracking.extract_status_lkup (extract_status_id, extract_status_name) VALUES (7, 'error');
342+
343+
INSERT INTO process_tracking.process_status_lkup (process_status_id, process_status_name) VALUES (1, 'running');
344+
INSERT INTO process_tracking.process_status_lkup (process_status_id, process_status_name) VALUES (2, 'completed');
345+
INSERT INTO process_tracking.process_status_lkup (process_status_id, process_status_name) VALUES (3, 'failed');

0 commit comments

Comments
 (0)