Merge pull request #35 from OpenDataAlex/process_tracker_python-14

Process tracker python 14

Author: OpenDataAlex

.idea/watcherTasks.xml

@@ -5,9 +5,9 @@ verify_ssl = true
 
 [dev-packages]
 coverage="*"
+coveralls="*"
 moto="*"
 python-coveralls="*"
-coveralls="*"
 
 [packages]
 boto3="*"

Pipfile

@@ -4,6 +4,10 @@ Data integration process management made easy!
 
 [![Coverage Status](https://coveralls.io/repos/github/OpenDataAlex/process_tracker_python/badge.svg?branch=master)](https://coveralls.io/github/OpenDataAlex/process_tracker_python?branch=master)
 [![Build Status](https://travis-ci.org/OpenDataAlex/process_tracker_python.svg?branch=master)](https://travis-ci.org/OpenDataAlex/process_tracker_python)
+[![Downloads](https://pepy.tech/badge/processtracker)](https://pepy.tech/project/processtracker)
+[![PyPI version](https://badge.fury.io/py/processtracker.svg)](https://badge.fury.io/py/processtracker)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black)
+[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
 This is the Python implementation of the ProcessTracker framework.  ProcessTracker builds a standard framework that is
 tool agnostic.  If you are working with data integration/cleansing processes within Python (i.e. using PySpark, Pandas, etc.)

README.md

@@ -76,6 +76,18 @@ create index extract_status_id
 create index location_type
 	on location_lkup (location_type);
 
+create table process_tracker.extract_dependency
+(
+	parent_extract_id int not null,
+	child_extract_id int not null,
+	primary key (parent_extract_id, child_extract_id),
+	constraint extract_dependency_fk01
+		foreign key (parent_extract_id) references process_tracker.extract_tracking (extract_id),
+	constraint extract_dependency_fk02
+		foreign key (child_extract_id) references process_tracker.extract_tracking (extract_id)
+)
+comment 'Table tracking interdependencies between extract files.';
+
 create table process_status_lkup
 (
 	process_status_id int auto_increment

dbscripts/mysql_process_tracker.sql

@@ -258,6 +258,9 @@ comment on table location_type_lkup is 'Listing of location types';
 
 alter table location_type_lkup owner to pt_admin;
 
+create unique index location_type_lkup_udx01
+	on location_type_lkup (location_type_name);
+
 create table location_lkup
 (
 	location_id serial not null
@@ -332,9 +335,6 @@ comment on table extract_process_tracking is 'Showing which processes have impac
 
 alter table extract_process_tracking owner to pt_admin;
 
-create unique index location_type_lkup_udx01
-	on location_type_lkup (location_type_name);
-
 create table process_source
 (
 	source_id integer not null
@@ -382,3 +382,20 @@ alter table system_lkup owner to pt_admin;
 
 create unique index system_lkup_system_key_uindex
 	on system_lkup (system_key);
+
+create table extract_dependency
+(
+	parent_extract_id integer not null
+		constraint extract_dependency_fk01
+			references extract_tracking,
+	child_extract_id integer not null
+		constraint extract_dependency_fk02
+			references extract_tracking,
+	constraint extract_dependency_pk
+		primary key (parent_extract_id, child_extract_id)
+);
+
+comment on table extract_dependency is 'Table tracking interdependencies between extract files.';
+
+alter table extract_dependency owner to pt_admin;
+

dbscripts/postgresql_process_tracker.sql

@@ -1,4 +1,4 @@
 from process_tracker.extract_tracker import ExtractTracker
 from process_tracker.process_tracker import ProcessTracker
 
-__version__ = '0.2.0'
+__version__ = "0.2.0"

process_tracker/__init__.py

@@ -8,7 +8,7 @@
 from process_tracker.utilities.logging import console
 
 data_store = DataStore()
-logger = logging.getLogger('Process Tracker')
+logger = logging.getLogger("Process Tracker")
 logger.addHandler(console)
 
 
@@ -42,8 +42,8 @@ def main():
 
 
 @main.command()
-@click.option('-t', '--topic', help='The topic being created')
-@click.option('-n', '--name', help='The name for the topic.')
+@click.option("-t", "--topic", help="The topic being created")
+@click.option("-n", "--name", help="The name for the topic.")
 def create(topic, name):
     """
     Create an item that is within the valid topics list.
@@ -52,13 +52,13 @@ def create(topic, name):
     :param name: The name of the topic item to be added.
     :type name: string
     """
-    click.echo('Attempting to create %s with name %s' % (topic, name))
+    click.echo("Attempting to create %s with name %s" % (topic, name))
     data_store.topic_creator(topic=topic, name=name)
 
 
 @main.command()
-@click.option('-t', '--topic', help='The topic being created')
-@click.option('-n', '--name', help='The name for the topic.')
+@click.option("-t", "--topic", help="The topic being created")
+@click.option("-n", "--name", help="The name for the topic.")
 def delete(topic, name):
     """
     Delete an item that is within the valid topics list and not a pre-loaded item.
@@ -67,15 +67,17 @@ def delete(topic, name):
     :param name: The name of the topic item to be deleted.
     :type name: string
     """
-    click.echo('Attempting to delete %s with name %s' % (topic, name))
+    click.echo("Attempting to delete %s with name %s" % (topic, name))
     data_store.topic_deleter(topic=topic, name=name)
 
 
 @main.command()
-@click.option('-t', '--topic', help='The topic being created')
-@click.option('-i', '--initial-name', help='The name that needs to be changed.')
-@click.option('-n', '--name', help='The new name for the topic.')
+@click.option("-t", "--topic", help="The topic being created")
+@click.option("-i", "--initial-name", help="The name that needs to be changed.")
+@click.option("-n", "--name", help="The new name for the topic.")
 def update(topic, initial_name, name):
 
-    click.echo('Attempting to update %s with name %s to %s' % (topic, initial_name, name))
+    click.echo(
+        "Attempting to update %s with name %s to %s" % (topic, initial_name, name)
+    )
     data_store.topic_updater(topic=topic, initial_name=initial_name, name=name)