Merge pull request #36 from Satvik-Singh192/test/tableplus

chore: verified data load in tableplus

Author: Dheerajyadav1

.gitignore

@@ -6,4 +6,12 @@ venv/
 .venv
 ENV/
 env.bak/
-venv.bak/
+venv.bak/
+
+etl_data.db
+
+# Ignore Python cache files
+__pycache__/
+*.pyc
+*.pyo
+*.pyd

app/__pycache__/__init__.cpython-310.pyc

@@ -21,20 +21,17 @@ def load(df: pd.DataFrame, db_path: str = "etl_data.db", table_name: str = "proc
     # Ensure directory exists
     db_dir = os.path.dirname(db_path)
     if db_dir and not os.path.exists(db_dir):
-        os.makedirs(db_dir)
-    
+        os.makedirs(db_dir) 
     conn = None
     try:
         # Connect to database
         conn = sqlite3.connect(db_path)
         cursor = conn.cursor()
 
         # TODO (Find & Fix): Table creation and schema logic missing
-        
-        # Idempotency check (should avoid duplicate inserts)
         cursor.execute(f"""
         CREATE TABLE IF NOT EXISTS {table_name} (
-            employee_id INTEGER PRIMARY KEY, 
+            employee_id TEXT PRIMARY KEY, 
             name TEXT,
             email TEXT,
             age INTEGER,
@@ -50,15 +47,11 @@ def load(df: pd.DataFrame, db_path: str = "etl_data.db", table_name: str = "proc
         )
         """)
 
-        data_to_insert = [tuple(row) for row in df.itertuples(index=False, name=None)]
         placeholders = ", ".join(["?"] * len(df.columns))
         column_names = ", ".join(df.columns)
-        sql_query = f"INSERT OR IGNORE INTO {table_name} ({column_names}) VALUES ({placeholders})"
-        cursor.executemany(sql_query, data_to_insert)
+        sql_query = f"INSERT OR REPLACE INTO {table_name} ({column_names}) VALUES ({placeholders})"  
+        cursor.executemany(sql_query, df.itertuples(index=False, name=None))
         conn.commit()
-        # TODO (Find & Fix): Bulk insert without checking for duplicates
-
-        
     except sqlite3.Error as e:
         if conn:
             conn.rollback()

app/etl/__pycache__/__init__.cpython-310.pyc

@@ -1,8 +1,13 @@
+import os
+
 from app.etl.extract import extract
 from app.etl.transform import transform
 from app.etl.load import load
 
-def run_pipeline(csv_path: str = "data.csv", db_path: str = "etl_data.db"):
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+data_path = os.path.join(BASE_DIR, "data.csv")
+
+def run_pipeline(csv_path: str =data_path, db_path: str = "etl_data.db"):
     """
     Run the complete ETL pipeline.