nebuleair_pro_4g/sqlite/flush_old_data.py

'''
  ____   ___  _     _ _       
 / ___| / _ \| |   (_) |_ ___ 
 \___ \| | | | |   | | __/ _ \
  ___) | |_| | |___| | ||  __/
 |____/ \__\_\_____|_|\__\___|
                              
Script to flush (delete) data from a sqlite database

/usr/bin/python3 /var/www/nebuleair_pro_4g/sqlite/flush_old_data.py

Script that is triggered by a systemd
sudo systemctl status nebuleair-db-cleanup-data.service

Available table are

data_NPM
data_NPM_5channels
data_BME280
data_envea
timestamp_table
data_MPPT
data_NOISE
data_WIND

'''

import sqlite3
import datetime
import sys


def table_exists(cursor, table_name):
    """Check if a table exists in the database"""
    try:
        cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table_name,))
        return cursor.fetchone() is not None
    except sqlite3.Error as e:
        print(f"[ERROR] Failed to check if table '{table_name}' exists: {e}")
        return False


def get_table_count(cursor, table_name):
    """Get the number of records in a table"""
    try:
        cursor.execute(f"SELECT COUNT(*) FROM {table_name}")
        return cursor.fetchone()[0]
    except sqlite3.Error as e:
        print(f"[WARNING] Could not get count for table '{table_name}': {e}")
        return 0


def delete_old_records(cursor, table_name, cutoff_date_str):
    """Delete old records from a specific table"""
    try:
        # First check how many records will be deleted
        cursor.execute(f"SELECT COUNT(*) FROM {table_name} WHERE timestamp < ?", (cutoff_date_str,))
        records_to_delete = cursor.fetchone()[0]
        
        if records_to_delete == 0:
            print(f"[INFO] No old records to delete from '{table_name}'")
            return True
        
        # Delete the records
        cursor.execute(f"DELETE FROM {table_name} WHERE timestamp < ?", (cutoff_date_str,))
        deleted_count = cursor.rowcount
        
        print(f"[SUCCESS] Deleted {deleted_count} old records from '{table_name}'")
        return True
        
    except sqlite3.Error as e:
        print(f"[ERROR] Failed to delete records from '{table_name}': {e}")
        return False


def main():
    try:
        # Connect to the SQLite database
        print("[INFO] Connecting to database...")
        conn = sqlite3.connect("/var/www/nebuleair_pro_4g/sqlite/sensors.db")
        cursor = conn.cursor()
        
        # Check database connection
        cursor.execute("SELECT sqlite_version()")
        version = cursor.fetchone()[0]
        print(f"[INFO] Connected to SQLite version: {version}")

        # GET RTC TIME from SQLite
        print("[INFO] Getting timestamp from database...")
        
        # First check if timestamp_table exists
        if not table_exists(cursor, "timestamp_table"):
            print("[ERROR] timestamp_table does not exist!")
            return False

        cursor.execute("SELECT * FROM timestamp_table LIMIT 1")
        row = cursor.fetchone()

        if not row:
            print("[ERROR] No timestamp found in timestamp_table.")
            return False

        rtc_time_str = row[1]  # Assuming timestamp is stored as TEXT (YYYY-MM-DD HH:MM:SS)
        print(f"[INFO] Last recorded timestamp: {rtc_time_str}")

        # Convert last_updated to a datetime object
        try:
            last_updated = datetime.datetime.strptime(rtc_time_str, "%Y-%m-%d %H:%M:%S")
        except ValueError as e:
            print(f"[ERROR] Invalid timestamp format: {e}")
            return False

        # Calculate the cutoff date (60 days before last_updated)
        cutoff_date = last_updated - datetime.timedelta(days=60)
        cutoff_date_str = cutoff_date.strftime("%Y-%m-%d %H:%M:%S")

        print(f"[INFO] Deleting records older than: {cutoff_date_str}")

        # List of tables to delete old data from
        tables_to_clean = [
            "data_NPM", 
            "data_NPM_5channels", 
            "data_BME280", 
            "data_envea",
            "data_WIND", 
            "data_MPPT", 
            "data_NOISE"
        ]

        # Check which tables actually exist
        existing_tables = []
        missing_tables = []
        
        for table in tables_to_clean:
            if table_exists(cursor, table):
                existing_tables.append(table)
                record_count = get_table_count(cursor, table)
                print(f"[INFO] Table '{table}' exists with {record_count} records")
            else:
                missing_tables.append(table)
                print(f"[WARNING] Table '{table}' does not exist - skipping")

        if missing_tables:
            print(f"[INFO] Missing tables: {', '.join(missing_tables)}")
        
        if not existing_tables:
            print("[WARNING] No tables found to clean!")
            return True

        # Loop through existing tables and delete old data
        successful_deletions = 0
        failed_deletions = 0
        
        for table in existing_tables:
            if delete_old_records(cursor, table, cutoff_date_str):
                successful_deletions += 1
            else:
                failed_deletions += 1

        # Commit changes before running VACUUM
        print("[INFO] Committing changes...")
        conn.commit()
        print("[SUCCESS] Changes committed successfully!")

        # Only run VACUUM if at least some deletions were successful
        if successful_deletions > 0:
            print("[INFO] Running VACUUM to optimize database space...")
            try:
                cursor.execute("VACUUM")
                print("[SUCCESS] Database optimized successfully!")
            except sqlite3.Error as e:
                print(f"[WARNING] VACUUM failed: {e}")

        # Summary
        print(f"\n[SUMMARY]")
        print(f"Tables processed successfully: {successful_deletions}")
        print(f"Tables with errors: {failed_deletions}")
        print(f"Tables skipped (missing): {len(missing_tables)}")
        
        if failed_deletions == 0:
            print("[SUCCESS] Old data flushed successfully!")
            return True
        else:
            print("[WARNING] Some operations failed - check logs above")
            return False

    except sqlite3.Error as e:
        print(f"[ERROR] Database error: {e}")
        return False
    except Exception as e:
        print(f"[ERROR] Unexpected error: {e}")
        return False
    finally:
        # Always close the database connection
        if 'conn' in locals():
            conn.close()
            print("[INFO] Database connection closed")


if __name__ == "__main__":
    success = main()
    sys.exit(0 if success else 1)