update
This commit is contained in:
Your Name
@@ -9,6 +9,9 @@ Script to flush (delete) data from a sqlite database
|
||||
|
||||
/usr/bin/python3 /var/www/nebuleair_pro_4g/sqlite/flush_old_data.py
|
||||
|
||||
Script that is triggered by a systemd
|
||||
sudo systemctl status nebuleair-db-cleanup-data.service
|
||||
|
||||
Available table are
|
||||
|
||||
data_NPM
|
||||
@@ -17,56 +20,183 @@ data_BME280
|
||||
data_envea
|
||||
timestamp_table
|
||||
data_MPPT
|
||||
data_NOISE
|
||||
data_WIND
|
||||
|
||||
'''
|
||||
|
||||
import sqlite3
|
||||
import datetime
|
||||
import sys
|
||||
|
||||
|
||||
# Connect to the SQLite database
|
||||
conn = sqlite3.connect("/var/www/nebuleair_pro_4g/sqlite/sensors.db")
|
||||
cursor = conn.cursor()
|
||||
|
||||
#GET RTC TIME from SQlite
|
||||
cursor.execute("SELECT * FROM timestamp_table LIMIT 1")
|
||||
row = cursor.fetchone() # Get the first (and only) row
|
||||
|
||||
if row:
|
||||
rtc_time_str = row[1] # Assuming timestamp is stored as TEXT (YYYY-MM-DD HH:MM:SS)
|
||||
print(f"[INFO] Last recorded timestamp: {rtc_time_str}")
|
||||
|
||||
# Convert last_updated to a datetime object
|
||||
last_updated = datetime.datetime.strptime(rtc_time_str, "%Y-%m-%d %H:%M:%S")
|
||||
|
||||
# Calculate the cutoff date (3 months before last_updated)
|
||||
cutoff_date = last_updated - datetime.timedelta(days=60)
|
||||
cutoff_date_str = cutoff_date.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
print(f"[INFO] Deleting records older than: {cutoff_date_str}")
|
||||
|
||||
# List of tables to delete old data from
|
||||
tables_to_clean = ["data_NPM", "data_NPM_5channels", "data_BME280", "data_envea","data_WIND", "data_MPPT"]
|
||||
|
||||
# Loop through each table and delete old data
|
||||
for table in tables_to_clean:
|
||||
delete_query = f"DELETE FROM {table} WHERE timestamp < ?"
|
||||
cursor.execute(delete_query, (cutoff_date_str,))
|
||||
print(f"[INFO] Deleted old records from {table}")
|
||||
|
||||
# **Commit changes before running VACUUM**
|
||||
conn.commit()
|
||||
print("[INFO] Changes committed successfully!")
|
||||
|
||||
# Now it's safe to run VACUUM
|
||||
print("[INFO] Running VACUUM to optimize database space...")
|
||||
cursor.execute("VACUUM")
|
||||
|
||||
print("[SUCCESS] Old data flushed successfully!")
|
||||
|
||||
else:
|
||||
print("[ERROR] No timestamp found in timestamp_table.")
|
||||
def table_exists(cursor, table_name):
|
||||
"""Check if a table exists in the database"""
|
||||
try:
|
||||
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table_name,))
|
||||
return cursor.fetchone() is not None
|
||||
except sqlite3.Error as e:
|
||||
print(f"[ERROR] Failed to check if table '{table_name}' exists: {e}")
|
||||
return False
|
||||
|
||||
|
||||
# Close the database connection
|
||||
conn.close()
|
||||
def get_table_count(cursor, table_name):
|
||||
"""Get the number of records in a table"""
|
||||
try:
|
||||
cursor.execute(f"SELECT COUNT(*) FROM {table_name}")
|
||||
return cursor.fetchone()[0]
|
||||
except sqlite3.Error as e:
|
||||
print(f"[WARNING] Could not get count for table '{table_name}': {e}")
|
||||
return 0
|
||||
|
||||
|
||||
def delete_old_records(cursor, table_name, cutoff_date_str):
|
||||
"""Delete old records from a specific table"""
|
||||
try:
|
||||
# First check how many records will be deleted
|
||||
cursor.execute(f"SELECT COUNT(*) FROM {table_name} WHERE timestamp < ?", (cutoff_date_str,))
|
||||
records_to_delete = cursor.fetchone()[0]
|
||||
|
||||
if records_to_delete == 0:
|
||||
print(f"[INFO] No old records to delete from '{table_name}'")
|
||||
return True
|
||||
|
||||
# Delete the records
|
||||
cursor.execute(f"DELETE FROM {table_name} WHERE timestamp < ?", (cutoff_date_str,))
|
||||
deleted_count = cursor.rowcount
|
||||
|
||||
print(f"[SUCCESS] Deleted {deleted_count} old records from '{table_name}'")
|
||||
return True
|
||||
|
||||
except sqlite3.Error as e:
|
||||
print(f"[ERROR] Failed to delete records from '{table_name}': {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
# Connect to the SQLite database
|
||||
print("[INFO] Connecting to database...")
|
||||
conn = sqlite3.connect("/var/www/nebuleair_pro_4g/sqlite/sensors.db")
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check database connection
|
||||
cursor.execute("SELECT sqlite_version()")
|
||||
version = cursor.fetchone()[0]
|
||||
print(f"[INFO] Connected to SQLite version: {version}")
|
||||
|
||||
# GET RTC TIME from SQLite
|
||||
print("[INFO] Getting timestamp from database...")
|
||||
|
||||
# First check if timestamp_table exists
|
||||
if not table_exists(cursor, "timestamp_table"):
|
||||
print("[ERROR] timestamp_table does not exist!")
|
||||
return False
|
||||
|
||||
cursor.execute("SELECT * FROM timestamp_table LIMIT 1")
|
||||
row = cursor.fetchone()
|
||||
|
||||
if not row:
|
||||
print("[ERROR] No timestamp found in timestamp_table.")
|
||||
return False
|
||||
|
||||
rtc_time_str = row[1] # Assuming timestamp is stored as TEXT (YYYY-MM-DD HH:MM:SS)
|
||||
print(f"[INFO] Last recorded timestamp: {rtc_time_str}")
|
||||
|
||||
# Convert last_updated to a datetime object
|
||||
try:
|
||||
last_updated = datetime.datetime.strptime(rtc_time_str, "%Y-%m-%d %H:%M:%S")
|
||||
except ValueError as e:
|
||||
print(f"[ERROR] Invalid timestamp format: {e}")
|
||||
return False
|
||||
|
||||
# Calculate the cutoff date (60 days before last_updated)
|
||||
cutoff_date = last_updated - datetime.timedelta(days=60)
|
||||
cutoff_date_str = cutoff_date.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
print(f"[INFO] Deleting records older than: {cutoff_date_str}")
|
||||
|
||||
# List of tables to delete old data from
|
||||
tables_to_clean = [
|
||||
"data_NPM",
|
||||
"data_NPM_5channels",
|
||||
"data_BME280",
|
||||
"data_envea",
|
||||
"data_WIND",
|
||||
"data_MPPT",
|
||||
"data_NOISE"
|
||||
]
|
||||
|
||||
# Check which tables actually exist
|
||||
existing_tables = []
|
||||
missing_tables = []
|
||||
|
||||
for table in tables_to_clean:
|
||||
if table_exists(cursor, table):
|
||||
existing_tables.append(table)
|
||||
record_count = get_table_count(cursor, table)
|
||||
print(f"[INFO] Table '{table}' exists with {record_count} records")
|
||||
else:
|
||||
missing_tables.append(table)
|
||||
print(f"[WARNING] Table '{table}' does not exist - skipping")
|
||||
|
||||
if missing_tables:
|
||||
print(f"[INFO] Missing tables: {', '.join(missing_tables)}")
|
||||
|
||||
if not existing_tables:
|
||||
print("[WARNING] No tables found to clean!")
|
||||
return True
|
||||
|
||||
# Loop through existing tables and delete old data
|
||||
successful_deletions = 0
|
||||
failed_deletions = 0
|
||||
|
||||
for table in existing_tables:
|
||||
if delete_old_records(cursor, table, cutoff_date_str):
|
||||
successful_deletions += 1
|
||||
else:
|
||||
failed_deletions += 1
|
||||
|
||||
# Commit changes before running VACUUM
|
||||
print("[INFO] Committing changes...")
|
||||
conn.commit()
|
||||
print("[SUCCESS] Changes committed successfully!")
|
||||
|
||||
# Only run VACUUM if at least some deletions were successful
|
||||
if successful_deletions > 0:
|
||||
print("[INFO] Running VACUUM to optimize database space...")
|
||||
try:
|
||||
cursor.execute("VACUUM")
|
||||
print("[SUCCESS] Database optimized successfully!")
|
||||
except sqlite3.Error as e:
|
||||
print(f"[WARNING] VACUUM failed: {e}")
|
||||
|
||||
# Summary
|
||||
print(f"\n[SUMMARY]")
|
||||
print(f"Tables processed successfully: {successful_deletions}")
|
||||
print(f"Tables with errors: {failed_deletions}")
|
||||
print(f"Tables skipped (missing): {len(missing_tables)}")
|
||||
|
||||
if failed_deletions == 0:
|
||||
print("[SUCCESS] Old data flushed successfully!")
|
||||
return True
|
||||
else:
|
||||
print("[WARNING] Some operations failed - check logs above")
|
||||
return False
|
||||
|
||||
except sqlite3.Error as e:
|
||||
print(f"[ERROR] Database error: {e}")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"[ERROR] Unexpected error: {e}")
|
||||
return False
|
||||
finally:
|
||||
# Always close the database connection
|
||||
if 'conn' in locals():
|
||||
conn.close()
|
||||
print("[INFO] Database connection closed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
success = main()
|
||||
sys.exit(0 if success else 1)
|
||||
Reference in New Issue
Block a user