diff --git a/NPM/get_data_modbus_v2.py b/NPM/get_data_modbus_v2.py index 556bcae..c60e6e7 100644 --- a/NPM/get_data_modbus_v2.py +++ b/NPM/get_data_modbus_v2.py @@ -60,7 +60,7 @@ ser = serial.Serial( parity=serial.PARITY_EVEN, stopbits=serial.STOPBITS_ONE, bytesize=serial.EIGHTBITS, - timeout = 0.5 + timeout = 2 ) # Define Modbus CRC-16 function @@ -89,7 +89,7 @@ while True: try: byte_data = ser.readline() formatted = ''.join(f'\\x{byte:02x}' for byte in byte_data) - #print(formatted) + print(formatted) # Register base (56 = 0x38) REGISTER_START = 56 @@ -125,10 +125,10 @@ while True: pm25_10s = extract_value(byte_data, 58, 1000, round_to=1) pm10_10s = extract_value(byte_data, 60, 1000, round_to=1) - #print("10 sec concentration:") - #print(f"PM1: {pm1_10s}") - #print(f"PM2.5: {pm25_10s}") - #print(f"PM10: {pm10_10s}") + print("10 sec concentration:") + print(f"PM1: {pm1_10s}") + print(f"PM2.5: {pm25_10s}") + print(f"PM10: {pm10_10s}") # 1-min PM Concentration pm1_1min = extract_value(byte_data, 68, 1000, round_to=1) @@ -147,11 +147,11 @@ while True: channel_4 = extract_value(byte_data, 134, round_to=0) # 2.5 - 5.0μm channel_5 = extract_value(byte_data, 136, round_to=0) # 5.0 - 10.0μm - #print(f"Channel 1 (0.2->0.5): {channel_1}") - #print(f"Channel 2 (0.5->1.0): {channel_2}") - #print(f"Channel 3 (1.0->2.5): {channel_3}") - #print(f"Channel 4 (2.5->5.0): {channel_4}") - #print(f"Channel 5 (5.0->10.): {channel_5}") + print(f"Channel 1 (0.2->0.5): {channel_1}") + print(f"Channel 2 (0.5->1.0): {channel_2}") + print(f"Channel 3 (1.0->2.5): {channel_3}") + print(f"Channel 4 (2.5->5.0): {channel_4}") + print(f"Channel 5 (5.0->10.): {channel_5}") # Retrieve relative humidity from register 106 (0x6A) diff --git a/NPM/get_data_modbus_v3.py b/NPM/get_data_modbus_v3.py new file mode 100644 index 0000000..740b482 --- /dev/null +++ b/NPM/get_data_modbus_v3.py @@ -0,0 +1,179 @@ +''' + _ _ ____ __ __ + | \ | | _ \| \/ | + | \| | |_) | |\/| | + | |\ | __/| | | | + |_| \_|_| |_| |_| + +Script to get NPM data via Modbus + +Improved version with data stream lenght check + +/usr/bin/python3 /var/www/nebuleair_pro_4g/NPM/get_data_modbus_v3.py + +Modbus RTU +[Slave Address][Function Code][Starting Address][Quantity of Registers][CRC] + +Pour récupérer + les concentrations en PM1, PM10 et PM2.5 (a partir du registre 0x38) + les 5 cannaux + la température et l'humidité à l'intérieur du capteur +Donnée actualisée toutes les 10 secondes + +Request +\x01\x03\x00\x38\x00\x55\...\... + +\x01 Slave Address (slave device address) +\x03 Function code (read multiple holding registers) +\x00\x38 Starting Address (The request starts reading from holding register address x38 or 56) +\x00\x55 Quantity of Registers (Requests to read x55 or 85 consecutive registers starting from address 56) +\...\... Cyclic Redundancy Check (checksum ) + +''' +import serial +import requests +import json +import sys +import crcmod +import sqlite3 +import time + +# Connect to the SQLite database +conn = sqlite3.connect("/var/www/nebuleair_pro_4g/sqlite/sensors.db") +cursor = conn.cursor() + +def load_config(config_file): + try: + with open(config_file, 'r') as file: + config_data = json.load(file) + return config_data + except Exception as e: + print(f"Error loading config file: {e}") + return {} + +# Load the configuration data +config_file = '/var/www/nebuleair_pro_4g/config.json' +config = load_config(config_file) +npm_solo_port = config.get('NPM_solo_port', '') #port du NPM solo + +#GET RTC TIME from SQlite +cursor.execute("SELECT * FROM timestamp_table LIMIT 1") +row = cursor.fetchone() # Get the first (and only) row +rtc_time_str = row[1] # '2025-02-07 12:30:45' + +# Initialize serial port +ser = serial.Serial( + port=npm_solo_port, + baudrate=115200, + parity=serial.PARITY_EVEN, + stopbits=serial.STOPBITS_ONE, + bytesize=serial.EIGHTBITS, + timeout=2 +) + +# Define Modbus CRC-16 function +crc16 = crcmod.predefined.mkPredefinedCrcFun('modbus') + +# Request frame without CRC +data = b'\x01\x03\x00\x38\x00\x55' + +# Calculate and append CRC +crc = crc16(data) +crc_low = crc & 0xFF +crc_high = (crc >> 8) & 0xFF +request = data + bytes([crc_low, crc_high]) + +# Clear serial buffer before sending +ser.flushInput() + +# Send request +ser.write(request) +time.sleep(0.2) # Wait for sensor to respond + +# Read response +response_length = 2 + 1 + (2 * 85) + 2 # Address + Function + Data + CRC +byte_data = ser.read(response_length) + +# Validate response length +if len(byte_data) < response_length: + print("[ERROR] Incomplete response received:", byte_data.hex()) + exit() + +# Verify CRC +received_crc = int.from_bytes(byte_data[-2:], byteorder='little') +calculated_crc = crc16(byte_data[:-2]) + +if received_crc != calculated_crc: + print("[ERROR] CRC check failed! Corrupted data received.") + exit() + +# Convert response to hex for debugging +formatted = ''.join(f'\\x{byte:02x}' for byte in byte_data) +#print("Response:", formatted) + +# Extract and print PM values +def extract_value(byte_data, register, scale=1, single_register=False, round_to=None): + REGISTER_START = 56 + offset = (register - REGISTER_START) * 2 + 3 + + if single_register: + value = int.from_bytes(byte_data[offset:offset+2], byteorder='big') + else: + lsw = int.from_bytes(byte_data[offset:offset+2], byteorder='big') + msw = int.from_bytes(byte_data[offset+2:offset+4], byteorder='big') + value = (msw << 16) | lsw + + value = value / scale + + if round_to == 0: + return int(value) + elif round_to is not None: + return round(value, round_to) + else: + return value + +pm1_10s = extract_value(byte_data, 56, 1000, round_to=1) +pm25_10s = extract_value(byte_data, 58, 1000, round_to=1) +pm10_10s = extract_value(byte_data, 60, 1000, round_to=1) + +#print("10 sec concentration:") +#print(f"PM1: {pm1_10s}") +#print(f"PM2.5: {pm25_10s}") +#print(f"PM10: {pm10_10s}") + +# Extract values for 5 channels +channel_1 = extract_value(byte_data, 128, round_to=0) # 0.2 - 0.5μm +channel_2 = extract_value(byte_data, 130, round_to=0) # 0.5 - 1.0μm +channel_3 = extract_value(byte_data, 132, round_to=0) # 1.0 - 2.5μm +channel_4 = extract_value(byte_data, 134, round_to=0) # 2.5 - 5.0μm +channel_5 = extract_value(byte_data, 136, round_to=0) # 5.0 - 10.0μm + +#print(f"Channel 1 (0.2->0.5): {channel_1}") +#print(f"Channel 2 (0.5->1.0): {channel_2}") +#print(f"Channel 3 (1.0->2.5): {channel_3}") +#print(f"Channel 4 (2.5->5.0): {channel_4}") +#print(f"Channel 5 (5.0->10.): {channel_5}") + + +# Retrieve relative humidity from register 106 (0x6A) +relative_humidity = extract_value(byte_data, 106, 100, single_register=True) +# Retrieve temperature from register 106 (0x6A) +temperature = extract_value(byte_data, 107, 100, single_register=True) + +print(f"Internal Relative Humidity: {relative_humidity} %") +print(f"Internal temperature: {temperature} °C") + + + +cursor.execute(''' +INSERT INTO data_NPM_5channels (timestamp,PM_ch1, PM_ch2, PM_ch3, PM_ch4, PM_ch5) VALUES (?,?,?,?,?,?)''' +, (rtc_time_str,channel_1,channel_2,channel_3,channel_4,channel_5)) + +cursor.execute(''' +INSERT INTO data_NPM (timestamp,PM1, PM25, PM10, temp_npm, hum_npm) VALUES (?,?,?,?,?,?)''' +, (rtc_time_str,pm1_10s,pm25_10s,pm10_10s,temperature,relative_humidity )) + +# Commit and close the connection +conn.commit() + +conn.close() diff --git a/config.json.dist b/config.json.dist index f1999d5..11da996 100755 --- a/config.json.dist +++ b/config.json.dist @@ -3,11 +3,12 @@ "loop_log": true, "boot_log": true, "modem_config_mode": false, - "NPM/get_data_modbus_v2.py":false, + "NPM/get_data_modbus_v3.py":false, "loop/SARA_send_data_v2.py": true, "RTC/save_to_db.py": true, "BME280/get_data_v2.py": true, "envea/read_value_v2.py": true, + "sqlite/flush_old_data.py": true, "deviceID": "XXXX", "deviceName": "NebuleAir-proXXX", "SaraR4_baudrate": 115200, diff --git a/master.py b/master.py index f3e0ad4..eeb5391 100644 --- a/master.py +++ b/master.py @@ -81,11 +81,12 @@ def run_script(script_name, interval, delay=0): # Define scripts and their execution intervals (seconds) SCRIPTS = [ - ("RTC/save_to_db.py", 1, 0), # SAVE RTC time every 1 second, no delay - ("NPM/get_data_modbus_v2.py", 10, 0), # Get NPM data (modbus 5 channels) every 10s, with 2s delay - ("envea/read_value_v2.py", 10, 0), # Get NPM data (modbus 5 channels) every 10s, with 2s delay - ("loop/SARA_send_data_v2.py", 60, 1), # Send data every 60 seconds, with 2s delay - ("BME280/get_data_v2.py", 120, 0) # Get BME280 data every 120 seconds, no delay + ("RTC/save_to_db.py", 1, 0), # SAVE RTC time every 1 second, no delay + ("NPM/get_data_modbus_v3.py", 10, 0), # Get NPM data (modbus 5 channels) every 10s, with 2s delay + ("envea/read_value_v2.py", 10, 0), # Get NPM data (modbus 5 channels) every 10s, with 2s delay + ("loop/SARA_send_data_v2.py", 60, 1), # Send data every 60 seconds, with 2s delay + ("BME280/get_data_v2.py", 120, 0), # Get BME280 data every 120 seconds, no delay + ("sqlite/flush_old_data.py", 86400, 0) # flush old data inside db every day () ] # Start threads for enabled scripts diff --git a/sqlite/flush_old_data.py b/sqlite/flush_old_data.py new file mode 100644 index 0000000..f372a54 --- /dev/null +++ b/sqlite/flush_old_data.py @@ -0,0 +1,71 @@ +''' + ____ ___ _ _ _ + / ___| / _ \| | (_) |_ ___ + \___ \| | | | | | | __/ _ \ + ___) | |_| | |___| | || __/ + |____/ \__\_\_____|_|\__\___| + +Script to flush (delete) data from a sqlite database + +/usr/bin/python3 /var/www/nebuleair_pro_4g/sqlite/flush_old_data.py + +Available table are + +data_NPM +data_NPM_5channels +data_BME280 +data_envea +timestamp_table + +''' + +import sqlite3 +import datetime + + +# Connect to the SQLite database +conn = sqlite3.connect("/var/www/nebuleair_pro_4g/sqlite/sensors.db") +cursor = conn.cursor() + +#GET RTC TIME from SQlite +cursor.execute("SELECT * FROM timestamp_table LIMIT 1") +row = cursor.fetchone() # Get the first (and only) row + +if row: + rtc_time_str = row[1] # Assuming timestamp is stored as TEXT (YYYY-MM-DD HH:MM:SS) + print(f"[INFO] Last recorded timestamp: {rtc_time_str}") + + # Convert last_updated to a datetime object + last_updated = datetime.datetime.strptime(rtc_time_str, "%Y-%m-%d %H:%M:%S") + + # Calculate the cutoff date (3 months before last_updated) + cutoff_date = last_updated - datetime.timedelta(days=60) + cutoff_date_str = cutoff_date.strftime("%Y-%m-%d %H:%M:%S") + + print(f"[INFO] Deleting records older than: {cutoff_date_str}") + + # List of tables to delete old data from + tables_to_clean = ["data_NPM", "data_NPM_5channels", "data_BME280", "data_envea"] + + # Loop through each table and delete old data + for table in tables_to_clean: + delete_query = f"DELETE FROM {table} WHERE timestamp < ?" + cursor.execute(delete_query, (cutoff_date_str,)) + print(f"[INFO] Deleted old records from {table}") + + # **Commit changes before running VACUUM** + conn.commit() + print("[INFO] Changes committed successfully!") + + # Now it's safe to run VACUUM + print("[INFO] Running VACUUM to optimize database space...") + cursor.execute("VACUUM") + + print("[SUCCESS] Old data flushed successfully!") + +else: + print("[ERROR] No timestamp found in timestamp_table.") + + +# Close the database connection +conn.close()