update

NPM/get_data_modbus_v2.py

@@ -60,7 +60,7 @@ ser = serial.Serial(
     parity=serial.PARITY_EVEN,
     stopbits=serial.STOPBITS_ONE,
     bytesize=serial.EIGHTBITS,
-    timeout = 0.5
+    timeout = 2
 )
 
 # Define Modbus CRC-16 function
@@ -89,7 +89,7 @@ while True:
     try:
         byte_data = ser.readline()
         formatted = ''.join(f'\\x{byte:02x}' for byte in byte_data)
-        #print(formatted)
+        print(formatted)
 
         # Register base (56 = 0x38)
         REGISTER_START = 56
@@ -125,10 +125,10 @@ while True:
         pm25_10s = extract_value(byte_data, 58, 1000, round_to=1)
         pm10_10s = extract_value(byte_data, 60, 1000, round_to=1)
 
-        #print("10 sec concentration:")
-        #print(f"PM1: {pm1_10s}")
-        #print(f"PM2.5: {pm25_10s}")
-        #print(f"PM10: {pm10_10s}")
+        print("10 sec concentration:")
+        print(f"PM1: {pm1_10s}")
+        print(f"PM2.5: {pm25_10s}")
+        print(f"PM10: {pm10_10s}")
 
         # 1-min PM Concentration
         pm1_1min = extract_value(byte_data, 68, 1000, round_to=1)
@@ -147,11 +147,11 @@ while True:
         channel_4 = extract_value(byte_data, 134,  round_to=0)  # 2.5 - 5.0μm
         channel_5 = extract_value(byte_data, 136,  round_to=0)  # 5.0 - 10.0μm
 
-        #print(f"Channel 1 (0.2->0.5): {channel_1}")
-        #print(f"Channel 2 (0.5->1.0): {channel_2}")
-        #print(f"Channel 3 (1.0->2.5): {channel_3}")
-        #print(f"Channel 4 (2.5->5.0): {channel_4}")
-        #print(f"Channel 5 (5.0->10.): {channel_5}")
+        print(f"Channel 1 (0.2->0.5): {channel_1}")
+        print(f"Channel 2 (0.5->1.0): {channel_2}")
+        print(f"Channel 3 (1.0->2.5): {channel_3}")
+        print(f"Channel 4 (2.5->5.0): {channel_4}")
+        print(f"Channel 5 (5.0->10.): {channel_5}")
 
         
         # Retrieve relative humidity from register 106 (0x6A)

NPM/get_data_modbus_v3.py

@@ -0,0 +1,179 @@
+'''
+  _   _ ____  __  __ 
+ | \ | |  _ \|  \/  |
+ |  \| | |_) | |\/| |
+ | |\  |  __/| |  | |
+ |_| \_|_|   |_|  |_|
+
+Script to get NPM data via Modbus
+
+Improved version with data stream lenght check
+
+/usr/bin/python3 /var/www/nebuleair_pro_4g/NPM/get_data_modbus_v3.py
+
+Modbus RTU
+[Slave Address][Function Code][Starting Address][Quantity of Registers][CRC]
+
+Pour récupérer
+ les concentrations en PM1, PM10 et PM2.5 (a partir du registre 0x38)
+ les 5 cannaux
+ la température et l'humidité à l'intérieur du capteur
+Donnée actualisée toutes les 10 secondes
+
+Request
+\x01\x03\x00\x38\x00\x55\...\...
+
+\x01        Slave Address (slave device address)
+\x03        Function code (read multiple holding registers)
+\x00\x38    Starting Address (The request starts reading from holding register address x38 or 56)
+\x00\x55    Quantity of Registers (Requests to read x55 or 85 consecutive registers starting from address 56)
+\...\...    Cyclic Redundancy Check (checksum )
+
+'''
+import serial
+import requests
+import json
+import sys
+import crcmod
+import sqlite3
+import time
+
+# Connect to the SQLite database
+conn = sqlite3.connect("/var/www/nebuleair_pro_4g/sqlite/sensors.db")
+cursor = conn.cursor()
+
+def load_config(config_file):
+    try:
+        with open(config_file, 'r') as file:
+            config_data = json.load(file)
+        return config_data
+    except Exception as e:
+        print(f"Error loading config file: {e}")
+        return {}
+
+# Load the configuration data
+config_file = '/var/www/nebuleair_pro_4g/config.json'
+config = load_config(config_file)
+npm_solo_port = config.get('NPM_solo_port', '')  #port du NPM solo
+
+#GET RTC TIME from SQlite
+cursor.execute("SELECT * FROM timestamp_table LIMIT 1")
+row = cursor.fetchone()  # Get the first (and only) row
+rtc_time_str = row[1]  # '2025-02-07 12:30:45'
+
+# Initialize serial port
+ser = serial.Serial(
+    port=npm_solo_port,
+    baudrate=115200,
+    parity=serial.PARITY_EVEN,
+    stopbits=serial.STOPBITS_ONE,
+    bytesize=serial.EIGHTBITS,
+    timeout=2
+)
+
+# Define Modbus CRC-16 function
+crc16 = crcmod.predefined.mkPredefinedCrcFun('modbus')
+
+# Request frame without CRC
+data = b'\x01\x03\x00\x38\x00\x55'
+
+# Calculate and append CRC
+crc = crc16(data)
+crc_low = crc & 0xFF
+crc_high = (crc >> 8) & 0xFF
+request = data + bytes([crc_low, crc_high])
+
+# Clear serial buffer before sending
+ser.flushInput()
+
+# Send request
+ser.write(request)
+time.sleep(0.2)  # Wait for sensor to respond
+
+# Read response
+response_length = 2 + 1 + (2 * 85) + 2  # Address + Function + Data + CRC
+byte_data = ser.read(response_length)
+
+# Validate response length
+if len(byte_data) < response_length:
+    print("[ERROR] Incomplete response received:", byte_data.hex())
+    exit()
+
+# Verify CRC
+received_crc = int.from_bytes(byte_data[-2:], byteorder='little')
+calculated_crc = crc16(byte_data[:-2])
+
+if received_crc != calculated_crc:
+    print("[ERROR] CRC check failed! Corrupted data received.")
+    exit()
+
+# Convert response to hex for debugging
+formatted = ''.join(f'\\x{byte:02x}' for byte in byte_data)
+#print("Response:", formatted)
+
+# Extract and print PM values
+def extract_value(byte_data, register, scale=1, single_register=False, round_to=None):
+    REGISTER_START = 56
+    offset = (register - REGISTER_START) * 2 + 3
+
+    if single_register:
+        value = int.from_bytes(byte_data[offset:offset+2], byteorder='big')
+    else:
+        lsw = int.from_bytes(byte_data[offset:offset+2], byteorder='big')  
+        msw = int.from_bytes(byte_data[offset+2:offset+4], byteorder='big')  
+        value = (msw << 16) | lsw
+
+    value = value / scale
+
+    if round_to == 0:
+        return int(value)
+    elif round_to is not None:
+        return round(value, round_to)
+    else:
+        return value
+
+pm1_10s = extract_value(byte_data, 56, 1000, round_to=1)
+pm25_10s = extract_value(byte_data, 58, 1000, round_to=1)
+pm10_10s = extract_value(byte_data, 60, 1000, round_to=1)
+
+#print("10 sec concentration:")
+#print(f"PM1: {pm1_10s}")
+#print(f"PM2.5: {pm25_10s}")
+#print(f"PM10: {pm10_10s}")
+
+# Extract values for 5 channels
+channel_1 = extract_value(byte_data, 128,  round_to=0)  # 0.2 - 0.5μm
+channel_2 = extract_value(byte_data, 130,  round_to=0)  # 0.5 - 1.0μm
+channel_3 = extract_value(byte_data, 132,  round_to=0)  # 1.0 - 2.5μm
+channel_4 = extract_value(byte_data, 134,  round_to=0)  # 2.5 - 5.0μm
+channel_5 = extract_value(byte_data, 136,  round_to=0)  # 5.0 - 10.0μm
+
+#print(f"Channel 1 (0.2->0.5): {channel_1}")
+#print(f"Channel 2 (0.5->1.0): {channel_2}")
+#print(f"Channel 3 (1.0->2.5): {channel_3}")
+#print(f"Channel 4 (2.5->5.0): {channel_4}")
+#print(f"Channel 5 (5.0->10.): {channel_5}")
+
+        
+# Retrieve relative humidity from register 106 (0x6A)
+relative_humidity = extract_value(byte_data, 106, 100, single_register=True)
+# Retrieve temperature from register 106 (0x6A)
+temperature = extract_value(byte_data, 107, 100, single_register=True)
+
+print(f"Internal Relative Humidity: {relative_humidity} %")
+print(f"Internal temperature: {temperature} °C")
+
+
+
+cursor.execute('''
+INSERT INTO data_NPM_5channels (timestamp,PM_ch1, PM_ch2, PM_ch3, PM_ch4, PM_ch5) VALUES (?,?,?,?,?,?)'''
+, (rtc_time_str,channel_1,channel_2,channel_3,channel_4,channel_5))
+
+cursor.execute('''
+INSERT INTO data_NPM (timestamp,PM1, PM25, PM10, temp_npm, hum_npm) VALUES (?,?,?,?,?,?)'''
+, (rtc_time_str,pm1_10s,pm25_10s,pm10_10s,temperature,relative_humidity  ))
+
+# Commit and close the connection
+conn.commit()
+
+conn.close()

config.json.dist

@@ -3,11 +3,12 @@
   "loop_log": true,
   "boot_log": true,
   "modem_config_mode": false,
-  "NPM/get_data_modbus_v2.py":false,
+  "NPM/get_data_modbus_v3.py":false,
   "loop/SARA_send_data_v2.py": true,
   "RTC/save_to_db.py": true,
   "BME280/get_data_v2.py": true,
   "envea/read_value_v2.py": true,
+  "sqlite/flush_old_data.py": true,
   "deviceID": "XXXX",
   "deviceName": "NebuleAir-proXXX",
   "SaraR4_baudrate": 115200,

master.py

@@ -82,10 +82,11 @@ def run_script(script_name, interval, delay=0):
 # Define scripts and their execution intervals (seconds)
 SCRIPTS = [
     ("RTC/save_to_db.py", 1, 0),                # SAVE RTC time every 1 second, no delay
-    ("NPM/get_data_modbus_v2.py", 10, 0),         # Get NPM data (modbus 5 channels) every 10s, with 2s delay
+    ("NPM/get_data_modbus_v3.py", 10, 0),       # Get NPM data (modbus 5 channels) every 10s, with 2s delay
     ("envea/read_value_v2.py", 10, 0),          # Get NPM data (modbus 5 channels) every 10s, with 2s delay
     ("loop/SARA_send_data_v2.py", 60, 1),       # Send data every 60 seconds, with 2s delay
-    ("BME280/get_data_v2.py", 120, 0)          # Get BME280 data every 120 seconds, no delay
+    ("BME280/get_data_v2.py", 120, 0),          # Get BME280 data every 120 seconds, no delay
+    ("sqlite/flush_old_data.py", 86400, 0)        # flush old data inside db every day ()
 ]
 
 # Start threads for enabled scripts

sqlite/flush_old_data.py

@@ -0,0 +1,71 @@
+'''
+  ____   ___  _     _ _       
+ / ___| / _ \| |   (_) |_ ___ 
+ \___ \| | | | |   | | __/ _ \
+  ___) | |_| | |___| | ||  __/
+ |____/ \__\_\_____|_|\__\___|
+                              
+Script to flush (delete) data from a sqlite database
+
+/usr/bin/python3 /var/www/nebuleair_pro_4g/sqlite/flush_old_data.py
+
+Available table are
+
+data_NPM
+data_NPM_5channels
+data_BME280
+data_envea
+timestamp_table
+
+'''
+
+import sqlite3
+import datetime
+
+
+# Connect to the SQLite database
+conn = sqlite3.connect("/var/www/nebuleair_pro_4g/sqlite/sensors.db")
+cursor = conn.cursor()
+
+#GET RTC TIME from SQlite
+cursor.execute("SELECT * FROM timestamp_table LIMIT 1")
+row = cursor.fetchone()  # Get the first (and only) row
+
+if row:
+    rtc_time_str = row[1]  # Assuming timestamp is stored as TEXT (YYYY-MM-DD HH:MM:SS)
+    print(f"[INFO] Last recorded timestamp: {rtc_time_str}")
+
+    # Convert last_updated to a datetime object
+    last_updated = datetime.datetime.strptime(rtc_time_str, "%Y-%m-%d %H:%M:%S")
+
+    # Calculate the cutoff date (3 months before last_updated)
+    cutoff_date = last_updated - datetime.timedelta(days=60)
+    cutoff_date_str = cutoff_date.strftime("%Y-%m-%d %H:%M:%S")
+
+    print(f"[INFO] Deleting records older than: {cutoff_date_str}")
+
+    # List of tables to delete old data from
+    tables_to_clean = ["data_NPM", "data_NPM_5channels", "data_BME280", "data_envea"]
+
+    # Loop through each table and delete old data
+    for table in tables_to_clean:
+        delete_query = f"DELETE FROM {table} WHERE timestamp < ?"
+        cursor.execute(delete_query, (cutoff_date_str,))
+        print(f"[INFO] Deleted old records from {table}")
+
+    # **Commit changes before running VACUUM**
+    conn.commit()
+    print("[INFO] Changes committed successfully!")
+
+    # Now it's safe to run VACUUM
+    print("[INFO] Running VACUUM to optimize database space...")
+    cursor.execute("VACUUM")
+
+    print("[SUCCESS] Old data flushed successfully!")
+
+else:
+    print("[ERROR] No timestamp found in timestamp_table.")
+
+
+# Close the database connection
+conn.close()