Merge remote-tracking branch 'refs/remotes/origin/main'

NPM/get_data_modbus_loop.py

@@ -1,8 +1,10 @@
 '''
 Loop to run every minutes
+
 * * * * * /usr/bin/python3 /var/www/nebuleair_pro_4g/NPM/get_data_modbus_loop.py ttyAMA5
 
-saves data to a json file /var/www/nebuleair_pro_4g/NPM/data/data.json
+saves data (avaerage) to a json file /var/www/nebuleair_pro_4g/NPM/data/data.json
+saves data (all) to a sqlite database
 
 first time running the script?
 sudo mkdir /var/www/nebuleair_pro_4g/NPM/data
@@ -15,6 +17,10 @@ import crcmod
 import time
 import json
 import os
+import sqlite3
+import smbus2  # For RTC DS3231
+from datetime import datetime
+
 
 # Ensure a port argument is provided
 if len(sys.argv) < 2:
@@ -54,14 +60,44 @@ request = data + bytes([crc_low, crc_high])
 # Log request frame
 print(f"Request frame: {request.hex()}")
 
+# Initialize SQLite database
+conn = sqlite3.connect("/var/www/nebuleair_pro_4g/sqlite/sensors.db")
+cursor = conn.cursor()
+
+# RTC Module (DS3231) Setup
+RTC_I2C_ADDR = 0x68  # DS3231 I2C Address
+bus = smbus2.SMBus(1)
+
+def bcd_to_dec(bcd):
+    return (bcd // 16 * 10) + (bcd % 16)
+
+def get_rtc_time():
+    """Reads time from RTC module (DS3231)"""
+    try:
+        data = bus.read_i2c_block_data(RTC_I2C_ADDR, 0x00, 7)
+        seconds = bcd_to_dec(data[0] & 0x7F)
+        minutes = bcd_to_dec(data[1])
+        hours = bcd_to_dec(data[2] & 0x3F)
+        day = bcd_to_dec(data[4])
+        month = bcd_to_dec(data[5])
+        year = bcd_to_dec(data[6]) + 2000
+        return datetime(year, month, day, hours, minutes, seconds).strftime("%Y-%m-%d %H:%M:%S")
+    except Exception as e:
+        print(f"RTC Read Error: {e}")
+        return "N/A"
+
 # Initialize storage for averaging
 num_samples = 6
 channel_sums = [0] * 5  # 5 channels
 
+#do not start immediately to prevent multiple write/read over NPM serial port
+time.sleep(3)
+
 # Loop 6 times to collect data every 10 seconds
 for i in range(num_samples):
     print(f"\nIteration {i+1}/{num_samples}")
     ser.write(request)
+    rtc_timestamp = get_rtc_time()
 
     try:
         byte_data = ser.readline()
@@ -70,7 +106,7 @@ for i in range(num_samples):
 
         if len(byte_data) < 23:
             print("Incomplete response, skipping this sample.")
-            time.sleep(10)
+            time.sleep(9)
             continue
 
         # Extract and process the 5 channels
@@ -86,9 +122,19 @@ for i in range(num_samples):
             channel_sums[j] += channels[j]
 
         # Print collected values
+        print(f"Timestamp (RTC): {rtc_timestamp}")
         for j in range(5):
             print(f"Channel {j+1}: {channels[j]}")
 
+        
+        # Save the individual reading to the database
+        cursor.execute('''
+        INSERT INTO data (timestamp, PM_ch1, PM_ch2, PM_ch3, PM_ch4, PM_ch5)
+        VALUES (?, ?, ?, ?, ?, ?)
+        ''', (rtc_timestamp, channels[0], channels[1], channels[2], channels[3], channels[4]))
+
+        conn.commit()
+
     except Exception as e:
         print(f"Error reading data: {e}")
 

README.md

@@ -82,6 +82,9 @@ sudo chmod 777 /dev/i2c-1
 
 Attention: sometimes activation with config.txt do not work, you need to activate i2c with `sudo raspi-config` and go to "Interface" -> I2C -> enable.
 
+It is possible to manage raspi-config only with cli: `sudo raspi-config nonint do_i2c 0`
+
+
 I2C addresses: use `sudo i2cdetect -y 1` to check the connected devices.
 
 ### BME280

config.json.dist

@@ -12,6 +12,7 @@
   "i2C_sound": false,
   "i2c_BME": false,
   "i2c_RTC": false,
+  "local_storage": false,
   "sshTunnel_port": 59228,
   "npm1_status": "connected",
   "SARA_R4_general_status": "connected",

loop/1_NPM/send_data.py

@@ -185,6 +185,7 @@ i2C_sound_config = config.get('i2C_sound', False)      #présence du capteur son
 send_aircarto = config.get('send_aircarto', True)      #envoi sur AirCarto (data.nebuleair.fr)
 send_uSpot = config.get('send_uSpot', False)           #envoi sur MicroSpot ()
 npm_5channel = config.get('NextPM_5channels', False)   #5 canaux du NPM
+local_storage = config.get('local_storage', False)   #enregistrement en local des data
 
 envea_sondes = config.get('envea_sondes', [])
 connected_envea_sondes = [sonde for sonde in envea_sondes if sonde.get('connected', False)]
@@ -236,7 +237,7 @@ if connected_envea_sondes:
 
 
 
-def read_complete_response(serial_connection, timeout=2, end_of_response_timeout=2, wait_for_line=None):
+def read_complete_response(serial_connection, timeout=2, end_of_response_timeout=2, wait_for_line=None, debug=True):
     response = bytearray()
     serial_connection.timeout = timeout
     end_time = time.time() + end_of_response_timeout
@@ -253,17 +254,17 @@ def read_complete_response(serial_connection, timeout=2, end_of_response_timeout
             if wait_for_line:
                 decoded_response = response.decode('utf-8', errors='replace')
                 if wait_for_line in decoded_response:
-                    print(f"[DEBUG] 🔎Found target line: {wait_for_line}")
+                    if debug: print(f"[DEBUG] 🔎Found target line: {wait_for_line}")
                     break
         elif time.time() > end_time:
-            print(f"[DEBUG] Timeout reached. No more data received.")
+            if debug: print(f"[DEBUG] Timeout reached. No more data received.")
             break
         time.sleep(0.1)  # Short sleep to prevent busy waiting
     # Final response and debug output
     total_elapsed_time = time.time() - start_time
-    print(f"[DEBUG] ⏱️ elapsed time: {total_elapsed_time:.2f}s. ⏱️")
+    if debug: print(f"[DEBUG] ⏱️ elapsed time: {total_elapsed_time:.2f}s. ⏱️")
     # Check if the elapsed time exceeded 10 seconds
-    if total_elapsed_time > 10:
+    if total_elapsed_time > 10 and debug:
         print(f"[ALERT] 🚨 The operation took too long🚨")
         print(f'<span style="color: red;font-weight: bold;">[ALERT] ⚠️{total_elapsed_time:.2f}s⚠️</span>')
 
@@ -458,14 +459,14 @@ try:
     print("Open JSON:")
     command = f'AT+UDWNFILE="sensordata_csv.json",{size_of_string}\r'
     ser_sara.write(command.encode('utf-8'))
-    response_SARA_1 = read_complete_response(ser_sara, wait_for_line=">")
+    response_SARA_1 = read_complete_response(ser_sara, wait_for_line=">", debug=False)
     print(response_SARA_1)
     time.sleep(1)
 
     #2. Write to shell
     print("Write data to memory:")
     ser_sara.write(csv_string.encode())
-    response_SARA_2 = read_complete_response(ser_sara, wait_for_line="OK")
+    response_SARA_2 = read_complete_response(ser_sara, wait_for_line="OK", debug=False)
     print(response_SARA_2)
 
     #3. Send to endpoint (with device ID)
@@ -553,7 +554,7 @@ try:
                 print("Getting error code (11->Server connection error, 73->Secure socket connect error)")
                 command = f'AT+UHTTPER={aircarto_profile_id}\r'
                 ser_sara.write(command.encode('utf-8'))    
-                response_SARA_9 = read_complete_response(ser_sara, wait_for_line="OK")
+                response_SARA_9 = read_complete_response(ser_sara, wait_for_line="OK", debug=False)
                 print('<p class="text-danger-emphasis">')
                 print(response_SARA_9)
                 print("</p>")
@@ -590,7 +591,7 @@ try:
                 #4. Read reply from server   
                 print("Reply from server:")                                                                         
                 ser_sara.write(b'AT+URDFILE="server_response.txt"\r')              
-                response_SARA_4 = read_complete_response(ser_sara, wait_for_line="OK")
+                response_SARA_4 = read_complete_response(ser_sara, wait_for_line="OK", debug=False)
                 print('<p class="text-success">')
                 print(response_SARA_4)
                 print('</p>')
@@ -605,7 +606,7 @@ try:
     #5. empty json
     print("Empty SARA memory:")
     ser_sara.write(b'AT+UDELFILE="sensordata_csv.json"\r')
-    response_SARA_5 = read_complete_response(ser_sara, wait_for_line="OK")
+    response_SARA_5 = read_complete_response(ser_sara, wait_for_line="OK", debug=False)
     print(response_SARA_5)
 
     '''
@@ -622,7 +623,7 @@ try:
         print("SET URL")
         command = f'AT+UHTTP={uSpot_profile_id},1,"api-prod.uspot.probesys.net"\r'
         ser_sara.write((command + '\r').encode('utf-8'))
-        response_SARA_5 = read_complete_response(ser_sara, wait_for_line="OK")
+        response_SARA_5 = read_complete_response(ser_sara, wait_for_line="OK", debug=False)
         print(response_SARA_5)
         time.sleep(1)
 
@@ -631,7 +632,7 @@ try:
         print("SET SSL")
         command = f'AT+UHTTP={uSpot_profile_id},6,0\r'
         ser_sara.write(command.encode('utf-8'))
-        response_SARA_5 = read_complete_response(ser_sara, wait_for_line="OK")
+        response_SARA_5 = read_complete_response(ser_sara, wait_for_line="OK", debug=False)
         print(response_SARA_5)
         time.sleep(1)
 
@@ -640,7 +641,7 @@ try:
         print("SET PORT")
         command = f'AT+UHTTP={uSpot_profile_id},5,81\r'
         ser_sara.write((command + '\r').encode('utf-8'))
-        response_SARA_55 = read_complete_response(ser_sara, wait_for_line="OK")
+        response_SARA_55 = read_complete_response(ser_sara, wait_for_line="OK", debug=False)
         print(response_SARA_55)
         time.sleep(1)
 
@@ -654,14 +655,14 @@ try:
         size_of_string = len(payload_string)
         command = f'AT+UDWNFILE="sensordata_json.json",{size_of_string}\r'
         ser_sara.write((command + '\r').encode('utf-8'))
-        response_SARA_1 = read_complete_response(ser_sara, wait_for_line=">")
+        response_SARA_1 = read_complete_response(ser_sara, wait_for_line=">", debug=False)
         print(response_SARA_1)
         time.sleep(1)
 
         #2. Write to shell
         print("Write to memory:")
         ser_sara.write(payload_string.encode())
-        response_SARA_2 = read_complete_response(ser_sara, wait_for_line="OK")
+        response_SARA_2 = read_complete_response(ser_sara, wait_for_line="OK", debug=False)
         print(response_SARA_2)
 
         #step 4: trigger the request (http_command=1 for GET and http_command=1 for POST)
@@ -680,7 +681,7 @@ try:
         print("****")
         print("Read reply from server")
         ser_sara.write(b'AT+URDFILE="http.resp"\r')              
-        response_SARA_7 = read_complete_response(ser_sara, wait_for_line="OK")        
+        response_SARA_7 = read_complete_response(ser_sara, wait_for_line="OK", debug=False)        
         print('<p class="text-success">')
         print(response_SARA_7)
         print('</p>')
@@ -689,20 +690,16 @@ try:
         #5. empty json
         print("Empty SARA memory:")
         ser_sara.write(b'AT+UDELFILE="sensordata_json.json"\r')
-        response_SARA_8 = read_complete_response(ser_sara, wait_for_line="OK")
+        response_SARA_8 = read_complete_response(ser_sara, wait_for_line="OK", debug=False)
         print(response_SARA_8)
         
 
-    
-
     # Calculate and print the elapsed time
     elapsed_time = time.time() - start_time_script
     print(f"Elapsed time: {elapsed_time:.2f} seconds")
     print("<hr>")
 
 
-
-
 except Exception as e:
     print("An error occurred:", e)
     traceback.print_exc()  # This prints the full traceback

sqlite/create_db.py

@@ -2,6 +2,8 @@
 Script to create a sqlite database
 /usr/bin/python3 /var/www/nebuleair_pro_4g/sqlite/create_db.py
 
+
+
 '''
 
 import sqlite3
@@ -13,10 +15,22 @@ cursor = conn.cursor()
 # Create a table for storing sensor data
 cursor.execute("""
 CREATE TABLE IF NOT EXISTS data (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    timestamp TEXT,
-    timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+    sensor_id TEXT,
-    sensor TEXT,
+    PM1 REAL,
-    value REAL
+    PM25 REAL,
+    PM10 REAL,
+    temp REAL,
+    hum REAL,
+    press REAL,
+    no2 REAL,
+    h2s REAL,
+    o3 REAL,
+    PM_ch1 INTEGER,
+    PM_ch2 INTEGER,
+    PM_ch3 INTEGER,
+    PM_ch4 INTEGER,
+    PM_ch5 INTEGER
 )
 """)
 

sqlite/write.py

@@ -11,10 +11,15 @@ conn = sqlite3.connect("/var/www/nebuleair_pro_4g/sqlite/sensors.db")
 cursor = conn.cursor()
 
 # Insert a sample temperature reading
-sensor_name = "temperature"
+timestamp = "2025-10-11"
-sensor_value = 25.3
+sensor_name = "NebuleAir-pro020"
+PM1 = 25.3
+PM25 = 18.3
+PM10 = 9.3
 
-cursor.execute("INSERT INTO data (sensor, value) VALUES (?, ?)", (sensor_name, sensor_value))
+cursor.execute('''
+INSERT INTO data (timestamp, sensor_id, PM1, PM25, PM10) VALUES (?,?,?,?,?)'''
+, (timestamp, sensor_name,PM1,PM25,PM10))
 
 # Commit and close the connection
 conn.commit()