update

NPM/get_data_v2.py

@@ -1,11 +1,12 @@
 '''
-  _   _ ____  __  __ 
+  ____  _____ _   _ ____   ___  ____  ____  
- | \ | |  _ \|  \/  |
+ / ___|| ____| \ | / ___| / _ \|  _ \/ ___| 
- |  \| | |_) | |\/| |
+ \___ \|  _| |  \| \___ \| | | | |_) \___ \ 
- | |\  |  __/| |  | |
+  ___) | |___| |\  |___) | |_| |  _ < ___) |
- |_| \_|_|   |_|  |_|
+ |____/|_____|_| \_|____/ \___/|_| \_\____/ 
                                             
-Script to get NPM values
+                     
+Script to get SENSORS values
 And store them inside sqlite database
 Uses RTC module for timing
 /usr/bin/python3 /var/www/nebuleair_pro_4g/NPM/get_data_v2.py
@@ -73,6 +74,7 @@ ser.write(b'\x81\x12\x6D')      #data60s
 
 while True:
     try:
+        #print("Start get_data_v2.py script")
         byte_data = ser.readline()
         #print(byte_data)
         stateByte = int.from_bytes(byte_data[2:3], byteorder='big')
@@ -86,8 +88,6 @@ while True:
         #print(f"PM10: {PM10}")
         #create JSON
         data = {
-            'capteurID': 'nebuleairpro1',
-            'sondeID':'USB2',
             'PM1': PM1,
             'PM25': PM25,
             'PM10': PM10,
@@ -101,7 +101,7 @@ while True:
             'laserError' : Statebits[7]
         }
         json_data = json.dumps(data)
-        print(json_data)
+        #print(json_data)
 
         #GET RTC TIME
         # Read RTC time
@@ -111,7 +111,7 @@ while True:
 
         if rtc_time:
             rtc_time_str = rtc_time.strftime('%Y-%m-%d %H:%M:%S')
-            print(rtc_time_str)
+            #print(rtc_time_str)
         else:
             print("Error! RTC module not connected")
             rtc_time_str = "1970-01-01 00:00:00"  # Default fallback time
@@ -125,7 +125,7 @@ while True:
         # Commit and close the connection
         conn.commit()
 
-        print("Sensor data saved successfully!")
+        #print("Sensor data saved successfully!")
 
         break # Exit loop after successful execution
     except KeyboardInterrupt:

html/index.html

@@ -308,9 +308,36 @@ window.onload = function() {
                     data: {
                         labels: labels,
                         datasets: [
-                            { label: "PM1", data: PM1, borderColor: "red", fill: false },
+                            {
-                            { label: "PM2.5", data: PM25, borderColor: "blue", fill: false },
+                                label: "PM1",
-                            { label: "PM10", data: PM10, borderColor: "green", fill: false }
+                                data: PM1,
+                                borderColor: "rgba(0, 51, 153, 1)",
+                                backgroundColor: "rgba(0, 51, 153, 0.2)", // Very light blue background
+                                fill: true,
+                                tension: 0.4, // Smooth curves
+                                pointRadius: 2, // Larger points
+                                pointHoverRadius: 6 // Bigger hover points
+                            },
+                            {
+                                label: "PM2.5",
+                                data: PM25,
+                                borderColor: "rgba(30, 144, 255, 1)",
+                                backgroundColor: "rgba(30, 144, 255, 0.2)", // Very light medium blue background
+                                fill: true,
+                                tension: 0.4,
+                                pointRadius: 2,
+                                pointHoverRadius: 6
+                            },
+                            {
+                                label: "PM10",
+                                data: PM10,
+                                borderColor: "rgba(135, 206, 250, 1)", 
+                                backgroundColor: "rgba(135, 206, 250, 0.2)", // Very light blue background
+                                fill: true,
+                                tension: 0.4,
+                                pointRadius: 2,
+                                pointHoverRadius: 6
+                            }
                         ]
                     },
                     options: {
@@ -325,11 +352,17 @@ window.onload = function() {
                         x: {
                             title: {
                                 display: true,
-                                text: 'Time'
+                                text: 'Time (UTC)',
+                                font: {
+                                    size: 16,
+                                    family: 'Arial, sans-serif'
+                                },
+                                color: '#4A4A4A'
                             },
                             ticks: {
                               autoSkip: true,
                               maxTicksLimit: 5,
+                              color: '#4A4A4A',
                               callback: function(value, index) {
                                   // Access the correct label from the `labels` array
                                   const label = labels[index]; // Use the original `labels` array
@@ -338,6 +371,9 @@ window.onload = function() {
                                   }
                                   return value; // Fallback for invalid labels
                               }
+                          },
+                          grid: {
+                              display: false // Remove gridlines for a cleaner look
                           }
 
 
@@ -345,7 +381,12 @@ window.onload = function() {
                         y: {
                             title: {
                                 display: true,
-                                text: 'Values (µg/m³)'
+                                text: 'Values (µg/m³)',
+                                font: {
+                                    size: 16,
+                                    family: 'Arial, sans-serif'
+                                },
+                                color: '#4A4A4A'
                             }
                         }
                     }

html/launcher.php

@@ -16,8 +16,10 @@ if ($type == "get_npm_sqlite_data") {
         // Fetch the last 30 records
         $stmt = $db->query("SELECT timestamp, PM1, PM25, PM10 FROM data ORDER BY timestamp DESC LIMIT 30");
         $data = $stmt->fetchAll(PDO::FETCH_ASSOC);
+        $reversedData = array_reverse($data); // Reverse the order
 
-        echo json_encode($data);
+    
+        echo json_encode($reversedData);
     } catch (PDOException $e) {
         echo json_encode(["error" => $e->getMessage()]);
     }

html/logs.html

@@ -56,7 +56,7 @@
           <div class="col-lg-6 col-12">
             <div class="card" style="height: 80vh;">
               <div class="card-header">
-                Loop logs  <button type="submit" class="btn btn-secondary btn-sm" onclick="clear_loopLogs()">Clear</button>
+                Master logs  <button type="submit" class="btn btn-secondary btn-sm" onclick="clear_loopLogs()">Clear</button>
 
               </div>
               <div class="card-body overflow-auto" id="card_loop_content">
@@ -110,7 +110,7 @@
     const loop_card_content = document.getElementById('card_loop_content');
     const boot_card_content = document.getElementById('card_boot_content');
 
-          fetch('../logs/loop.log')
+          fetch('../logs/master.log')
             .then((response) => {
               if (!response.ok) {
                 throw new Error('Failed to fetch the log file.');

loop/1_NPM/send_data.py

@@ -78,9 +78,8 @@ import re
 import os
 import traceback
 import sys
-from threading import Thread
-
 import RPi.GPIO as GPIO
+from threading import Thread
 from adafruit_bme280 import basic as adafruit_bme280
 
 # Record the start time of the script
@@ -95,7 +94,6 @@ if uptime_seconds < 120:
     print(f"System just booted ({uptime_seconds:.2f} seconds uptime), skipping execution.")
     sys.exit()
 
-url_nebuleair="data.nebuleair.fr"
 payload_csv = [None] * 20
 payload_json = {
     "nebuleairid": "82D25549434",
@@ -185,7 +183,6 @@ i2C_sound_config = config.get('i2C_sound', False)      #présence du capteur son
 send_aircarto = config.get('send_aircarto', True)      #envoi sur AirCarto (data.nebuleair.fr)
 send_uSpot = config.get('send_uSpot', False)           #envoi sur MicroSpot ()
 npm_5channel = config.get('NextPM_5channels', False)   #5 canaux du NPM
-local_storage = config.get('local_storage', False)   #enregistrement en local des data
 
 envea_sondes = config.get('envea_sondes', [])
 connected_envea_sondes = [sonde for sonde in envea_sondes if sonde.get('connected', False)]

loop/SARA_send_data_v2.py

@@ -0,0 +1,320 @@
+"""
+  ____    _    ____      _      ____                 _   ____        _        
+ / ___|  / \  |  _ \    / \    / ___|  ___ _ __   __| | |  _ \  __ _| |_ __ _ 
+ \___ \ / _ \ | |_) |  / _ \   \___ \ / _ \ '_ \ / _` | | | | |/ _` | __/ _` |
+  ___) / ___ \|  _ <  / ___ \   ___) |  __/ | | | (_| | | |_| | (_| | || (_| |
+ |____/_/   \_\_| \_\/_/   \_\ |____/ \___|_| |_|\__,_| |____/ \__,_|\__\__,_|
+                                                                              
+Main loop to gather data from sensor inside SQLite database:
+
+* NPM
+* Envea
+* I2C BME280
+* Noise sensor
+
+and send it to AirCarto servers via SARA R4 HTTP post requests
+also send the timestamp (already stored inside the DB) !
+
+/usr/bin/python3 /var/www/nebuleair_pro_4g/loop/SARA_send_data_v2.py
+
+
+ATTENTION:
+    # This script is triggered every minutes by /var/www/nebuleair_pro_4g/master.py (as a service)
+
+CSV PAYLOAD (AirCarto Servers)
+    Endpoint:
+    data.nebuleair.fr
+    /pro_4G/data.php?sensor_id={device_id}&timestamp={rtc_module_time}
+
+    ATTENTION : do not change order !
+    CSV size: 18 
+    {PM1},{PM25},{PM10},{temp},{hum},{press},{avg_noise},{max_noise},{min_noise},{envea_no2},{envea_h2s},{envea_o3},{4g_signal_quality}
+    0   -> PM1 (μg/m3)
+    1   -> PM25 (μg/m3)
+    2   -> PM10 (μg/m3)
+    3   -> temp
+    4   -> hum
+    5   -> press
+    6   -> avg_noise
+    7   -> max_noise
+    8   -> min_noise
+    9   -> envea_no2
+    10  -> envea_h2s
+    11  -> envea_o3
+    12  -> 4G signal quality,
+    13  -> PM 0.2μm to 0.5μm quantity (Nb/L)
+    14  -> PM 0.5μm to 1.0μm quantity (Nb/L)
+    15  -> PM 1.0μm to 2.5μm quantity (Nb/L)
+    16  -> PM 2.5μm to 5.0μm quantity (Nb/L)
+    17  -> PM 5.0μm to 10μm quantity (Nb/L)
+
+JSON PAYLOAD (Micro-Spot Servers)
+    Same as NebuleAir wifi
+    Endpoint: 
+        api-prod.uspot.probesys.net 
+        nebuleair?token=2AFF6dQk68daFZ
+        port 443
+
+    {"nebuleairid": "82D25549434",
+     "software_version": "ModuleAirV2-V1-042022",
+      "sensordatavalues":
+      [
+      {"value_type":"NPM_P0","value":"1.54"},
+      {"value_type":"NPM_P1","value":"1.54"},
+      {"value_type":"NPM_P2","value":"1.54"},
+      {"value_type":"NPM_N1","value":"0.02"},
+      {"value_type":"NPM_N10","value":"0.02"},
+      {"value_type":"NPM_N25","value":"0.02"},
+      {"value_type":"MHZ16_CO2","value":"793.00"},
+      {"value_type":"SGP40_VOC","value":"29915.00"},
+      {"value_type":"samples","value":"134400"},
+      {"value_type":"min_micro","value":"137"},
+      {"value_type":"max_micro","value":"155030"},
+      {"value_type":"interval","value":"145000"},
+      {"value_type":"signal","value":"-80"},
+      {"value_type":"latitude","value":"43.2964"},
+      {"value_type":"longitude","value":"5.36978"},
+      {"value_type":"state_npm","value":"State: 00000000"},
+      {"value_type":"BME280_temperature","value":"28.47"},
+    {"value_type":"BME280_humidity","value":"28.47"},
+      {"value_type":"BME280_pressure","value":"28.47"},
+      {"value_type":"CAIRSENS_NO2","value":"54"},
+      {"value_type":"CAIRSENS_H2S","value":"54"},
+      {"value_type":"CAIRSENS_O3","value":"54"}
+      ]
+      }
+"""
+
+import board
+import json
+import serial
+import time
+import busio
+import re
+import os
+import traceback
+import sys
+import sqlite3
+import RPi.GPIO as GPIO
+from threading import Thread
+
+# Record the start time of the script
+start_time_script = time.time()
+
+# Check system uptime
+with open('/proc/uptime', 'r') as f:
+    uptime_seconds = float(f.readline().split()[0])
+
+# Skip execution if uptime is less than 2 minutes (120 seconds)
+if uptime_seconds < 120:
+    print(f"System just booted ({uptime_seconds:.2f} seconds uptime), skipping execution.")
+    sys.exit()
+
+#Payload CSV to be sent to data.nebuleair.fr
+payload_csv = [None] * 20
+#Payload JSON to be sent to uSpot
+payload_json = {
+    "nebuleairid": "XXX",
+    "software_version": "ModuleAirV2-V1-042022",
+    "sensordatavalues": []  # Empty list to start with
+}
+
+# SARA R4 UHTTPC profile IDs
+aircarto_profile_id = 0
+uSpot_profile_id = 1
+
+# database connection
+conn = sqlite3.connect("/var/www/nebuleair_pro_4g/sqlite/sensors.db")
+cursor = conn.cursor()
+
+def blink_led(pin, blink_count, delay=1):
+    """
+    Blink an LED on a specified GPIO pin.
+
+    Args:
+        pin (int): GPIO pin number (BCM mode) to which the LED is connected.
+        blink_count (int): Number of times the LED should blink.
+        delay (float): Time in seconds for the LED to stay ON or OFF (default is 1 second).
+    """
+    # GPIO setup
+    GPIO.setwarnings(False)
+    GPIO.setmode(GPIO.BCM)  # Use BCM numbering
+    GPIO.setup(pin, GPIO.OUT)  # Set the specified pin as an output
+
+    try:
+        for _ in range(blink_count):
+            GPIO.output(pin, GPIO.HIGH)  # Turn the LED on
+            #print(f"LED on GPIO {pin} is ON")
+            time.sleep(delay)  # Wait for the specified delay
+            GPIO.output(pin, GPIO.LOW)  # Turn the LED off
+            #print(f"LED on GPIO {pin} is OFF")
+            time.sleep(delay)  # Wait for the specified delay
+    finally:
+        GPIO.cleanup(pin)  # Clean up the specific pin to reset its state
+        print(f"GPIO {pin} cleaned up")
+
+#get data from config
+def load_config(config_file):
+    try:
+        with open(config_file, 'r') as file:
+            config_data = json.load(file)
+        return config_data
+    except Exception as e:
+        print(f"Error loading config file: {e}")
+        return {}
+
+#Fonction pour mettre à jour le JSON de configuration
+def update_json_key(file_path, key, value):
+    """
+    Updates a specific key in a JSON file with a new value.
+
+    :param file_path: Path to the JSON file.
+    :param key: The key to update in the JSON file.
+    :param value: The new value to assign to the key.
+    """
+    try:
+        # Load the existing data
+        with open(file_path, "r") as file:
+            data = json.load(file)
+        
+        # Check if the key exists in the JSON file
+        if key in data:
+            data[key] = value  # Update the key with the new value
+        else:
+            print(f"Key '{key}' not found in the JSON file.")
+            return
+        
+        # Write the updated data back to the file
+        with open(file_path, "w") as file:
+            json.dump(data, file, indent=2)  # Use indent for pretty printing
+        
+        print(f"updating '{key}' to '{value}'.")
+    except Exception as e:
+        print(f"Error updating the JSON file: {e}")
+
+# Define the config file path
+config_file = '/var/www/nebuleair_pro_4g/config.json'
+
+# Load the configuration data
+config = load_config(config_file)
+baudrate = config.get('SaraR4_baudrate', 115200)       #baudrate du sara R4
+device_id = config.get('deviceID', '').upper()         #device ID en maj
+need_to_log = config.get('loop_log', False)            #inscription des logs
+send_aircarto = config.get('send_aircarto', True)      #envoi sur AirCarto (data.nebuleair.fr)
+send_uSpot = config.get('send_uSpot', False)           #envoi sur MicroSpot ()
+selected_networkID = config.get('SARA_R4_neworkID', '') 
+
+#update device id in the payload json
+payload_json["nebuleairid"] = device_id
+
+ser_sara = serial.Serial(
+    port='/dev/ttyAMA2',
+    baudrate=baudrate, #115200 ou 9600
+    parity=serial.PARITY_NONE, #PARITY_NONE, PARITY_EVEN or PARITY_ODD
+    stopbits=serial.STOPBITS_ONE,
+    bytesize=serial.EIGHTBITS,
+    timeout = 2
+)
+
+def read_complete_response(serial_connection, timeout=2, end_of_response_timeout=2, wait_for_line=None, debug=True):
+    '''
+    Fonction très importante !!!
+    '''
+    response = bytearray()
+    serial_connection.timeout = timeout
+    end_time = time.time() + end_of_response_timeout
+    start_time = time.time()
+ 
+    while True:
+        elapsed_time = time.time() - start_time  # Time since function start
+        if serial_connection.in_waiting > 0:
+            data = serial_connection.read(serial_connection.in_waiting)
+            response.extend(data)
+            end_time = time.time() + end_of_response_timeout  # Reset timeout on new data
+
+            # Decode and check for the specific line
+            if wait_for_line:
+                decoded_response = response.decode('utf-8', errors='replace')
+                if wait_for_line in decoded_response:
+                    if debug: print(f"[DEBUG] 🔎Found target line: {wait_for_line}")
+                    break
+        elif time.time() > end_time:
+            if debug: print(f"[DEBUG] Timeout reached. No more data received.")
+            break
+        time.sleep(0.1)  # Short sleep to prevent busy waiting
+    # Final response and debug output
+    total_elapsed_time = time.time() - start_time
+    if debug: print(f"[DEBUG] ⏱️ elapsed time: {total_elapsed_time:.2f}s. ⏱️")
+    # Check if the elapsed time exceeded 10 seconds
+    if total_elapsed_time > 10 and debug:
+        print(f"[ALERT] 🚨 The operation took too long🚨")
+        print(f'<span style="color: red;font-weight: bold;">[ALERT] ⚠️{total_elapsed_time:.2f}s⚠️</span>')
+
+    return response.decode('utf-8', errors='replace')
+
+try:
+    print('<h3>START LOOP</h3>')
+    print("Getting NPM values")
+    # Retrieve the last sensor readings
+    cursor.execute("SELECT * FROM data ORDER BY timestamp DESC LIMIT 1")
+    last_row = cursor.fetchone()
+    # Display the result
+    if last_row:
+        pm1_value = last_row[1]  # Adjust the index based on the column order in your table
+        print("Last available row:", last_row)
+    else:
+        print("No data available in the database.")
+
+
+    print("Verify SARA R4 connection")
+    
+    # Getting the LTE Signal
+    print("-> Getting LTE signal <-")
+    ser_sara.write(b'AT+CSQ\r')
+    response2 = read_complete_response(ser_sara, wait_for_line="OK")
+    print('<p class="text-danger-emphasis">')
+    print(response2)
+    print("</p>")
+    match = re.search(r'\+CSQ:\s*(\d+),', response2)
+    if match:
+        signal_quality = int(match.group(1))
+        payload_csv[12]=signal_quality
+        time.sleep(0.1)
+
+        # On vérifie si le signal n'est pas à 99 pour déconnexion
+        # si c'est le cas on essaie de se reconnecter
+        if signal_quality == 99:
+            print('<span style="color: red;font-weight: bold;">⚠️ATTENTION: Signal Quality indicates no signal (99)⚠️</span>')
+            print("TRY TO RECONNECT:")
+            command = f'AT+COPS=1,2,"{selected_networkID}"\r'
+            ser_sara.write(command.encode('utf-8'))
+            responseReconnect = read_complete_response(ser_sara, timeout=20, end_of_response_timeout=20)
+            print('<p class="text-danger-emphasis">')
+            print(responseReconnect)
+            print("</p>")
+            
+            print('🛑STOP LOOP🛑')
+            print("<hr>")
+
+            #on arrete le script pas besoin de continuer
+            sys.exit()
+        else:
+            print("Signal Quality:", signal_quality)
+
+    
+    '''
+    SEND TO AIRCARTO
+    '''
+
+
+    
+    
+    
+    # Calculate and print the elapsed time
+    elapsed_time = time.time() - start_time_script
+    print(f"Elapsed time: {elapsed_time:.2f} seconds")
+    print("<hr>")
+
+except Exception as e:
+    print("An error occurred:", e)
+    traceback.print_exc()  # This prints the full traceback

master.py

@@ -44,6 +44,10 @@ Check the service status:
     sudo systemctl status master_nebuleair.service
 
 
+Specific scripts can be disabled with config.json
+    Exemple: stop gathering data from NPM
+    Exemple: stop sending data with SARA R4
+
 '''
 import time
 import threading
@@ -71,8 +75,8 @@ def run_script(script_name, interval):
 
 # Define scripts and their execution intervals (seconds)
 SCRIPTS = [
-    ("NPM/get_data_v2.py", 60),   # Runs every 60 seconds
+    ("NPM/get_data_v2.py", 60),     # Get NPM data every 60s
-    ("tests/script2.py", 10),   # Runs every 10 seconds
+    ("loop/SARA_send_data_v2.py", 60),   # Runs every 60 seconds
     ("tests/script3.py", 10),   # Runs every 10 seconds
 ]
 

sqlite/create_db.py

@@ -15,19 +15,44 @@ import sqlite3
 conn = sqlite3.connect("/var/www/nebuleair_pro_4g/sqlite/sensors.db")
 cursor = conn.cursor()
 
-# Create a table for storing sensor data
+# Create a table 1
 cursor.execute("""
-CREATE TABLE IF NOT EXISTS data (
+CREATE TABLE IF NOT EXISTS data_NPM (
     timestamp TEXT,
     PM1 REAL,
     PM25 REAL,
     PM10 REAL,
-    temp REAL,
+    temp_npm REAL,
-    hum REAL,
+    hum_npm REAL
-    press REAL,
+)
+""")
+
+# Create a table 2
+cursor.execute("""
+CREATE TABLE IF NOT EXISTS data_BME280 (
+    timestamp TEXT,
+    temperature REAL,
+    humidity REAL,
+    pressure REAL
+)
+""")
+
+# Create a table 3
+cursor.execute("""
+CREATE TABLE IF NOT EXISTS data_envea (
+    timestamp TEXT,
     no2 REAL,
     h2s REAL,
-    o3 REAL,
+    nh3 REAL,
+    co REAL,
+    o3 REAL
+)
+""")
+
+# Create a table 4
+cursor.execute("""
+CREATE TABLE IF NOT EXISTS data_NPM_5channels (
+    timestamp TEXT,
     PM_ch1 INTEGER,
     PM_ch2 INTEGER,
     PM_ch3 INTEGER,
@@ -36,6 +61,8 @@ CREATE TABLE IF NOT EXISTS data (
 )
 """)
 
+
+
 # Commit and close the connection
 conn.commit()
 conn.close()

sqlite/read.py

@@ -20,6 +20,8 @@ cursor = conn.cursor()
 cursor.execute("SELECT * FROM data ORDER BY timestamp DESC LIMIT 10")
 
 rows = cursor.fetchall()
+rows.reverse()  # Reverse the order in Python (to get ascending order)
+
 
 # Display the results
 for row in rows: