diff --git a/NPM/get_data_v2.py b/NPM/get_data_v2.py index fa9797c..ab7321e 100644 --- a/NPM/get_data_v2.py +++ b/NPM/get_data_v2.py @@ -1,11 +1,12 @@ ''' - _ _ ____ __ __ - | \ | | _ \| \/ | - | \| | |_) | |\/| | - | |\ | __/| | | | - |_| \_|_| |_| |_| + ____ _____ _ _ ____ ___ ____ ____ + / ___|| ____| \ | / ___| / _ \| _ \/ ___| + \___ \| _| | \| \___ \| | | | |_) \___ \ + ___) | |___| |\ |___) | |_| | _ < ___) | + |____/|_____|_| \_|____/ \___/|_| \_\____/ + -Script to get NPM values +Script to get SENSORS values And store them inside sqlite database Uses RTC module for timing /usr/bin/python3 /var/www/nebuleair_pro_4g/NPM/get_data_v2.py @@ -73,6 +74,7 @@ ser.write(b'\x81\x12\x6D') #data60s while True: try: + #print("Start get_data_v2.py script") byte_data = ser.readline() #print(byte_data) stateByte = int.from_bytes(byte_data[2:3], byteorder='big') @@ -86,8 +88,6 @@ while True: #print(f"PM10: {PM10}") #create JSON data = { - 'capteurID': 'nebuleairpro1', - 'sondeID':'USB2', 'PM1': PM1, 'PM25': PM25, 'PM10': PM10, @@ -101,7 +101,7 @@ while True: 'laserError' : Statebits[7] } json_data = json.dumps(data) - print(json_data) + #print(json_data) #GET RTC TIME # Read RTC time @@ -111,7 +111,7 @@ while True: if rtc_time: rtc_time_str = rtc_time.strftime('%Y-%m-%d %H:%M:%S') - print(rtc_time_str) + #print(rtc_time_str) else: print("Error! RTC module not connected") rtc_time_str = "1970-01-01 00:00:00" # Default fallback time @@ -125,7 +125,7 @@ while True: # Commit and close the connection conn.commit() - print("Sensor data saved successfully!") + #print("Sensor data saved successfully!") break # Exit loop after successful execution except KeyboardInterrupt: diff --git a/html/index.html b/html/index.html index fe1aa80..581b30d 100755 --- a/html/index.html +++ b/html/index.html @@ -308,9 +308,36 @@ window.onload = function() { data: { labels: labels, datasets: [ - { label: "PM1", data: PM1, borderColor: "red", fill: false }, - { label: "PM2.5", data: PM25, borderColor: "blue", fill: false }, - { label: "PM10", data: PM10, borderColor: "green", fill: false } + { + label: "PM1", + data: PM1, + borderColor: "rgba(0, 51, 153, 1)", + backgroundColor: "rgba(0, 51, 153, 0.2)", // Very light blue background + fill: true, + tension: 0.4, // Smooth curves + pointRadius: 2, // Larger points + pointHoverRadius: 6 // Bigger hover points + }, + { + label: "PM2.5", + data: PM25, + borderColor: "rgba(30, 144, 255, 1)", + backgroundColor: "rgba(30, 144, 255, 0.2)", // Very light medium blue background + fill: true, + tension: 0.4, + pointRadius: 2, + pointHoverRadius: 6 + }, + { + label: "PM10", + data: PM10, + borderColor: "rgba(135, 206, 250, 1)", + backgroundColor: "rgba(135, 206, 250, 0.2)", // Very light blue background + fill: true, + tension: 0.4, + pointRadius: 2, + pointHoverRadius: 6 + } ] }, options: { @@ -325,11 +352,17 @@ window.onload = function() { x: { title: { display: true, - text: 'Time' + text: 'Time (UTC)', + font: { + size: 16, + family: 'Arial, sans-serif' + }, + color: '#4A4A4A' }, ticks: { autoSkip: true, maxTicksLimit: 5, + color: '#4A4A4A', callback: function(value, index) { // Access the correct label from the `labels` array const label = labels[index]; // Use the original `labels` array @@ -338,6 +371,9 @@ window.onload = function() { } return value; // Fallback for invalid labels } + }, + grid: { + display: false // Remove gridlines for a cleaner look } @@ -345,7 +381,12 @@ window.onload = function() { y: { title: { display: true, - text: 'Values (µg/m³)' + text: 'Values (µg/m³)', + font: { + size: 16, + family: 'Arial, sans-serif' + }, + color: '#4A4A4A' } } } diff --git a/html/launcher.php b/html/launcher.php index 2352be6..b5b7a04 100755 --- a/html/launcher.php +++ b/html/launcher.php @@ -16,8 +16,10 @@ if ($type == "get_npm_sqlite_data") { // Fetch the last 30 records $stmt = $db->query("SELECT timestamp, PM1, PM25, PM10 FROM data ORDER BY timestamp DESC LIMIT 30"); $data = $stmt->fetchAll(PDO::FETCH_ASSOC); + $reversedData = array_reverse($data); // Reverse the order + - echo json_encode($data); + echo json_encode($reversedData); } catch (PDOException $e) { echo json_encode(["error" => $e->getMessage()]); } @@ -94,50 +96,50 @@ if ($type == "clear_loopLogs") { if ($type == "database_size") { -// Path to the SQLite database file -$databasePath = '/var/www/nebuleair_pro_4g/sqlite/sensors.db'; + // Path to the SQLite database file + $databasePath = '/var/www/nebuleair_pro_4g/sqlite/sensors.db'; -// Check if the file exists -if (file_exists($databasePath)) { - try { - // Connect to the SQLite database - $db = new PDO("sqlite:$databasePath"); + // Check if the file exists + if (file_exists($databasePath)) { + try { + // Connect to the SQLite database + $db = new PDO("sqlite:$databasePath"); - // Get the file size in bytes - $fileSizeBytes = filesize($databasePath); + // Get the file size in bytes + $fileSizeBytes = filesize($databasePath); - // Convert the file size to human-readable formats - $fileSizeKilobytes = $fileSizeBytes / 1024; // KB - $fileSizeMegabytes = $fileSizeKilobytes / 1024; // MB + // Convert the file size to human-readable formats + $fileSizeKilobytes = $fileSizeBytes / 1024; // KB + $fileSizeMegabytes = $fileSizeKilobytes / 1024; // MB - // Query the number of records in the `data` table - $query = "SELECT COUNT(*) AS total_records FROM data"; - $result = $db->query($query); - $recordCount = $result ? $result->fetch(PDO::FETCH_ASSOC)['total_records'] : 0; + // Query the number of records in the `data` table + $query = "SELECT COUNT(*) AS total_records FROM data"; + $result = $db->query($query); + $recordCount = $result ? $result->fetch(PDO::FETCH_ASSOC)['total_records'] : 0; - // Prepare the JSON response - $data = [ - 'path' => $databasePath, - 'size_bytes' => $fileSizeBytes, - 'size_kilobytes' => round($fileSizeKilobytes, 2), - 'size_megabytes' => round($fileSizeMegabytes, 2), - 'data_table_records' => $recordCount - ]; + // Prepare the JSON response + $data = [ + 'path' => $databasePath, + 'size_bytes' => $fileSizeBytes, + 'size_kilobytes' => round($fileSizeKilobytes, 2), + 'size_megabytes' => round($fileSizeMegabytes, 2), + 'data_table_records' => $recordCount + ]; - // Output the JSON response - echo json_encode($data, JSON_PRETTY_PRINT); - } catch (PDOException $e) { - // Handle database connection errors + // Output the JSON response + echo json_encode($data, JSON_PRETTY_PRINT); + } catch (PDOException $e) { + // Handle database connection errors + echo json_encode([ + 'error' => 'Database query failed: ' . $e->getMessage() + ]); + } + } else { + // Handle error if the file doesn't exist echo json_encode([ - 'error' => 'Database query failed: ' . $e->getMessage() + 'error' => 'Database file not found', + 'path' => $databasePath ]); - } -} else { - // Handle error if the file doesn't exist - echo json_encode([ - 'error' => 'Database file not found', - 'path' => $databasePath - ]); } diff --git a/html/logs.html b/html/logs.html index 454dd14..d73fdcf 100755 --- a/html/logs.html +++ b/html/logs.html @@ -56,7 +56,7 @@
- Loop logs + Master logs
@@ -110,7 +110,7 @@ const loop_card_content = document.getElementById('card_loop_content'); const boot_card_content = document.getElementById('card_boot_content'); - fetch('../logs/loop.log') + fetch('../logs/master.log') .then((response) => { if (!response.ok) { throw new Error('Failed to fetch the log file.'); diff --git a/loop/1_NPM/send_data.py b/loop/1_NPM/send_data.py index 522089e..4edecdd 100755 --- a/loop/1_NPM/send_data.py +++ b/loop/1_NPM/send_data.py @@ -78,9 +78,8 @@ import re import os import traceback import sys -from threading import Thread - import RPi.GPIO as GPIO +from threading import Thread from adafruit_bme280 import basic as adafruit_bme280 # Record the start time of the script @@ -95,7 +94,6 @@ if uptime_seconds < 120: print(f"System just booted ({uptime_seconds:.2f} seconds uptime), skipping execution.") sys.exit() -url_nebuleair="data.nebuleair.fr" payload_csv = [None] * 20 payload_json = { "nebuleairid": "82D25549434", @@ -185,7 +183,6 @@ i2C_sound_config = config.get('i2C_sound', False) #présence du capteur son send_aircarto = config.get('send_aircarto', True) #envoi sur AirCarto (data.nebuleair.fr) send_uSpot = config.get('send_uSpot', False) #envoi sur MicroSpot () npm_5channel = config.get('NextPM_5channels', False) #5 canaux du NPM -local_storage = config.get('local_storage', False) #enregistrement en local des data envea_sondes = config.get('envea_sondes', []) connected_envea_sondes = [sonde for sonde in envea_sondes if sonde.get('connected', False)] diff --git a/loop/SARA_send_data_v2.py b/loop/SARA_send_data_v2.py new file mode 100644 index 0000000..8f64685 --- /dev/null +++ b/loop/SARA_send_data_v2.py @@ -0,0 +1,320 @@ +""" + ____ _ ____ _ ____ _ ____ _ + / ___| / \ | _ \ / \ / ___| ___ _ __ __| | | _ \ __ _| |_ __ _ + \___ \ / _ \ | |_) | / _ \ \___ \ / _ \ '_ \ / _` | | | | |/ _` | __/ _` | + ___) / ___ \| _ < / ___ \ ___) | __/ | | | (_| | | |_| | (_| | || (_| | + |____/_/ \_\_| \_\/_/ \_\ |____/ \___|_| |_|\__,_| |____/ \__,_|\__\__,_| + +Main loop to gather data from sensor inside SQLite database: + +* NPM +* Envea +* I2C BME280 +* Noise sensor + +and send it to AirCarto servers via SARA R4 HTTP post requests +also send the timestamp (already stored inside the DB) ! + +/usr/bin/python3 /var/www/nebuleair_pro_4g/loop/SARA_send_data_v2.py + + +ATTENTION: + # This script is triggered every minutes by /var/www/nebuleair_pro_4g/master.py (as a service) + +CSV PAYLOAD (AirCarto Servers) + Endpoint: + data.nebuleair.fr + /pro_4G/data.php?sensor_id={device_id}×tamp={rtc_module_time} + + ATTENTION : do not change order ! + CSV size: 18 + {PM1},{PM25},{PM10},{temp},{hum},{press},{avg_noise},{max_noise},{min_noise},{envea_no2},{envea_h2s},{envea_o3},{4g_signal_quality} + 0 -> PM1 (μg/m3) + 1 -> PM25 (μg/m3) + 2 -> PM10 (μg/m3) + 3 -> temp + 4 -> hum + 5 -> press + 6 -> avg_noise + 7 -> max_noise + 8 -> min_noise + 9 -> envea_no2 + 10 -> envea_h2s + 11 -> envea_o3 + 12 -> 4G signal quality, + 13 -> PM 0.2μm to 0.5μm quantity (Nb/L) + 14 -> PM 0.5μm to 1.0μm quantity (Nb/L) + 15 -> PM 1.0μm to 2.5μm quantity (Nb/L) + 16 -> PM 2.5μm to 5.0μm quantity (Nb/L) + 17 -> PM 5.0μm to 10μm quantity (Nb/L) + +JSON PAYLOAD (Micro-Spot Servers) + Same as NebuleAir wifi + Endpoint: + api-prod.uspot.probesys.net + nebuleair?token=2AFF6dQk68daFZ + port 443 + + {"nebuleairid": "82D25549434", + "software_version": "ModuleAirV2-V1-042022", + "sensordatavalues": + [ + {"value_type":"NPM_P0","value":"1.54"}, + {"value_type":"NPM_P1","value":"1.54"}, + {"value_type":"NPM_P2","value":"1.54"}, + {"value_type":"NPM_N1","value":"0.02"}, + {"value_type":"NPM_N10","value":"0.02"}, + {"value_type":"NPM_N25","value":"0.02"}, + {"value_type":"MHZ16_CO2","value":"793.00"}, + {"value_type":"SGP40_VOC","value":"29915.00"}, + {"value_type":"samples","value":"134400"}, + {"value_type":"min_micro","value":"137"}, + {"value_type":"max_micro","value":"155030"}, + {"value_type":"interval","value":"145000"}, + {"value_type":"signal","value":"-80"}, + {"value_type":"latitude","value":"43.2964"}, + {"value_type":"longitude","value":"5.36978"}, + {"value_type":"state_npm","value":"State: 00000000"}, + {"value_type":"BME280_temperature","value":"28.47"}, + {"value_type":"BME280_humidity","value":"28.47"}, + {"value_type":"BME280_pressure","value":"28.47"}, + {"value_type":"CAIRSENS_NO2","value":"54"}, + {"value_type":"CAIRSENS_H2S","value":"54"}, + {"value_type":"CAIRSENS_O3","value":"54"} + ] + } +""" + +import board +import json +import serial +import time +import busio +import re +import os +import traceback +import sys +import sqlite3 +import RPi.GPIO as GPIO +from threading import Thread + +# Record the start time of the script +start_time_script = time.time() + +# Check system uptime +with open('/proc/uptime', 'r') as f: + uptime_seconds = float(f.readline().split()[0]) + +# Skip execution if uptime is less than 2 minutes (120 seconds) +if uptime_seconds < 120: + print(f"System just booted ({uptime_seconds:.2f} seconds uptime), skipping execution.") + sys.exit() + +#Payload CSV to be sent to data.nebuleair.fr +payload_csv = [None] * 20 +#Payload JSON to be sent to uSpot +payload_json = { + "nebuleairid": "XXX", + "software_version": "ModuleAirV2-V1-042022", + "sensordatavalues": [] # Empty list to start with +} + +# SARA R4 UHTTPC profile IDs +aircarto_profile_id = 0 +uSpot_profile_id = 1 + +# database connection +conn = sqlite3.connect("/var/www/nebuleair_pro_4g/sqlite/sensors.db") +cursor = conn.cursor() + +def blink_led(pin, blink_count, delay=1): + """ + Blink an LED on a specified GPIO pin. + + Args: + pin (int): GPIO pin number (BCM mode) to which the LED is connected. + blink_count (int): Number of times the LED should blink. + delay (float): Time in seconds for the LED to stay ON or OFF (default is 1 second). + """ + # GPIO setup + GPIO.setwarnings(False) + GPIO.setmode(GPIO.BCM) # Use BCM numbering + GPIO.setup(pin, GPIO.OUT) # Set the specified pin as an output + + try: + for _ in range(blink_count): + GPIO.output(pin, GPIO.HIGH) # Turn the LED on + #print(f"LED on GPIO {pin} is ON") + time.sleep(delay) # Wait for the specified delay + GPIO.output(pin, GPIO.LOW) # Turn the LED off + #print(f"LED on GPIO {pin} is OFF") + time.sleep(delay) # Wait for the specified delay + finally: + GPIO.cleanup(pin) # Clean up the specific pin to reset its state + print(f"GPIO {pin} cleaned up") + +#get data from config +def load_config(config_file): + try: + with open(config_file, 'r') as file: + config_data = json.load(file) + return config_data + except Exception as e: + print(f"Error loading config file: {e}") + return {} + +#Fonction pour mettre à jour le JSON de configuration +def update_json_key(file_path, key, value): + """ + Updates a specific key in a JSON file with a new value. + + :param file_path: Path to the JSON file. + :param key: The key to update in the JSON file. + :param value: The new value to assign to the key. + """ + try: + # Load the existing data + with open(file_path, "r") as file: + data = json.load(file) + + # Check if the key exists in the JSON file + if key in data: + data[key] = value # Update the key with the new value + else: + print(f"Key '{key}' not found in the JSON file.") + return + + # Write the updated data back to the file + with open(file_path, "w") as file: + json.dump(data, file, indent=2) # Use indent for pretty printing + + print(f"updating '{key}' to '{value}'.") + except Exception as e: + print(f"Error updating the JSON file: {e}") + +# Define the config file path +config_file = '/var/www/nebuleair_pro_4g/config.json' + +# Load the configuration data +config = load_config(config_file) +baudrate = config.get('SaraR4_baudrate', 115200) #baudrate du sara R4 +device_id = config.get('deviceID', '').upper() #device ID en maj +need_to_log = config.get('loop_log', False) #inscription des logs +send_aircarto = config.get('send_aircarto', True) #envoi sur AirCarto (data.nebuleair.fr) +send_uSpot = config.get('send_uSpot', False) #envoi sur MicroSpot () +selected_networkID = config.get('SARA_R4_neworkID', '') + +#update device id in the payload json +payload_json["nebuleairid"] = device_id + +ser_sara = serial.Serial( + port='/dev/ttyAMA2', + baudrate=baudrate, #115200 ou 9600 + parity=serial.PARITY_NONE, #PARITY_NONE, PARITY_EVEN or PARITY_ODD + stopbits=serial.STOPBITS_ONE, + bytesize=serial.EIGHTBITS, + timeout = 2 +) + +def read_complete_response(serial_connection, timeout=2, end_of_response_timeout=2, wait_for_line=None, debug=True): + ''' + Fonction très importante !!! + ''' + response = bytearray() + serial_connection.timeout = timeout + end_time = time.time() + end_of_response_timeout + start_time = time.time() + + while True: + elapsed_time = time.time() - start_time # Time since function start + if serial_connection.in_waiting > 0: + data = serial_connection.read(serial_connection.in_waiting) + response.extend(data) + end_time = time.time() + end_of_response_timeout # Reset timeout on new data + + # Decode and check for the specific line + if wait_for_line: + decoded_response = response.decode('utf-8', errors='replace') + if wait_for_line in decoded_response: + if debug: print(f"[DEBUG] 🔎Found target line: {wait_for_line}") + break + elif time.time() > end_time: + if debug: print(f"[DEBUG] Timeout reached. No more data received.") + break + time.sleep(0.1) # Short sleep to prevent busy waiting + # Final response and debug output + total_elapsed_time = time.time() - start_time + if debug: print(f"[DEBUG] ⏱️ elapsed time: {total_elapsed_time:.2f}s. ⏱️") + # Check if the elapsed time exceeded 10 seconds + if total_elapsed_time > 10 and debug: + print(f"[ALERT] 🚨 The operation took too long🚨") + print(f'[ALERT] ⚠️{total_elapsed_time:.2f}s⚠️') + + return response.decode('utf-8', errors='replace') + +try: + print('

START LOOP

') + print("Getting NPM values") + # Retrieve the last sensor readings + cursor.execute("SELECT * FROM data ORDER BY timestamp DESC LIMIT 1") + last_row = cursor.fetchone() + # Display the result + if last_row: + pm1_value = last_row[1] # Adjust the index based on the column order in your table + print("Last available row:", last_row) + else: + print("No data available in the database.") + + + print("Verify SARA R4 connection") + + # Getting the LTE Signal + print("-> Getting LTE signal <-") + ser_sara.write(b'AT+CSQ\r') + response2 = read_complete_response(ser_sara, wait_for_line="OK") + print('

') + print(response2) + print("

") + match = re.search(r'\+CSQ:\s*(\d+),', response2) + if match: + signal_quality = int(match.group(1)) + payload_csv[12]=signal_quality + time.sleep(0.1) + + # On vérifie si le signal n'est pas à 99 pour déconnexion + # si c'est le cas on essaie de se reconnecter + if signal_quality == 99: + print('⚠️ATTENTION: Signal Quality indicates no signal (99)⚠️') + print("TRY TO RECONNECT:") + command = f'AT+COPS=1,2,"{selected_networkID}"\r' + ser_sara.write(command.encode('utf-8')) + responseReconnect = read_complete_response(ser_sara, timeout=20, end_of_response_timeout=20) + print('

') + print(responseReconnect) + print("

") + + print('🛑STOP LOOP🛑') + print("
") + + #on arrete le script pas besoin de continuer + sys.exit() + else: + print("Signal Quality:", signal_quality) + + + ''' + SEND TO AIRCARTO + ''' + + + + + + # Calculate and print the elapsed time + elapsed_time = time.time() - start_time_script + print(f"Elapsed time: {elapsed_time:.2f} seconds") + print("
") + +except Exception as e: + print("An error occurred:", e) + traceback.print_exc() # This prints the full traceback \ No newline at end of file diff --git a/master.py b/master.py index b768661..b965208 100644 --- a/master.py +++ b/master.py @@ -44,6 +44,10 @@ Check the service status: sudo systemctl status master_nebuleair.service +Specific scripts can be disabled with config.json + Exemple: stop gathering data from NPM + Exemple: stop sending data with SARA R4 + ''' import time import threading @@ -71,8 +75,8 @@ def run_script(script_name, interval): # Define scripts and their execution intervals (seconds) SCRIPTS = [ - ("NPM/get_data_v2.py", 60), # Runs every 60 seconds - ("tests/script2.py", 10), # Runs every 10 seconds + ("NPM/get_data_v2.py", 60), # Get NPM data every 60s + ("loop/SARA_send_data_v2.py", 60), # Runs every 60 seconds ("tests/script3.py", 10), # Runs every 10 seconds ] diff --git a/sqlite/create_db.py b/sqlite/create_db.py index b0bbe25..01a5c09 100755 --- a/sqlite/create_db.py +++ b/sqlite/create_db.py @@ -15,19 +15,44 @@ import sqlite3 conn = sqlite3.connect("/var/www/nebuleair_pro_4g/sqlite/sensors.db") cursor = conn.cursor() -# Create a table for storing sensor data +# Create a table 1 cursor.execute(""" -CREATE TABLE IF NOT EXISTS data ( +CREATE TABLE IF NOT EXISTS data_NPM ( timestamp TEXT, PM1 REAL, PM25 REAL, PM10 REAL, - temp REAL, - hum REAL, - press REAL, + temp_npm REAL, + hum_npm REAL +) +""") + +# Create a table 2 +cursor.execute(""" +CREATE TABLE IF NOT EXISTS data_BME280 ( + timestamp TEXT, + temperature REAL, + humidity REAL, + pressure REAL +) +""") + +# Create a table 3 +cursor.execute(""" +CREATE TABLE IF NOT EXISTS data_envea ( + timestamp TEXT, no2 REAL, h2s REAL, - o3 REAL, + nh3 REAL, + co REAL, + o3 REAL +) +""") + +# Create a table 4 +cursor.execute(""" +CREATE TABLE IF NOT EXISTS data_NPM_5channels ( + timestamp TEXT, PM_ch1 INTEGER, PM_ch2 INTEGER, PM_ch3 INTEGER, @@ -36,6 +61,8 @@ CREATE TABLE IF NOT EXISTS data ( ) """) + + # Commit and close the connection conn.commit() conn.close() diff --git a/sqlite/read.py b/sqlite/read.py index 84fdf50..947fe6a 100755 --- a/sqlite/read.py +++ b/sqlite/read.py @@ -20,6 +20,8 @@ cursor = conn.cursor() cursor.execute("SELECT * FROM data ORDER BY timestamp DESC LIMIT 10") rows = cursor.fetchall() +rows.reverse() # Reverse the order in Python (to get ascending order) + # Display the results for row in rows: